Coverage for test/test_vars_qaqc_processor.py: 100%
150 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-23 02:22 +0000
1from unittest.mock import patch
3from application.util.functions import parse_datetime
4from application.qaqc.vars.vars_qaqc_processor import VarsQaqcProcessor
5from test.data.vars_responses import ex_23060001, ex_23060002
6from test.util.mock_response import MockResponse
8VARS_DIVE_QUERY_URL = 'http://hurlstor.soest.hawaii.edu:8086/query/dive'
9VARS_PHYLOGENY_URL = 'http://hurlstor.soest.hawaii.edu:8083/v1/phylogeny/up'
12def mocked_requests_get(*args, **kwargs):
13 return MockResponse(url=kwargs.get('url'))
16class TestVarsQaqcProcessor:
17 def test_init(self):
18 qaqc_processor = VarsQaqcProcessor(
19 sequence_names=['Deep Discoverer 23060001'],
20 vars_dive_url=VARS_DIVE_QUERY_URL,
21 vars_phylogeny_url=VARS_PHYLOGENY_URL,
22 )
23 assert qaqc_processor.sequence_names == ['Deep Discoverer 23060001']
24 assert qaqc_processor.videos == []
25 assert qaqc_processor.working_records == []
26 assert qaqc_processor.final_records == []
27 assert len(qaqc_processor.phylogeny.keys()) > 0
29 @patch('requests.get', side_effect=mocked_requests_get)
30 def test_fetch_annotations(self, _):
31 qaqc_processor = VarsQaqcProcessor(
32 sequence_names=['Deep Discoverer 23060001'],
33 vars_dive_url=VARS_DIVE_QUERY_URL,
34 vars_phylogeny_url=VARS_PHYLOGENY_URL,
35 )
36 assert qaqc_processor.fetch_annotations('Deep Discoverer 23060001') == ex_23060001['annotations']
37 assert qaqc_processor.videos == [
38 {
39 'start_timestamp': parse_datetime('2023-08-24T18:30:00Z'),
40 'uri': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_01/EX2306_01_20230824T183000Z.m4v',
41 'sequence_name': 'Deep Discoverer 23060001',
42 'video_reference_uuid': 'dda3dc62-9f78-4dbb-91cd-5015026e0434',
43 },
44 {
45 'start_timestamp': parse_datetime('2023-08-24T20:30:00Z'),
46 'uri': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_01/EX2306_01_20230824T203000Z.m4v',
47 'sequence_name': 'Deep Discoverer 23060001',
48 'video_reference_uuid': 'd955c4ef-94e0-4f0d-83f5-d0144a09a933',
49 },
50 ]
52 @patch('requests.get', side_effect=mocked_requests_get)
53 def test_find_duplicate_associations(self, _):
54 qaqc_processor_okay = VarsQaqcProcessor(
55 sequence_names=['Deep Discoverer 23060001'],
56 vars_dive_url=VARS_DIVE_QUERY_URL,
57 vars_phylogeny_url=VARS_PHYLOGENY_URL,
58 )
59 qaqc_processor_problems = VarsQaqcProcessor(
60 sequence_names=['Deep Discoverer 23060002'],
61 vars_dive_url=VARS_DIVE_QUERY_URL,
62 vars_phylogeny_url=VARS_PHYLOGENY_URL,
63 )
64 qaqc_processor_okay.find_duplicate_associations()
65 qaqc_processor_problems.find_duplicate_associations()
66 assert qaqc_processor_okay.working_records == []
67 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0]]
69 @patch('requests.get', side_effect=mocked_requests_get)
70 def test_find_missing_s1(self, _):
71 qaqc_processor_okay = VarsQaqcProcessor(
72 sequence_names=['Deep Discoverer 23060001'],
73 vars_dive_url=VARS_DIVE_QUERY_URL,
74 vars_phylogeny_url=VARS_PHYLOGENY_URL,
75 )
76 qaqc_processor_problems = VarsQaqcProcessor(
77 sequence_names=['Deep Discoverer 23060002'],
78 vars_dive_url=VARS_DIVE_QUERY_URL,
79 vars_phylogeny_url=VARS_PHYLOGENY_URL,
80 )
81 qaqc_processor_okay.find_missing_s1()
82 qaqc_processor_problems.find_missing_s1()
83 assert qaqc_processor_okay.working_records == []
84 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]]
86 @patch('requests.get', side_effect=mocked_requests_get)
87 def test_find_identical_s1_s2(self, _):
88 qaqc_processor_okay = VarsQaqcProcessor(
89 sequence_names=['Deep Discoverer 23060001'],
90 vars_dive_url=VARS_DIVE_QUERY_URL,
91 vars_phylogeny_url=VARS_PHYLOGENY_URL,
92 )
93 qaqc_processor_problems = VarsQaqcProcessor(
94 sequence_names=['Deep Discoverer 23060002'],
95 vars_dive_url=VARS_DIVE_QUERY_URL,
96 vars_phylogeny_url=VARS_PHYLOGENY_URL,
97 )
98 qaqc_processor_okay.find_identical_s1_s2()
99 qaqc_processor_problems.find_identical_s1_s2()
100 assert qaqc_processor_okay.working_records == []
101 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2]]
103 @patch('requests.get', side_effect=mocked_requests_get)
104 def test_find_duplicate_s2(self, _):
105 qaqc_processor_okay = VarsQaqcProcessor(
106 sequence_names=['Deep Discoverer 23060001'],
107 vars_dive_url=VARS_DIVE_QUERY_URL,
108 vars_phylogeny_url=VARS_PHYLOGENY_URL,
109 )
110 qaqc_processor_problems = VarsQaqcProcessor(
111 sequence_names=['Deep Discoverer 23060002'],
112 vars_dive_url=VARS_DIVE_QUERY_URL,
113 vars_phylogeny_url=VARS_PHYLOGENY_URL,
114 )
115 qaqc_processor_okay.find_duplicate_s2()
116 qaqc_processor_problems.find_duplicate_s2()
117 assert qaqc_processor_okay.working_records == []
118 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]]
120 @patch('requests.get', side_effect=mocked_requests_get)
121 def test_find_missing_upon_substrate(self, _):
122 qaqc_processor_okay = VarsQaqcProcessor(
123 sequence_names=['Deep Discoverer 23060001'],
124 vars_dive_url=VARS_DIVE_QUERY_URL,
125 vars_phylogeny_url=VARS_PHYLOGENY_URL,
126 )
127 qaqc_processor_problems = VarsQaqcProcessor(
128 sequence_names=['Deep Discoverer 23060002'],
129 vars_dive_url=VARS_DIVE_QUERY_URL,
130 vars_phylogeny_url=VARS_PHYLOGENY_URL,
131 )
132 qaqc_processor_okay.find_missing_upon_substrate()
133 qaqc_processor_problems.find_missing_upon_substrate()
134 assert qaqc_processor_okay.working_records == []
135 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0]]
137 @patch('requests.get', side_effect=mocked_requests_get)
138 def test_find_mismatched_substrates(self, _):
139 qaqc_processor_okay = VarsQaqcProcessor(
140 sequence_names=['Deep Discoverer 23060001'],
141 vars_dive_url=VARS_DIVE_QUERY_URL,
142 vars_phylogeny_url=VARS_PHYLOGENY_URL,
143 )
144 qaqc_processor_problems = VarsQaqcProcessor(
145 sequence_names=['Deep Discoverer 23060002'],
146 vars_dive_url=VARS_DIVE_QUERY_URL,
147 vars_phylogeny_url=VARS_PHYLOGENY_URL,
148 )
149 qaqc_processor_okay.find_mismatched_substrates()
150 qaqc_processor_problems.find_mismatched_substrates()
151 assert qaqc_processor_okay.working_records == []
152 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][3], ex_23060002['annotations'][5]]
154 @patch('requests.get', side_effect=mocked_requests_get)
155 def test_find_missing_upon(self, _):
156 qaqc_processor_okay = VarsQaqcProcessor(
157 sequence_names=['Deep Discoverer 23060001'],
158 vars_dive_url=VARS_DIVE_QUERY_URL,
159 vars_phylogeny_url=VARS_PHYLOGENY_URL,
160 )
161 qaqc_processor_problems = VarsQaqcProcessor(
162 sequence_names=['Deep Discoverer 23060002'],
163 vars_dive_url=VARS_DIVE_QUERY_URL,
164 vars_phylogeny_url=VARS_PHYLOGENY_URL,
165 )
166 qaqc_processor_okay.find_missing_upon()
167 qaqc_processor_problems.find_missing_upon()
168 assert qaqc_processor_okay.working_records == []
169 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][3]]
171 @patch('requests.get', side_effect=mocked_requests_get)
172 def test_get_num_records_missing_ancillary_data(self, _):
173 qaqc_processor_okay = VarsQaqcProcessor(
174 sequence_names=['Deep Discoverer 23060001'],
175 vars_dive_url=VARS_DIVE_QUERY_URL,
176 vars_phylogeny_url=VARS_PHYLOGENY_URL,
177 )
178 qaqc_processor_problems = VarsQaqcProcessor(
179 sequence_names=['Deep Discoverer 23060002'],
180 vars_dive_url=VARS_DIVE_QUERY_URL,
181 vars_phylogeny_url=VARS_PHYLOGENY_URL,
182 )
183 assert qaqc_processor_okay.get_num_records_missing_ancillary_data() == 0
184 assert qaqc_processor_problems.get_num_records_missing_ancillary_data() == 2
186 @patch('requests.get', side_effect=mocked_requests_get)
187 def test_find_missing_ancillary_data(self, _):
188 qaqc_processor_okay = VarsQaqcProcessor(
189 sequence_names=['Deep Discoverer 23060001'],
190 vars_dive_url=VARS_DIVE_QUERY_URL,
191 vars_phylogeny_url=VARS_PHYLOGENY_URL,
192 )
193 qaqc_processor_problems = VarsQaqcProcessor(
194 sequence_names=['Deep Discoverer 23060002'],
195 vars_dive_url=VARS_DIVE_QUERY_URL,
196 vars_phylogeny_url=VARS_PHYLOGENY_URL,
197 )
198 qaqc_processor_okay.find_missing_ancillary_data()
199 qaqc_processor_problems.find_missing_ancillary_data()
200 assert qaqc_processor_okay.working_records == []
201 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]]
203 @patch('requests.get', side_effect=mocked_requests_get)
204 def test_find_id_refs_different_concept_name(self, _):
205 qaqc_processor_okay = VarsQaqcProcessor(
206 sequence_names=['Deep Discoverer 23060001'],
207 vars_dive_url=VARS_DIVE_QUERY_URL,
208 vars_phylogeny_url=VARS_PHYLOGENY_URL,
209 )
210 qaqc_processor_problems = VarsQaqcProcessor(
211 sequence_names=['Deep Discoverer 23060002'],
212 vars_dive_url=VARS_DIVE_QUERY_URL,
213 vars_phylogeny_url=VARS_PHYLOGENY_URL,
214 )
215 qaqc_processor_okay.find_id_refs_different_concept_name()
216 qaqc_processor_problems.find_id_refs_different_concept_name()
217 assert qaqc_processor_okay.working_records == []
218 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]]
220 @patch('requests.get', side_effect=mocked_requests_get)
221 def test_find_id_refs_conflicting_associations(self, _):
222 qaqc_processor_okay = VarsQaqcProcessor(
223 sequence_names=['Deep Discoverer 23060001'],
224 vars_dive_url=VARS_DIVE_QUERY_URL,
225 vars_phylogeny_url=VARS_PHYLOGENY_URL,
226 )
227 qaqc_processor_problems = VarsQaqcProcessor(
228 sequence_names=['Deep Discoverer 23060002'],
229 vars_dive_url=VARS_DIVE_QUERY_URL,
230 vars_phylogeny_url=VARS_PHYLOGENY_URL,
231 )
232 qaqc_processor_okay.find_id_refs_conflicting_associations()
233 qaqc_processor_problems.find_id_refs_conflicting_associations()
234 assert qaqc_processor_okay.working_records == []
235 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]]
237 @patch('requests.get', side_effect=mocked_requests_get)
238 def test_find_blank_associations(self, _):
239 qaqc_processor_okay = VarsQaqcProcessor(
240 sequence_names=['Deep Discoverer 23060001'],
241 vars_dive_url=VARS_DIVE_QUERY_URL,
242 vars_phylogeny_url=VARS_PHYLOGENY_URL,
243 )
244 qaqc_processor_problems = VarsQaqcProcessor(
245 sequence_names=['Deep Discoverer 23060002'],
246 vars_dive_url=VARS_DIVE_QUERY_URL,
247 vars_phylogeny_url=VARS_PHYLOGENY_URL,
248 )
249 qaqc_processor_okay.find_blank_associations()
250 qaqc_processor_problems.find_blank_associations()
251 assert qaqc_processor_okay.working_records == []
252 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0], ex_23060002['annotations'][1]]
254 @patch('requests.get', side_effect=mocked_requests_get)
255 def test_find_suspicious_hosts(self, _):
256 qaqc_processor_okay = VarsQaqcProcessor(
257 sequence_names=['Deep Discoverer 23060001'],
258 vars_dive_url=VARS_DIVE_QUERY_URL,
259 vars_phylogeny_url=VARS_PHYLOGENY_URL,
260 )
261 qaqc_processor_problems = VarsQaqcProcessor(
262 sequence_names=['Deep Discoverer 23060002'],
263 vars_dive_url=VARS_DIVE_QUERY_URL,
264 vars_phylogeny_url=VARS_PHYLOGENY_URL,
265 )
266 qaqc_processor_okay.find_suspicious_hosts()
267 qaqc_processor_problems.find_suspicious_hosts()
268 assert qaqc_processor_okay.working_records == []
269 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]]
271 @patch('requests.get', side_effect=mocked_requests_get)
272 def test_find_missing_expected_association(self, _):
273 qaqc_processor_okay = VarsQaqcProcessor(
274 sequence_names=['Deep Discoverer 23060001'],
275 vars_dive_url=VARS_DIVE_QUERY_URL,
276 vars_phylogeny_url=VARS_PHYLOGENY_URL,
277 )
278 qaqc_processor_problems = VarsQaqcProcessor(
279 sequence_names=['Deep Discoverer 23060002'],
280 vars_dive_url=VARS_DIVE_QUERY_URL,
281 vars_phylogeny_url=VARS_PHYLOGENY_URL,
282 )
283 qaqc_processor_okay.find_missing_expected_association()
284 qaqc_processor_problems.find_missing_expected_association()
285 assert qaqc_processor_okay.final_records == []
286 assert qaqc_processor_problems.final_records == [
287 {
288 'observation_uuid': '006fb032-13b5-4517-136c-11aa9597e81e',
289 'concept': 'Hydroidolina',
290 'associations': ex_23060002['annotations'][0]['associations'],
291 'activity': 'cruise',
292 'annotator': 'Nikki Cunanan',
293 'depth': 4255.0,
294 'phylum': 'Cnidaria',
295 'class': 'Hydrozoa',
296 'order': None,
297 'family': None,
298 'genus': None,
299 'species': None,
300 'identity_reference': '50',
301 'image_url': '',
302 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=3725',
303 'recorded_timestamp': '25 Aug 23 20:52:05 UTC',
304 'video_sequence_name': 'Deep Discoverer 23060002',
305 }
306 ]
308 @patch('requests.get', side_effect=mocked_requests_get)
309 def test_find_long_host_associate_time_diff(self, _):
310 qaqc_processor_okay = VarsQaqcProcessor(
311 sequence_names=['Deep Discoverer 23060001'],
312 vars_dive_url=VARS_DIVE_QUERY_URL,
313 vars_phylogeny_url=VARS_PHYLOGENY_URL,
314 )
315 qaqc_processor_problems = VarsQaqcProcessor(
316 sequence_names=['Deep Discoverer 23060002'],
317 vars_dive_url=VARS_DIVE_QUERY_URL,
318 vars_phylogeny_url=VARS_PHYLOGENY_URL,
319 )
320 qaqc_processor_okay.find_long_host_associate_time_diff()
321 qaqc_processor_problems.find_long_host_associate_time_diff()
322 assert qaqc_processor_okay.final_records == []
323 assert qaqc_processor_problems.final_records == [
324 {
325 'observation_uuid': '01f3e954-b793-40a3-6166-88f24898e81e',
326 'concept': 'Pomacentridae',
327 'associations': ex_23060002['annotations'][1]['associations'],
328 'activity': 'cruise',
329 'annotator': 'Nikki Cunanan',
330 'depth': 4256.0,
331 'phylum': 'Chordata',
332 'class': 'Actinopterygii',
333 'order': 'Perciformes',
334 'family': 'Pomacentridae',
335 'genus': None,
336 'species': None,
337 'identity_reference': None,
338 'image_url': '',
339 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=4543',
340 'recorded_timestamp': '25 Aug 23 21:05:43 UTC',
341 'video_sequence_name': 'Deep Discoverer 23060002',
342 'status': 'Host not found in previous records'
343 },
344 {
345 'observation_uuid': '02dfd7f4-c834-433d-4960-9577c98ce81e',
346 'concept': 'Hydroidolina',
347 'associations': ex_23060002['annotations'][2]['associations'],
348 'activity': 'cruise',
349 'annotator': 'Nikki Cunanan',
350 'depth': None,
351 'phylum': 'Cnidaria',
352 'class': 'Hydrozoa',
353 'order': None,
354 'family': None,
355 'genus': None,
356 'species': None,
357 'identity_reference': '13',
358 'image_url': '',
359 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=2435',
360 'recorded_timestamp': '25 Aug 23 20:30:35 UTC',
361 'video_sequence_name': 'Deep Discoverer 23060002',
362 'status': 'Time between record and closest previous matching host record greater than one minute (95 seconds)'
363 },
364 {
365 'observation_uuid': '0983d9f1-d28a-482e-0160-6d3df753e91e',
366 'concept': 'AssociateConcept',
367 'associations': ex_23060002['annotations'][4]['associations'],
368 'activity': 'stationary',
369 'annotator': 'Nikki Cunanan',
370 'depth': 4260.0,
371 'phylum': None,
372 'class': None,
373 'order': None,
374 'family': None, 'genus': None,
375 'species': None,
376 'identity_reference': None,
377 'image_url': '',
378 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=2941',
379 'recorded_timestamp': '25 Aug 23 20:39:01 UTC',
380 'video_sequence_name': 'Deep Discoverer 23060002',
381 'status': 'Time between record and closest previous matching host record greater than five minutes (10 mins, 0 seconds)'
382 },
383 ]
385 @patch('requests.get', side_effect=mocked_requests_get)
386 def test_find_num_bounding_boxes(self, _):
387 qaqc_processor = VarsQaqcProcessor(
388 sequence_names=['Deep Discoverer 23060001'],
389 vars_dive_url=VARS_DIVE_QUERY_URL,
390 vars_phylogeny_url=VARS_PHYLOGENY_URL,
391 )
392 qaqc_processor.find_num_bounding_boxes()
393 assert qaqc_processor.final_records == [{
394 'bounding_box_counts': {
395 'Pomacentridae': {'annos': 5, 'boxes': 1},
396 'none': {'annos': 1, 'boxes': 0}
397 },
398 'total_count_annos': 6,
399 'total_count_boxes': 1,
400 }]
402 @patch('requests.get', side_effect=mocked_requests_get)
403 def test_find_unique_fields(self, _):
404 qaqc_processor = VarsQaqcProcessor(
405 sequence_names=['Deep Discoverer 23060001'],
406 vars_dive_url=VARS_DIVE_QUERY_URL,
407 vars_phylogeny_url=VARS_PHYLOGENY_URL,
408 )
409 qaqc_processor.find_unique_fields()
410 assert qaqc_processor.final_records == [
411 {
412 'concept-names': {
413 'Pomacentridae': {
414 'individuals': 5,
415 'records': 5,
416 },
417 'none': {
418 'individuals': 1,
419 'records': 1,
420 }
421 },
422 },
423 {
424 'concept-upon-combinations': {
425 'Pomacentridae:bed': {
426 'individuals': 2,
427 'records': 2,
428 },
429 'Pomacentridae:sed': {
430 'individuals': 3,
431 'records': 3,
432 },
433 'none:None': {
434 'individuals': 1,
435 'records': 1,
436 }
437 },
438 },
439 {
440 'substrate-combinations': {
441 '': {
442 'individuals': 1,
443 'records': 1,
444 },
445 'bed, bou, sed': {
446 'individuals': 1,
447 'records': 1,
448 },
449 'bed, sed': {
450 'individuals': 1,
451 'records': 1,
452 },
453 'mantra, sed': {
454 'individuals': 1,
455 'records': 1,
456 },
457 'sed': {
458 'individuals': 2,
459 'records': 2,
460 },
461 },
462 },
463 {
464 'comments': {
465 None: {
466 'individuals': 3,
467 'records': 3,
468 },
469 'Added for review: Don Draper': {
470 'individuals': 1,
471 'records': 1,
472 },
473 'Added for review: Jon Snow; This is a weird lookin sponge thing!': {
474 'individuals': 1,
475 'records': 1,
476 },
477 'this is a comment': {
478 'individuals': 1,
479 'records': 1,
480 },
481 },
482 },
483 {
484 'condition-comments': {
485 None: {
486 'individuals': 6,
487 'records': 6,
488 },
489 },
490 },
491 {
492 'megahabitats': {
493 None: {
494 'individuals': 5,
495 'records': 5,
496 },
497 'continental shelf': {
498 'individuals': 1,
499 'records': 1,
500 },
501 },
502 },
503 {
504 'habitats': {
505 None: {
506 'individuals': 5,
507 'records': 5,
508 },
509 'slope': {
510 'individuals': 1,
511 'records': 1,
512 },
513 },
514 },
515 {
516 'habitat-comments': {
517 None: {
518 'individuals': 5,
519 'records': 5,
520 },
521 'loose talus': {
522 'individuals': 1,
523 'records': 1,
524 },
525 },
526 },
527 {
528 'identity-certainty': {
529 None: {
530 'individuals': 4,
531 'records': 4,
532 },
533 'maybe': {
534 'individuals': 2,
535 'records': 2,
536 },
537 },
538 },
539 {
540 'occurrence-remarks': {
541 None: {
542 'individuals': 4,
543 'records': 4,
544 },
545 'bottom in sight': {
546 'individuals': 1,
547 'records': 1,
548 },
549 'in water column on descent': {
550 'individuals': 1,
551 'records': 1,
552 },
553 },
554 },
555 ]