Coverage for application / qaqc / vars / vars_qaqc_processor.py: 89%
422 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-07 06:46 +0000
1import requests
2import sys
4from application.util.functions import *
5from application.image_review.vars.vars_annotation_processor import VarsAnnotationProcessor
8class VarsQaqcProcessor(VarsAnnotationProcessor):
9 """
10 Filters and formats annotations for the various DARC QA/QC checks.
11 """
13 def __init__(self, sequence_names: list, vars_dive_url: str, vars_phylogeny_url: str):
14 super().__init__(sequence_names, vars_dive_url, vars_phylogeny_url)
15 self.videos = []
16 self.load_phylogeny()
18 def fetch_annotations(self, seq_name):
19 """
20 Fetches annotations for a given sequence name from VARS
21 """
22 print(f'Fetching annotations for sequence {seq_name} from VARS...', end='')
23 sys.stdout.flush()
25 res = requests.get(url=f'{self.vars_dive_url}/{seq_name.replace(" ", "%20")}')
26 dive_json = res.json()
27 print('fetched!')
29 for video in dive_json['media']:
30 if 'urn:imagecollection:org' not in video['uri']:
31 self.videos.append({
32 'start_timestamp': parse_datetime(video['start_timestamp']),
33 'uri': video['uri'].replace('http://hurlstor.soest.hawaii.edu/videoarchive', 'https://hurlvideo.soest.hawaii.edu'),
34 'sequence_name': video['video_sequence_name'],
35 'video_reference_uuid': video['video_reference_uuid'],
36 })
37 return dive_json['annotations']
39 def find_duplicate_associations(self):
40 """
41 Finds annotations that have more than one of the same association besides s2
42 """
43 for name in self.sequence_names:
44 for annotation in self.fetch_annotations(name):
45 if annotation.get('group') == 'localization':
46 continue
47 # get list of associations
48 association_set = set()
49 duplicate_associations = False
50 for association in annotation['associations']:
51 name = association['link_name']
52 if name not in association_set:
53 if name != 's2':
54 association_set.add(name)
55 else:
56 duplicate_associations = True
57 break
58 if duplicate_associations:
59 self.working_records.append(annotation)
60 self.sort_records(self.process_working_records(self.videos))
62 def find_missing_s1(self):
63 """
64 Finds annotations that are missing s1 (ignores 'none' records)
65 """
66 for name in self.sequence_names:
67 for annotation in self.fetch_annotations(name):
68 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
69 continue
70 s1 = get_association(annotation, 's1')
71 if not s1:
72 self.working_records.append(annotation)
73 self.sort_records(self.process_working_records(self.videos))
75 def find_identical_s1_s2(self):
76 """
77 Finds annotations that have an s2 association that is the same as its s1 association
78 """
79 for name in self.sequence_names:
80 for annotation in self.fetch_annotations(name):
81 if annotation.get('group') == 'localization':
82 continue
83 s2s = []
84 s1 = ''
85 for association in annotation['associations']:
86 if association['link_name'] == 's1':
87 s1 = association['to_concept']
88 elif association['link_name'] == 's2':
89 s2s.append(association['to_concept'])
90 if s1 in s2s:
91 self.working_records.append(annotation)
92 self.sort_records(self.process_working_records(self.videos))
94 def find_duplicate_s2(self):
95 """
96 Finds annotations that have multiple s2 associations with the same value
97 """
98 for name in self.sequence_names:
99 for annotation in self.fetch_annotations(name):
100 if annotation.get('group') == 'localization':
101 continue
102 duplicate_s2s = False
103 s2_set = set()
104 for association in annotation['associations']:
105 if association['link_name'] == 's2':
106 if association['to_concept'] in s2_set:
107 duplicate_s2s = True
108 break
109 else:
110 s2_set.add(association['to_concept'])
111 if duplicate_s2s:
112 self.working_records.append(annotation)
113 self.sort_records(self.process_working_records(self.videos))
115 def find_missing_upon_substrate(self):
116 """
117 Finds annotations that have an upon association that is not an organism, but the 'upon' is not present in s1 or
118 any s2
119 """
120 for name in self.sequence_names:
121 for annotation in self.fetch_annotations(name):
122 if annotation.get('group') == 'localization':
123 continue
124 upon = None
125 missing_upon = False
126 for association in annotation['associations']:
127 if association['link_name'] == 'upon':
128 if (association['to_concept'] and association['to_concept'][0].isupper()) \
129 or association['to_concept'].startswith('dead'):
130 # 'upon' is an organism, don't need it to be in s1/s2
131 pass
132 else:
133 # 'upon' should be in s1 or s2
134 upon = association['to_concept']
135 break
136 if upon:
137 missing_upon = True
138 for association in annotation['associations']:
139 if (association['link_name'] == 's1' or association['link_name'] == 's2') \
140 and association['to_concept'] == upon:
141 missing_upon = False
142 break
143 if missing_upon:
144 self.working_records.append(annotation)
145 self.sort_records(self.process_working_records(self.videos))
147 def find_mismatched_substrates(self):
148 """
149 Finds annotations that occur at the same timestamp (same second) but have different substrates
150 """
151 for name in self.sequence_names:
152 annotations_with_same_timestamp = {}
153 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
154 # loop through all annotations, add ones with same timestamp to dict
155 i = 0
156 while i < len(sorted_annotations) - 2:
157 if sorted_annotations[i].get('group') == 'localization':
158 i += 1
159 continue
160 base_timestamp = sorted_annotations[i]['recorded_timestamp'][:19]
161 base_annotation = sorted_annotations[i]
162 i += 1
163 while sorted_annotations[i]['recorded_timestamp'][:19] == base_timestamp:
164 if sorted_annotations[i].get('group') != 'localization':
165 if base_timestamp not in annotations_with_same_timestamp.keys():
166 annotations_with_same_timestamp[base_timestamp] = [base_annotation]
167 annotations_with_same_timestamp[base_timestamp].append(sorted_annotations[i])
168 i += 1
169 # loop through each annotation that shares the same timestamp, compare substrates
170 for timestamp_key in annotations_with_same_timestamp.keys():
171 base_substrates = {'s2': set()}
172 check_substrates = {'s2': set()}
173 for association in annotations_with_same_timestamp[timestamp_key][0]['associations']:
174 if association['link_name'] == 's1':
175 base_substrates['s1'] = association['to_concept']
176 if association['link_name'] == 's2':
177 base_substrates['s2'].add(association['to_concept'])
178 for i in range(1, len(annotations_with_same_timestamp[timestamp_key])):
179 for association in annotations_with_same_timestamp[timestamp_key][i]['associations']:
180 if association['link_name'] == 's1':
181 check_substrates['s1'] = association['to_concept']
182 if association['link_name'] == 's2':
183 check_substrates['s2'].add(association['to_concept'])
184 if base_substrates != check_substrates:
185 for annotation in annotations_with_same_timestamp[timestamp_key]:
186 self.working_records.append(annotation)
187 self.sort_records(self.process_working_records(self.videos))
189 def find_missing_upon(self):
190 """
191 Finds annotations that are missing upon (ignores 'none' records)
192 """
193 for name in self.sequence_names:
194 for annotation in self.fetch_annotations(name):
195 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
196 continue
197 if not get_association(annotation, 'upon'):
198 self.working_records.append(annotation)
199 self.sort_records(self.process_working_records(self.videos))
201 def get_num_records_missing_ancillary_data(self):
202 """
203 Finds number of annotations that are missing ancillary data
204 """
205 num_records_missing = 0
206 for name in self.sequence_names:
207 for annotation in self.fetch_annotations(name):
208 if 'ancillary_data' not in annotation.keys():
209 num_records_missing += 1
210 return num_records_missing
212 def find_missing_ancillary_data(self):
213 """
214 Finds annotations that are missing ancillary data (can be very slow)
215 """
216 for name in self.sequence_names:
217 for annotation in self.fetch_annotations(name):
218 if 'ancillary_data' not in annotation.keys():
219 self.working_records.append(annotation)
220 self.sort_records(self.process_working_records(self.videos))
222 def find_id_refs_different_concept_name(self):
223 """
224 Finds annotations with the same ID reference that have different concept names
225 """
226 for name in self.sequence_names:
227 id_ref_names = {} # dict of {id_ref: {name_1, name_2}} to check for more than one name
228 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
229 for annotation in self.fetch_annotations(name):
230 if annotation.get('group') == 'localization':
231 continue
232 for association in annotation['associations']:
233 if association['link_name'] == 'identity-reference':
234 if association['link_value'] not in id_ref_names.keys():
235 id_ref_names[association['link_value']] = set()
236 id_ref_annotations[association['link_value']] = []
237 id_ref_names[association['link_value']].add(annotation['concept'])
238 id_ref_annotations[association['link_value']].append(annotation)
239 break
240 for id_ref, name_set in id_ref_names.items():
241 if len(name_set) > 1:
242 for annotation in id_ref_annotations[id_ref]:
243 self.working_records.append(annotation)
244 self.sort_records(self.process_working_records(self.videos))
246 def find_id_refs_conflicting_associations(self):
247 """
248 Finds annotations with the same ID reference that have conflicting associations
249 """
250 to_concepts = ['s1', 's2', 'upon', 'size', 'habitat', 'megahabitat', 'sampled-by']
251 for name in self.sequence_names:
252 id_ref_associations = {} # dict of {id_ref: {ass_1_name: ass_1_val, ass_2_name: ass_2_val}}
253 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
254 for annotation in self.fetch_annotations(name):
255 if annotation.get('group') == 'localization':
256 continue
257 id_ref = get_association(annotation, 'identity-reference')
258 if id_ref:
259 current_id_ref = id_ref['link_value']
260 if current_id_ref not in id_ref_associations.keys():
261 id_ref_associations[current_id_ref] = {
262 'flag': False, # we'll set this to true if we find any conflicting associations
263 's2': set(), # s2, sampled-by, and sample-reference are allowed to have
264 'sampled-by': set(), # more than one association
265 'sample-reference': set(),
266 }
267 id_ref_annotations[current_id_ref] = [annotation]
268 # populate id_ref dict with all associations
269 for ass in annotation['associations']:
270 if ass['link_name'] == 'guide-photo':
271 pass
272 elif ass['link_name'] == 's2' or ass['link_name'] == 'sampled-by':
273 id_ref_associations[current_id_ref][ass['link_name']].add(ass['to_concept'])
274 elif ass['link_name'] == 'sample-reference':
275 id_ref_associations[current_id_ref][ass['link_name']].add(ass['link_value'])
276 else:
277 id_ref_associations[current_id_ref][ass['link_name']] = \
278 ass['link_value'] if ass['link_name'] not in to_concepts else ass['to_concept']
279 else:
280 # check current association values vs those saved
281 id_ref_annotations[current_id_ref].append(annotation)
282 temp_s2_set = set()
283 temp_sampled_by_set = set()
284 temp_sample_ref_set = set()
285 for ass in annotation['associations']:
286 if ass['link_name'] == 'guide-photo':
287 pass
288 elif ass['link_name'] == 's2':
289 temp_s2_set.add(ass['to_concept'])
290 elif ass['link_name'] == 'sampled-by':
291 temp_sampled_by_set.add(ass['to_concept'])
292 elif ass['link_name'] == 'sample-reference':
293 temp_sample_ref_set.add(ass['link_value'])
294 else:
295 if ass['link_name'] in to_concepts:
296 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
297 # cases like 'guide-photo' will only be present on one record
298 if id_ref_associations[current_id_ref][ass['link_name']] != ass['to_concept']:
299 id_ref_associations[current_id_ref]['flag'] = True
300 break
301 else:
302 id_ref_associations[current_id_ref][ass['link_name']] = ass['to_concept']
303 else:
304 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
305 if id_ref_associations[current_id_ref][ass['link_name']] != ass['link_value']:
306 id_ref_associations[current_id_ref]['flag'] = True
307 break
308 else:
309 id_ref_associations[current_id_ref][ass['link_name']] = ass['link_value']
310 if temp_s2_set != id_ref_associations[current_id_ref]['s2'] \
311 or temp_sampled_by_set != id_ref_associations[current_id_ref]['sampled-by'] \
312 or temp_sample_ref_set != id_ref_associations[current_id_ref]['sample-reference']:
313 id_ref_associations[current_id_ref]['flag'] = True
314 for id_ref in id_ref_associations.keys():
315 if id_ref_associations[id_ref]['flag']:
316 for annotation in id_ref_annotations[id_ref]:
317 self.working_records.append(annotation)
318 self.sort_records(self.process_working_records(self.videos))
320 def find_blank_associations(self):
321 """
322 Finds all records that have associations with a link value of ""
323 """
324 for name in self.sequence_names:
325 for annotation in self.fetch_annotations(name):
326 if annotation.get('group') == 'localization':
327 continue
328 for association in annotation['associations']:
329 if association['link_value'] == '' and association['to_concept'] == 'self':
330 self.working_records.append(annotation)
331 self.sort_records(self.process_working_records(self.videos))
333 def find_suspicious_hosts(self):
334 """
335 Finds annotations that have an upon that is the same concept as itself
336 """
337 for name in self.sequence_names:
338 for annotation in self.fetch_annotations(name):
339 if annotation.get('group') == 'localization':
340 continue
341 upon = get_association(annotation, 'upon')
342 if upon and upon['to_concept'] == annotation['concept']:
343 self.working_records.append(annotation)
344 self.sort_records(self.process_working_records(self.videos))
346 def find_missing_expected_association(self):
347 """
348 Finds annotations that are expected to be upon another organism, but are not. This is a very slow test because
349 before it can begin, we must retrieve the taxa from VARS for every record (unlike the other tests, we can't
350 filter beforehand).
352 If more concepts need to be added for this check, simply add them to the appropriate list below:
354 Example: To add the order 'order123' to the list, change the declaration below from:
356 orders = ['Comatulida']
358 to:
360 orders = ['Comatulida', 'order123']
362 If a list does not exist, declare a new list and add it to the conditional:
364 Example: To add the subfamily 'subfam123' to the check, add a new list named 'subfamilies':
366 subfamilies = ['subfam123']
368 Then add the new list to the conditional:
370 ...
371 or ('family' in record.keys() and record['family'] in families)
372 or ('subfamily' in record.keys() and record['subfamily'] in subfamilies) <<< ADD THIS LINE
373 or ('genus' in record.keys() and record['genus'] in genera)
374 ...
376 If you want the new addition to be highlighted in the table on the webpage, add the name to the ranksToHighlight
377 list in vars/qaqc.js, at ~line 340
378 """
379 classes = ['Ophiuroidea']
380 orders = ['Comatulida']
381 infraorders = ['Anomura', 'Caridea']
382 families = ['Goniasteridae', 'Poecilasmatidae', 'Parazoanthidae', 'Tubulariidae', 'Amphianthidae', 'Actinoscyphiidae']
383 genera = ['Henricia']
384 concepts = ['Hydroidolina']
385 for name in self.sequence_names:
386 for annotation in self.fetch_annotations(name):
387 if annotation.get('group') == 'localization':
388 continue
389 self.working_records.append(annotation)
390 self.sort_records(self.process_working_records(self.videos))
391 temp_records = self.final_records
392 self.final_records = []
393 for record in temp_records:
394 if record.get('class') in classes \
395 or record.get('order') in orders \
396 or record.get('infraorder') in infraorders \
397 or record.get('family') in families \
398 or record.get('genus') in genera \
399 or record.get('concept') in concepts:
400 upon = get_association(record, 'upon')
401 if upon and upon['to_concept'][0].islower() and 'dead' not in upon['to_concept']:
402 self.final_records.append(record)
404 def find_long_host_associate_time_diff(self):
405 greater_than_one_min = {}
406 greater_than_five_mins = {}
407 not_found = []
408 for name in self.sequence_names:
409 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
410 for i in range(len(sorted_annotations)):
411 associate_record = sorted_annotations[i]
412 upon = get_association(sorted_annotations[i], 'upon')
413 if upon and upon['to_concept'] and upon['to_concept'][0].isupper():
414 # the associate's 'upon' is an organism
415 host_concept_name = upon['to_concept']
416 observation_time = extract_recorded_datetime(associate_record)
417 found = False
418 for j in range(i + 10, -1, -1):
419 """
420 Checks backward, looking for the most recent host w/ matching name. We start at i + 10 because
421 there can be multiple records with the exact same timestamp, and one of those records could be
422 the 'upon'
423 """
424 # to catch index out of range exception
425 while j >= len(sorted_annotations):
426 j -= 1
427 host_record = sorted_annotations[j]
428 host_time = extract_recorded_datetime(host_record)
429 if host_time > observation_time or i == j:
430 # host record won't be recorded after associate record, ignore this record
431 # i == j: record shouldn't be associated with itself, ignore
432 pass
433 else:
434 if host_record['concept'] == host_concept_name:
435 # the host record's name is equal to the host concept name (associate's 'upon' name)
436 found = True
437 time_diff = observation_time - host_time
438 if time_diff.seconds > 300:
439 greater_than_five_mins[associate_record['observation_uuid']] = time_diff
440 self.working_records.append(associate_record)
441 elif time_diff.seconds > 60:
442 greater_than_one_min[associate_record['observation_uuid']] = time_diff
443 self.working_records.append(associate_record)
444 break
445 if not found:
446 not_found.append(associate_record['observation_uuid'])
447 self.working_records.append(associate_record)
448 self.sort_records(self.process_working_records(self.videos))
449 for uuid in greater_than_one_min.keys():
450 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
451 'Time between record and closest previous matching host record greater than one minute ' \
452 f'({greater_than_one_min[uuid].seconds} seconds)'
453 for uuid in greater_than_five_mins.keys():
454 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
455 'Time between record and closest previous matching host record greater than five minutes ' \
456 f'({greater_than_five_mins[uuid].seconds // 60 % 60} mins, {greater_than_five_mins[uuid].seconds % 60} seconds)'
457 for uuid in not_found:
458 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
459 f'Host not found in previous records'
461 def find_num_bounding_boxes(self):
462 """
463 Finds the number of bounding boxes and total annotation count for each unique concept.
464 """
465 bounding_box_counts = {}
466 total_count_annos = 0
467 total_count_boxes = 0
468 for name in self.sequence_names:
469 for annotation in self.fetch_annotations(name):
470 total_count_annos += 1
471 if annotation['concept'] not in bounding_box_counts.keys():
472 bounding_box_counts[annotation['concept']] = {
473 'boxes': 0,
474 'annos': 0,
475 }
476 bounding_box_counts[annotation['concept']]['annos'] += 1
477 if get_association(annotation, 'bounding box'):
478 total_count_boxes += 1
479 bounding_box_counts[annotation['concept']]['boxes'] += 1
480 sorted_box_counts = dict(sorted(bounding_box_counts.items()))
481 self.final_records.append({
482 'total_count_annos': total_count_annos,
483 'total_count_boxes': total_count_boxes,
484 'bounding_box_counts': sorted_box_counts,
485 })
487 def find_localizations_without_bounding_boxes(self):
488 """
489 Finds records that are in the "localization" group but do not contain a bounding box association. Also finds
490 records that have a bounding box association but are not in the "localization" group.
491 """
492 for name in self.sequence_names:
493 for annotation in self.fetch_annotations(name):
494 has_box = False
495 for association in annotation['associations']:
496 if association['link_name'] == 'bounding box':
497 has_box = True
498 break
499 if annotation.get('group') == 'localization':
500 if not has_box:
501 self.working_records.append(annotation)
502 elif has_box:
503 self.working_records.append(annotation)
504 self.sort_records(self.process_working_records(self.videos))
506 def find_unique_fields(self):
507 def load_dict(field_name, unique_dict, individual_count):
508 if field_name not in unique_dict.keys():
509 unique_dict[field_name] = {}
510 unique_dict[field_name]['records'] = 1
511 unique_dict[field_name]['individuals'] = individual_count
512 else:
513 unique_dict[field_name]['records'] += 1
514 unique_dict[field_name]['individuals'] += individual_count
516 unique_concept_names = {}
517 unique_concept_upons = {}
518 unique_substrate_combinations = {}
519 unique_comments = {}
520 unique_condition_comments = {}
521 unique_megahabitats = {}
522 unique_habitats = {}
523 unique_habitat_comments = {}
524 unique_id_certainty = {}
525 unique_occurrence_remarks = {}
527 for name in self.sequence_names:
528 for annotation in self.fetch_annotations(name):
529 substrates = []
530 upon = None
531 comment = None
532 condition_comment = None
533 megahabitat = None
534 habitat = None
535 habitat_comment = None
536 id_certainty = None
537 occurrence_remark = None
538 individual_count = 1
540 for association in annotation['associations']:
541 match association['link_name']:
542 case 's1' | 's2':
543 substrates.append(association['to_concept'])
544 case 'upon':
545 upon = association['to_concept']
546 case 'comment':
547 comment = association['link_value']
548 case 'condition-comment':
549 condition_comment = association['link_value']
550 case 'megahabitat':
551 megahabitat = association['to_concept']
552 case 'habitat':
553 habitat = association['to_concept']
554 case 'habitat-comment':
555 habitat_comment = association['link_value']
556 case 'identity-certainty':
557 id_certainty = association['link_value']
558 case 'occurrence-remark':
559 occurrence_remark = association['link_value']
560 case 'population-quantity':
561 if association['link_value'] != '':
562 individual_count = int(association['link_value'])
563 case 'categorical-abundance':
564 match association['link_value']:
565 case '11-20':
566 individual_count = 15
567 case '21-50':
568 individual_count = 35
569 case '51-100':
570 individual_count = 75
571 case '\u003e100':
572 individual_count = 100
574 if substrates is not None:
575 substrates.sort()
576 substrates = ', '.join(substrates)
578 load_dict(annotation['concept'], unique_concept_names, individual_count)
579 load_dict(f'{annotation["concept"]}:{upon}', unique_concept_upons, individual_count)
580 load_dict(substrates, unique_substrate_combinations, individual_count)
581 load_dict(comment, unique_comments, individual_count)
582 load_dict(condition_comment, unique_condition_comments, individual_count)
583 load_dict(megahabitat, unique_megahabitats, individual_count)
584 load_dict(habitat, unique_habitats, individual_count)
585 load_dict(habitat_comment, unique_habitat_comments, individual_count)
586 load_dict(id_certainty, unique_id_certainty, individual_count)
587 load_dict(occurrence_remark, unique_occurrence_remarks, individual_count)
589 self.final_records.append({'concept-names': unique_concept_names})
590 self.final_records.append({'concept-upon-combinations': unique_concept_upons})
591 self.final_records.append({'substrate-combinations': unique_substrate_combinations})
592 self.final_records.append({'comments': unique_comments})
593 self.final_records.append({'condition-comments': unique_condition_comments})
594 self.final_records.append({'megahabitats': unique_megahabitats})
595 self.final_records.append({'habitats': unique_habitats})
596 self.final_records.append({'habitat-comments': unique_habitat_comments})
597 self.final_records.append({'identity-certainty': unique_id_certainty})
598 self.final_records.append({'occurrence-remarks': unique_occurrence_remarks})