Coverage for application/qaqc/vars/vars_qaqc_processor.py: 89%
422 statements
« prev ^ index » next coverage.py v7.10.4, created at 2025-08-17 17:51 +0000
1import requests
2import sys
4from application.util.functions import *
5from application.image_review.vars.vars_annotation_processor import VarsAnnotationProcessor
8class VarsQaqcProcessor(VarsAnnotationProcessor):
9 """
10 Filters and formats annotations for the various DARC QA/QC checks.
11 """
13 def __init__(self, sequence_names: list, vars_dive_url: str, vars_phylogeny_url: str):
14 super().__init__(sequence_names, vars_dive_url, vars_phylogeny_url)
15 self.videos = []
16 self.load_phylogeny()
18 def fetch_annotations(self, seq_name):
19 """
20 Fetches annotations for a given sequence name from VARS
21 """
22 print(f'Fetching annotations for sequence {seq_name} from VARS...', end='')
23 sys.stdout.flush()
25 res = requests.get(url=f'{self.vars_dive_url}/{seq_name.replace(" ", "%20")}')
26 dive_json = res.json()
27 print('fetched!')
29 for video in dive_json['media']:
30 if 'urn:imagecollection:org' not in video['uri']:
31 self.videos.append({
32 'start_timestamp': parse_datetime(video['start_timestamp']),
33 'uri': video['uri'].replace('http://hurlstor.soest.hawaii.edu/videoarchive', 'https://hurlvideo.soest.hawaii.edu'),
34 'sequence_name': video['video_sequence_name'],
35 'video_reference_uuid': video['video_reference_uuid'],
36 })
37 return dive_json['annotations']
39 def find_duplicate_associations(self):
40 """
41 Finds annotations that have more than one of the same association besides s2
42 """
43 for name in self.sequence_names:
44 for annotation in self.fetch_annotations(name):
45 if annotation.get('group') == 'localization':
46 continue
47 # get list of associations
48 association_set = set()
49 duplicate_associations = False
50 for association in annotation['associations']:
51 name = association['link_name']
52 if name not in association_set:
53 if name != 's2':
54 association_set.add(name)
55 else:
56 duplicate_associations = True
57 break
58 if duplicate_associations:
59 self.working_records.append(annotation)
60 self.sort_records(self.process_working_records(self.videos))
62 def find_missing_s1(self):
63 """
64 Finds annotations that are missing s1 (ignores 'none' records)
65 """
66 for name in self.sequence_names:
67 for annotation in self.fetch_annotations(name):
68 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
69 continue
70 s1 = get_association(annotation, 's1')
71 if not s1:
72 self.working_records.append(annotation)
73 self.sort_records(self.process_working_records(self.videos))
75 def find_identical_s1_s2(self):
76 """
77 Finds annotations that have an s2 association that is the same as its s1 association
78 """
79 for name in self.sequence_names:
80 for annotation in self.fetch_annotations(name):
81 if annotation.get('group') == 'localization':
82 continue
83 s2s = []
84 s1 = ''
85 for association in annotation['associations']:
86 if association['link_name'] == 's1':
87 s1 = association['to_concept']
88 elif association['link_name'] == 's2':
89 s2s.append(association['to_concept'])
90 if s1 in s2s:
91 self.working_records.append(annotation)
92 self.sort_records(self.process_working_records(self.videos))
94 def find_duplicate_s2(self):
95 """
96 Finds annotations that have multiple s2 associations with the same value
97 """
98 for name in self.sequence_names:
99 for annotation in self.fetch_annotations(name):
100 if annotation.get('group') == 'localization':
101 continue
102 duplicate_s2s = False
103 s2_set = set()
104 for association in annotation['associations']:
105 if association['link_name'] == 's2':
106 if association['to_concept'] in s2_set:
107 duplicate_s2s = True
108 break
109 else:
110 s2_set.add(association['to_concept'])
111 if duplicate_s2s:
112 self.working_records.append(annotation)
113 self.sort_records(self.process_working_records(self.videos))
115 def find_missing_upon_substrate(self):
116 """
117 Finds annotations that have an upon association that is not an organism, but the 'upon' is not present in s1 or
118 any s2
119 """
120 for name in self.sequence_names:
121 for annotation in self.fetch_annotations(name):
122 if annotation.get('group') == 'localization':
123 continue
124 upon = None
125 missing_upon = False
126 for association in annotation['associations']:
127 if association['link_name'] == 'upon':
128 if (association['to_concept'] and association['to_concept'][0].isupper()) \
129 or association['to_concept'].startswith('dead'):
130 # 'upon' is an organism, don't need it to be in s1/s2
131 pass
132 else:
133 # 'upon' should be in s1 or s2
134 upon = association['to_concept']
135 break
136 if upon:
137 missing_upon = True
138 for association in annotation['associations']:
139 if (association['link_name'] == 's1' or association['link_name'] == 's2') \
140 and association['to_concept'] == upon:
141 missing_upon = False
142 break
143 if missing_upon:
144 self.working_records.append(annotation)
145 self.sort_records(self.process_working_records(self.videos))
147 def find_mismatched_substrates(self):
148 """
149 Finds annotations that occur at the same timestamp (same second) but have different substrates
150 """
151 for name in self.sequence_names:
152 annotations_with_same_timestamp = {}
153 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
154 # loop through all annotations, add ones with same timestamp to dict
155 i = 0
156 while i < len(sorted_annotations) - 2:
157 if sorted_annotations[i].get('group') == 'localization':
158 i += 1
159 continue
160 base_timestamp = sorted_annotations[i]['recorded_timestamp'][:19]
161 base_annotation = sorted_annotations[i]
162 i += 1
163 while sorted_annotations[i]['recorded_timestamp'][:19] == base_timestamp:
164 if sorted_annotations[i].get('group') != 'localization':
165 if base_timestamp not in annotations_with_same_timestamp.keys():
166 annotations_with_same_timestamp[base_timestamp] = [base_annotation]
167 annotations_with_same_timestamp[base_timestamp].append(sorted_annotations[i])
168 i += 1
169 # loop through each annotation that shares the same timestamp, compare substrates
170 for timestamp_key in annotations_with_same_timestamp.keys():
171 base_substrates = {'s2': set()}
172 check_substrates = {'s2': set()}
173 for association in annotations_with_same_timestamp[timestamp_key][0]['associations']:
174 if association['link_name'] == 's1':
175 base_substrates['s1'] = association['to_concept']
176 if association['link_name'] == 's2':
177 base_substrates['s2'].add(association['to_concept'])
178 for i in range(1, len(annotations_with_same_timestamp[timestamp_key])):
179 for association in annotations_with_same_timestamp[timestamp_key][i]['associations']:
180 if association['link_name'] == 's1':
181 check_substrates['s1'] = association['to_concept']
182 if association['link_name'] == 's2':
183 check_substrates['s2'].add(association['to_concept'])
184 if base_substrates != check_substrates:
185 for annotation in annotations_with_same_timestamp[timestamp_key]:
186 self.working_records.append(annotation)
187 self.sort_records(self.process_working_records(self.videos))
189 def find_missing_upon(self):
190 """
191 Finds annotations that are missing upon (ignores 'none' records)
192 """
193 for name in self.sequence_names:
194 for annotation in self.fetch_annotations(name):
195 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
196 continue
197 if not get_association(annotation, 'upon'):
198 self.working_records.append(annotation)
199 self.sort_records(self.process_working_records(self.videos))
201 def get_num_records_missing_ancillary_data(self):
202 """
203 Finds number of annotations that are missing ancillary data
204 """
205 num_records_missing = 0
206 for name in self.sequence_names:
207 for annotation in self.fetch_annotations(name):
208 if 'ancillary_data' not in annotation.keys():
209 num_records_missing += 1
210 return num_records_missing
212 def find_missing_ancillary_data(self):
213 """
214 Finds annotations that are missing ancillary data (can be very slow)
215 """
216 for name in self.sequence_names:
217 for annotation in self.fetch_annotations(name):
218 if 'ancillary_data' not in annotation.keys():
219 self.working_records.append(annotation)
220 self.sort_records(self.process_working_records(self.videos))
222 def find_id_refs_different_concept_name(self):
223 """
224 Finds annotations with the same ID reference that have different concept names
225 """
226 for name in self.sequence_names:
227 id_ref_names = {} # dict of {id_ref: {name_1, name_2}} to check for more than one name
228 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
229 for annotation in self.fetch_annotations(name):
230 if annotation.get('group') == 'localization':
231 continue
232 for association in annotation['associations']:
233 if association['link_name'] == 'identity-reference':
234 if association['link_value'] not in id_ref_names.keys():
235 id_ref_names[association['link_value']] = set()
236 id_ref_annotations[association['link_value']] = []
237 id_ref_names[association['link_value']].add(annotation['concept'])
238 id_ref_annotations[association['link_value']].append(annotation)
239 break
240 for id_ref, name_set in id_ref_names.items():
241 if len(name_set) > 1:
242 for annotation in id_ref_annotations[id_ref]:
243 self.working_records.append(annotation)
244 self.sort_records(self.process_working_records(self.videos))
246 def find_id_refs_conflicting_associations(self):
247 """
248 Finds annotations with the same ID reference that have conflicting associations
249 """
250 to_concepts = ['s1', 's2', 'upon', 'size', 'habitat', 'megahabitat', 'sampled-by']
251 for name in self.sequence_names:
252 id_ref_associations = {} # dict of {id_ref: {ass_1_name: ass_1_val, ass_2_name: ass_2_val}}
253 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
254 for annotation in self.fetch_annotations(name):
255 if annotation.get('group') == 'localization':
256 continue
257 id_ref = get_association(annotation, 'identity-reference')
258 if id_ref:
259 current_id_ref = id_ref['link_value']
260 if current_id_ref not in id_ref_associations.keys():
261 id_ref_associations[current_id_ref] = {
262 'flag': False, # we'll set this to true if we find any conflicting associations
263 's2': set(), # s2, sampled-by, and sample-reference are allowed to have
264 'sampled-by': set(), # more than one association
265 'sample-reference': set(),
266 }
267 id_ref_annotations[current_id_ref] = [annotation]
268 # populate id_ref dict with all associations
269 for ass in annotation['associations']:
270 if ass['link_name'] == 'guide-photo':
271 pass
272 elif ass['link_name'] == 's2' or ass['link_name'] == 'sampled-by':
273 id_ref_associations[current_id_ref][ass['link_name']].add(ass['to_concept'])
274 elif ass['link_name'] == 'sample-reference':
275 id_ref_associations[current_id_ref][ass['link_name']].add(ass['link_value'])
276 else:
277 id_ref_associations[current_id_ref][ass['link_name']] = \
278 ass['link_value'] if ass['link_name'] not in to_concepts else ass['to_concept']
279 else:
280 # check current association values vs those saved
281 id_ref_annotations[current_id_ref].append(annotation)
282 temp_s2_set = set()
283 temp_sampled_by_set = set()
284 temp_sample_ref_set = set()
285 for ass in annotation['associations']:
286 if ass['link_name'] == 'guide-photo':
287 pass
288 elif ass['link_name'] == 's2':
289 temp_s2_set.add(ass['to_concept'])
290 elif ass['link_name'] == 'sampled-by':
291 temp_sampled_by_set.add(ass['to_concept'])
292 elif ass['link_name'] == 'sample-reference':
293 temp_sample_ref_set.add(ass['link_value'])
294 else:
295 if ass['link_name'] in to_concepts:
296 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
297 # cases like 'guide-photo' will only be present on one record
298 if id_ref_associations[current_id_ref][ass['link_name']] != ass['to_concept']:
299 id_ref_associations[current_id_ref]['flag'] = True
300 break
301 else:
302 id_ref_associations[current_id_ref][ass['link_name']] = ass['to_concept']
303 else:
304 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
305 if id_ref_associations[current_id_ref][ass['link_name']] != ass['link_value']:
306 id_ref_associations[current_id_ref]['flag'] = True
307 break
308 else:
309 id_ref_associations[current_id_ref][ass['link_name']] = ass['link_value']
310 if temp_s2_set != id_ref_associations[current_id_ref]['s2'] \
311 or temp_sampled_by_set != id_ref_associations[current_id_ref]['sampled-by'] \
312 or temp_sample_ref_set != id_ref_associations[current_id_ref]['sample-reference']:
313 id_ref_associations[current_id_ref]['flag'] = True
314 for id_ref in id_ref_associations.keys():
315 if id_ref_associations[id_ref]['flag']:
316 for annotation in id_ref_annotations[id_ref]:
317 self.working_records.append(annotation)
318 self.sort_records(self.process_working_records(self.videos))
320 def find_blank_associations(self):
321 """
322 Finds all records that have associations with a link value of ""
323 """
324 for name in self.sequence_names:
325 for annotation in self.fetch_annotations(name):
326 if annotation.get('group') == 'localization':
327 continue
328 for association in annotation['associations']:
329 if association['link_value'] == '' and association['to_concept'] == 'self':
330 self.working_records.append(annotation)
331 self.sort_records(self.process_working_records(self.videos))
333 def find_suspicious_hosts(self):
334 """
335 Finds annotations that have an upon that is the same concept as itself
336 """
337 for name in self.sequence_names:
338 for annotation in self.fetch_annotations(name):
339 if annotation.get('group') == 'localization':
340 continue
341 upon = get_association(annotation, 'upon')
342 if upon and upon['to_concept'] == annotation['concept']:
343 self.working_records.append(annotation)
344 self.sort_records(self.process_working_records(self.videos))
346 def find_missing_expected_association(self):
347 """
348 Finds annotations that are expected to be upon another organism, but are not. This is a very slow test because
349 before it can begin, we must retrieve the taxa from VARS for every record (unlike the other tests, we can't
350 filter beforehand).
352 If more concepts need to be added for this check, simply add them to the appropriate list below:
354 Example: To add the order 'order123' to the list, change the declaration below from:
356 orders = ['Comatulida']
358 to:
360 orders = ['Comatulida', 'order123']
362 If a list does not exist, declare a new list and add it to the conditional:
364 Example: To add the subfamily 'subfam123' to the check, add a new list named 'subfamilies':
366 subfamilies = ['subfam123']
368 Then add the new list to the conditional:
370 ...
371 or ('family' in record.keys() and record['family'] in families)
372 or ('subfamily' in record.keys() and record['subfamily'] in subfamilies) <<< ADD THIS LINE
373 or ('genus' in record.keys() and record['genus'] in genera)
374 ...
376 If you want the new addition to be highlighted in the table on the webpage, add the name to the ranksToHighlight
377 list in vars/qaqc.js, at ~line 340
378 """
379 classes = ['Ophiuroidea']
380 orders = ['Comatulida']
381 infraorders = ['Anomura', 'Caridea']
382 families = ['Goniasteridae', 'Poecilasmatidae', 'Parazoanthidae', 'Tubulariidae', 'Amphianthidae', 'Actinoscyphiidae']
383 genera = ['Henricia']
384 concepts = ['Hydroidolina']
385 for name in self.sequence_names:
386 for annotation in self.fetch_annotations(name):
387 if annotation.get('group') == 'localization':
388 continue
389 self.working_records.append(annotation)
390 self.sort_records(self.process_working_records(self.videos))
391 temp_records = self.final_records
392 self.final_records = []
393 for record in temp_records:
394 if record.get('class') in classes \
395 or record.get('order') in orders \
396 or record.get('infraorder') in infraorders \
397 or record.get('family') in families \
398 or record.get('genus') in genera \
399 or record.get('concept') in concepts:
400 upon = get_association(record, 'upon')
401 if upon and upon['to_concept'][0].islower() and 'dead' not in upon['to_concept']:
402 self.final_records.append(record)
404 def find_long_host_associate_time_diff(self):
405 greater_than_one_min = {}
406 greater_than_five_mins = {}
407 not_found = []
408 for name in self.sequence_names:
409 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
410 for i in range(len(sorted_annotations)):
411 associate_record = sorted_annotations[i]
412 upon = get_association(sorted_annotations[i], 'upon')
413 if upon and upon['to_concept'] and upon['to_concept'][0].isupper():
414 # the associate's 'upon' is an organism
415 host_concept_name = upon['to_concept']
416 observation_time = extract_recorded_datetime(associate_record)
417 found = False
418 for j in range(i + 10, -1, -1):
419 """
420 Checks backward, looking for the most recent host w/ matching name. We start at i + 10 because
421 there can be multiple records with the exact same timestamp, and one of those records could be
422 the 'upon'
423 """
424 # to catch index out of range exception
425 while j >= len(sorted_annotations):
426 j -= 1
427 host_record = sorted_annotations[j]
428 host_time = extract_recorded_datetime(host_record)
429 if host_time > observation_time or i == j:
430 # host record won't be recorded after associate record, ignore this record
431 # i == j: record shouldn't be associated with itself, ignore
432 pass
433 else:
434 if host_record['concept'] == host_concept_name:
435 # the host record's name is equal to the host concept name (associate's 'upon' name)
436 found = True
437 time_diff = observation_time - host_time
438 if time_diff.seconds > 300:
439 greater_than_five_mins[associate_record['observation_uuid']] = time_diff
440 self.working_records.append(associate_record)
441 elif time_diff.seconds > 60:
442 greater_than_one_min[associate_record['observation_uuid']] = time_diff
443 self.working_records.append(associate_record)
444 break
445 if not found:
446 not_found.append(associate_record['observation_uuid'])
447 self.working_records.append(associate_record)
448 self.sort_records(self.process_working_records(self.videos))
449 for uuid in greater_than_one_min.keys():
450 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
451 'Time between record and closest previous matching host record greater than one minute ' \
452 f'({greater_than_one_min[uuid].seconds} seconds)'
453 for uuid in greater_than_five_mins.keys():
454 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
455 'Time between record and closest previous matching host record greater than five minutes ' \
456 f'({greater_than_five_mins[uuid].seconds // 60 % 60} mins, {greater_than_five_mins[uuid].seconds % 60} seconds)'
457 for uuid in not_found:
458 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
459 f'Host not found in previous records'
461 def find_num_bounding_boxes(self):
462 bounding_box_counts = {}
463 total_count_annos = 0
464 total_count_boxes = 0
465 for name in self.sequence_names:
466 for annotation in self.fetch_annotations(name):
467 total_count_annos += 1
468 if annotation['concept'] not in bounding_box_counts.keys():
469 bounding_box_counts[annotation['concept']] = {
470 'boxes': 0,
471 'annos': 0,
472 }
473 bounding_box_counts[annotation['concept']]['annos'] += 1
474 if get_association(annotation, 'bounding box'):
475 total_count_boxes += 1
476 bounding_box_counts[annotation['concept']]['boxes'] += 1
477 sorted_box_counts = dict(sorted(bounding_box_counts.items()))
478 self.final_records.append({
479 'total_count_annos': total_count_annos,
480 'total_count_boxes': total_count_boxes,
481 'bounding_box_counts': sorted_box_counts,
482 })
484 def find_localizations_without_bounding_boxes(self):
485 """
486 Finds records that are in the "localization" group but do not contain a bounding box association. Also finds
487 records that have a bounding box association but are not in the "localization" group.
488 """
489 for name in self.sequence_names:
490 for annotation in self.fetch_annotations(name):
491 has_box = False
492 for association in annotation['associations']:
493 if association['link_name'] == 'bounding box':
494 has_box = True
495 break
496 if annotation.get('group') == 'localization':
497 if not has_box:
498 self.working_records.append(annotation)
499 elif has_box:
500 self.working_records.append(annotation)
501 self.sort_records(self.process_working_records(self.videos))
503 def find_unique_fields(self):
504 def load_dict(field_name, unique_dict, individual_count):
505 if field_name not in unique_dict.keys():
506 unique_dict[field_name] = {}
507 unique_dict[field_name]['records'] = 1
508 unique_dict[field_name]['individuals'] = individual_count
509 else:
510 unique_dict[field_name]['records'] += 1
511 unique_dict[field_name]['individuals'] += individual_count
513 unique_concept_names = {}
514 unique_concept_upons = {}
515 unique_substrate_combinations = {}
516 unique_comments = {}
517 unique_condition_comments = {}
518 unique_megahabitats = {}
519 unique_habitats = {}
520 unique_habitat_comments = {}
521 unique_id_certainty = {}
522 unique_occurrence_remarks = {}
524 for name in self.sequence_names:
525 for annotation in self.fetch_annotations(name):
526 substrates = []
527 upon = None
528 comment = None
529 condition_comment = None
530 megahabitat = None
531 habitat = None
532 habitat_comment = None
533 id_certainty = None
534 occurrence_remark = None
535 individual_count = 1
537 for association in annotation['associations']:
538 match association['link_name']:
539 case 's1' | 's2':
540 substrates.append(association['to_concept'])
541 case 'upon':
542 upon = association['to_concept']
543 case 'comment':
544 comment = association['link_value']
545 case 'condition-comment':
546 condition_comment = association['link_value']
547 case 'megahabitat':
548 megahabitat = association['to_concept']
549 case 'habitat':
550 habitat = association['to_concept']
551 case 'habitat-comment':
552 habitat_comment = association['link_value']
553 case 'identity-certainty':
554 id_certainty = association['link_value']
555 case 'occurrence-remark':
556 occurrence_remark = association['link_value']
557 case 'population-quantity':
558 if association['link_value'] != '':
559 individual_count = int(association['link_value'])
560 case 'categorical-abundance':
561 match association['link_value']:
562 case '11-20':
563 individual_count = 15
564 case '21-50':
565 individual_count = 35
566 case '51-100':
567 individual_count = 75
568 case '\u003e100':
569 individual_count = 100
571 if substrates is not None:
572 substrates.sort()
573 substrates = ', '.join(substrates)
575 load_dict(annotation['concept'], unique_concept_names, individual_count)
576 load_dict(f'{annotation["concept"]}:{upon}', unique_concept_upons, individual_count)
577 load_dict(substrates, unique_substrate_combinations, individual_count)
578 load_dict(comment, unique_comments, individual_count)
579 load_dict(condition_comment, unique_condition_comments, individual_count)
580 load_dict(megahabitat, unique_megahabitats, individual_count)
581 load_dict(habitat, unique_habitats, individual_count)
582 load_dict(habitat_comment, unique_habitat_comments, individual_count)
583 load_dict(id_certainty, unique_id_certainty, individual_count)
584 load_dict(occurrence_remark, unique_occurrence_remarks, individual_count)
586 self.final_records.append({'concept-names': unique_concept_names})
587 self.final_records.append({'concept-upon-combinations': unique_concept_upons})
588 self.final_records.append({'substrate-combinations': unique_substrate_combinations})
589 self.final_records.append({'comments': unique_comments})
590 self.final_records.append({'condition-comments': unique_condition_comments})
591 self.final_records.append({'megahabitats': unique_megahabitats})
592 self.final_records.append({'habitats': unique_habitats})
593 self.final_records.append({'habitat-comments': unique_habitat_comments})
594 self.final_records.append({'identity-certainty': unique_id_certainty})
595 self.final_records.append({'occurrence-remarks': unique_occurrence_remarks})