Coverage for application/qaqc/vars/vars_qaqc_processor.py: 89%
424 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-23 02:22 +0000
1import requests
2import sys
4from application.util.functions import *
5from application.image_review.vars.vars_annotation_processor import VarsAnnotationProcessor
8class VarsQaqcProcessor(VarsAnnotationProcessor):
9 """
10 Filters and formats annotations for the various DARC QA/QC checks.
11 """
13 def __init__(self, sequence_names: list, vars_dive_url: str, vars_phylogeny_url: str):
14 super().__init__(sequence_names, vars_dive_url, vars_phylogeny_url)
15 self.videos = []
16 self.load_phylogeny()
18 def fetch_annotations(self, seq_name):
19 """
20 Fetches annotations for a given sequence name from VARS
21 """
22 print(f'Fetching annotations for sequence {seq_name} from VARS...', end='')
23 sys.stdout.flush()
25 res = requests.get(url=f'{self.vars_dive_url}/{seq_name.replace(" ", "%20")}')
26 dive_json = res.json()
27 print('fetched!')
29 for video in dive_json['media']:
30 if 'urn:imagecollection:org' not in video['uri']:
31 self.videos.append({
32 'start_timestamp': parse_datetime(video['start_timestamp']),
33 'uri': video['uri'].replace('http://hurlstor.soest.hawaii.edu/videoarchive', 'https://hurlvideo.soest.hawaii.edu'),
34 'sequence_name': video['video_sequence_name'],
35 'video_reference_uuid': video['video_reference_uuid'],
36 })
37 return dive_json['annotations']
39 def find_duplicate_associations(self):
40 """
41 Finds annotations that have more than one of the same association besides s2
42 """
43 for name in self.sequence_names:
44 for annotation in self.fetch_annotations(name):
45 if annotation.get('group') == 'localization':
46 continue
47 # get list of associations
48 association_set = set()
49 duplicate_associations = False
50 for association in annotation['associations']:
51 name = association['link_name']
52 if name not in association_set:
53 if name != 's2':
54 association_set.add(name)
55 else:
56 duplicate_associations = True
57 break
58 if duplicate_associations:
59 self.working_records.append(annotation)
60 self.sort_records(self.process_working_records(self.videos))
62 def find_missing_s1(self):
63 """
64 Finds annotations that are missing s1 (ignores 'none' records)
65 """
66 for name in self.sequence_names:
67 for annotation in self.fetch_annotations(name):
68 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
69 continue
70 s1 = get_association(annotation, 's1')
71 if not s1:
72 self.working_records.append(annotation)
73 self.sort_records(self.process_working_records(self.videos))
75 def find_identical_s1_s2(self):
76 """
77 Finds annotations that have an s2 association that is the same as its s1 association
78 """
79 for name in self.sequence_names:
80 for annotation in self.fetch_annotations(name):
81 if annotation.get('group') == 'localization':
82 continue
83 s2s = []
84 s1 = ''
85 for association in annotation['associations']:
86 if association['link_name'] == 's1':
87 s1 = association['to_concept']
88 elif association['link_name'] == 's2':
89 s2s.append(association['to_concept'])
90 if s1 in s2s:
91 self.working_records.append(annotation)
92 self.sort_records(self.process_working_records(self.videos))
94 def find_duplicate_s2(self):
95 """
96 Finds annotations that have multiple s2 associations with the same value
97 """
98 for name in self.sequence_names:
99 for annotation in self.fetch_annotations(name):
100 if annotation.get('group') == 'localization':
101 continue
102 duplicate_s2s = False
103 s2_set = set()
104 for association in annotation['associations']:
105 if association['link_name'] == 's2':
106 if association['to_concept'] in s2_set:
107 duplicate_s2s = True
108 break
109 else:
110 s2_set.add(association['to_concept'])
111 if duplicate_s2s:
112 self.working_records.append(annotation)
113 self.sort_records(self.process_working_records(self.videos))
115 def find_missing_upon_substrate(self):
116 """
117 Finds annotations that have an upon association that is not an organism, but the 'upon' is not present in s1 or
118 any s2
119 """
120 for name in self.sequence_names:
121 for annotation in self.fetch_annotations(name):
122 if annotation.get('group') == 'localization':
123 continue
124 upon = None
125 missing_upon = False
126 for association in annotation['associations']:
127 if association['link_name'] == 'upon':
128 if (association['to_concept'] and association['to_concept'][0].isupper()) \
129 or association['to_concept'].startswith('dead'):
130 # 'upon' is an organism, don't need it to be in s1/s2
131 pass
132 else:
133 # 'upon' should be in s1 or s2
134 upon = association['to_concept']
135 break
136 if upon:
137 missing_upon = True
138 for association in annotation['associations']:
139 if (association['link_name'] == 's1' or association['link_name'] == 's2') \
140 and association['to_concept'] == upon:
141 missing_upon = False
142 break
143 if missing_upon:
144 self.working_records.append(annotation)
145 self.sort_records(self.process_working_records(self.videos))
147 def find_mismatched_substrates(self):
148 """
149 Finds annotations that occur at the same timestamp (same second) but have different substrates
150 """
151 for name in self.sequence_names:
152 annotations_with_same_timestamp = {}
153 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
154 # loop through all annotations, add ones with same timestamp to dict
155 i = 0
156 while i < len(sorted_annotations) - 2:
157 if sorted_annotations[i].get('group') == 'localization':
158 i += 1
159 continue
160 base_timestamp = sorted_annotations[i]['recorded_timestamp'][:19]
161 base_annotation = sorted_annotations[i]
162 i += 1
163 while sorted_annotations[i]['recorded_timestamp'][:19] == base_timestamp:
164 if sorted_annotations[i].get('group') != 'localization':
165 if base_timestamp not in annotations_with_same_timestamp.keys():
166 annotations_with_same_timestamp[base_timestamp] = [base_annotation]
167 annotations_with_same_timestamp[base_timestamp].append(sorted_annotations[i])
168 i += 1
169 # loop through each annotation that shares the same timestamp, compare substrates
170 for timestamp_key in annotations_with_same_timestamp.keys():
171 base_substrates = {'s2': set()}
172 check_substrates = {'s2': set()}
173 for association in annotations_with_same_timestamp[timestamp_key][0]['associations']:
174 if association['link_name'] == 's1':
175 base_substrates['s1'] = association['to_concept']
176 if association['link_name'] == 's2':
177 base_substrates['s2'].add(association['to_concept'])
178 for i in range(1, len(annotations_with_same_timestamp[timestamp_key])):
179 for association in annotations_with_same_timestamp[timestamp_key][i]['associations']:
180 if association['link_name'] == 's1':
181 check_substrates['s1'] = association['to_concept']
182 if association['link_name'] == 's2':
183 check_substrates['s2'].add(association['to_concept'])
184 if base_substrates != check_substrates:
185 for annotation in annotations_with_same_timestamp[timestamp_key]:
186 self.working_records.append(annotation)
187 self.sort_records(self.process_working_records(self.videos))
189 def find_missing_upon(self):
190 """
191 Finds annotations that are missing upon (ignores 'none' records)
192 """
193 for name in self.sequence_names:
194 for annotation in self.fetch_annotations(name):
195 if annotation['concept'] == 'none' or annotation.get('group') == 'localization':
196 continue
197 if not get_association(annotation, 'upon'):
198 self.working_records.append(annotation)
199 self.sort_records(self.process_working_records(self.videos))
201 def get_num_records_missing_ancillary_data(self):
202 """
203 Finds number of annotations that are missing ancillary data
204 """
205 num_records_missing = 0
206 for name in self.sequence_names:
207 for annotation in self.fetch_annotations(name):
208 if 'ancillary_data' not in annotation.keys():
209 num_records_missing += 1
210 return num_records_missing
212 def find_missing_ancillary_data(self):
213 """
214 Finds annotations that are missing ancillary data (can be very slow)
215 """
216 for name in self.sequence_names:
217 for annotation in self.fetch_annotations(name):
218 if annotation.get('group') == 'localization':
219 continue
220 if 'ancillary_data' not in annotation.keys():
221 self.working_records.append(annotation)
222 self.sort_records(self.process_working_records(self.videos))
224 def find_id_refs_different_concept_name(self):
225 """
226 Finds annotations with the same ID reference that have different concept names
227 """
228 for name in self.sequence_names:
229 id_ref_names = {} # dict of {id_ref: {name_1, name_2}} to check for more than one name
230 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
231 for annotation in self.fetch_annotations(name):
232 if annotation.get('group') == 'localization':
233 continue
234 for association in annotation['associations']:
235 if association['link_name'] == 'identity-reference':
236 if association['link_value'] not in id_ref_names.keys():
237 id_ref_names[association['link_value']] = set()
238 id_ref_annotations[association['link_value']] = []
239 id_ref_names[association['link_value']].add(annotation['concept'])
240 id_ref_annotations[association['link_value']].append(annotation)
241 break
242 for id_ref, name_set in id_ref_names.items():
243 if len(name_set) > 1:
244 for annotation in id_ref_annotations[id_ref]:
245 self.working_records.append(annotation)
246 self.sort_records(self.process_working_records(self.videos))
248 def find_id_refs_conflicting_associations(self):
249 """
250 Finds annotations with the same ID reference that have conflicting associations
251 """
252 to_concepts = ['s1', 's2', 'upon', 'size', 'habitat', 'megahabitat', 'sampled-by']
253 for name in self.sequence_names:
254 id_ref_associations = {} # dict of {id_ref: {ass_1_name: ass_1_val, ass_2_name: ass_2_val}}
255 id_ref_annotations = {} # dict of all annotations per id_ref: {id_ref: [annotation_1, annotation_2]}
256 for annotation in self.fetch_annotations(name):
257 if annotation.get('group') == 'localization':
258 continue
259 id_ref = get_association(annotation, 'identity-reference')
260 if id_ref:
261 current_id_ref = id_ref['link_value']
262 if current_id_ref not in id_ref_associations.keys():
263 id_ref_associations[current_id_ref] = {
264 'flag': False, # we'll set this to true if we find any conflicting associations
265 's2': set(), # s2, sampled-by, and sample-reference are allowed to have
266 'sampled-by': set(), # more than one association
267 'sample-reference': set(),
268 }
269 id_ref_annotations[current_id_ref] = [annotation]
270 # populate id_ref dict with all associations
271 for ass in annotation['associations']:
272 if ass['link_name'] == 'guide-photo':
273 pass
274 elif ass['link_name'] == 's2' or ass['link_name'] == 'sampled-by':
275 id_ref_associations[current_id_ref][ass['link_name']].add(ass['to_concept'])
276 elif ass['link_name'] == 'sample-reference':
277 id_ref_associations[current_id_ref][ass['link_name']].add(ass['link_value'])
278 else:
279 id_ref_associations[current_id_ref][ass['link_name']] = \
280 ass['link_value'] if ass['link_name'] not in to_concepts else ass['to_concept']
281 else:
282 # check current association values vs those saved
283 id_ref_annotations[current_id_ref].append(annotation)
284 temp_s2_set = set()
285 temp_sampled_by_set = set()
286 temp_sample_ref_set = set()
287 for ass in annotation['associations']:
288 if ass['link_name'] == 'guide-photo':
289 pass
290 elif ass['link_name'] == 's2':
291 temp_s2_set.add(ass['to_concept'])
292 elif ass['link_name'] == 'sampled-by':
293 temp_sampled_by_set.add(ass['to_concept'])
294 elif ass['link_name'] == 'sample-reference':
295 temp_sample_ref_set.add(ass['link_value'])
296 else:
297 if ass['link_name'] in to_concepts:
298 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
299 # cases like 'guide-photo' will only be present on one record
300 if id_ref_associations[current_id_ref][ass['link_name']] != ass['to_concept']:
301 id_ref_associations[current_id_ref]['flag'] = True
302 break
303 else:
304 id_ref_associations[current_id_ref][ass['link_name']] = ass['to_concept']
305 else:
306 if ass['link_name'] in id_ref_associations[current_id_ref].keys():
307 if id_ref_associations[current_id_ref][ass['link_name']] != ass['link_value']:
308 id_ref_associations[current_id_ref]['flag'] = True
309 break
310 else:
311 id_ref_associations[current_id_ref][ass['link_name']] = ass['link_value']
312 if temp_s2_set != id_ref_associations[current_id_ref]['s2'] \
313 or temp_sampled_by_set != id_ref_associations[current_id_ref]['sampled-by'] \
314 or temp_sample_ref_set != id_ref_associations[current_id_ref]['sample-reference']:
315 id_ref_associations[current_id_ref]['flag'] = True
316 for id_ref in id_ref_associations.keys():
317 if id_ref_associations[id_ref]['flag']:
318 for annotation in id_ref_annotations[id_ref]:
319 self.working_records.append(annotation)
320 self.sort_records(self.process_working_records(self.videos))
322 def find_blank_associations(self):
323 """
324 Finds all records that have associations with a link value of ""
325 """
326 for name in self.sequence_names:
327 for annotation in self.fetch_annotations(name):
328 if annotation.get('group') == 'localization':
329 continue
330 for association in annotation['associations']:
331 if association['link_value'] == '' and association['to_concept'] == 'self':
332 self.working_records.append(annotation)
333 self.sort_records(self.process_working_records(self.videos))
335 def find_suspicious_hosts(self):
336 """
337 Finds annotations that have an upon that is the same concept as itself
338 """
339 for name in self.sequence_names:
340 for annotation in self.fetch_annotations(name):
341 if annotation.get('group') == 'localization':
342 continue
343 upon = get_association(annotation, 'upon')
344 if upon and upon['to_concept'] == annotation['concept']:
345 self.working_records.append(annotation)
346 self.sort_records(self.process_working_records(self.videos))
348 def find_missing_expected_association(self):
349 """
350 Finds annotations that are expected to be upon another organism, but are not. This is a very slow test because
351 before it can begin, we must retrieve the taxa from VARS for every record (unlike the other tests, we can't
352 filter beforehand).
354 If more concepts need to be added for this check, simply add them to the appropriate list below:
356 Example: To add the order 'order123' to the list, change the declaration below from:
358 orders = ['Comatulida']
360 to:
362 orders = ['Comatulida', 'order123']
364 If a list does not exist, declare a new list and add it to the conditional:
366 Example: To add the subfamily 'subfam123' to the check, add a new list named 'subfamilies':
368 subfamilies = ['subfam123']
370 Then add the new list to the conditional:
372 ...
373 or ('family' in record.keys() and record['family'] in families)
374 or ('subfamily' in record.keys() and record['subfamily'] in subfamilies) <<< ADD THIS LINE
375 or ('genus' in record.keys() and record['genus'] in genera)
376 ...
378 If you want the new addition to be highlighted in the table on the webpage, add the name to the ranksToHighlight
379 list in vars/qaqc.js, at ~line 340
380 """
381 classes = ['Ophiuroidea']
382 orders = ['Comatulida']
383 infraorders = ['Anomura', 'Caridea']
384 families = ['Goniasteridae', 'Poecilasmatidae', 'Parazoanthidae', 'Tubulariidae', 'Amphianthidae', 'Actinoscyphiidae']
385 genera = ['Henricia']
386 concepts = ['Hydroidolina']
387 for name in self.sequence_names:
388 for annotation in self.fetch_annotations(name):
389 if annotation.get('group') == 'localization':
390 continue
391 self.working_records.append(annotation)
392 self.sort_records(self.process_working_records(self.videos))
393 temp_records = self.final_records
394 self.final_records = []
395 for record in temp_records:
396 if record.get('class') in classes \
397 or record.get('order') in orders \
398 or record.get('infraorder') in infraorders \
399 or record.get('family') in families \
400 or record.get('genus') in genera \
401 or record.get('concept') in concepts:
402 upon = get_association(record, 'upon')
403 if upon and upon['to_concept'][0].islower() and 'dead' not in upon['to_concept']:
404 self.final_records.append(record)
406 def find_long_host_associate_time_diff(self):
407 greater_than_one_min = {}
408 greater_than_five_mins = {}
409 not_found = []
410 for name in self.sequence_names:
411 sorted_annotations = sorted(self.fetch_annotations(name), key=lambda d: d['recorded_timestamp'])
412 for i in range(len(sorted_annotations)):
413 associate_record = sorted_annotations[i]
414 upon = get_association(sorted_annotations[i], 'upon')
415 if upon and upon['to_concept'] and upon['to_concept'][0].isupper():
416 # the associate's 'upon' is an organism
417 host_concept_name = upon['to_concept']
418 observation_time = extract_recorded_datetime(associate_record)
419 found = False
420 for j in range(i + 10, -1, -1):
421 """
422 Checks backward, looking for the most recent host w/ matching name. We start at i + 10 because
423 there can be multiple records with the exact same timestamp, and one of those records could be
424 the 'upon'
425 """
426 # to catch index out of range exception
427 while j >= len(sorted_annotations):
428 j -= 1
429 host_record = sorted_annotations[j]
430 host_time = extract_recorded_datetime(host_record)
431 if host_time > observation_time or i == j:
432 # host record won't be recorded after associate record, ignore this record
433 # i == j: record shouldn't be associated with itself, ignore
434 pass
435 else:
436 if host_record['concept'] == host_concept_name:
437 # the host record's name is equal to the host concept name (associate's 'upon' name)
438 found = True
439 time_diff = observation_time - host_time
440 if time_diff.seconds > 300:
441 greater_than_five_mins[associate_record['observation_uuid']] = time_diff
442 self.working_records.append(associate_record)
443 elif time_diff.seconds > 60:
444 greater_than_one_min[associate_record['observation_uuid']] = time_diff
445 self.working_records.append(associate_record)
446 break
447 if not found:
448 not_found.append(associate_record['observation_uuid'])
449 self.working_records.append(associate_record)
450 self.sort_records(self.process_working_records(self.videos))
451 for uuid in greater_than_one_min.keys():
452 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
453 'Time between record and closest previous matching host record greater than one minute ' \
454 f'({greater_than_one_min[uuid].seconds} seconds)'
455 for uuid in greater_than_five_mins.keys():
456 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
457 'Time between record and closest previous matching host record greater than five minutes ' \
458 f'({greater_than_five_mins[uuid].seconds // 60 % 60} mins, {greater_than_five_mins[uuid].seconds % 60} seconds)'
459 for uuid in not_found:
460 next((x for x in self.final_records if x['observation_uuid'] == uuid), None)['status'] = \
461 f'Host not found in previous records'
463 def find_num_bounding_boxes(self):
464 bounding_box_counts = {}
465 total_count_annos = 0
466 total_count_boxes = 0
467 for name in self.sequence_names:
468 for annotation in self.fetch_annotations(name):
469 total_count_annos += 1
470 if annotation['concept'] not in bounding_box_counts.keys():
471 bounding_box_counts[annotation['concept']] = {
472 'boxes': 0,
473 'annos': 0,
474 }
475 bounding_box_counts[annotation['concept']]['annos'] += 1
476 if get_association(annotation, 'bounding box'):
477 total_count_boxes += 1
478 bounding_box_counts[annotation['concept']]['boxes'] += 1
479 sorted_box_counts = dict(sorted(bounding_box_counts.items()))
480 self.final_records.append({
481 'total_count_annos': total_count_annos,
482 'total_count_boxes': total_count_boxes,
483 'bounding_box_counts': sorted_box_counts,
484 })
486 def find_localizations_without_bounding_boxes(self):
487 """
488 Finds records that are in the "localization" group but do not contain a bounding box association. Also finds
489 records that have a bounding box association but are not in the "localization" group.
490 """
491 for name in self.sequence_names:
492 for annotation in self.fetch_annotations(name):
493 has_box = False
494 for association in annotation['associations']:
495 if association['link_name'] == 'bounding box':
496 has_box = True
497 break
498 if annotation.get('group') == 'localization':
499 if not has_box:
500 self.working_records.append(annotation)
501 elif has_box:
502 self.working_records.append(annotation)
503 self.sort_records(self.process_working_records(self.videos))
505 def find_unique_fields(self):
506 def load_dict(field_name, unique_dict, individual_count):
507 if field_name not in unique_dict.keys():
508 unique_dict[field_name] = {}
509 unique_dict[field_name]['records'] = 1
510 unique_dict[field_name]['individuals'] = individual_count
511 else:
512 unique_dict[field_name]['records'] += 1
513 unique_dict[field_name]['individuals'] += individual_count
515 unique_concept_names = {}
516 unique_concept_upons = {}
517 unique_substrate_combinations = {}
518 unique_comments = {}
519 unique_condition_comments = {}
520 unique_megahabitats = {}
521 unique_habitats = {}
522 unique_habitat_comments = {}
523 unique_id_certainty = {}
524 unique_occurrence_remarks = {}
526 for name in self.sequence_names:
527 for annotation in self.fetch_annotations(name):
528 substrates = []
529 upon = None
530 comment = None
531 condition_comment = None
532 megahabitat = None
533 habitat = None
534 habitat_comment = None
535 id_certainty = None
536 occurrence_remark = None
537 individual_count = 1
539 for association in annotation['associations']:
540 match association['link_name']:
541 case 's1' | 's2':
542 substrates.append(association['to_concept'])
543 case 'upon':
544 upon = association['to_concept']
545 case 'comment':
546 comment = association['link_value']
547 case 'condition-comment':
548 condition_comment = association['link_value']
549 case 'megahabitat':
550 megahabitat = association['to_concept']
551 case 'habitat':
552 habitat = association['to_concept']
553 case 'habitat-comment':
554 habitat_comment = association['link_value']
555 case 'identity-certainty':
556 id_certainty = association['link_value']
557 case 'occurrence-remark':
558 occurrence_remark = association['link_value']
559 case 'population-quantity':
560 if association['link_value'] != '':
561 individual_count = int(association['link_value'])
562 case 'categorical-abundance':
563 match association['link_value']:
564 case '11-20':
565 individual_count = 15
566 case '21-50':
567 individual_count = 35
568 case '51-100':
569 individual_count = 75
570 case '\u003e100':
571 individual_count = 100
573 if substrates is not None:
574 substrates.sort()
575 substrates = ', '.join(substrates)
577 load_dict(annotation['concept'], unique_concept_names, individual_count)
578 load_dict(f'{annotation["concept"]}:{upon}', unique_concept_upons, individual_count)
579 load_dict(substrates, unique_substrate_combinations, individual_count)
580 load_dict(comment, unique_comments, individual_count)
581 load_dict(condition_comment, unique_condition_comments, individual_count)
582 load_dict(megahabitat, unique_megahabitats, individual_count)
583 load_dict(habitat, unique_habitats, individual_count)
584 load_dict(habitat_comment, unique_habitat_comments, individual_count)
585 load_dict(id_certainty, unique_id_certainty, individual_count)
586 load_dict(occurrence_remark, unique_occurrence_remarks, individual_count)
588 self.final_records.append({'concept-names': unique_concept_names})
589 self.final_records.append({'concept-upon-combinations': unique_concept_upons})
590 self.final_records.append({'substrate-combinations': unique_substrate_combinations})
591 self.final_records.append({'comments': unique_comments})
592 self.final_records.append({'condition-comments': unique_condition_comments})
593 self.final_records.append({'megahabitats': unique_megahabitats})
594 self.final_records.append({'habitats': unique_habitats})
595 self.final_records.append({'habitat-comments': unique_habitat_comments})
596 self.final_records.append({'identity-certainty': unique_id_certainty})
597 self.final_records.append({'occurrence-remarks': unique_occurrence_remarks})