Coverage for application/qaqc/tator/routes.py: 11%
194 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-06-23 02:22 +0000
1"""
2Tator-specific QA/QC endpoints
4/qaqc/tator/checklist [GET, PATCH]
5/qaqc/tator/check/<check> [GET]
6/qaqc/tator/attracted-list [GET]
7/qaqc/tator/attracted [POST]
8/qaqc/tator/attracted/<concept> [PATCH, DELETE]
9"""
11from io import BytesIO
13import tator
14import requests
15from flask import current_app, flash, redirect, render_template, request, send_file, session
17from . import tator_qaqc_bp
18from .tator_qaqc_processor import TatorQaqcProcessor
19from ...util.tator_localization_type import TatorLocalizationType
22# view QA/QC checklist for a specified project & section
23@tator_qaqc_bp.get('/checklist')
24def tator_qaqc_checklist():
25 if not request.args.get('project') or not request.args.get('section') or not request.args.getlist('deployment'):
26 flash('Please select a project, section, and deployment', 'info')
27 return redirect('/')
28 project_id = int(request.args.get('project'))
29 section_id = int(request.args.get('section'))
30 if 'tator_token' not in session.keys():
31 flash('Please log in to Tator', 'info')
32 return redirect('/')
33 try:
34 api = tator.get_api(
35 host=current_app.config.get('TATOR_URL'),
36 token=session['tator_token'],
37 )
38 section_name = api.get_section(id=section_id).name
39 except tator.openapi.tator_openapi.exceptions.ApiException:
40 flash('Please log in to Tator', 'info')
41 return redirect('/')
42 media_ids = []
43 localizations = []
44 individual_count = 0
45 deployments = request.args.getlist('deployment')
46 for deployment in deployments:
47 media_ids += session[f'{project_id}_{section_id}'][deployment]
48 with requests.get(
49 url=f'{current_app.config.get("DARC_REVIEW_URL")}/tator-qaqc-checklist/{"&".join(request.args.getlist("deployment"))}',
50 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
51 ) as checklist_res:
52 if checklist_res.status_code == 200:
53 checklist = checklist_res.json()
54 else:
55 print('ERROR: Unable to get QAQC checklist from external review server')
56 checklist = {}
57 # REST is much faster than Python API for large queries
58 # adding too many media ids results in a query that is too long, so we have to break it up
59 for i in range(0, len(media_ids), 300):
60 chunk = media_ids[i:i + 300]
61 res = requests.get(
62 url=f'{current_app.config.get("TATOR_URL")}/rest/Localizations/{project_id}?media_id={",".join(map(str, chunk))}',
63 headers={
64 'Content-Type': 'application/json',
65 'Authorization': f'Token {session["tator_token"]}',
66 })
67 localizations += res.json()
68 for localization in localizations:
69 if localization['type'] == TatorLocalizationType.DOT.value:
70 individual_count += 1
71 if localization['attributes']['Categorical Abundance'] != '--':
72 match localization['attributes']['Categorical Abundance']:
73 case '20-49':
74 individual_count += 35
75 case '50-99':
76 individual_count += 75
77 case '100-999':
78 individual_count += 500
79 case '1000+':
80 individual_count += 1000
81 data = {
82 'title': section_name,
83 'tab_title': deployments[0] if len(deployments) == 1 else f'{deployments[0]} - {deployments[-1].split("_")[-1]}',
84 'localization_count': len(localizations),
85 'individual_count': individual_count,
86 'checklist': checklist,
87 }
88 return render_template('qaqc/tator/qaqc-checklist.html', data=data)
91# update tator qaqc checklist
92@tator_qaqc_bp.patch('/checklist')
93def patch_tator_qaqc_checklist():
94 req_json = request.json
95 deployments = req_json.get('deployments')
96 if not deployments:
97 return {}, 400
98 req_json.pop('deployments')
99 res = requests.patch(
100 url=f'{current_app.config.get("DARC_REVIEW_URL")}/tator-qaqc-checklist/{deployments}',
101 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
102 json=req_json,
103 )
104 return res.json(), res.status_code
107# individual qaqc checks
108@tator_qaqc_bp.get('/check/<check>')
109def tator_qaqc(check):
110 if 'tator_token' not in session.keys():
111 flash('Please log in to Tator', 'info')
112 return redirect('/')
113 try:
114 api = tator.get_api(
115 host=current_app.config.get('TATOR_URL'),
116 token=session['tator_token'],
117 )
118 except tator.openapi.tator_openapi.exceptions.ApiException:
119 flash('Please log in to Tator', 'info')
120 return redirect('/')
121 # get comments and image references from external review db
122 comments = {}
123 image_refs = {}
124 if not request.args.get('project') or not request.args.get('section') or not request.args.getlist('deployment'):
125 flash('Please select a project, section, and deployment', 'info')
126 return redirect('/')
127 project_id = int(request.args.get('project'))
128 section_id = int(request.args.get('section'))
129 deployments = request.args.getlist('deployment')
130 try:
131 for deployment in deployments:
132 comment_res = requests.get(
133 url=f'{current_app.config.get("DARC_REVIEW_URL")}/comment/sequence/{deployment}',
134 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
135 )
136 if comment_res.status_code != 200:
137 raise requests.exceptions.ConnectionError
138 comments |= comment_res.json() # merge dicts
139 image_ref_res = requests.get(f'{current_app.config.get("DARC_REVIEW_URL")}/image-reference/quick')
140 if image_ref_res.status_code != 200:
141 raise requests.exceptions.ConnectionError
142 image_refs = image_ref_res.json()
143 except requests.exceptions.ConnectionError:
144 print('\nERROR: unable to connect to external review server\n')
145 tab_title = deployments[0] if len(deployments) == 1 else f'{deployments[0]} - {deployments[-1].split("_")[-1]}'
146 data = {
147 'concepts': session.get('vars_concepts', []),
148 'title': check.replace('-', ' ').title(),
149 'tab_title': f'{tab_title} {check.replace("-", " ").title()}',
150 'reviewers': session.get('reviewers', []),
151 'comments': comments,
152 'image_refs': image_refs,
153 }
154 if check == 'media-attributes':
155 # the one case where we don't want to initialize a TatorQaqcProcessor (no need to fetch localizations)
156 media_attributes = {}
157 for deployment in request.args.getlist('deployment'):
158 media_attributes[deployment] = []
159 res = requests.get( # REST API is much faster than Python API for large queries
160 url=f'{current_app.config.get("TATOR_URL")}/rest/Medias/{project_id}?section={section_id}&attribute_contains=%24name%3A%3A{deployment}',
161 headers={
162 'Content-Type': 'application/json',
163 'Authorization': f'Token {session["tator_token"]}',
164 })
165 if res.status_code != 200:
166 raise tator.openapi.tator_openapi.exceptions.ApiException
167 for media in res.json():
168 media_attributes[deployment].append(media)
169 data['page_title'] = 'Media attributes'
170 data['media_attributes'] = media_attributes
171 return render_template('qaqc/tator/qaqc-tables.html', data=data)
172 qaqc_annos = TatorQaqcProcessor(
173 project_id=project_id,
174 section_id=section_id,
175 api=api,
176 deployment_list=request.args.getlist('deployment'),
177 darc_review_url=current_app.config.get('DARC_REVIEW_URL'),
178 tator_url=current_app.config.get('TATOR_URL'),
179 )
180 qaqc_annos.fetch_localizations()
181 qaqc_annos.load_phylogeny()
182 match check:
183 case 'names-accepted':
184 qaqc_annos.check_names_accepted()
185 data['page_title'] = 'Scientific names/tentative IDs not accepted in WoRMS'
186 case 'missing-qualifier':
187 qaqc_annos.check_missing_qualifier()
188 data['page_title'] = 'Records classified higher than species missing qualifier'
189 case 'stet-missing-reason':
190 qaqc_annos.check_stet_reason()
191 data['page_title'] = 'Records with a qualifier of \'stet\' missing \'Reason\''
192 case 'attracted-not-attracted':
193 attracted_concepts = requests.get(url=f'{current_app.config.get("DARC_REVIEW_URL")}/attracted').json()
194 qaqc_annos.check_attracted_not_attracted(attracted_concepts)
195 data['page_title'] = 'Attracted/not attracted match expected taxa list'
196 data['attracted_concepts'] = attracted_concepts
197 case 'exists-in-image-references':
198 qaqc_annos.check_exists_in_image_references(image_refs)
199 data['page_title'] = 'Records that do not exist in image references'
200 case 'same-name-qualifier':
201 qaqc_annos.check_same_name_qualifier()
202 data['page_title'] = 'Records with the same scientific name/tentative ID but different qualifiers'
203 case 'non-target-not-attracted':
204 qaqc_annos.check_non_target_not_attracted()
205 data['page_title'] = '"Non-target" records marked as "attracted"'
206 case 'all-tentative-ids':
207 qaqc_annos.get_all_tentative_ids()
208 data['page_title'] = 'Records with a tentative ID (also checks phylogeny vs. scientific name)'
209 case 'notes-and-remarks':
210 qaqc_annos.get_all_notes_and_remarks()
211 data['page_title'] = 'Records with notes and/or remarks'
212 case 're-examined':
213 qaqc_annos.get_re_examined()
214 data['page_title'] = 'Records marked "to be re-examined"'
215 case 'unique-taxa':
216 qaqc_annos.get_unique_taxa()
217 data['page_title'] = 'All unique taxa'
218 data['unique_taxa'] = qaqc_annos.final_records
219 return render_template('qaqc/tator/qaqc-tables.html', data=data)
220 case 'summary':
221 qaqc_annos.get_summary()
222 data['page_title'] = 'Summary'
223 data['annotations'] = qaqc_annos.final_records
224 return render_template('qaqc/tator/qaqc-tables.html', data=data)
225 case 'max-n':
226 qaqc_annos.get_max_n()
227 data['page_title'] = 'Max N'
228 data['max_n'] = qaqc_annos.final_records
229 return render_template('qaqc/tator/qaqc-tables.html', data=data)
230 case 'tofa':
231 qaqc_annos.get_tofa()
232 data['page_title'] = 'Time of First Arrival'
233 data['tofa'] = qaqc_annos.final_records
234 return render_template('qaqc/tator/qaqc-tables.html', data=data)
235 case 'image-guide':
236 presentation_data = BytesIO()
237 qaqc_annos.download_image_guide(current_app).save(presentation_data)
238 presentation_data.seek(0)
239 return send_file(presentation_data, as_attachment=True, download_name='image-guide.pptx')
240 case _:
241 return render_template('errors/404.html', err=''), 404
242 data['annotations'] = qaqc_annos.final_records
243 return render_template('qaqc/tator/qaqc.html', data=data)
246# view list of saved attracted/non-attracted taxa
247@tator_qaqc_bp.get('/attracted-list')
248def attracted_list():
249 res = requests.get(url=f'{current_app.config.get("DARC_REVIEW_URL")}/attracted')
250 return render_template('qaqc/tator/attracted-list.html', attracted_concepts=res.json()), 200
253# add a new concept to the attracted collection
254@tator_qaqc_bp.post('/attracted')
255def add_attracted():
256 res = requests.post(
257 url=f'{current_app.config.get("DARC_REVIEW_URL")}/attracted',
258 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
259 data={
260 'scientific_name': request.values.get('concept'),
261 'attracted': request.values.get('attracted'),
262 },
263 )
264 if res.status_code == 201:
265 flash(f'Added {request.values.get("concept")}', 'success')
266 else:
267 flash(f'Failed to add {request.values.get("concept")}', 'danger')
268 return res.json(), res.status_code
271# update an existing attracted concept
272@tator_qaqc_bp.patch('/attracted/<concept>')
273def update_attracted(concept):
274 res = requests.patch(
275 url=f'{current_app.config.get("DARC_REVIEW_URL")}/attracted/{concept}',
276 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
277 data={
278 'attracted': request.values.get('attracted'),
279 }
280 )
281 if res.status_code == 200:
282 flash(f'Updated {concept}', 'success')
283 return res.json(), res.status_code
286# delete an attracted concept
287@tator_qaqc_bp.delete('/attracted/<concept>')
288def delete_attracted(concept):
289 res = requests.delete(
290 url=f'{current_app.config.get("DARC_REVIEW_URL")}/attracted/{concept}',
291 headers=current_app.config.get('DARC_REVIEW_HEADERS'),
292 )
293 if res.status_code == 200:
294 flash(f'Deleted {concept}', 'success')
295 return res.json(), res.status_code