Coverage for test / test_vars_qaqc_processor.py: 100%

1from unittest.mock import patch 

2 

3from application.util.functions import parse_datetime 

4from application.vars.vars_qaqc_processor import VarsQaqcProcessor 

5from test.data.vars_responses import ex_23060001, ex_23060002 

6from test.util.mock_response import MockResponse 

7 

8 

9def mocked_requests_get(*args, **kwargs): 

10 return MockResponse(url=kwargs.get('url')) 

11 

12 

13class TestVarsQaqcProcessor: 

14 def test_init(self): 

15 qaqc_processor = VarsQaqcProcessor( 

16 sequence_names=['Deep Discoverer 23060001'], 

17 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

18 vars_kb_url=MockResponse.VARS_KB_URL, 

19 ) 

20 assert qaqc_processor.sequence_names == ['Deep Discoverer 23060001'] 

21 assert qaqc_processor.videos == [] 

22 assert qaqc_processor.working_records == [] 

23 assert qaqc_processor.final_records == [] 

24 assert len(qaqc_processor.phylogeny.data.keys()) > 0 

25 

26 @patch('requests.get', side_effect=mocked_requests_get) 

27 def test_find_duplicate_associations(self, _): 

28 qaqc_processor_okay = VarsQaqcProcessor( 

29 sequence_names=['Deep Discoverer 23060001'], 

30 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

31 vars_kb_url=MockResponse.VARS_KB_URL, 

32 ) 

33 qaqc_processor_problems = VarsQaqcProcessor( 

34 sequence_names=['Deep Discoverer 23060002'], 

35 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

36 vars_kb_url=MockResponse.VARS_KB_URL, 

37 ) 

38 qaqc_processor_okay.find_duplicate_associations() 

39 qaqc_processor_problems.find_duplicate_associations() 

40 assert qaqc_processor_okay.working_records == [] 

41 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0]] 

42 

43 @patch('requests.get', side_effect=mocked_requests_get) 

44 def test_find_missing_s1(self, _): 

45 qaqc_processor_okay = VarsQaqcProcessor( 

46 sequence_names=['Deep Discoverer 23060001'], 

47 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

48 vars_kb_url=MockResponse.VARS_KB_URL, 

49 ) 

50 qaqc_processor_problems = VarsQaqcProcessor( 

51 sequence_names=['Deep Discoverer 23060002'], 

52 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

53 vars_kb_url=MockResponse.VARS_KB_URL, 

54 ) 

55 qaqc_processor_okay.find_missing_s1() 

56 qaqc_processor_problems.find_missing_s1() 

57 assert qaqc_processor_okay.working_records == [] 

58 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]] 

59 

60 @patch('requests.get', side_effect=mocked_requests_get) 

61 def test_find_identical_s1_s2(self, _): 

62 qaqc_processor_okay = VarsQaqcProcessor( 

63 sequence_names=['Deep Discoverer 23060001'], 

64 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

65 vars_kb_url=MockResponse.VARS_KB_URL, 

66 ) 

67 qaqc_processor_problems = VarsQaqcProcessor( 

68 sequence_names=['Deep Discoverer 23060002'], 

69 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

70 vars_kb_url=MockResponse.VARS_KB_URL, 

71 ) 

72 qaqc_processor_okay.find_identical_s1_s2() 

73 qaqc_processor_problems.find_identical_s1_s2() 

74 assert qaqc_processor_okay.working_records == [] 

75 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2]] 

76 

77 @patch('requests.get', side_effect=mocked_requests_get) 

78 def test_find_duplicate_s2(self, _): 

79 qaqc_processor_okay = VarsQaqcProcessor( 

80 sequence_names=['Deep Discoverer 23060001'], 

81 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

82 vars_kb_url=MockResponse.VARS_KB_URL, 

83 ) 

84 qaqc_processor_problems = VarsQaqcProcessor( 

85 sequence_names=['Deep Discoverer 23060002'], 

86 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

87 vars_kb_url=MockResponse.VARS_KB_URL, 

88 ) 

89 qaqc_processor_okay.find_duplicate_s2() 

90 qaqc_processor_problems.find_duplicate_s2() 

91 assert qaqc_processor_okay.working_records == [] 

92 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]] 

93 

94 @patch('requests.get', side_effect=mocked_requests_get) 

95 def test_find_missing_upon_substrate(self, _): 

96 qaqc_processor_okay = VarsQaqcProcessor( 

97 sequence_names=['Deep Discoverer 23060001'], 

98 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

99 vars_kb_url=MockResponse.VARS_KB_URL, 

100 ) 

101 qaqc_processor_problems = VarsQaqcProcessor( 

102 sequence_names=['Deep Discoverer 23060002'], 

103 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

104 vars_kb_url=MockResponse.VARS_KB_URL, 

105 ) 

106 qaqc_processor_okay.find_missing_upon_substrate() 

107 qaqc_processor_problems.find_missing_upon_substrate() 

108 assert qaqc_processor_okay.working_records == [] 

109 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0]] 

110 

111 @patch('requests.get', side_effect=mocked_requests_get) 

112 def test_find_mismatched_substrates(self, _): 

113 qaqc_processor_okay = VarsQaqcProcessor( 

114 sequence_names=['Deep Discoverer 23060001'], 

115 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

116 vars_kb_url=MockResponse.VARS_KB_URL, 

117 ) 

118 qaqc_processor_problems = VarsQaqcProcessor( 

119 sequence_names=['Deep Discoverer 23060002'], 

120 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

121 vars_kb_url=MockResponse.VARS_KB_URL, 

122 ) 

123 qaqc_processor_okay.find_mismatched_substrates() 

124 qaqc_processor_problems.find_mismatched_substrates() 

125 assert qaqc_processor_okay.working_records == [] 

126 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][3], ex_23060002['annotations'][5]] 

127 

128 @patch('requests.get', side_effect=mocked_requests_get) 

129 def test_find_missing_upon(self, _): 

130 qaqc_processor_okay = VarsQaqcProcessor( 

131 sequence_names=['Deep Discoverer 23060001'], 

132 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

133 vars_kb_url=MockResponse.VARS_KB_URL, 

134 ) 

135 qaqc_processor_problems = VarsQaqcProcessor( 

136 sequence_names=['Deep Discoverer 23060002'], 

137 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

138 vars_kb_url=MockResponse.VARS_KB_URL, 

139 ) 

140 qaqc_processor_okay.find_missing_upon() 

141 qaqc_processor_problems.find_missing_upon() 

142 assert qaqc_processor_okay.working_records == [] 

143 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][3]] 

144 

145 @patch('requests.get', side_effect=mocked_requests_get) 

146 def test_get_num_records_missing_ancillary_data(self, _): 

147 qaqc_processor_okay = VarsQaqcProcessor( 

148 sequence_names=['Deep Discoverer 23060001'], 

149 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

150 vars_kb_url=MockResponse.VARS_KB_URL, 

151 ) 

152 qaqc_processor_problems = VarsQaqcProcessor( 

153 sequence_names=['Deep Discoverer 23060002'], 

154 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

155 vars_kb_url=MockResponse.VARS_KB_URL, 

156 ) 

157 assert qaqc_processor_okay.get_num_records_missing_ancillary_data() == 0 

158 assert qaqc_processor_problems.get_num_records_missing_ancillary_data() == 2 

159 

160 @patch('requests.get', side_effect=mocked_requests_get) 

161 def test_find_missing_ancillary_data(self, _): 

162 qaqc_processor_okay = VarsQaqcProcessor( 

163 sequence_names=['Deep Discoverer 23060001'], 

164 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

165 vars_kb_url=MockResponse.VARS_KB_URL, 

166 ) 

167 qaqc_processor_problems = VarsQaqcProcessor( 

168 sequence_names=['Deep Discoverer 23060002'], 

169 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

170 vars_kb_url=MockResponse.VARS_KB_URL, 

171 ) 

172 qaqc_processor_okay.find_missing_ancillary_data() 

173 qaqc_processor_problems.find_missing_ancillary_data() 

174 assert qaqc_processor_okay.working_records == [] 

175 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]] 

176 

177 @patch('requests.get', side_effect=mocked_requests_get) 

178 def test_find_id_refs_different_concept_name(self, _): 

179 qaqc_processor_okay = VarsQaqcProcessor( 

180 sequence_names=['Deep Discoverer 23060001'], 

181 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

182 vars_kb_url=MockResponse.VARS_KB_URL, 

183 ) 

184 qaqc_processor_problems = VarsQaqcProcessor( 

185 sequence_names=['Deep Discoverer 23060002'], 

186 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

187 vars_kb_url=MockResponse.VARS_KB_URL, 

188 ) 

189 qaqc_processor_okay.find_id_refs_different_concept_name() 

190 qaqc_processor_problems.find_id_refs_different_concept_name() 

191 assert qaqc_processor_okay.working_records == [] 

192 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]] 

193 

194 @patch('requests.get', side_effect=mocked_requests_get) 

195 def test_find_id_refs_conflicting_associations(self, _): 

196 qaqc_processor_okay = VarsQaqcProcessor( 

197 sequence_names=['Deep Discoverer 23060001'], 

198 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

199 vars_kb_url=MockResponse.VARS_KB_URL, 

200 ) 

201 qaqc_processor_problems = VarsQaqcProcessor( 

202 sequence_names=['Deep Discoverer 23060002'], 

203 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

204 vars_kb_url=MockResponse.VARS_KB_URL, 

205 ) 

206 qaqc_processor_okay.find_id_refs_conflicting_associations() 

207 qaqc_processor_problems.find_id_refs_conflicting_associations() 

208 assert qaqc_processor_okay.working_records == [] 

209 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][2], ex_23060002['annotations'][3]] 

210 

211 @patch('requests.get', side_effect=mocked_requests_get) 

212 def test_find_blank_associations(self, _): 

213 qaqc_processor_okay = VarsQaqcProcessor( 

214 sequence_names=['Deep Discoverer 23060001'], 

215 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

216 vars_kb_url=MockResponse.VARS_KB_URL, 

217 ) 

218 qaqc_processor_problems = VarsQaqcProcessor( 

219 sequence_names=['Deep Discoverer 23060002'], 

220 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

221 vars_kb_url=MockResponse.VARS_KB_URL, 

222 ) 

223 qaqc_processor_okay.find_blank_associations() 

224 qaqc_processor_problems.find_blank_associations() 

225 assert qaqc_processor_okay.working_records == [] 

226 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][0], ex_23060002['annotations'][1]] 

227 

228 @patch('requests.get', side_effect=mocked_requests_get) 

229 def test_find_suspicious_hosts(self, _): 

230 qaqc_processor_okay = VarsQaqcProcessor( 

231 sequence_names=['Deep Discoverer 23060001'], 

232 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

233 vars_kb_url=MockResponse.VARS_KB_URL, 

234 ) 

235 qaqc_processor_problems = VarsQaqcProcessor( 

236 sequence_names=['Deep Discoverer 23060002'], 

237 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

238 vars_kb_url=MockResponse.VARS_KB_URL, 

239 ) 

240 qaqc_processor_okay.find_suspicious_hosts() 

241 qaqc_processor_problems.find_suspicious_hosts() 

242 assert qaqc_processor_okay.working_records == [] 

243 assert qaqc_processor_problems.working_records == [ex_23060002['annotations'][1]] 

244 

245 @patch('requests.get', side_effect=mocked_requests_get) 

246 def test_find_missing_expected_association(self, _): 

247 qaqc_processor_okay = VarsQaqcProcessor( 

248 sequence_names=['Deep Discoverer 23060001'], 

249 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

250 vars_kb_url=MockResponse.VARS_KB_URL, 

251 ) 

252 qaqc_processor_problems = VarsQaqcProcessor( 

253 sequence_names=['Deep Discoverer 23060002'], 

254 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

255 vars_kb_url=MockResponse.VARS_KB_URL, 

256 ) 

257 qaqc_processor_okay.find_missing_expected_association() 

258 qaqc_processor_problems.find_missing_expected_association() 

259 assert qaqc_processor_okay.final_records == [] 

260 assert qaqc_processor_problems.final_records == [ 

261 { 

262 'observation_uuid': '006fb032-13b5-4517-136c-11aa9597e81e', 

263 'concept': 'Hydroidolina', 

264 'associations': ex_23060002['annotations'][0]['associations'], 

265 'activity': 'cruise', 

266 'annotator': 'Nikki Cunanan', 

267 'depth': 4255.0, 

268 'phylum': 'Cnidaria', 

269 'class': 'Hydrozoa', 

270 'order': None, 

271 'family': None, 

272 'genus': None, 

273 'species': None, 

274 'identity_reference': '50', 

275 'image_url': '', 

276 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=3725', 

277 'recorded_timestamp': '25 Aug 23 20:52:05 UTC', 

278 'video_sequence_name': 'Deep Discoverer 23060002', 

279 } 

280 ] 

281 

282 @patch('requests.get', side_effect=mocked_requests_get) 

283 def test_find_long_host_associate_time_diff(self, _): 

284 qaqc_processor_okay = VarsQaqcProcessor( 

285 sequence_names=['Deep Discoverer 23060001'], 

286 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

287 vars_kb_url=MockResponse.VARS_KB_URL, 

288 ) 

289 qaqc_processor_problems = VarsQaqcProcessor( 

290 sequence_names=['Deep Discoverer 23060002'], 

291 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

292 vars_kb_url=MockResponse.VARS_KB_URL, 

293 ) 

294 qaqc_processor_okay.find_long_host_associate_time_diff() 

295 qaqc_processor_problems.find_long_host_associate_time_diff() 

296 assert qaqc_processor_okay.final_records == [] 

297 assert qaqc_processor_problems.final_records == [ 

298 { 

299 'observation_uuid': '01f3e954-b793-40a3-6166-88f24898e81e', 

300 'concept': 'Pomacentridae', 

301 'associations': ex_23060002['annotations'][1]['associations'], 

302 'activity': 'cruise', 

303 'annotator': 'Nikki Cunanan', 

304 'depth': 4256.0, 

305 'phylum': 'Chordata', 

306 'class': 'Actinopterygii', 

307 'order': 'Perciformes', 

308 'family': 'Pomacentridae', 

309 'genus': None, 

310 'species': None, 

311 'identity_reference': None, 

312 'image_url': '', 

313 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=4543', 

314 'recorded_timestamp': '25 Aug 23 21:05:43 UTC', 

315 'video_sequence_name': 'Deep Discoverer 23060002', 

316 'status': 'Host not found in previous records' 

317 }, 

318 { 

319 'observation_uuid': '02dfd7f4-c834-433d-4960-9577c98ce81e', 

320 'concept': 'Hydroidolina', 

321 'associations': ex_23060002['annotations'][2]['associations'], 

322 'activity': 'cruise', 

323 'annotator': 'Nikki Cunanan', 

324 'depth': None, 

325 'phylum': 'Cnidaria', 

326 'class': 'Hydrozoa', 

327 'order': None, 

328 'family': None, 

329 'genus': None, 

330 'species': None, 

331 'identity_reference': '13', 

332 'image_url': '', 

333 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=2435', 

334 'recorded_timestamp': '25 Aug 23 20:30:35 UTC', 

335 'video_sequence_name': 'Deep Discoverer 23060002', 

336 'status': 'Time between record and closest previous matching host record greater than one minute (95 seconds)' 

337 }, 

338 { 

339 'observation_uuid': '0983d9f1-d28a-482e-0160-6d3df753e91e', 

340 'concept': 'AssociateConcept', 

341 'associations': ex_23060002['annotations'][4]['associations'], 

342 'activity': 'stationary', 

343 'annotator': 'Nikki Cunanan', 

344 'depth': 4260.0, 

345 'phylum': None, 

346 'class': None, 

347 'order': None, 

348 'family': None, 'genus': None, 

349 'species': None, 

350 'identity_reference': None, 

351 'image_url': '', 

352 'video_url': 'https://hurlvideo.soest.hawaii.edu/D2/2023/EX2306_02/EX2306_02_20230825T195000Z.m4v#t=2941', 

353 'recorded_timestamp': '25 Aug 23 20:39:01 UTC', 

354 'video_sequence_name': 'Deep Discoverer 23060002', 

355 'status': 'Time between record and closest previous matching host record greater than five minutes (10 mins, 0 seconds)' 

356 }, 

357 ] 

358 

359 @patch('requests.get', side_effect=mocked_requests_get) 

360 def test_find_num_bounding_boxes(self, _): 

361 qaqc_processor = VarsQaqcProcessor( 

362 sequence_names=['Deep Discoverer 23060001'], 

363 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

364 vars_kb_url=MockResponse.VARS_KB_URL, 

365 ) 

366 qaqc_processor.find_num_bounding_boxes() 

367 assert qaqc_processor.final_records == [{ 

368 'bounding_box_counts': { 

369 'Pomacentridae': {'annos': 5, 'boxes': 1}, 

370 'none': {'annos': 1, 'boxes': 0} 

371 }, 

372 'total_count_annos': 6, 

373 'total_count_boxes': 1, 

374 }] 

375 

376 @patch('requests.get', side_effect=mocked_requests_get) 

377 def test_find_unique_fields(self, _): 

378 qaqc_processor = VarsQaqcProcessor( 

379 sequence_names=['Deep Discoverer 23060001'], 

380 vars_charybdis_url=MockResponse.VARS_CHARYBDIS_URL, 

381 vars_kb_url=MockResponse.VARS_KB_URL, 

382 ) 

383 qaqc_processor.find_unique_fields() 

384 assert qaqc_processor.final_records == [ 

385 { 

386 'concept-names': { 

387 'Pomacentridae': { 

388 'individuals': 5, 

389 'records': 5, 

390 }, 

391 'none': { 

392 'individuals': 1, 

393 'records': 1, 

394 } 

395 }, 

396 }, 

397 { 

398 'concept-upon-combinations': { 

399 'Pomacentridae:bed': { 

400 'individuals': 2, 

401 'records': 2, 

402 }, 

403 'Pomacentridae:sed': { 

404 'individuals': 3, 

405 'records': 3, 

406 }, 

407 'none:None': { 

408 'individuals': 1, 

409 'records': 1, 

410 } 

411 }, 

412 }, 

413 { 

414 'substrate-combinations': { 

415 '': { 

416 'individuals': 1, 

417 'records': 1, 

418 }, 

419 'bed, bou, sed': { 

420 'individuals': 1, 

421 'records': 1, 

422 }, 

423 'bed, sed': { 

424 'individuals': 1, 

425 'records': 1, 

426 }, 

427 'mantra, sed': { 

428 'individuals': 1, 

429 'records': 1, 

430 }, 

431 'sed': { 

432 'individuals': 2, 

433 'records': 2, 

434 }, 

435 }, 

436 }, 

437 { 

438 'comments': { 

439 None: { 

440 'individuals': 3, 

441 'records': 3, 

442 }, 

443 'Added for review: Don Draper': { 

444 'individuals': 1, 

445 'records': 1, 

446 }, 

447 'Added for review: Jon Snow; This is a weird lookin sponge thing!': { 

448 'individuals': 1, 

449 'records': 1, 

450 }, 

451 'this is a comment': { 

452 'individuals': 1, 

453 'records': 1, 

454 }, 

455 }, 

456 }, 

457 { 

458 'condition-comments': { 

459 None: { 

460 'individuals': 6, 

461 'records': 6, 

462 }, 

463 }, 

464 }, 

465 { 

466 'megahabitats': { 

467 None: { 

468 'individuals': 5, 

469 'records': 5, 

470 }, 

471 'continental shelf': { 

472 'individuals': 1, 

473 'records': 1, 

474 }, 

475 }, 

476 }, 

477 { 

478 'habitats': { 

479 None: { 

480 'individuals': 5, 

481 'records': 5, 

482 }, 

483 'slope': { 

484 'individuals': 1, 

485 'records': 1, 

486 }, 

487 }, 

488 }, 

489 { 

490 'habitat-comments': { 

491 None: { 

492 'individuals': 5, 

493 'records': 5, 

494 }, 

495 'loose talus': { 

496 'individuals': 1, 

497 'records': 1, 

498 }, 

499 }, 

500 }, 

501 { 

502 'identity-certainty': { 

503 None: { 

504 'individuals': 4, 

505 'records': 4, 

506 }, 

507 'maybe': { 

508 'individuals': 2, 

509 'records': 2, 

510 }, 

511 }, 

512 }, 

513 { 

514 'occurrence-remarks': { 

515 None: { 

516 'individuals': 4, 

517 'records': 4, 

518 }, 

519 'bottom in sight': { 

520 'individuals': 1, 

521 'records': 1, 

522 }, 

523 'in water column on descent': { 

524 'individuals': 1, 

525 'records': 1, 

526 }, 

527 }, 

528 }, 

529 ]