diff --git a/medcat-trainer/webapp/api/api/metrics.py b/medcat-trainer/webapp/api/api/metrics.py index b421e8381..5c989635a 100644 --- a/medcat-trainer/webapp/api/api/metrics.py +++ b/medcat-trainer/webapp/api/api/metrics.py @@ -173,31 +173,63 @@ def concept_summary(self, extra_cui_filter=None): def enrich_medcat_metrics(self, examples): """ - Add the user prop to the medcat output metrics. Can potentially add more later for each of the categories + Add the user prop to the medcat output metrics. Can potentially add more later for each of the categories. + + get_stats examples use snake_case keys (document_id). Older payloads used + spaced keys ("document id"). Accept both so FP/FN/TP dialogs can be enriched. """ - for tp in [i for e_i in examples['tp'].values() for i in e_i]: - try: - ann = AnnotatedEntity.objects.get(project_id=tp['project id'], document_id=tp['document id'], - start_ind=tp['start'], end_ind=tp['end']) - tp['user'] = ann.user.username - except: - tp['user'] = None - for fp in (i for e_i in examples['fp'].values() for i in e_i): - try: - ann = AnnotatedEntity.objects.get(project_id=fp['project id'], document_id=fp['document id'], - start_ind=fp['start'], end_ind=fp['end']) - fp['user'] = ann.user.username - except: - fp['user'] = None - for fn in (i for e_i in examples['fn'].values() for i in e_i): - try: - ann = AnnotatedEntity.objects.get(project_id=fn['project id'], document_id=fn['document id'], - start_ind=fn['start'], end_ind=fn['end']) - fn['user'] = ann.user.username - except: - fn['user'] = None + if not isinstance(examples, dict): + return examples + for kind in ('tp', 'fp', 'fn'): + grouped = examples.get(kind) or {} + if not isinstance(grouped, dict): + continue + for example_list in grouped.values(): + if not isinstance(example_list, list): + continue + for example in example_list: + if isinstance(example, dict): + self._attach_example_user(example) return examples + @staticmethod + def _example_value(example, *keys): + for key in keys: + if key in example and example[key] is not None: + return example[key] + return None + + @classmethod + def _normalize_example_keys(cls, example): + """Expose both legacy spaced keys and current snake_case keys on one example.""" + aliases = ( + ('project id', 'project_id'), + ('project name', 'project_name'), + ('document id', 'document_id'), + ('document name', 'document_name'), + ('source value', 'source_value'), + ) + for legacy, modern in aliases: + if legacy not in example and modern in example: + example[legacy] = example[modern] + elif modern not in example and legacy in example: + example[modern] = example[legacy] + if 'acc' not in example and example.get('confidence') is not None: + example['acc'] = example['confidence'] + return example + + def _attach_example_user(self, example): + self._normalize_example_keys(example) + project_id = self._example_value(example, 'project id', 'project_id') + document_id = self._example_value(example, 'document id', 'document_id') + try: + ann = AnnotatedEntity.objects.get(project_id=project_id, document_id=document_id, + start_ind=example.get('start'), end_ind=example.get('end')) + example['user'] = ann.user.username if getattr(ann, 'user', None) else None + except Exception: + example['user'] = None + return example + def user_stats(self, by_user: bool = True): """ Summary of user annotation work done diff --git a/medcat-trainer/webapp/api/api/tests/test_metrics.py b/medcat-trainer/webapp/api/api/tests/test_metrics.py index f938d529e..8bc5681c2 100644 --- a/medcat-trainer/webapp/api/api/tests/test_metrics.py +++ b/medcat-trainer/webapp/api/api/tests/test_metrics.py @@ -5,6 +5,8 @@ are exercised by passing cat=None. """ +from unittest.mock import MagicMock, patch + import pandas as pd from django.test import TestCase @@ -154,6 +156,80 @@ def test_rename_meta_value_when_specified(self): self.assertEqual(pm.annotations[0]['Existence'], 'Yes') +class EnrichMedcatMetricsTests(TestCase): + def _examples(self): + return { + 'tp': {}, + 'fp': { + 'C001': [{ + 'project_id': 4, + 'document_id': 9, + 'project_name': 'Ward round', + 'document_name': 'Note 9', + 'source_value': 'asthma', + 'text': 'history of asthma today', + 'start': 11, + 'end': 17, + 'cui': 'C001', + 'acc': 0.5, + }] + }, + 'fn': { + 'C002': [{ + 'project id': 4, + 'document id': 9, + 'source value': 'diabetes', + 'text': 'diabetes mellitus', + 'start': 0, + 'end': 8, + 'cui': 'C002', + 'acc': 1, + }] + }, + } + + @patch('api.metrics.AnnotatedEntity.objects.get') + def test_snake_case_examples_are_normalized_and_assigned_a_user(self, mock_get): + ann = MagicMock() + ann.user.username = 'alice' + mock_get.return_value = ann + pm = ProjectMetrics(_build_export(), cat=None) + + examples = pm.enrich_medcat_metrics(self._examples()) + fp = examples['fp']['C001'][0] + + self.assertEqual(fp['user'], 'alice') + self.assertEqual(fp['project id'], 4) + self.assertEqual(fp['document id'], 9) + self.assertEqual(fp['document name'], 'Note 9') + self.assertEqual(fp['source value'], 'asthma') + mock_get.assert_any_call(project_id=4, document_id=9, start_ind=11, end_ind=17) + + @patch('api.metrics.AnnotatedEntity.objects.get') + def test_legacy_spaced_keys_still_resolve_the_annotation(self, mock_get): + ann = MagicMock() + ann.user.username = 'bob' + mock_get.return_value = ann + pm = ProjectMetrics(_build_export(), cat=None) + + examples = pm.enrich_medcat_metrics(self._examples()) + fn = examples['fn']['C002'][0] + + self.assertEqual(fn['user'], 'bob') + self.assertEqual(fn['project_id'], 4) + self.assertEqual(fn['document_id'], 9) + self.assertEqual(fn['source_value'], 'diabetes') + mock_get.assert_any_call(project_id=4, document_id=9, start_ind=0, end_ind=8) + + @patch('api.metrics.AnnotatedEntity.objects.get', side_effect=LookupError('missing')) + def test_missing_annotation_still_normalizes_keys(self, _mock_get): + pm = ProjectMetrics(_build_export(), cat=None) + examples = pm.enrich_medcat_metrics(self._examples()) + fp = examples['fp']['C001'][0] + self.assertIsNone(fp['user']) + self.assertEqual(fp['source value'], 'asthma') + + class EmptyExportTests(TestCase): def test_handles_empty_documents_without_error(self): export = {'projects': [{'id': 1, 'name': 'empty', 'documents': [], diff --git a/medcat-trainer/webapp/frontend/src/components/anns/AnnoResult.vue b/medcat-trainer/webapp/frontend/src/components/anns/AnnoResult.vue index a053e626a..965f2394e 100644 --- a/medcat-trainer/webapp/frontend/src/components/anns/AnnoResult.vue +++ b/medcat-trainer/webapp/frontend/src/components/anns/AnnoResult.vue @@ -1,17 +1,18 @@ - {{result['document name']}} - {{result.cui}} - {{result['source value']}} - {{Number(result.acc).toFixed(3)}} + {{ documentName }} + {{ result && result.cui }} + {{ sourceValue }} + {{ accuracy }} - +