diff --git a/medcat/medcat/stats/common.py b/medcat/medcat/stats/common.py index 3a496c712..2cbc444d9 100644 --- a/medcat/medcat/stats/common.py +++ b/medcat/medcat/stats/common.py @@ -28,6 +28,7 @@ class PredictedAnnotation(TypedDict): end: int cui: str text: str + detected_name: str confidence: float raw: MutableEntity no_tokens: int diff --git a/medcat/medcat/stats/stats.py b/medcat/medcat/stats/stats.py index 739c5e4e1..bb7f83914 100644 --- a/medcat/medcat/stats/stats.py +++ b/medcat/medcat/stats/stats.py @@ -282,7 +282,9 @@ def _extract_predictions( 'start': ent.base.start_char_index, 'end': ent.base.end_char_index, 'cui': ent.cui, - 'text': ent.base.text, + 'text': doc['text'][max(0, ent.base.start_char_index): + ent.base.end_char_index], + 'detected_name': ent.detected_name, 'context': doc['text'][max(0, ent.base.start_char_index - 60): ent.base.end_char_index + 60], 'document_id': doc['id'], diff --git a/medcat/medcat/tokenizing/spacy_impl/tokens.py b/medcat/medcat/tokenizing/spacy_impl/tokens.py index fb6595d2d..e7afc0953 100644 --- a/medcat/medcat/tokenizing/spacy_impl/tokens.py +++ b/medcat/medcat/tokenizing/spacy_impl/tokens.py @@ -1,5 +1,5 @@ from typing import Iterator, Union, Optional, overload, cast, Any -from bisect import bisect_left, bisect_right +from bisect import bisect_left import logging from spacy.tokens import Token as SpacyToken @@ -236,7 +236,7 @@ def get_tokens(self, start_index: int, end_index: int ) -> list[MutableToken]: char_indices = self._ensure_char_indices() lo = bisect_left(char_indices, start_index) - hi = bisect_right(char_indices, end_index) + hi = bisect_left(char_indices, end_index) return [Token(self._delegate[i]) for i in range(lo, hi)] def set_addon_data(self, path: str, val: Any) -> None: diff --git a/medcat/tests/stats/test_stats.py b/medcat/tests/stats/test_stats.py index 4f42005f1..d46daf778 100644 --- a/medcat/tests/stats/test_stats.py +++ b/medcat/tests/stats/test_stats.py @@ -54,6 +54,7 @@ class DummyEntity: base: SimpleNamespace = field(init=False) def __post_init__(self): + self.detected_name = self.text self.base = SimpleNamespace( text=self.text, start_char_index=self.start_char_index, diff --git a/medcat/tests/utils/test_training_utils.py b/medcat/tests/utils/test_training_utils.py index 1c02e4dcf..289646243 100644 --- a/medcat/tests/utils/test_training_utils.py +++ b/medcat/tests/utils/test_training_utils.py @@ -27,6 +27,7 @@ class _FakeEntity: def __init__(self, start: int, end: int, text: str, cui: str): self.base = _FakeEntityBase(start, end, text) self.cui = cui + self.detected_name = text self.context_similarity = 1.0 self.id = 0