fix(medcat): fix tokenisation - and correct PredicatedAnnotations text

medcat/medcat/stats/common.py

@@ -28,6 +28,7 @@ class PredictedAnnotation(TypedDict):
     end: int
     cui: str
     text: str
+    detected_name: str
     confidence: float
     raw: MutableEntity
     no_tokens: int

medcat/medcat/stats/stats.py

@@ -282,7 +282,9 @@ def _extract_predictions(
                 'start': ent.base.start_char_index,
                 'end': ent.base.end_char_index,
                 'cui': ent.cui,
-                'text': ent.base.text,
+                'text': doc['text'][max(0, ent.base.start_char_index):
+                                       ent.base.end_char_index],
+                'detected_name': ent.detected_name,
                 'context': doc['text'][max(0, ent.base.start_char_index - 60):
                                        ent.base.end_char_index + 60],
                 'document_id': doc['id'],

medcat/medcat/tokenizing/spacy_impl/tokens.py

@@ -1,5 +1,5 @@
 from typing import Iterator, Union, Optional, overload, cast, Any
-from bisect import bisect_left, bisect_right
+from bisect import bisect_left
 import logging
 
 from spacy.tokens import Token as SpacyToken
@@ -236,7 +236,7 @@ def get_tokens(self, start_index: int, end_index: int
                    ) -> list[MutableToken]:
         char_indices = self._ensure_char_indices()
         lo = bisect_left(char_indices, start_index)
-        hi = bisect_right(char_indices, end_index)
+        hi = bisect_left(char_indices, end_index)
         return [Token(self._delegate[i]) for i in range(lo, hi)]
 
     def set_addon_data(self, path: str, val: Any) -> None:

medcat/tests/stats/test_stats.py

@@ -54,6 +54,7 @@ class DummyEntity:
     base: SimpleNamespace = field(init=False)
 
     def __post_init__(self):
+        self.detected_name = self.text
         self.base = SimpleNamespace(
             text=self.text,
             start_char_index=self.start_char_index,

medcat/tests/utils/test_training_utils.py

@@ -27,6 +27,7 @@ class _FakeEntity:
     def __init__(self, start: int, end: int, text: str, cui: str):
         self.base = _FakeEntityBase(start, end, text)
         self.cui = cui
+        self.detected_name = text
         self.context_similarity = 1.0
         self.id = 0