From f65a261333098c5921b630a862672300b871a862 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Fri, 16 Oct 2026 20:53:31 +0100 Subject: [PATCH 1/5] fixing tokenization of gold label entities and adding detected name to stats --- medcat/medcat/stats/stats.py | 4 +++- medcat/medcat/tokenizing/spacy_impl/tokens.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/medcat/medcat/stats/stats.py b/medcat/medcat/stats/stats.py index ae75d4d42..7419dcb9d 100644 --- a/medcat/medcat/stats/stats.py +++ b/medcat/medcat/stats/stats.py @@ -299,7 +299,9 @@ def _extract_predictions( 'start': ent.base.start_char_index, 'end': ent.base.end_char_index, 'cui': ent.cui, - 'text': ent.base.text, + 'text': doc['text'][max(0, ent.base.start_char_index): + ent.base.end_char_index], + 'detected_name': ent.detected_name, 'context': doc['text'][max(0, ent.base.start_char_index - 60): ent.base.end_char_index + 60], 'document_id': doc['id'], diff --git a/medcat/medcat/tokenizing/spacy_impl/tokens.py b/medcat/medcat/tokenizing/spacy_impl/tokens.py index fb6595d2d..aa3e90617 100644 --- a/medcat/medcat/tokenizing/spacy_impl/tokens.py +++ b/medcat/medcat/tokenizing/spacy_impl/tokens.py @@ -236,7 +236,7 @@ def get_tokens(self, start_index: int, end_index: int ) -> list[MutableToken]: char_indices = self._ensure_char_indices() lo = bisect_left(char_indices, start_index) - hi = bisect_right(char_indices, end_index) + hi = bisect_left(char_indices, end_index) return [Token(self._delegate[i]) for i in range(lo, hi)] def set_addon_data(self, path: str, val: Any) -> None: From eb4562e119707fb6e832acab1cc59c1449b208b1 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Fri, 16 Oct 2026 21:09:14 +0100 Subject: [PATCH 2/5] mypy! --- medcat/medcat/stats/stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/medcat/medcat/stats/stats.py b/medcat/medcat/stats/stats.py index 7419dcb9d..e40e96837 100644 --- a/medcat/medcat/stats/stats.py +++ b/medcat/medcat/stats/stats.py @@ -135,6 +135,7 @@ class PredictedAnnotation(TypedDict): end: int cui: str text: str + detected_name: str confidence: float raw: MutableEntity no_tokens: int From 2315bf97498cf0cecea565362e2e36dce193ffc5 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Fri, 16 Oct 2026 21:29:19 +0100 Subject: [PATCH 3/5] mypy fix --- medcat/medcat/tokenizing/spacy_impl/tokens.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medcat/medcat/tokenizing/spacy_impl/tokens.py b/medcat/medcat/tokenizing/spacy_impl/tokens.py index aa3e90617..e7afc0953 100644 --- a/medcat/medcat/tokenizing/spacy_impl/tokens.py +++ b/medcat/medcat/tokenizing/spacy_impl/tokens.py @@ -1,5 +1,5 @@ from typing import Iterator, Union, Optional, overload, cast, Any -from bisect import bisect_left, bisect_right +from bisect import bisect_left import logging from spacy.tokens import Token as SpacyToken From 12d291eb28787d7e27d3be758652c791bdb59349 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Mon, 19 Oct 2026 11:35:50 +0100 Subject: [PATCH 4/5] fixed testing --- medcat/tests/utils/test_training_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/medcat/tests/utils/test_training_utils.py b/medcat/tests/utils/test_training_utils.py index 1c02e4dcf..289646243 100644 --- a/medcat/tests/utils/test_training_utils.py +++ b/medcat/tests/utils/test_training_utils.py @@ -27,6 +27,7 @@ class _FakeEntity: def __init__(self, start: int, end: int, text: str, cui: str): self.base = _FakeEntityBase(start, end, text) self.cui = cui + self.detected_name = text self.context_similarity = 1.0 self.id = 0 From 2b253910f72c007e475c14807b5d952190ba7390 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Mon, 19 Oct 2026 12:21:12 +0100 Subject: [PATCH 5/5] fixed test set up --- medcat/tests/stats/test_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/medcat/tests/stats/test_stats.py b/medcat/tests/stats/test_stats.py index 4f42005f1..d46daf778 100644 --- a/medcat/tests/stats/test_stats.py +++ b/medcat/tests/stats/test_stats.py @@ -54,6 +54,7 @@ class DummyEntity: base: SimpleNamespace = field(init=False) def __post_init__(self): + self.detected_name = self.text self.base = SimpleNamespace( text=self.text, start_char_index=self.start_char_index,