From 0dc21697ab5c9b3f2cac95cc93e05acc54f039ea Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 15 Oct 2026 20:18:34 +0000 Subject: [PATCH 1/7] feat(medcat-trainer): Add text and name contains filters to documents API Co-authored-by: tomolopolis --- .../webapp/api/api/tests/test_views.py | 36 +++++++++++++++++++ medcat-trainer/webapp/api/api/views.py | 11 +++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/medcat-trainer/webapp/api/api/tests/test_views.py b/medcat-trainer/webapp/api/api/tests/test_views.py index a0e896827..4e358e935 100644 --- a/medcat-trainer/webapp/api/api/tests/test_views.py +++ b/medcat-trainer/webapp/api/api/tests/test_views.py @@ -442,3 +442,39 @@ def test_anonymous_cannot_create_group(self): resp = client.post('/api/project-groups/', {'name': 'nope'}, format='json') self.assertIn(resp.status_code, (401, 403)) + +@override_settings(MEDIA_ROOT='/tmp/mct-tests-views') +class DocumentViewSetFilterTests(TestCase): + def setUp(self): + self.user = create_user(username='docsearch') + self.client = _auth_client(self.user) + self.project = create_basic_project(name='doc-search-proj') + self.other_project = create_basic_project(name='doc-search-other') + create_document(self.project, name='note-1', text='Patient reports Chest Pain today.') + create_document(self.project, name='note-2', text='No fever or cough.') + create_document(self.project, name='note-3', text='chest pain resolved') + create_document(self.other_project, name='note-x', text='chest pain elsewhere') + + def _names(self, query): + resp = self.client.get(f'/api/documents/?{query}') + self.assertEqual(resp.status_code, 200) + return [d['name'] for d in resp.json()['results']] + + def test_filter_by_text_span_is_case_insensitive_and_scoped_to_dataset(self): + names = self._names(f'dataset={self.project.dataset.id}&text__icontains=chest%20pain') + self.assertEqual(names, ['note-1', 'note-3']) + + def test_filter_by_text_span_no_match(self): + self.assertEqual( + self._names(f'dataset={self.project.dataset.id}&text__icontains=diabetes'), [], + ) + + def test_filter_by_name_contains(self): + self.assertEqual( + self._names(f'dataset={self.project.dataset.id}&name__icontains=TE-2'), ['note-2'], + ) + + def test_filter_by_dataset_still_works(self): + names = self._names(f'dataset={self.other_project.dataset.id}') + self.assertEqual(names, ['note-x']) + diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py index 388f4a954..d0a150a90 100644 --- a/medcat-trainer/webapp/api/api/views.py +++ b/medcat-trainer/webapp/api/api/views.py @@ -144,11 +144,20 @@ class MetaAnnotationViewSet(viewsets.ModelViewSet): filterset_fields = ['id', 'annotated_entity', 'validated'] +class DocumentFilter(drf.FilterSet): + name__icontains = drf.CharFilter(field_name='name', lookup_expr='icontains') + text__icontains = drf.CharFilter(field_name='text', lookup_expr='icontains') + + class Meta: + model = Document + fields = ['dataset'] + + class DocumentViewSet(viewsets.ModelViewSet): permission_classes = [permissions.IsAuthenticated] queryset = Document.objects.all() serializer_class = DocumentSerializer - filterset_fields = ['dataset'] + filterset_class = DocumentFilter class EntityViewSet(viewsets.ModelViewSet): From 62800657547918073f4fa878e6d3cf256d16399a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 15 Oct 2026 20:18:34 +0000 Subject: [PATCH 2/7] feat(medcat-trainer): Search clinical notes by text span in annotate view Co-authored-by: tomolopolis --- .../src/components/common/DocumentSummary.vue | 166 +++++++++++++++++- .../tests/components/DocumentSummary.spec.ts | 117 ++++++++++++ .../frontend/src/views/TrainAnnotations.vue | 3 +- 3 files changed, 277 insertions(+), 9 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue index 55917a7e0..6976efa37 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue @@ -5,13 +5,27 @@
- + + +
+ +
+ Searching... + {{ textSearchError }} + {{ textSearchCount }} matching note{{ textSearchCount === 1 ? '' : 's' }} +
-
-
+
{{ textSnippet(doc).pre }}{{ textSnippet(doc).match }}{{ textSnippet(doc).post }}
+
{{doc.text === 'nan' ? '' : (limitText(doc.text) || '')}}
-
+
+
+ More Results +
+
+
More Docs
@@ -43,10 +64,14 @@ @@ -244,4 +371,27 @@ $width: 175px; width: 100%; } +.search-mode { + display: flex; + gap: 4px; + margin: 4px 0; + + .btn { + flex: 1; + padding: 0 4px; + font-size: 10pt; + } +} + +.search-status { + font-size: 9pt; + opacity: 0.7; + padding: 2px 4px; +} + +.search-match { + background: $warning; + font-weight: bold; +} + diff --git a/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts b/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts index 54db19729..d5048247e 100644 --- a/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts @@ -118,6 +118,123 @@ describe('DocumentSummary.vue', () => { expect(wrapper.findAll('.doc')).toHaveLength(1) }) + describe('text search', () => { + type TextSearchVm = { + searching: boolean + searchMode: string + searchDocs: (e: { target: { value: string } }) => void + setSearchMode: (mode: string) => void + textSearchResults: { id: number }[] + } + + const textResults = [ + { id: 7, name: 'note-far', text: 'History: patient has chest pain on exertion.' } + ] + + const mountWithHttp = (mockGet: ReturnType, props = {}) => + mount(DocumentSummary, { + props: { ...defaultProps, datasetId: 5, ...props }, + global: { + stubs: { 'font-awesome-icon': true, 'v-tooltip': true }, + mocks: { $http: { get: mockGet } } + } + }) + + const flush = () => new Promise(resolve => originalSetTimeout(resolve, 0)) + + beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('queries the API for docs containing the span and lists the results', async () => { + const mockGet = vi.fn().mockResolvedValue({ + data: { count: 1, next: null, results: textResults } + }) + const wrapper = mountWithHttp(mockGet) + const vm = wrapper.vm as unknown as TextSearchVm + vm.searching = true + await wrapper.vm.$nextTick() + await wrapper.find('.search-mode-text').trigger('click') + + vm.searchDocs({ target: { value: 'chest pain' } }) + vm.searchDocs({ target: { value: 'chest pain' } }) + expect(mockGet).not.toHaveBeenCalled() + vi.advanceTimersByTime(300) + await flush() + await wrapper.vm.$nextTick() + + expect(mockGet).toHaveBeenCalledTimes(1) + expect(mockGet).toHaveBeenCalledWith('/api/documents/?text__icontains=chest+pain&dataset=5') + const docEls = wrapper.findAll('.doc') + expect(docEls).toHaveLength(1) + expect(wrapper.find('.search-match').text()).toBe('chest pain') + expect(wrapper.find('.search-status').text()).toContain('1 matching note') + + await docEls[0].trigger('click') + expect(wrapper.emitted('request:loadDoc')?.[0]).toEqual([textResults[0]]) + }) + + it('pages through further results with More Results', async () => { + const mockGet = vi.fn() + .mockResolvedValueOnce({ + data: { count: 2, next: 'http://host/api/documents/?dataset=5&page=2&text__icontains=pain', results: textResults } + }) + .mockResolvedValueOnce({ + data: { count: 2, next: null, results: [{ id: 8, name: 'n8', text: 'pain' }] } + }) + const wrapper = mountWithHttp(mockGet) + const vm = wrapper.vm as unknown as TextSearchVm + vm.searchMode = 'text' + vm.searchDocs({ target: { value: 'pain' } }) + vi.advanceTimersByTime(300) + await flush() + await wrapper.vm.$nextTick() + + await wrapper.find('.more-docs').trigger('click') + await flush() + await wrapper.vm.$nextTick() + expect(mockGet).toHaveBeenLastCalledWith('/api/documents/?dataset=5&page=2&text__icontains=pain') + expect(wrapper.findAll('.doc')).toHaveLength(2) + expect(wrapper.find('.more-docs').exists()).toBe(false) + }) + + it('ignores stale responses from superseded searches', async () => { + let resolveFirst: (v: unknown) => void = () => {} + const mockGet = vi.fn() + .mockImplementationOnce(() => new Promise(resolve => { resolveFirst = resolve })) + .mockResolvedValueOnce({ data: { count: 1, next: null, results: textResults } }) + const wrapper = mountWithHttp(mockGet) + const vm = wrapper.vm as unknown as TextSearchVm + vm.searchMode = 'text' + vm.searchDocs({ target: { value: 'fever' } }) + vi.advanceTimersByTime(300) + vm.searchDocs({ target: { value: 'chest' } }) + vi.advanceTimersByTime(300) + await flush() + resolveFirst({ data: { count: 1, next: null, results: [{ id: 99, name: 'stale', text: 'fever' }] } }) + await flush() + await wrapper.vm.$nextTick() + expect(vm.textSearchResults.map(d => d.id)).toEqual([7]) + }) + + it('switching back to name mode filters loaded docs without calling the API', async () => { + const mockGet = vi.fn() + const wrapper = mountWithHttp(mockGet) + const vm = wrapper.vm as unknown as TextSearchVm + vm.searchMode = 'text' + vm.searchDocs({ target: { value: 'note-b' } }) + vm.setSearchMode('name') + vi.advanceTimersByTime(300) + await wrapper.vm.$nextTick() + expect(mockGet).not.toHaveBeenCalled() + expect(wrapper.findAll('.doc')).toHaveLength(1) + }) + }) + it('treats nan text as empty in preview', () => { const wrapper = mountComponent() const docWithNan = wrapper.findAll('.note-summary')[2] diff --git a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue index a39def5b3..df552bdfd 100644 --- a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue +++ b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue @@ -24,7 +24,8 @@
- Date: Thu, 15 Oct 2026 20:34:33 +0000 Subject: [PATCH 3/7] feat(medcat-trainer): Index dataset documents into Solr on upload and add Solr backed document search endpoint Co-authored-by: tomolopolis --- .../webapp/api/api/admin/actions.py | 10 + medcat-trainer/webapp/api/api/admin/models.py | 1 + .../webapp/api/api/document_search.py | 216 +++++++++++++++ medcat-trainer/webapp/api/api/signals.py | 15 +- medcat-trainer/webapp/api/api/solr_utils.py | 4 +- .../api/api/tests/test_document_search.py | 257 ++++++++++++++++++ .../webapp/api/api/tests/test_solr_utils.py | 13 + medcat-trainer/webapp/api/api/views.py | 51 ++++ medcat-trainer/webapp/api/core/settings.py | 3 + medcat-trainer/webapp/api/core/urls.py | 1 + 10 files changed, 568 insertions(+), 3 deletions(-) create mode 100644 medcat-trainer/webapp/api/api/document_search.py create mode 100644 medcat-trainer/webapp/api/api/tests/test_document_search.py diff --git a/medcat-trainer/webapp/api/api/admin/actions.py b/medcat-trainer/webapp/api/api/admin/actions.py index 07541b5d5..5cce84afe 100644 --- a/medcat-trainer/webapp/api/api/admin/actions.py +++ b/medcat-trainer/webapp/api/api/admin/actions.py @@ -10,6 +10,7 @@ from django.http import HttpResponse from rest_framework.exceptions import PermissionDenied +from api.document_search import index_dataset_task from api.models import AnnotatedEntity, MetaAnnotation, EntityRelation, Document, ConceptDB from api.solr_utils import drop_collection, import_all_concepts, solr_collection_name from api.utils import clear_cdb_cnf_addons @@ -353,6 +354,15 @@ def clone_projects(modeladmin, request, queryset): project_copy.save() +def index_datasets_for_search(modeladmin, request, queryset): + for dataset in queryset: + index_dataset_task(dataset.id) + modeladmin.message_user(request, f'Scheduled text search indexing for {queryset.count()} dataset(s)') + + +index_datasets_for_search.short_description = 'Index selected datasets for text search' + + def dataset_document_counts(dataset): return f'{Document.objects.filter(dataset=dataset).count()}' diff --git a/medcat-trainer/webapp/api/api/admin/models.py b/medcat-trainer/webapp/api/api/admin/models.py index b0c8cf254..71de65841 100644 --- a/medcat-trainer/webapp/api/api/admin/models.py +++ b/medcat-trainer/webapp/api/api/admin/models.py @@ -37,6 +37,7 @@ class DatasetAdmin(ReportErrorModelAdminMixin, admin.ModelAdmin): model = Dataset form = DatasetForm list_display = ['name', 'create_time', 'description', dataset_document_counts] + actions = [index_datasets_for_search] class ProjectAnnotateEntitiesAdmin(admin.ModelAdmin): diff --git a/medcat-trainer/webapp/api/api/document_search.py b/medcat-trainer/webapp/api/api/document_search.py new file mode 100644 index 000000000..c95a49b4a --- /dev/null +++ b/medcat-trainer/webapp/api/api/document_search.py @@ -0,0 +1,216 @@ +"""Full text search of dataset documents, backed by a single shared Solr collection. + +Every document is indexed with its ``dataset_id`` so a search can be scoped to one dataset, +and a dataset can be (re-)indexed or removed with a delete-by-query. +""" +import logging +import re +from typing import Dict, Iterable, List, Optional, Tuple + +import requests +from background_task import background +from opentelemetry import trace +from opentelemetry.semconv.attributes import server_attributes +from opentelemetry.semconv._incubating.attributes import service_attributes + +from api.models import Document +from core.settings import SOLR_HOST, SOLR_PORT, DOCUMENT_SEARCH_COLLECTION + +logger = logging.getLogger(__name__) +tracer = trace.get_tracer("medcat-trainer") + +solr_trace_attributes = { + service_attributes.SERVICE_PEER_NAME: "medcat-trainer-solr", + server_attributes.SERVER_ADDRESS: SOLR_HOST, + server_attributes.SERVER_PORT: int(SOLR_PORT), +} + +INDEX_BATCH_SIZE = 500 +REQUEST_TIMEOUT_SECS = 30 + +# Control characters can't appear in sanitised document text, so they are safe highlight delimiters. +HIGHLIGHT_PRE = '\x02' +HIGHLIGHT_POST = '\x03' +_HIGHLIGHT_RE = re.compile(f'({HIGHLIGHT_PRE}.*?{HIGHLIGHT_POST})', re.DOTALL) + +DOCUMENT_FIELDS = [ + {'name': 'dataset_id', 'type': 'plong', 'indexed': True, 'stored': True}, + {'name': 'doc_id', 'type': 'plong', 'indexed': True, 'stored': True}, + {'name': 'name', 'type': 'text_general', 'indexed': True, 'stored': True, 'multiValued': False}, + {'name': 'text', 'type': 'text_general', 'indexed': True, 'stored': True, 'multiValued': False}, +] + + +class DocumentSearchUnavailable(Exception): + """Raised when Solr can't serve a search for a dataset, so callers can fall back to the DB.""" + + +def _base_url() -> str: + return f'http://{SOLR_HOST}:{SOLR_PORT}/solr' + + +def _collection_url() -> str: + return f'{_base_url()}/{DOCUMENT_SEARCH_COLLECTION}' + + +def _list_collections() -> List[str]: + resp = requests.get(f'{_base_url()}/admin/collections', params={'action': 'LIST'}, + timeout=REQUEST_TIMEOUT_SECS) + if resp.status_code != 200: + raise DocumentSearchUnavailable(f'Failed to list Solr collections: {resp.text}') + return resp.json().get('collections', []) + + +def _ensure_schema_fields(): + resp = requests.get(f'{_collection_url()}/schema/fields', timeout=REQUEST_TIMEOUT_SECS) + if resp.status_code != 200: + raise DocumentSearchUnavailable(f'Failed to read schema of {DOCUMENT_SEARCH_COLLECTION}: {resp.text}') + existing = {f['name'] for f in resp.json().get('fields', [])} + missing = [f for f in DOCUMENT_FIELDS if f['name'] not in existing] + if missing: + resp = requests.post(f'{_collection_url()}/schema', json={'add-field': missing}, + timeout=REQUEST_TIMEOUT_SECS) + if resp.status_code != 200: + raise DocumentSearchUnavailable(f'Failed to add fields to {DOCUMENT_SEARCH_COLLECTION}: {resp.text}') + + +@tracer.start_as_current_span("ensure_document_collection", attributes=solr_trace_attributes) +def ensure_document_collection(): + if DOCUMENT_SEARCH_COLLECTION not in _list_collections(): + resp = requests.get(f'{_base_url()}/admin/collections', + params={'action': 'CREATE', 'name': DOCUMENT_SEARCH_COLLECTION, 'numShards': 1}, + timeout=REQUEST_TIMEOUT_SECS * 4) + # another indexing task may have created it concurrently + if resp.status_code != 200 and DOCUMENT_SEARCH_COLLECTION not in _list_collections(): + raise DocumentSearchUnavailable(f'Failed to create {DOCUMENT_SEARCH_COLLECTION}: {resp.text}') + _ensure_schema_fields() + + +def _post_update(payload, commit=False): + params = {'commit': 'true'} if commit else None + resp = requests.post(f'{_collection_url()}/update', json=payload, params=params, + timeout=REQUEST_TIMEOUT_SECS * 4) + if resp.status_code != 200: + raise DocumentSearchUnavailable(f'Failed to update {DOCUMENT_SEARCH_COLLECTION}: {resp.text}') + + +def _solr_doc(doc: Dict) -> Dict: + return { + 'id': str(doc['id']), + 'doc_id': doc['id'], + 'dataset_id': doc['dataset_id'], + 'name': doc['name'], + 'text': doc['text'], + } + + +def _batched(items: Iterable, size: int) -> Iterable[List]: + batch = [] + for item in items: + batch.append(item) + if len(batch) == size: + yield batch + batch = [] + if batch: + yield batch + + +@tracer.start_as_current_span("index_dataset_documents", attributes=solr_trace_attributes) +def index_dataset_documents(dataset_id: int) -> int: + """Replace every indexed document of the dataset with its current documents. Returns the number indexed.""" + ensure_document_collection() + _post_update({'delete': {'query': f'dataset_id:{int(dataset_id)}'}}) + docs = Document.objects.filter(dataset_id=dataset_id).order_by('id') \ + .values('id', 'dataset_id', 'name', 'text').iterator(chunk_size=INDEX_BATCH_SIZE) + total = 0 + for batch in _batched(docs, INDEX_BATCH_SIZE): + _post_update([_solr_doc(d) for d in batch]) + total += len(batch) + _post_update({'commit': {}}) + logger.info('Indexed %s documents of dataset %s into Solr collection %s', + total, dataset_id, DOCUMENT_SEARCH_COLLECTION) + return total + + +@tracer.start_as_current_span("remove_dataset_documents", attributes=solr_trace_attributes) +def remove_dataset_documents(dataset_id: int): + if DOCUMENT_SEARCH_COLLECTION not in _list_collections(): + return + _post_update({'delete': {'query': f'dataset_id:{int(dataset_id)}'}}, commit=True) + + +@background(schedule=1) +def index_dataset_task(dataset_id: int): + index_dataset_documents(dataset_id) + + +@background(schedule=1) +def remove_dataset_task(dataset_id: int): + remove_dataset_documents(dataset_id) + + +def _select(params: Dict) -> Dict: + try: + resp = requests.get(f'{_collection_url()}/select', params=params, timeout=REQUEST_TIMEOUT_SECS) + except requests.RequestException as e: + raise DocumentSearchUnavailable(f'Solr not reachable: {e}') from e + if resp.status_code != 200: + raise DocumentSearchUnavailable(f'Solr search of {DOCUMENT_SEARCH_COLLECTION} failed: {resp.text}') + return resp.json() + + +def indexed_document_count(dataset_id: int) -> int: + resp = _select({'q': '*:*', 'fq': f'dataset_id:{int(dataset_id)}', 'rows': 0}) + return resp['response']['numFound'] + + +def _phrase_query(text: str) -> str: + escaped = text.replace('\\', '\\\\').replace('"', '\\"') + return f'text:"{escaped}"' + + +def highlight_segments(snippet: str) -> List[Dict]: + segments = [] + for part in _HIGHLIGHT_RE.split(snippet): + if not part: + continue + if part.startswith(HIGHLIGHT_PRE) and part.endswith(HIGHLIGHT_POST): + segments.append({'text': part[len(HIGHLIGHT_PRE):-len(HIGHLIGHT_POST)], 'match': True}) + else: + segments.append({'text': part, 'match': False}) + return segments + + +@tracer.start_as_current_span("search_dataset_documents", attributes=solr_trace_attributes) +def search_dataset_documents(dataset_id: int, text: str, start: int, rows: int, + expected_doc_count: Optional[int] = None) -> Tuple[int, List[Tuple[int, Optional[List]]]]: + """Phrase search the dataset's documents. + + Returns the total number of matches and, for the requested page, (document id, highlight segments) pairs. + Raises DocumentSearchUnavailable if Solr is unavailable or (when ``expected_doc_count`` is given) the + dataset is not yet fully indexed. + """ + if expected_doc_count is not None and indexed_document_count(dataset_id) != expected_doc_count: + raise DocumentSearchUnavailable(f'Dataset {dataset_id} is not fully indexed in {DOCUMENT_SEARCH_COLLECTION}') + resp = _select({ + 'q': _phrase_query(text), + 'fq': f'dataset_id:{int(dataset_id)}', + 'fl': 'id,doc_id', + 'sort': 'score desc,doc_id asc', + 'start': start, + 'rows': rows, + 'hl': 'true', + 'hl.method': 'unified', + 'hl.fl': 'text', + 'hl.snippets': 1, + 'hl.fragsize': 160, + 'hl.maxAnalyzedChars': 1000000, + 'hl.tag.pre': HIGHLIGHT_PRE, + 'hl.tag.post': HIGHLIGHT_POST, + }) + highlighting = resp.get('highlighting', {}) + hits = [] + for d in resp['response']['docs']: + snippets = highlighting.get(d['id'], {}).get('text') or [] + hits.append((int(d['doc_id']), highlight_segments(snippets[0]) if snippets else None)) + return resp['response']['numFound'], hits diff --git a/medcat-trainer/webapp/api/api/signals.py b/medcat-trainer/webapp/api/api/signals.py index c2a97489f..df8745417 100644 --- a/medcat-trainer/webapp/api/api/signals.py +++ b/medcat-trainer/webapp/api/api/signals.py @@ -4,11 +4,13 @@ import shutil from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.db import transaction from django.db.models.fields.files import FileField from django.db.models.signals import post_save, post_delete, pre_save, m2m_changed, pre_delete from django.dispatch import receiver from api.data_utils import dataset_from_file, delete_orphan_docs, upload_projects_export +from api.document_search import index_dataset_task, remove_dataset_task from api.extensions import ( annotation_created, annotation_deleted, @@ -28,7 +30,7 @@ ProjectFields, ProjectGroup, ) -from core.settings import MEDIA_ROOT +from core.settings import MEDIA_ROOT, DOCUMENT_SEARCH_INDEX_ENABLED logger = logging.getLogger(__name__) @@ -37,6 +39,17 @@ @receiver(post_save, sender=Dataset) def save_dataset(sender, instance, **kwargs): dataset_from_file(instance) + if DOCUMENT_SEARCH_INDEX_ENABLED: + dataset_id = instance.id + # schedule after commit so the worker sees the dataset's documents + transaction.on_commit(lambda: index_dataset_task(dataset_id)) + + +@receiver(post_delete, sender=Dataset) +def remove_dataset_from_search_index(sender, instance, **kwargs): + if DOCUMENT_SEARCH_INDEX_ENABLED: + dataset_id = instance.id + transaction.on_commit(lambda: remove_dataset_task(dataset_id)) @receiver(pre_save, sender=Dataset) diff --git a/medcat-trainer/webapp/api/api/solr_utils.py b/medcat-trainer/webapp/api/api/solr_utils.py index fe84b5917..6e76d529b 100644 --- a/medcat-trainer/webapp/api/api/solr_utils.py +++ b/medcat-trainer/webapp/api/api/solr_utils.py @@ -12,7 +12,7 @@ from opentelemetry.semconv.attributes import server_attributes from opentelemetry.semconv._incubating.attributes import service_attributes from api.models import ConceptDB -from core.settings import SOLR_HOST, SOLR_PORT +from core.settings import SOLR_HOST, SOLR_PORT, DOCUMENT_SEARCH_COLLECTION SOLR_INDEX_SCHEMA = {} @@ -67,7 +67,7 @@ def collections_available(cdbs: List[int]): logger.info(f'Retrieving all SOLR collections: {url}') resp = requests.get(url) if resp.status_code == 200: - collections = json.loads(resp.text)['collections'] + collections = [c for c in json.loads(resp.text)['collections'] if c != DOCUMENT_SEARCH_COLLECTION] # cache schema field types for col in collections: _cache_solr_collection_schema_types(col) diff --git a/medcat-trainer/webapp/api/api/tests/test_document_search.py b/medcat-trainer/webapp/api/api/tests/test_document_search.py new file mode 100644 index 000000000..154ad8e2a --- /dev/null +++ b/medcat-trainer/webapp/api/api/tests/test_document_search.py @@ -0,0 +1,257 @@ +"""Tests for the Solr backed document text search: indexing, the search endpoint and dataset signals. + +Solr HTTP calls are mocked, except in ``LiveSolrDocumentSearchTests`` which runs against a real +SolrCloud instance when ``MCT_SOLR_LIVE_TESTS=1`` (host / port from CONCEPT_SEARCH_SERVICE_HOST / _PORT). +""" + +import json +import os +import tempfile +import unittest +import uuid +from unittest.mock import MagicMock, patch + +from background_task.models import Task +from django.core.files.base import ContentFile +from django.test import TestCase, override_settings +from rest_framework.test import APIClient + +from .. import document_search +from ..models import Dataset, Document +from ._helpers import create_basic_project, create_document, create_user, make_csv_file + +COLLECTION = document_search.DOCUMENT_SEARCH_COLLECTION + + +def _resp(status=200, body=None): + return MagicMock(status_code=status, text=json.dumps(body or {}), json=lambda: body or {}) + + +def _auth_client(user): + client = APIClient() + client.force_authenticate(user=user) + return client + + +class HighlightSegmentsTests(TestCase): + def test_splits_marked_spans(self): + pre, post = document_search.HIGHLIGHT_PRE, document_search.HIGHLIGHT_POST + segments = document_search.highlight_segments(f'has {pre}chest pain{post} and {pre}fever{post}') + self.assertEqual(segments, [ + {'text': 'has ', 'match': False}, + {'text': 'chest pain', 'match': True}, + {'text': ' and ', 'match': False}, + {'text': 'fever', 'match': True}, + ]) + + def test_phrase_query_escapes_quotes_and_backslashes(self): + self.assertEqual(document_search._phrase_query('say "hi" \\o/'), 'text:"say \\"hi\\" \\\\o/"') + + +@override_settings(MEDIA_ROOT='/tmp/mct-tests-doc-search') +class IndexDatasetDocumentsTests(TestCase): + def setUp(self): + self.project = create_basic_project(name='index-proj') + self.docs = [create_document(self.project, name=f'n{i}', text=f'text {i}') for i in range(3)] + + @patch('api.document_search.requests.post') + @patch('api.document_search.requests.get') + def test_creates_collection_and_fields_then_replaces_dataset_docs(self, mock_get, mock_post): + def get_side_effect(url, params=None, **kwargs): + if url.endswith('/admin/collections') and params['action'] == 'LIST': + return _resp(body={'collections': []}) + if url.endswith('/admin/collections') and params['action'] == 'CREATE': + return _resp() + if url.endswith('/schema/fields'): + return _resp(body={'fields': [{'name': 'id'}, {'name': 'name'}]}) + raise AssertionError(f'unexpected GET {url}') + mock_get.side_effect = get_side_effect + mock_post.return_value = _resp() + + with patch.object(document_search, 'INDEX_BATCH_SIZE', 2): + total = document_search.index_dataset_documents(self.project.dataset.id) + + self.assertEqual(total, 3) + create_call = [c for c in mock_get.call_args_list if c.kwargs.get('params', {}).get('action') == 'CREATE'] + self.assertEqual(create_call[0].kwargs['params']['name'], COLLECTION) + + posts = [(c.args[0], c.kwargs.get('json')) for c in mock_post.call_args_list] + schema_url, schema_body = posts[0] + self.assertTrue(schema_url.endswith(f'/{COLLECTION}/schema')) + self.assertEqual([f['name'] for f in schema_body['add-field']], ['dataset_id', 'doc_id', 'text']) + + self.assertEqual(posts[1][1], {'delete': {'query': f'dataset_id:{self.project.dataset.id}'}}) + self.assertEqual([d['doc_id'] for d in posts[2][1]], [self.docs[0].id, self.docs[1].id]) + self.assertEqual([d['doc_id'] for d in posts[3][1]], [self.docs[2].id]) + self.assertEqual(posts[2][1][0], { + 'id': str(self.docs[0].id), 'doc_id': self.docs[0].id, 'dataset_id': self.project.dataset.id, + 'name': 'n0', 'text': 'text 0', + }) + self.assertEqual(posts[4][1], {'commit': {}}) + + @patch('api.document_search.requests.get') + def test_raises_when_solr_cannot_list_collections(self, mock_get): + mock_get.return_value = _resp(status=500) + with self.assertRaises(document_search.DocumentSearchUnavailable): + document_search.index_dataset_documents(self.project.dataset.id) + + +@override_settings(MEDIA_ROOT='/tmp/mct-tests-doc-search') +class SearchDocumentsViewTests(TestCase): + def setUp(self): + self.client = _auth_client(create_user(username='searcher')) + self.project = create_basic_project(name='search-proj') + self.dataset_id = self.project.dataset.id + self.d1 = create_document(self.project, name='n1', text='Patient has Chest Pain.') + self.d2 = create_document(self.project, name='n2', text='No fever.') + self.d3 = create_document(self.project, name='n3', text='chest pain resolved') + + def _get(self, **params): + return self.client.get('/api/search-documents/', params) + + @patch('api.document_search.requests.get') + def test_returns_solr_hits_in_solr_order_with_highlights(self, mock_get): + pre, post = document_search.HIGHLIGHT_PRE, document_search.HIGHLIGHT_POST + + def side_effect(url, params=None, **kwargs): + if params['q'] == '*:*': + return _resp(body={'response': {'numFound': 3, 'docs': []}}) + self.assertEqual(params['q'], 'text:"chest pain"') + self.assertEqual(params['fq'], f'dataset_id:{self.dataset_id}') + return _resp(body={ + 'response': {'numFound': 2, 'docs': [ + {'id': str(self.d3.id), 'doc_id': self.d3.id}, + {'id': str(self.d1.id), 'doc_id': self.d1.id}, + ]}, + 'highlighting': {str(self.d3.id): {'text': [f'{pre}chest pain{post} resolved']}}, + }) + mock_get.side_effect = side_effect + + resp = self._get(dataset=self.dataset_id, text='chest pain') + self.assertEqual(resp.status_code, 200) + body = resp.json() + self.assertEqual(body['search_backend'], 'solr') + self.assertEqual(body['count'], 2) + self.assertIsNone(body['next']) + self.assertEqual([r['id'] for r in body['results']], [self.d3.id, self.d1.id]) + self.assertEqual(body['results'][0]['text'], 'chest pain resolved') + self.assertEqual(body['results'][0]['highlight'][0], {'text': 'chest pain', 'match': True}) + self.assertIsNone(body['results'][1]['highlight']) + + @patch('api.document_search.requests.get') + def test_paginates(self, mock_get): + mock_get.side_effect = lambda url, params=None, **kw: _resp(body={ + 'response': {'numFound': 3 if params['q'] == '*:*' else 120, 'docs': []} + }) + body = self._get(dataset=self.dataset_id, text='pain', page=2).json() + self.assertEqual(mock_get.call_args.kwargs['params']['start'], 50) + self.assertEqual(body['next'], f'/api/search-documents/?dataset={self.dataset_id}&text=pain&page=3') + self.assertEqual(body['previous'], f'/api/search-documents/?dataset={self.dataset_id}&text=pain&page=1') + + @patch('api.document_search.requests.get') + def test_falls_back_to_db_when_dataset_not_fully_indexed(self, mock_get): + mock_get.return_value = _resp(body={'response': {'numFound': 1, 'docs': []}}) + body = self._get(dataset=self.dataset_id, text='chest pain').json() + self.assertEqual(body['search_backend'], 'db') + self.assertEqual([r['id'] for r in body['results']], [self.d1.id, self.d3.id]) + + @patch('api.document_search.requests.get') + def test_falls_back_to_db_when_solr_unreachable(self, mock_get): + import requests + mock_get.side_effect = requests.ConnectionError('down') + body = self._get(dataset=self.dataset_id, text='FEVER').json() + self.assertEqual(body['search_backend'], 'db') + self.assertEqual([r['id'] for r in body['results']], [self.d2.id]) + + def test_validates_params(self): + self.assertEqual(self._get(dataset=self.dataset_id, text=' ').status_code, 400) + self.assertEqual(self._get(dataset='abc', text='x').status_code, 400) + + def test_requires_authentication(self): + resp = APIClient().get('/api/search-documents/', {'dataset': self.dataset_id, 'text': 'x'}) + self.assertIn(resp.status_code, (401, 403)) + + +@override_settings(MEDIA_ROOT=tempfile.mkdtemp()) +class DatasetIndexSignalTests(TestCase): + def _upload_dataset(self): + csv_path = make_csv_file(tempfile.mkdtemp()) + with open(csv_path, 'rb') as fh: + ds = Dataset(name='index-signal-ds') + ds.original_file.save('index-signal-ds.csv', ContentFile(fh.read())) + return ds + + def test_upload_schedules_index_task_on_commit(self): + with self.captureOnCommitCallbacks(execute=True): + ds = self._upload_dataset() + tasks = Task.objects.filter(task_name='api.document_search.index_dataset_task') + self.assertEqual(tasks.count(), 1) + self.assertEqual(json.loads(tasks.get().task_params)[0], [ds.id]) + + def test_delete_schedules_remove_task(self): + ds = self._upload_dataset() + with self.captureOnCommitCallbacks(execute=True): + ds_id = ds.id + ds.delete() + task = Task.objects.get(task_name='api.document_search.remove_dataset_task') + self.assertEqual(json.loads(task.task_params)[0], [ds_id]) + + @patch('api.signals.DOCUMENT_SEARCH_INDEX_ENABLED', False) + def test_no_task_when_indexing_disabled(self): + with self.captureOnCommitCallbacks(execute=True): + self._upload_dataset() + self.assertFalse(Task.objects.filter(task_name__startswith='api.document_search').exists()) + + +@unittest.skipUnless(os.environ.get('MCT_SOLR_LIVE_TESTS') == '1', 'requires a running SolrCloud') +@override_settings(MEDIA_ROOT=tempfile.mkdtemp()) +class LiveSolrDocumentSearchTests(TestCase): + def setUp(self): + self.collection = f'mct_test_docs_{uuid.uuid4().hex[:8]}' + patcher = patch.object(document_search, 'DOCUMENT_SEARCH_COLLECTION', self.collection) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(lambda: document_search.requests.get( + f'{document_search._base_url()}/admin/collections', + params={'action': 'DELETE', 'name': self.collection})) + self.client = _auth_client(create_user(username='live-searcher')) + + def _upload(self, name, rows): + csv_path = make_csv_file(tempfile.mkdtemp(), rows=rows) + with open(csv_path, 'rb') as fh: + ds = Dataset(name=name) + ds.original_file.save(f'{name}.csv', ContentFile(fh.read())) + return ds + + def test_upload_index_and_search(self): + with self.captureOnCommitCallbacks(execute=True): + ds = self._upload('live-ds', [ + {'name': 'a', 'text': 'Patient reports CHEST pain, radiating to the left arm.'}, + {'name': 'b', 'text': 'No fever or cough. Pain in chest wall only.'}, + {'name': 'c', 'text': 'Chest pain resolved after GTN.'}, + ]) + other = self._upload('live-other', [{'name': 'x', 'text': 'chest pain elsewhere'}]) + scheduled = Task.objects.filter(task_name='api.document_search.index_dataset_task') + self.assertEqual(scheduled.count(), 2) + + # run the scheduled tasks as the background worker would + for task in scheduled: + args, kwargs = json.loads(task.task_params) + document_search.index_dataset_task.now(*args, **kwargs) + + body = self.client.get('/api/search-documents/', {'dataset': ds.id, 'text': 'chest pain'}).json() + self.assertEqual(body['search_backend'], 'solr') + names = sorted(r['name'] for r in body['results']) + self.assertEqual(names, ['a', 'c']) + for r in body['results']: + matched = ''.join(s['text'] for s in r['highlight'] if s['match']) + self.assertEqual(matched.lower(), 'chest pain') + + # re-indexing replaces rather than duplicates + document_search.index_dataset_documents(ds.id) + self.assertEqual(document_search.indexed_document_count(ds.id), 3) + + document_search.remove_dataset_documents(other.id) + self.assertEqual(document_search.indexed_document_count(other.id), 0) + self.assertEqual(document_search.indexed_document_count(ds.id), 3) + self.assertEqual(Document.objects.filter(dataset=ds).count(), 3) diff --git a/medcat-trainer/webapp/api/api/tests/test_solr_utils.py b/medcat-trainer/webapp/api/api/tests/test_solr_utils.py index 1df84d4d4..8dd8ed765 100644 --- a/medcat-trainer/webapp/api/api/tests/test_solr_utils.py +++ b/medcat-trainer/webapp/api/api/tests/test_solr_utils.py @@ -47,6 +47,19 @@ def side_effect(url, *args, **kwargs): self.assertEqual(response.status_code, 200) self.assertEqual(response.data['results']['1']['index_name'], 'my_id_1') + @patch('api.solr_utils.requests.get') + def test_excludes_document_search_collection(self, mock_get): + def side_effect(url, *args, **kwargs): + if 'admin/collections' in url: + collections = ['my_id_1', solr_utils.DOCUMENT_SEARCH_COLLECTION] + return MagicMock(status_code=200, text=json.dumps({'collections': collections})) + return MagicMock(text=json.dumps({'schema': {'fields': [{'name': 'cui', 'type': 'string'}]}})) + + mock_get.side_effect = side_effect + + response = solr_utils.collections_available([]) + self.assertEqual(list(response.data['results'].keys()), ['1']) + @patch('api.solr_utils.requests.get') def test_returns_500_when_solr_admin_unavailable(self, mock_get): mock_get.return_value = MagicMock(status_code=500, text='boom') diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py index d0a150a90..25abe9596 100644 --- a/medcat-trainer/webapp/api/api/views.py +++ b/medcat-trainer/webapp/api/api/views.py @@ -3,6 +3,7 @@ from smtplib import SMTPException from tempfile import NamedTemporaryFile, TemporaryDirectory from typing import Any +from urllib.parse import urlencode import shutil from background_task.models import Task, CompletedTask @@ -26,6 +27,7 @@ from .admin import download_projects_with_text, download_projects_without_text, \ import_concepts_from_cdb from .data_utils import upload_projects_export +from .document_search import DocumentSearchUnavailable, search_dataset_documents from .metrics import calculate_metrics from .model_cache import get_medcat, get_medcat_from_model_pack_id, get_cached_cdb, VOCAB_MAP, clear_cached_medcat, clear_cached_medcat_by_model_pack_id, is_model_pack_loaded, CAT_MAP, CDB_MAP, is_model_loaded from .permissions import * @@ -882,6 +884,55 @@ def search_solr(request): return search_collection(cdbs, query) +DOCUMENT_SEARCH_PAGE_SIZE = 50 + + +@api_view(http_method_names=['GET']) +@permission_classes([permissions.IsAuthenticated]) +def search_documents(request): + """Full text (phrase) search of a dataset's documents. + + Served from the Solr document index, falling back to a case-insensitive DB substring match + when Solr is unavailable or the dataset has not finished indexing. + """ + text = request.GET.get('text', '').strip() + try: + dataset_id = int(request.GET.get('dataset', '')) + page = max(1, int(request.GET.get('page', 1))) + except ValueError: + return Response({'message': 'dataset and page must be integers'}, 400) + if not text: + return Response({'message': 'text is required'}, 400) + + dataset_docs = Document.objects.filter(dataset_id=dataset_id) + start = (page - 1) * DOCUMENT_SEARCH_PAGE_SIZE + try: + count, hits = search_dataset_documents(dataset_id, text, start, DOCUMENT_SEARCH_PAGE_SIZE, + expected_doc_count=dataset_docs.count()) + docs_by_id = dataset_docs.in_bulk([doc_id for doc_id, _ in hits]) + results = [{**DocumentSerializer(docs_by_id[doc_id]).data, 'highlight': segments} + for doc_id, segments in hits if doc_id in docs_by_id] + backend = 'solr' + except DocumentSearchUnavailable as e: + logger.warning('Document search falling back to DB for dataset %s: %s', dataset_id, e) + matches = dataset_docs.filter(text__icontains=text).order_by('id') + count = matches.count() + page_docs = matches[start:start + DOCUMENT_SEARCH_PAGE_SIZE] + results = [{**DocumentSerializer(d).data, 'highlight': None} for d in page_docs] + backend = 'db' + + def page_url(p): + return f'/api/search-documents/?{urlencode({"dataset": dataset_id, "text": text, "page": p})}' + + return Response({ + 'count': count, + 'next': page_url(page + 1) if start + DOCUMENT_SEARCH_PAGE_SIZE < count else None, + 'previous': page_url(page - 1) if page > 1 else None, + 'results': results, + 'search_backend': backend, + }) + + @api_view(http_method_names=['POST']) def upload_deployment(request): deployment_export = request.data diff --git a/medcat-trainer/webapp/api/core/settings.py b/medcat-trainer/webapp/api/core/settings.py index 98c35f53d..a168091be 100644 --- a/medcat-trainer/webapp/api/core/settings.py +++ b/medcat-trainer/webapp/api/core/settings.py @@ -325,6 +325,9 @@ def resolve_secret_key(realm, secret_key): # Solr Concept Search settings SOLR_HOST = os.environ.get('CONCEPT_SEARCH_SERVICE_HOST', 'solr') SOLR_PORT = os.environ.get('CONCEPT_SEARCH_SERVICE_PORT', '8983') +# Solr collection holding the text of every dataset document, for full text document search +DOCUMENT_SEARCH_COLLECTION = os.environ.get('DOCUMENT_SEARCH_COLLECTION', 'medcattrainer_documents') +DOCUMENT_SEARCH_INDEX_ENABLED = os.environ.get('DOCUMENT_SEARCH_INDEX_ENABLED', '1').lower() in ('1', 'true', 'y', 'yes') SILENCED_SYSTEM_CHECKS = ['admin.E130'] diff --git a/medcat-trainer/webapp/api/core/urls.py b/medcat-trainer/webapp/api/core/urls.py index d43317dfd..045c7e225 100644 --- a/medcat-trainer/webapp/api/core/urls.py +++ b/medcat-trainer/webapp/api/core/urls.py @@ -30,6 +30,7 @@ path('api/health/', include('health_check.urls')), path('api/anno-conf/', api.views.get_anno_tool_conf), path('api/search-concepts/', api.views.search_solr), + path('api/search-documents/', api.views.search_documents), path('api/prepare-documents/', api.views.prepare_documents), path('api/prepare-documents-bg/', api.views.prepare_documents_bg), path('api/prep-docs-bg-tasks/', api.views.prepare_docs_bg_tasks), From dce1c569dc947dd5ceb76ada107059502f7d9974 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 15 Oct 2026 20:34:34 +0000 Subject: [PATCH 4/7] feat(medcat-trainer): Use Solr document search for clinical notes text search Co-authored-by: tomolopolis --- .../src/components/common/DocumentSummary.vue | 32 +++++++++++-------- .../tests/components/DocumentSummary.spec.ts | 30 +++++++++++++++-- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue index 6976efa37..a28194b4d 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue @@ -35,8 +35,9 @@ class="prepared-doc" icon="clipboard-check"> -
{{ textSnippet(doc).pre }}{{ textSnippet(doc).match }}{{ textSnippet(doc).post }}
+
{{ seg.text }}
{{doc.text === 'nan' ? '' : (limitText(doc.text) || '')}}
@@ -177,11 +178,8 @@ export default { this.textSearchLoading = true this.textSearchTimer = setTimeout(() => { this.textSearchTimer = null - const params = new URLSearchParams({ text__icontains: this.searchCrit }) - if (this.datasetId !== undefined && this.datasetId !== null) { - params.set('dataset', this.datasetId) - } - this.fetchTextResults(`/api/documents/?${params.toString()}`) + const params = new URLSearchParams({ dataset: this.datasetId, text: this.searchCrit }) + this.fetchTextResults(`/api/search-documents/?${params.toString()}`) }, TEXT_SEARCH_DEBOUNCE_MS) } else { this.filteredDocs = this.docs.filter(d => { @@ -224,11 +222,17 @@ export default { }, loadMoreTextResults () { if (this.textSearchNextUrl) { - this.fetchTextResults(`/api/documents/?${this.textSearchNextUrl.split('?').slice(-1)[0]}`) + this.fetchTextResults(`/api/search-documents/?${this.textSearchNextUrl.split('?').slice(-1)[0]}`) } }, textSnippet (doc) { - if (!this.isTextSearch || !doc.text) { + if (!this.isTextSearch) { + return null + } + if (doc.highlight && doc.highlight.length) { + return doc.highlight + } + if (!doc.text) { return null } const idx = doc.text.toLowerCase().indexOf(this.searchCrit.toLowerCase()) @@ -237,11 +241,11 @@ export default { } const end = idx + this.searchCrit.length const start = Math.max(0, idx - SNIPPET_CONTEXT_CHARS) - return { - pre: (start > 0 ? '...' : '') + doc.text.slice(start, idx), - match: doc.text.slice(idx, end), - post: doc.text.slice(end, end + SNIPPET_CONTEXT_CHARS) - } + return [ + { text: (start > 0 ? '...' : '') + doc.text.slice(start, idx), match: false }, + { text: doc.text.slice(idx, end), match: true }, + { text: doc.text.slice(end, end + SNIPPET_CONTEXT_CHARS), match: false } + ] }, limitText (value) { value = value.trim() diff --git a/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts b/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts index d5048247e..194697b1b 100644 --- a/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts @@ -168,7 +168,7 @@ describe('DocumentSummary.vue', () => { await wrapper.vm.$nextTick() expect(mockGet).toHaveBeenCalledTimes(1) - expect(mockGet).toHaveBeenCalledWith('/api/documents/?text__icontains=chest+pain&dataset=5') + expect(mockGet).toHaveBeenCalledWith('/api/search-documents/?dataset=5&text=chest+pain') const docEls = wrapper.findAll('.doc') expect(docEls).toHaveLength(1) expect(wrapper.find('.search-match').text()).toBe('chest pain') @@ -181,7 +181,7 @@ describe('DocumentSummary.vue', () => { it('pages through further results with More Results', async () => { const mockGet = vi.fn() .mockResolvedValueOnce({ - data: { count: 2, next: 'http://host/api/documents/?dataset=5&page=2&text__icontains=pain', results: textResults } + data: { count: 2, next: '/api/search-documents/?dataset=5&text=pain&page=2', results: textResults } }) .mockResolvedValueOnce({ data: { count: 2, next: null, results: [{ id: 8, name: 'n8', text: 'pain' }] } @@ -197,11 +197,35 @@ describe('DocumentSummary.vue', () => { await wrapper.find('.more-docs').trigger('click') await flush() await wrapper.vm.$nextTick() - expect(mockGet).toHaveBeenLastCalledWith('/api/documents/?dataset=5&page=2&text__icontains=pain') + expect(mockGet).toHaveBeenLastCalledWith('/api/search-documents/?dataset=5&text=pain&page=2') expect(wrapper.findAll('.doc')).toHaveLength(2) expect(wrapper.find('.more-docs').exists()).toBe(false) }) + it('renders the Solr highlight snippet when the result has one', async () => { + const highlighted = { + ...textResults[0], + highlight: [ + { text: 'History: patient has ', match: false }, + { text: 'chest pain', match: true }, + { text: ' on exertion.', match: false } + ] + } + const mockGet = vi.fn().mockResolvedValue({ + data: { count: 1, next: null, results: [highlighted] } + }) + const wrapper = mountWithHttp(mockGet) + const vm = wrapper.vm as unknown as TextSearchVm + vm.searchMode = 'text' + vm.searchDocs({ target: { value: 'chest pain' } }) + vi.advanceTimersByTime(300) + await flush() + await wrapper.vm.$nextTick() + + expect(wrapper.find('.search-match').text()).toBe('chest pain') + expect(wrapper.find('.note-summary').text()).toBe('History: patient has chest pain on exertion.') + }) + it('ignores stale responses from superseded searches', async () => { let resolveFirst: (v: unknown) => void = () => {} const mockGet = vi.fn() From 1a01342e0848cdf8364761357004be1003814fc7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 15 Oct 2026 20:34:34 +0000 Subject: [PATCH 5/7] docs(medcat-trainer): Document Solr document search settings Co-authored-by: tomolopolis --- medcat-trainer/docs/maintenance.md | 5 +++++ medcat-trainer/envs/env | 3 +++ medcat-trainer/envs/env-prod | 3 +++ 3 files changed, 11 insertions(+) diff --git a/medcat-trainer/docs/maintenance.md b/medcat-trainer/docs/maintenance.md index ee2290cc0..672df314c 100644 --- a/medcat-trainer/docs/maintenance.md +++ b/medcat-trainer/docs/maintenance.md @@ -26,6 +26,11 @@ Database migrations are applied automatically on container startup. - Container logs: `docker compose logs -f medcattrainer` - Concept search availability: verify Solr container and project concept import status. +- Document text search: each uploaded dataset is indexed into the Solr collection + `DOCUMENT_SEARCH_COLLECTION` (default `medcattrainer_documents`) by a background task. + Datasets uploaded before this was available can be indexed via the Django admin + Datasets action "Index selected datasets for text search". Until a dataset is fully + indexed, or if Solr is unavailable, text search falls back to a slower database match. ## Backup and restore (SQLite deployments) diff --git a/medcat-trainer/envs/env b/medcat-trainer/envs/env index 044eaa2ed..cad8b2e2c 100644 --- a/medcat-trainer/envs/env +++ b/medcat-trainer/envs/env @@ -33,6 +33,9 @@ MAX_DATASET_SIZE=10000 ### Solr Concept Search Conf ### CONCEPT_SEARCH_SERVICE_HOST=solr CONCEPT_SEARCH_SERVICE_PORT=8983 +# Solr collection used for full text search of dataset documents, (re-)indexed on dataset upload +DOCUMENT_SEARCH_COLLECTION=medcattrainer_documents +DOCUMENT_SEARCH_INDEX_ENABLED=1 ### DB backup dir ### # volume mount location, default docker host system volume location, this might be different in /etc/docker/daemon.json diff --git a/medcat-trainer/envs/env-prod b/medcat-trainer/envs/env-prod index 39bd1ff9f..fc004642d 100644 --- a/medcat-trainer/envs/env-prod +++ b/medcat-trainer/envs/env-prod @@ -26,6 +26,9 @@ MAX_DATASET_SIZE=10000 ### Solr Concept Search Conf ### CONCEPT_SEARCH_SERVICE_HOST=solr CONCEPT_SEARCH_SERVICE_PORT=8983 +# Solr collection used for full text search of dataset documents, (re-)indexed on dataset upload +DOCUMENT_SEARCH_COLLECTION=medcattrainer_documents +DOCUMENT_SEARCH_INDEX_ENABLED=1 ### DB backup dir - should be set ideally to a mounted / backed up drive ### # volume mount location, default docker host system volume location, this might be different in /etc/docker/daemon.json From 5fb9bbc205383e4a07040c3bc3894ad876f5734a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 22:22:43 +0000 Subject: [PATCH 6/7] feat(medcat-trainer): Put clinical notes summary in a resizable split pane Co-authored-by: tomolopolis --- .../src/components/common/DocumentSummary.vue | 16 +++++++++---- .../src/tests/views/TrainAnnotations.spec.ts | 23 +++++++++++++++++++ .../frontend/src/views/TrainAnnotations.vue | 15 +++++++----- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue index a28194b4d..19219af1a 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/DocumentSummary.vue @@ -269,8 +269,6 @@ export default { diff --git a/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json b/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json new file mode 100644 index 000000000..e44c47e3b --- /dev/null +++ b/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json @@ -0,0 +1 @@ +{"version":"3.2.7","results":[[":medcat-trainer/webapp/frontend/src/tests/components/DocumentSummary.spec.ts",{"duration":0,"failed":true}]]} \ No newline at end of file