From f556a996e46ab1f7424f6aefd152455a0e0cb8cb Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Thu, 3 Sep 2026 16:37:28 +0100
Subject: [PATCH 01/12] Docs: updated workflow docs.

---
 docs/deploy/workflows.md                      |  83 ++++++++++++--
 .../azure_blob_parquet_to_opensearch.md       |  90 +++++++++++++++
 .../workflows/database_ocr_to_opensearch.md   | 108 ++++++++++++++++++
 .../workflows/database_to_opensearch.md       |  92 +++++++++++++++
 .../filesystem_parquet_to_opensearch.md       |  86 ++++++++++++++
 .../medcat_annotations_to_opensearch.md       |  97 ++++++++++++++++
 mkdocs.yml                                    |   8 +-
 7 files changed, 555 insertions(+), 9 deletions(-)
 create mode 100644 docs/deploy/workflows/azure_blob_parquet_to_opensearch.md
 create mode 100644 docs/deploy/workflows/database_ocr_to_opensearch.md
 create mode 100644 docs/deploy/workflows/database_to_opensearch.md
 create mode 100644 docs/deploy/workflows/filesystem_parquet_to_opensearch.md
 create mode 100644 docs/deploy/workflows/medcat_annotations_to_opensearch.md

diff --git a/docs/deploy/workflows.md b/docs/deploy/workflows.md
index cfad5b08..c9ec32c6 100644
--- a/docs/deploy/workflows.md
+++ b/docs/deploy/workflows.md
@@ -1,10 +1,71 @@
-# Workflows
+# 🔀 Workflows
 
-This page contains current workflow guidance for NiFi 2.x based deployments.
+This section documents every maintained NiFi 2.x flow definition in
+`nifi/user_templates/`. Files below `nifi/user_templates/legacy/` are historical and
+are not covered.
 
-Current JSON templates are stored in `nifi/user_templates/`.
+## 📚 Maintained flow definitions
 
-## Cerner blob decompression
+| Flow definition | Source | Destination | Additional service |
+|---|---|---|---|
+| [Database documents to OpenSearch](workflows/database_to_opensearch.md) | PostgreSQL table | OpenSearch index | None |
+| [Database documents via OCR](workflows/database_ocr_to_opensearch.md) | PostgreSQL binary-document table | OpenSearch index | OCR service |
+| [MedCAT annotations to OpenSearch](workflows/medcat_annotations_to_opensearch.md) | Existing OpenSearch index | Annotation index | MedCAT service |
+| [Filesystem Parquet to OpenSearch](workflows/filesystem_parquet_to_opensearch.md) | Parquet files under `/data` | Index per filename | None |
+| [Azure Blob Parquet to OpenSearch](workflows/azure_blob_parquet_to_opensearch.md) | Azure Blob Storage | Index per blob prefix | Azure Storage |
+
+## 🚀 Prepare and import a flow
+
+From the repository root, initialize certificates and start the core data services:
+
+```bash
+make -C deploy init-security
+make -C deploy start-data-infra
+```
+
+Then open NiFi at `https://localhost:8443`. Keep the imported process group stopped
+while configuring it. Use NiFi's **Upload Process Group** or flow-definition import
+action and select the JSON file named by the relevant guide.
+
+Most bundled standard processors and controller services target NiFi 2.10.0; the Azure
+Blob flow contains Azure components from NiFi 2.7.2. The definitions include processor
+and controller-service settings, but they do not contain parameter contexts and NiFi
+omits sensitive property values from exports. Review component compatibility when
+importing them into another NiFi version.
+
+## 🔐 Post-import configuration
+
+Before starting an imported flow:
+
+1. Open the process group's **Controller Services** configuration.
+2. Enter all required sensitive values. Depending on the flow, these include database,
+   OpenSearch, keystore, truststore, or Azure Storage credentials.
+3. Confirm that service URLs, database tables, field names, certificate paths, and
+   target index names match the deployment.
+4. Enable the record readers/writers, SSL context, database pool, storage credentials,
+   and OpenSearch client used by the flow.
+5. Resolve every validation error, then start the process group.
+
+The bundled URLs and credentials are local-development defaults. Use deployment-specific
+values in production. Failure, retry, and error relationships terminate at funnels in
+these examples; inspect their queues instead of assuming failed FlowFiles were discarded.
+
+## 🔎 Verify OpenSearch output
+
+Check the expected index in OpenSearch Dashboards or query its count endpoint. For the
+default local OpenSearch deployment:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/<index>/_count'
+```
+
+Use the target index documented for the selected flow. A successful HTTP response does
+not prove that every FlowFile succeeded, so also inspect NiFi bulletins, provenance, and
+the queues connected to failure funnels.
+
+## 🗜️ Cerner blob decompression
 
 Use `CogStackJsonRecordDecompressCernerBlob` when source documents are stored as Cerner
 blob fragments, with one database row per blob sequence. The processor expects one
@@ -12,7 +73,7 @@ FlowFile to contain all blob rows for a single document, then sorts the fragment
 sequence number, validates the sequence, concatenates the binary payload, and extracts
 or decompresses the embedded document bytes.
 
-Recommended processor chain:
+### 🔗 Recommended processor chain
 
 ```text
 Fetch document IDs
@@ -27,7 +88,7 @@ that every blob sequence for a document remains in the same FlowFile. For Cerner
 rows, prefer a document-ID driven query where each `ExecuteSQLRecord` invocation fetches
 all sequences for one document.
 
-Example SQL shape:
+### 🧾 Example SQL shape
 
 ```sql
 SELECT
@@ -39,7 +100,7 @@ WHERE docid = ?
 ORDER BY blob_sequence_num
 ```
 
-`ExecuteSQLRecord` settings:
+### ⚙️ `ExecuteSQLRecord` settings
 
 - Set `Max Rows Per FlowFile` to `0`.
 - Set `Output Batch Size` to `0`.
@@ -51,6 +112,8 @@ ORDER BY blob_sequence_num
   SQL is configured on the processor, pass the document ID as a prepared-statement
   argument, for example with `sql.args.1.type` and `sql.args.1.value` attributes.
 
+### 📥 Required input fields
+
 Input records must include these fields unless you override the processor properties:
 
 | Field | Default property | Required behavior |
@@ -59,7 +122,7 @@ Input records must include these fields unless you override the processor proper
 | Blob sequence | `blob_sequence_order_field_name=blob_sequence_num` | Present, integer-like, unique by default, contiguous, and starting at `0` or `1`. |
 | Blob payload | `binary_field_name=binarydoc` | Present and non-empty. With the default `binary_field_source_encoding=base64`, this must be a base64 string. |
 
-Recommended processor properties:
+### 🛠️ Recommended processor properties
 
 | Property | Recommended value | Notes |
 |---|---|---|
@@ -70,6 +133,8 @@ Recommended processor properties:
 | `binary_field_source_encoding` | `base64` | Recommended for JSON FlowFiles. |
 | `output_mode` | `base64` | Recommended for JSON output and downstream processors. |
 
+### ⚠️ Failure routing
+
 The processor routes to `failure` when it detects unsafe input, including:
 
 - missing document ID, blob sequence, or blob payload fields;
@@ -79,6 +144,8 @@ The processor routes to `failure` when it detects unsafe input, including:
 - invalid base64 payloads;
 - payloads that cannot be extracted as embedded PDF/RTF bytes or decoded as Cerner LZW.
 
+### ✅ Successful output
+
 On success, the output is a JSON array containing one merged record. Non-binary fields
 are copied from the first input record, and the blob field contains the decompressed or
 extracted document bytes encoded according to `output_mode`. Useful FlowFile attributes
diff --git a/docs/deploy/workflows/azure_blob_parquet_to_opensearch.md b/docs/deploy/workflows/azure_blob_parquet_to_opensearch.md
new file mode 100644
index 00000000..56c2e9c8
--- /dev/null
+++ b/docs/deploy/workflows/azure_blob_parquet_to_opensearch.md
@@ -0,0 +1,90 @@
+# ☁️ Azure Blob Parquet to OpenSearch
+
+Template: `nifi/user_templates/azure_blobs_parquet_to_opensearch.json`
+
+[View the flow definition on GitHub](https://github.com/CogStack/CogStack-NiFi/blob/main/nifi/user_templates/azure_blobs_parquet_to_opensearch.json)
+
+## 🎯 Purpose
+
+This flow lists blobs in an Azure Storage container, fetches Parquet blobs, converts
+them to NDJSON, and writes their records to OpenSearch. The first path segment of each
+blob name becomes the destination index.
+
+## 🔗 Processor chain
+
+```text
+ListAzureBlobStorage_v12
+  -> FetchAzureBlobStorage_v12
+  -> RouteOnAttribute-parquetFiles
+  -> ExecuteStreamCommand-ParquetToJson
+  -> PutElasticsearchRecord
+```
+
+The list processor emits blob metadata, the fetch processor retrieves one blob per
+FlowFile, and `RouteOnAttribute` accepts filenames ending in `.parquet`. The conversion
+script reads the Parquet content from standard input and emits NDJSON to standard output.
+
+## 📋 Bundled defaults
+
+| Setting | Value |
+|---|---|
+| Azure endpoint suffix | `blob.core.windows.net` |
+| Credential mode | Account key |
+| Container name | `container_name` placeholder |
+| Listing strategy | Timestamps |
+| Polling schedule | `1 min` |
+| Initial listing target | All blobs |
+| Accepted filename suffix | `.parquet` |
+| Conversion command | `python3.11 convert_record_parquet_to_json.py` |
+| Script working directory | `/opt/nifi/user_scripts/processors/` |
+| Target index | `${azure.blobname:substringBefore('/'):toLower()}` |
+| Index operation | `create` |
+| OpenSearch bulk batch size | `10000` |
+
+The index expression expects blob names shaped like `<index>/<file>.parquet`. Review it
+before processing root-level blobs or prefixes containing characters that are invalid in
+OpenSearch index names.
+
+## ✅ Requirements
+
+- NiFi and OpenSearch are running and certificates are generated.
+- The NiFi container can reach the Azure Storage endpoint.
+- The Azure account, container, and selected authentication method allow blob listing
+  and reads.
+- The bundled Parquet conversion script and Python 3.11 are available in the NiFi image.
+- Azure, OpenSearch, keystore, and truststore sensitive values are entered after import.
+
+## ⚙️ Configure after import
+
+1. In `AzureStorageCredentialsControllerService_v12-RIO_BLOBS`, enter the storage
+   account name and account key, or select and configure another supported credential
+   type. Do not place production secrets in the flow JSON.
+2. Set the real container name on `ListAzureBlobStorage_v12`.
+3. Keep its **Record Writer** unset. The flow expects one listed blob per FlowFile so
+   `FetchAzureBlobStorage_v12` can use `${azure.container}` and `${azure.blobname}`.
+4. Confirm the route expression and conversion command path.
+5. Configure and enable the JSON reader/writer, SSL context, Azure credential service,
+   and OpenSearch client.
+6. Review the target-index expression and add a deterministic `ID Record Path` if the
+   Parquet data has a stable unique key.
+
+## ▶️ Run and verify
+
+Start with a dedicated test prefix and a small Parquet blob. Verify the derived index,
+then compare its count with the source file:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/<blob-prefix>/_count'
+```
+
+## ⚠️ State and failure handling
+
+The list processor keeps timestamp-based state and is configured to include all blobs
+on its initial listing. Test the scope before enabling it against a large container.
+Resetting listing state can fetch old blobs again. Because the writer uses `create`
+without a document ID, reprocessing can create duplicate logical records.
+
+Fetch failures, conversion non-zero exits, OpenSearch failures, retries, and per-record
+error responses are routed to funnel queues for inspection.
diff --git a/docs/deploy/workflows/database_ocr_to_opensearch.md b/docs/deploy/workflows/database_ocr_to_opensearch.md
new file mode 100644
index 00000000..17d1d15a
--- /dev/null
+++ b/docs/deploy/workflows/database_ocr_to_opensearch.md
@@ -0,0 +1,108 @@
+# 📄 Database documents via OCR
+
+Template: `nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json`
+
+[View the flow definition on GitHub](https://github.com/CogStack/CogStack-NiFi/blob/main/nifi/user_templates/opensearch_ingest_docs_db_ocr_service_to_es.json)
+
+## 🎯 Purpose
+
+This flow reads binary documents from PostgreSQL, sends one document at a time to the
+CogStack OCR service, merges the extracted text with the source metadata, and indexes
+the result in OpenSearch.
+
+## 🔗 Processor chain
+
+```text
+GenerateTableFetch-encounters
+  -> ExecuteSQLRecord
+  -> CogStackPrepareRecordForOcr
+  -> InvokeHTTP
+  -> CogStackParseCogStackServiceResult
+  -> SplitJson
+  -> EvaluateJsonPath
+  -> PutElasticsearchJson
+```
+
+`CogStackPrepareRecordForOcr` converts the Avro binary field to a base64 JSON request
+and preserves the remaining source fields in a footer. After OCR, the parser restores
+that footer, adds the extracted `text`, and exposes `id` as a FlowFile attribute for
+OpenSearch indexing.
+
+## 📋 Bundled defaults
+
+| Setting | Value |
+|---|---|
+| Database URL | `jdbc:postgresql://samples-db:5432/db_samples` |
+| Source table | `encounters` |
+| Incremental/partition column | `cid` |
+| Partition size | `1000` |
+| Rows per FlowFile | `1` |
+| Binary source field | `binarydocument` |
+| Document ID field | `id` |
+| OCR endpoint | `http://ocr-service:8090/api/process` |
+| Extracted text field | `text` |
+| OpenSearch endpoint | `https://elasticsearch-1:9200` |
+| Target index | `${generatetablefetch.tableName}` (`encounters` by default) |
+| Index operation | `index` |
+| OpenSearch identifier attribute | `id` |
+
+## ✅ Requirements
+
+- The core stack and sample database are running.
+- The OCR service is running. From the repository root use:
+
+  ```bash
+  make -C deploy start-ocr-services
+  ```
+
+- The source table contains a numeric incremental column, a unique document ID, and a
+  binary document column.
+- NiFi can resolve `ocr-service` on the Docker network, or the `InvokeHTTP` URL is
+  changed to a reachable endpoint.
+- Database, OpenSearch, keystore, and truststore passwords are entered after import.
+
+## ⚙️ Configure after import
+
+1. Configure and enable `DBCPConnectionPool` for the source database.
+2. Update `GenerateTableFetch-encounters` when the source table or incremental column
+   differs from `encounters.cid`.
+3. Set `CogStackPrepareRecordForOcr` properties to the actual source fields:
+
+   | Property | Bundled value |
+   |---|---|
+   | `process_flow_file_type` | `avro` |
+   | `binary_field_name` | `binarydocument` |
+   | `document_id_field_name` | `id` |
+   | `output_text_field_name` | `text` |
+   | `operation_mode` | `base64` |
+
+4. Verify the `InvokeHTTP` URL. For the text-extraction-only container on the same
+   Docker network, use `http://ocr-service-text-only:8090/api/process`.
+5. Keep **Request Body Enabled** set to `true`; the processor sends JSON rather than a
+   raw binary body.
+6. Configure the SSL context and OpenSearch client passwords, hosts, and certificate
+   paths. Verify that `EvaluateJsonPath` extracts the same ID field used by
+   `PutElasticsearchJson`.
+
+## ▶️ Run and verify
+
+Start the process group with a small source range first. Confirm that OCR responses
+reach `CogStackParseCogStackServiceResult`, that split records have an `id` attribute,
+and that the destination index count increases:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/encounters/_count'
+```
+
+The default flow writes OCR-enriched records to an index with the same name as the
+source table. Change the target index before starting if source and enriched documents
+must be kept separately.
+
+## ⚠️ Failure handling
+
+HTTP failures, parse failures, SQL failures, unmatched IDs, and OpenSearch errors are
+routed to separate funnel queues. Inspect the response body and NiFi bulletin before
+retrying. Clearing `GenerateTableFetch` state causes source rows to be submitted to OCR
+again and should be done only deliberately.
diff --git a/docs/deploy/workflows/database_to_opensearch.md b/docs/deploy/workflows/database_to_opensearch.md
new file mode 100644
index 00000000..3c6b611f
--- /dev/null
+++ b/docs/deploy/workflows/database_to_opensearch.md
@@ -0,0 +1,92 @@
+# 🗄️ Database documents to OpenSearch
+
+Template: `nifi/user_templates/opensearch_ingest_docs_db_to_es.json`
+
+[View the flow definition on GitHub](https://github.com/CogStack/CogStack-NiFi/blob/main/nifi/user_templates/opensearch_ingest_docs_db_to_es.json)
+
+## 🎯 Purpose
+
+This flow incrementally reads text documents from a PostgreSQL table and indexes each
+record in OpenSearch. The bundled configuration targets the sample
+`medical_reports_text` table and uses `docid` as the stable document identifier.
+
+## 🔗 Processor chain
+
+```text
+GenerateTableFetch-medical_reports_text
+  -> ExecuteSQLRecord
+  -> PutElasticsearchRecord
+```
+
+- `GenerateTableFetch` creates paginated SQL queries and tracks the maximum `docid` it
+  has seen.
+- `ExecuteSQLRecord` executes each query and writes embedded-schema Avro.
+- `PutElasticsearchRecord` reads the Avro records and indexes them using `/docid` as
+  the OpenSearch document ID.
+
+Failure and retry relationships are routed to funnels for inspection.
+
+## 📋 Bundled defaults
+
+| Setting | Value |
+|---|---|
+| Database URL | `jdbc:postgresql://samples-db:5432/db_samples` |
+| Database user | `test` |
+| JDBC driver | `/opt/nifi/drivers/postgresql-42.7.7.jar` |
+| Source table | `medical_reports_text` |
+| Incremental/partition column | `docid` |
+| Partition size | `10000` |
+| Returned columns | `sampleid`, `typeid`, `dct`, `filename`, `document`, and `docid` cast to text |
+| OpenSearch endpoint | `https://elasticsearch-1:9200` |
+| Target index | `${generatetablefetch.tableName}` (`medical_reports_text` by default) |
+| Index operation | `index` |
+| Document ID | `/docid` |
+| Bulk batch size | `10000` |
+
+## ✅ Requirements
+
+- The core stack is running; for the bundled sample database use
+  `make -C deploy start-data-infra`.
+- The source table has a numeric, indexed, monotonically increasing maximum-value
+  column. The sample schema provides integer primary key `docid`.
+- OpenSearch certificates exist and are mounted into NiFi.
+- Database and OpenSearch credentials are available. Sensitive passwords are not
+  included in the exported JSON.
+
+## ⚙️ Configure after import
+
+1. In `DBCPConnectionPool`, verify the JDBC URL, driver class, driver location, user,
+   and password. The bundled local values correspond to `security/env/users_database.env`.
+2. In `GenerateTableFetch`, change the table name, returned columns, partition column,
+   and maximum-value column when using another schema.
+3. In `PutElasticsearchRecord`, verify the target index and set `ID Record Path` to the
+   unique source field, including the leading `/` required by RecordPath.
+4. In `StandardSSLContextService`, enter the keystore, key, and truststore passwords and
+   confirm that the mounted files match the selected search backend.
+5. In `ElasticSearchClientServiceImpl`, set the correct hosts and credentials, then
+   enable every controller service.
+
+`Use Avro Logical Types` is enabled. If a target mapping rejects dates or timestamps,
+adjust the record conversion and OpenSearch mapping deliberately rather than disabling
+logical types without checking the resulting JSON.
+
+## ▶️ Run and verify
+
+Start the process group and watch the queues between all three processors. The default
+target is `medical_reports_text`:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/medical_reports_text/_count'
+```
+
+Because the flow uses `docid` and the `index` operation, processing the same source row
+again replaces the document with that ID instead of creating another generated ID.
+
+## ⚠️ State and recovery
+
+`GenerateTableFetch` stores incremental state. Clear its state only when intentionally
+re-reading older rows. Before doing so, confirm that the target document ID is stable and
+that re-indexing is safe. Check the failure funnels and NiFi bulletins before advancing
+or discarding queued FlowFiles.
diff --git a/docs/deploy/workflows/filesystem_parquet_to_opensearch.md b/docs/deploy/workflows/filesystem_parquet_to_opensearch.md
new file mode 100644
index 00000000..0e84f254
--- /dev/null
+++ b/docs/deploy/workflows/filesystem_parquet_to_opensearch.md
@@ -0,0 +1,86 @@
+# 📁 Filesystem Parquet to OpenSearch
+
+Template: `nifi/user_templates/opensearch_ingest_parquet_form_fs_to_es.json`
+
+[View the flow definition on GitHub](https://github.com/CogStack/CogStack-NiFi/blob/main/nifi/user_templates/opensearch_ingest_parquet_form_fs_to_es.json)
+
+## 🎯 Purpose
+
+This flow recursively discovers Parquet files in NiFi's `/data` directory, converts
+each file to newline-delimited JSON (NDJSON), and writes its records to an OpenSearch
+index derived from the filename.
+
+## 🔗 Processor chain
+
+```text
+GetFile
+  -> CogStackConvertParquetToJson
+  -> PutElasticsearchRecord
+```
+
+The custom Python processor reads Parquet in batches of 10,000 records and emits one
+compact JSON object per line. It adds `mime.type=application/x-ndjson` and a
+`record.count` FlowFile attribute.
+
+## 📋 Bundled defaults
+
+| Setting | Value |
+|---|---|
+| NiFi input directory | `/data/` |
+| Host directory | `data/` by default through the Compose mount |
+| Recursive scan | Enabled |
+| File filter | `.*\.parquet` |
+| Polling interval | `30 sec` |
+| Files per poll | `10` |
+| Keep source file | `true` |
+| Target index | `${filename:substringBefore('.parquet'):toLower()}` |
+| Index operation | `create` |
+| OpenSearch bulk batch size | `10000` |
+
+## ✅ Requirements
+
+- NiFi and OpenSearch are running and their generated certificates are available.
+- The NiFi Python environment includes `pyarrow`; the repository image installs the
+  extension dependencies.
+- Parquet files are readable by the NiFi container below `/data`.
+- OpenSearch and SSL controller-service passwords are entered after import.
+
+The standard deployment mounts `${NIFI_DATA_PATH:-../data/}` at `/data/`. With default
+settings, place input files under the repository's `data/` directory.
+
+## ⚙️ Configure after import
+
+1. Confirm the `GetFile` input directory and file filter. Use a mounted path; a host path
+   that is not visible inside the container will not work.
+2. Set **Keep Source File** to `false` for one-time ingestion and ensure NiFi can delete
+   the source file. When it is `true`, `GetFile` picks up the same file continually; use
+   a `ListFile`/`FetchFile` pattern or another archival strategy if files must remain.
+3. Configure the SSL context and `ElasticSearchClientServiceImpl` credentials and
+   certificate passwords.
+4. Review the index expression. `example.parquet` becomes index `example`. Files with
+   the same name in different subdirectories target the same index.
+5. Add an `ID Record Path` or another deterministic ID strategy when the source contains
+   a stable key and safe reprocessing is required.
+
+## ▶️ Run and verify
+
+Copy a small Parquet file into `data/`, start the process group, and verify that
+`record.count` matches the number of indexed rows:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/<filename-without-parquet>/_count'
+```
+
+## ⚠️ Reprocessing and failures
+
+The bundled **Keep Source File** value is `true`, so `GetFile` continually reprocesses
+the same files. The writer also uses `create` without a configured document ID, causing
+OpenSearch to generate new IDs and duplicate logical records. Change the retention
+behavior before starting the flow unless repeated test ingestion is intentional.
+
+See the
+[Apache NiFi `GetFile` documentation](https://nifi.apache.org/components/org.apache.nifi.processors.standard.GetFile/)
+for the processor's source-file behavior. Conversion and OpenSearch failures are routed
+to funnels; the writer also exposes detailed `error_responses` for rejected records.
diff --git a/docs/deploy/workflows/medcat_annotations_to_opensearch.md b/docs/deploy/workflows/medcat_annotations_to_opensearch.md
new file mode 100644
index 00000000..8bdf578e
--- /dev/null
+++ b/docs/deploy/workflows/medcat_annotations_to_opensearch.md
@@ -0,0 +1,97 @@
+# 🧠 MedCAT annotations to OpenSearch
+
+Template: `nifi/user_templates/opensearch_ingest_docs_annotations_medcat_service_to_es.json`
+
+[View the flow definition on GitHub](https://github.com/CogStack/CogStack-NiFi/blob/main/nifi/user_templates/opensearch_ingest_docs_annotations_medcat_service_to_es.json)
+
+## 🎯 Purpose
+
+This flow reads documents that already exist in OpenSearch, sends their text to the
+CogStack MedCAT service, expands the response into individual annotation documents,
+and writes those annotations to a separate OpenSearch index.
+
+## 🔗 Processor chain
+
+```text
+SearchElasticsearch
+  -> CogStackPrepareRecordForNlp
+  -> InvokeHTTP
+  -> CogStackParseCogStackServiceResult
+  -> SplitJson
+  -> EvaluateJsonPath-create_annotation_id_store_as_attritbute
+  -> PutElasticsearchJson
+```
+
+The preparation processor sends the document text plus a footer containing the source
+fields. The response parser creates an `annotation_id` by combining the source document
+ID and MedCAT annotation ID. `EvaluateJsonPath` copies it to a FlowFile attribute used
+as the destination document ID.
+
+## 📋 Bundled defaults
+
+| Setting | Value |
+|---|---|
+| Source OpenSearch index | `encounters` |
+| Source fields | `id`, `text` |
+| Search condition | Documents where `text` exists |
+| Pagination | Scroll, keep-alive `30 mins` |
+| Restart on finish | `false` |
+| MedCAT endpoint | `http://cogstack-medcat-service-production:5000/api/process_bulk` |
+| Document ID field | `id` |
+| Document text field | `text` |
+| Destination index | `encounters_annotations` |
+| Index operation | `index` |
+| Destination ID attribute | `annotation_id` |
+
+## ✅ Requirements
+
+- The source OpenSearch index exists and its documents contain stable `id` and `text`
+  fields.
+- The MedCAT service and its configured model are running:
+
+  ```bash
+  make -C deploy start-medcat-service
+  ```
+
+- NiFi can resolve `cogstack-medcat-service-production` on the Docker network, or the
+  `InvokeHTTP` URL is changed to a reachable MedCAT endpoint.
+- OpenSearch and SSL controller-service passwords are entered after import.
+
+## ⚙️ Configure after import
+
+1. Configure `ElasticSearchClientServiceImpl` with the correct hosts and credentials.
+   The bundled flow lists `elasticsearch-1` and `elasticsearch-2`.
+2. Configure and enable `StandardSSLContextService` using the generated deployment
+   certificates and passwords.
+3. Update the source index, `_source` fields, and query in `SearchElasticsearch`.
+4. Match `document_id_field_name` and `document_text_field_name` in
+   `CogStackPrepareRecordForNlp` and `CogStackParseCogStackServiceResult` to the source
+   mapping.
+5. Verify the MedCAT URL and keep **Request Body Enabled** set to `true`.
+6. Set the destination index in `PutElasticsearchJson`. Keep `annotation_id` as the
+   identifier unless another deterministic uniqueness scheme is required.
+
+The bundled parser uses normal annotation mode (`medcat_output_mode=not_set`), not
+de-identification mode. Each annotation becomes a separate destination document.
+
+## ▶️ Run and verify
+
+The search processor is configured for a single pass because **Restart On Finish** is
+`false`. Start with a restrictive query or a test index, then confirm the annotation
+index count:
+
+```bash
+curl --cacert security/certificates/elastic/opensearch/elastic-stack-ca.crt.pem \
+  -u '<user>:<password>' \
+  'https://localhost:9200/encounters_annotations/_count'
+```
+
+Using deterministic `annotation_id` values makes deliberate reprocessing idempotent for
+the same source document and MedCAT annotation IDs.
+
+## ⚠️ Failure handling
+
+Search, HTTP, preparation, parsing, splitting, identifier extraction, and OpenSearch
+failures are routed to funnels. Documents with no returned annotations do not create
+annotation records in the bundled non-de-identification mode. Inspect MedCAT responses
+and NiFi provenance when input counts and annotation counts differ.
diff --git a/mkdocs.yml b/mkdocs.yml
index bd6c54f2..75965af9 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -44,7 +44,13 @@ nav:
   - 📦 Deployment: deploy/deployment.md
   - 🧩 Services: deploy/services.md
   - 📛 Troubleshooting: deploy/troubleshooting.md
-  - Workflows: deploy/workflows.md
+  - 🔀 Workflows:
+      - Overview: deploy/workflows.md
+      - 🗄️ Database documents to OpenSearch: deploy/workflows/database_to_opensearch.md
+      - 📄 Database documents via OCR: deploy/workflows/database_ocr_to_opensearch.md
+      - 🧠 MedCAT annotations to OpenSearch: deploy/workflows/medcat_annotations_to_opensearch.md
+      - 📁 Filesystem Parquet to OpenSearch: deploy/workflows/filesystem_parquet_to_opensearch.md
+      - ☁️ Azure Blob Parquet to OpenSearch: deploy/workflows/azure_blob_parquet_to_opensearch.md
   - 💧 NiFi:
       - nifi/main.md
       - Development Guide: nifi/development_guide.md

From 5c25105bd64ee47fd02d3011f5408db73b5cf54e Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Thu, 3 Sep 2026 17:00:15 +0100
Subject: [PATCH 02/12] Docs: doc trim.

---
 docs/README.md                 |  25 ++--
 docs/deploy/configuration.md   | 142 +++++++++++++--------
 docs/deploy/deployment.md      | 199 ++++++-----------------------
 docs/deploy/main.md            |   4 +-
 docs/deploy/services.md        | 160 ++++++++++++------------
 docs/deploy/troubleshooting.md | 220 +++++++++++++++++++++------------
 docs/nifi/development_guide.md |   2 +-
 docs/nifi/main.md              |   4 +-
 mkdocs.yml                     |   2 +-
 9 files changed, 373 insertions(+), 385 deletions(-)

diff --git a/docs/README.md b/docs/README.md
index c450d852..7dc214b7 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,8 @@
-# CogStack NiFi Documentation
+# CogStack NiFi documentation
 
-The central documentation for cogstack, hosted on docs.cogstack.org
+The published documentation is hosted at
+[cogstack-nifi.readthedocs.io](https://cogstack-nifi.readthedocs.io/en/latest/).
+Run the commands below from the repository root.
 
 ## Setup
 
@@ -12,9 +14,7 @@ The central documentation for cogstack, hosted on docs.cogstack.org
 ### Installation
 
 ```bash
-uv venv --python 3.12 --allow-existing 
-source .venv/bin/activate
-uv sync --dev
+uv sync --project docs --frozen
 ```
 
 ## Usage
@@ -24,17 +24,24 @@ uv sync --dev
 To preview the documentation locally with live reload:
 
 ```bash
-uv run mkdocs serve
+uv run --project docs mkdocs serve
 ```
 
-The documentation will be available at `http://127.0.0.1:8000`
+The documentation will be available at `http://127.0.0.1:8000`.
 
 ### Build documentation
 
 To build the static site:
 
 ```bash
-uv run mkdocs build
+uv run --project docs mkdocs build --strict
 ```
 
-The built site will be in the `site/` directory.
+The built site will be in the repository's `site/` directory.
+
+Run the documentation-specific checks with:
+
+```bash
+python3 scripts/tests/lint_markdown.py
+python3 scripts/tests/check_docs_links.py
+```
diff --git a/docs/deploy/configuration.md b/docs/deploy/configuration.md
index 1ec2f06f..c4240be5 100644
--- a/docs/deploy/configuration.md
+++ b/docs/deploy/configuration.md
@@ -1,83 +1,123 @@
+# ⚙️ Configuration
 
+CogStack separates deployment configuration from local-development security
+settings:
 
+- `deploy/*.env` configures services, images, ports, resource limits, and the
+  shared Docker network.
+- `security/env/*.env` configures certificate generation and local-development
+  users.
+- Standalone services under `services/` keep additional environment files in
+  their own directories.
 
-## Environment variables
+The tracked values are development defaults. Keep deployment-specific values
+outside the repository or inject them with your platform's secret manager.
+Never commit production credentials or generated private keys.
 
-As mentioned above, environment variables have been made available after release 1.0.
-The variables are configurable, and are separated, into security and general env vars, furthermore, all services declared in the `services.yml` file have their variables in separate files.
-In most cases, modifying these variables should be the only thing that is needed in order to run a successful deployment.
+## Core deployment files
 
-Multiple files are available, split into two categories:
+The core Compose file, `deploy/services.yml`, loads these files:
 
-- Service configuration under `deploy/`.
-- Certificate and local-development credential configuration under `security/env/`.
+| File | Main purpose |
+|------|--------------|
+| `deploy/project.env` | Compose project naming and shared project settings |
+| `deploy/general.env` | General Docker logging and shared defaults |
+| `deploy/nifi.env` | NiFi image, ports, resources, proxy settings, and paths |
+| `deploy/elasticsearch.env` | Elasticsearch/OpenSearch, Dashboards/Kibana, and Beats |
+| `deploy/database.env` | Sample and production database resources |
+| `deploy/gitea.env` | Gitea image, ports, and application settings |
+| `deploy/nginx.env` | NiFi nginx proxy settings |
+| `deploy/telemetry.env` | Telemetry configuration |
+| `deploy/network_settings.env` | Hostnames and shared network settings |
 
-The variables declared in the `./deploy` folder are used in multiple config files, as follows:
-- `elasticsearch.env`, variables here are used in :
-    -   `./services/elasticsearch/config/(opensearch|elasticsearch).yml`
-    -   `./services/kibana/config/(opensearch|elasticsearch).yml` 
-    -   `./services/metricbeat/metricbeat.yml`
-    -   `./deploy/services.yml` in the following sections: `nifi`, `elasticsearch-1`, `elasticsearch-1`, `elasticsearch-3`, `kibana`, `metricbeat-1`,`metricbeat-2`
+The following security files are also loaded where required:
 
-- `nifi.env`, vars used in:
-    -   `./deploy/services.yml`, sections: `nifi`
-    -   `./nifi/conf/nifi.properties`
+- `security/env/certificates_general.env`
+- `security/env/certificates_elasticsearch.env`
+- `security/env/certificates_nifi.env`
+- `security/env/users_database.env`
+- `security/env/users_elasticsearch.env`
+- `security/env/users_nginx.env`
+- `security/env/users_nifi.env`
 
-- `jupyter.env`, vars used in:
-    -   `./deploy/services.yml`, sections: `jupyter`
+See the `env_file` anchors at the top of `deploy/services.yml` for the exact
+files consumed by each core service.
 
-- `nlp_service.env`, vars used in:
-    -   `./deploy/services.yml`, sections: `nlp-medcat-service-production`
+## Standalone service files
 
-- `database.env`, vars used in:
-    -   `./deploy/services.yml`, sections: `cogstack-databank-db`,  `samples-db`
+Services launched from their own Compose projects use environment files below
+their respective directories. The deployment helper currently loads, when
+present:
 
-- `general.env`, these vars are optional, declared any custom variables you want here, used in the `nifi` section
+- `services/cogstack-jupyter-hub/env/jupyter.env`
+- `services/ocr-service/env/ocr_service.env`
+- `services/cogstack-nlp/medcat-service/env/app.env`
+- `services/cogstack-nlp/medcat-service/env/medcat.env`
+- `services/cogstack-nlp/medcat-trainer/envs/env-prod`
 
-Additional environment files under `security/env/` configure certificate generation and local accounts:
+Check each service's Compose file for variant-specific files, such as MedCAT
+de-identification or OCR text-only overrides.
 
-- `certificates_elasticsearch.env`
-- `certificates_general.env`
-- `certificates_nifi.env`
-- `users_database.env`
-- `users_elasticsearch.env`
-- `users_nginx.env`
-- `users_nifi.env`
+## Loading configuration
 
+Make targets load the required files automatically. If you invoke Docker
+Compose directly, change to the deployment directory and load the same
+environment first:
 
-### Customization
-The tracked `.env` values are public development defaults. For a real deployment, keep deployment-specific copies outside the repository or inject the values through the platform's secret-management mechanism. Never commit production credentials.
+```bash
+cd deploy
+source ./export_env_vars.sh
+```
+
+To inspect the combined environment used by the Makefile:
+
+```bash
+make -C deploy show-env
+```
+
+Be careful when sharing this output because it includes development
+credentials and certificate passwords.
+
+## Deployment-specific overrides
+
+For a separate deployment, copy the tracked defaults to a protected location
+and edit the copies there:
 
 ```bash
 cp deploy/*.env /path/to/deployment-config/
 cp security/env/*.env /path/to/deployment-config/security/
 ```
 
-### Multiple deployments on the same machine
-When deploying multiple docker-compose projects on the same machine (e.g. for dev or testing), it can be useful to remove all containers, volume and network names from the docker-compose file, and let [Docker create names](https://docs.docker.com/compose/reference/envvars/#compose_project_name) based on `COMPOSE_PROJECT_NAME` in `deploy/.env`. Docker will automatically create a Docker network and makes sure that containers can find each other by container name.
+You must then adapt your Compose invocation or deployment tooling to load those
+copies. The repository Makefile always loads the paths listed in
+`deploy/export_env_vars.sh`.
 
-For example, when setting `COMPOSE_PROJECT_NAME=cogstack-prod`, Docker Compose will create a container named `cogstack-prod_elasticsearch-1_1` for the `elasticsearch-1` service. Within the NiFi container, which is running in the same Docker network, you can refer to that container using just the service name `elasticsearch-1`.
+## Multiple deployments on one host
 
-<br>
+The supplied Compose file uses explicit container and volume names in several
+places. Merely changing `COMPOSE_PROJECT_NAME` therefore does not isolate every
+resource. For concurrent deployments, use deployment-specific Compose
+overrides that remove or replace explicit names and assign distinct host ports.
 
-## <span style="color:red">Important security detail</span> 
+Within a Compose network, services should address one another by service name,
+for example `elasticsearch-1`, rather than by host-published ports.
 
-Generated certificates and private keys are stored under `security/certificates/` and ignored by Git. Create them with `make -C deploy init-security`. Do not reuse development certificates or public development credential defaults in production.
+## Security setup
 
-## Services
-Please note that all the services are deployed using [Docker](https://docker.io) engine and requires docker deamon to be running / functioning.
+Generated certificates and private keys are stored under
+`security/certificates/` and ignored by Git. Generate the local certificate sets
+before starting services:
 
-Please see [the available services](./services.md) for more details.
+```bash
+make -C deploy init-security
+```
 
+Do not reuse development certificates or tracked development credentials in
+production. See the [security overview](../security/main.md) for production
+guidance.
 
 ## Workflows
-Apache NiFi provides users the ability to build very large and complex data flows. 
-These data flows can be later saved as workflow *templates*, exported into XML format and shared with other users.
-We provide few example templates for ingesting the records from a database into Elasticsearch and to perform extraction of NLP annotations from documents.
-
-### Deployment using Makefile
-For deployments based on example workflows, please see
-[current workflows](./workflows.md) for more details.
 
-### Deployment using a custom Docker-compose
-When using a fork of this repository for a customized deployments, it can be useful to copy `services.yml` to a deployment-specific `docker-compose.yml`. In this Compose file you can specify the services you need for your instance and configure all parameters per service, as well as track this file in a branch in your own fork. This way you can use your own version control and rebase on `CogStack/CogStack-NiFi` master without running into merge conflicts.
+For settings required by the supplied NiFi templates, see the
+[workflow documentation](./workflows.md). For general service endpoints and
+ports, see [Services](./services.md).
diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md
index 32ac6b42..cc12302f 100644
--- a/docs/deploy/deployment.md
+++ b/docs/deploy/deployment.md
@@ -15,12 +15,15 @@ Make sure you have read the [Prerequisites](./main.md) section before proceeding
   - environment variables that apply **only to the services defined inside `services.yml`**.  
   - Security-related `.env` files (certificates, users) are under **`/security`**
 
-  These variables configure NiFi, Elasticsearch/OpenSearch, Kibana, Jupyter, Metricbeat, the sample DB, etc.
+  These variables configure core services such as NiFi, Elasticsearch/OpenSearch,
+  OpenSearch Dashboards or Kibana, Beats, and the databases.
 
-> **Important:** If you run `docker compose` directly (instead of `make`), first load the envs with:
+> **Important:** If you run `docker compose` directly (instead of `make`),
+> change to the deployment directory and load the environment first:
 >
 > ```bash
-> source ./deploy/export_env_vars.sh
+> cd deploy
+> source ./export_env_vars.sh
 > ```
 >
 > The Makefile targets already do this for you.
@@ -168,9 +171,20 @@ You can optionally create `gitea-admin-credentials` with `username` and `passwor
 
 ## 🧰 Makefile Command Overview
 
-A concise reference for controlling the full CogStack deployment stack (NiFi, Elasticsearch, JupyterHub, MedCAT, OCR-service, GitEA, Beats, DB, etc.).  
+A concise reference for controlling the CogStack deployment stack (NiFi,
+Elasticsearch, MedCAT, OCR, Gitea, Beats, databases, and optional services).
 All commands automatically load environment variables via `export_env_vars.sh`.
 
+Before running a `start-*` target on a fresh checkout, generate the local
+certificate sets:
+
+```bash
+make -C deploy init-security
+```
+
+The command is idempotent and leaves complete existing certificate sets
+unchanged.
+
 ### 🔎 Discover available Make targets
 
 You can list all available `deploy/Makefile` targets with descriptions:
@@ -273,7 +287,7 @@ make -C deploy remote-delete-service \
 | `make start-git-ea`             | Start GitEA                          |
 | `make start-production-db`      | Start Databank DB                    |
 | **`make start-data-infra`**     | Start NiFi + Elastic + Samples DB    |
-| **`make start-all`**            | Full stack: data infra + NLP + JupyterHub + OCR |
+| **`make start-all`**            | Start data infrastructure + MedCAT inference + both OCR services |
 
 ---
 
@@ -298,7 +312,7 @@ make -C deploy remote-delete-service \
 | `make stop-git-ea`              | Stop GitEA                          |
 | `make stop-production-db`       | Stop Databank DB                    |
 | **`make stop-data-infra`**      | Stop NiFi + Elastic + Samples       |
-| **`make stop-all`**             | Stop entire stack                   |
+| **`make stop-all`**             | Stop the service groups wired into the `stop-all` target |
 
 ---
 
@@ -338,43 +352,20 @@ make -C deploy remote-delete-service \
 
 ### 📝 Notes
 
-- All `start-*` commands use `docker compose -f services.yml` unless referencing a specific service’s Dockerfile.
-- `start-all` and `stop-all` act as the top-level orchestration entry points.
-- Environment variables are **always sourced** using the integrated `WITH_ENV` macro.
-
----
+- Core-service targets use `deploy/services.yml`; standalone-service targets
+  use the Compose files in their respective `services/` directories.
+- `start-all` currently starts `start-data-infra`, `start-medcat-service`, and
+  `start-ocr-services`. Start other optional services explicitly.
+- `stop-all` follows the dependencies declared in the Makefile; check
+  `make -C deploy help` before relying on it for a customised deployment.
+- Environment variables are sourced using the integrated `WITH_ENV` macro.
 
-If you want, I can also generate a **minimal cheat sheet**, or an **ASCII tree diagram** that shows how `start-all` expands into all services.
-
-## 🚀 Starting the Services
-
-All core services defined in `services.yml` can be started using the Makefile in the `deploy/` directory.
-
-Before running any `start-*` command on a fresh checkout, generate the
-deployment-local certificates:
-
-```bash
-make init-security
-```
+## 🚀 Common startup patterns
 
-This command is idempotent and leaves complete existing certificate sets
-unchanged. The examples below assume that you are in the `deploy/` directory.
-
-For most services in the `services` folder that are not part of the core stack defined in `services.yml` and are pulled from external git submodule repositories, the start-up process is the same.
-
-### ▶️ Start each service individually
-
-You can start individual components of the CogStack-NiFi stack using the `make start-*` commands.  
-Each target loads all required environment variables automatically via `export_env_vars.sh`.
-
-This is useful for:
-
-- debugging a single service  
-- restarting only one component after config changes  
-- running lightweight subsets of the stack  
-- isolating problems or logs per service  
-
----
+The commands below assume that you are in `deploy/`. From the repository root,
+use the same target with `make -C deploy`, for example
+`make -C deploy start-data-infra`. The command table above is the complete
+target reference.
 
 ### 🧩 Core NiFi Services
 
@@ -406,130 +397,18 @@ Starts:
 
 Ideal for running ingestion pipelines and ETL workflows.
 
----
-
-#### 🛢️ Elasticsearch / OpenSearch Services
-
-Please note that to switch from OpenSearch (Amazon open-source fork) to ElasticSearch you will need to change some environment variables, see the [configuration](./configuration.md) section.
-
-```bash
-make start-elastic
-```
-
-Starts the standard 2-node Elasticsearch cluster + Kibana.
-
-```bash
-make start-elastic-cluster
-```
-
-Starts all 3 ES nodes. Useful for testing clustering, sharding, and replication.
-
-```bash
-make start-elastic-1
-make start-elastic-2
-make start-elastic-3
-```
-
-Start individual Elasticsearch nodes for debugging or failure-scenario testing.
-
----
-
-#### 📈 Kibana
-
-```bash
-make start-kibana
-```
-
-Starts Kibana for inspecting logs, checking index mappings, monitoring ES health, and debugging pipelines.
-
----
-
-### 🗄️ Databases
-
-```bash
-make start-samples
-```
-
-Starts **samples-db**, the small example DB used for demo flows.
-
-```bash
-make start-production-db
-```
-
-Starts the **cogstack-databank-db** production database.
-
-Use when testing SQL ingestion or verifying DB-driven NiFi flows.
-
----
-
-### 📚 JupyterHub
-
-```bash
-make start-jupyter
-```
-
-Starts the CogStack JupyterHub instance. Used for notebooks, analysis, model testing, and visualisation.
-
----
-
-### 🧠 NLP Services (MedCAT Service & Trainer)
-
-```bash
-make start-medcat-service
-```
-
-Starts the MedCAT concept extraction inference API.
-
-```bash
-make start-medcat-service-deid
-```
-
-Starts the MedCAT DEID (de-identification) inference API.
-
-```bash
-make start-medcat-trainer
-```
-
-Starts the full MedCAT Trainer stack (Trainer UI + Solr + NGINX). Useful for annotation and supervised training tasks.
-
----
-
-### 📝 OCR Services
-
-```bash
-make start-ocr-services
-```
-
-Starts:
-
-- **ocr-service** — main OCR pipeline  
-- **ocr-service-text-only** — lightweight OCR/text extraction  
-
-Use for PDF ingestion, OCR debugging, and pipeline validation.
-
----
-
-### 🛠️ Miscellaneous Services (GIT EA)
-
-```bash
-make start-git-ea
-```
-
-Starts the internal Gitea Git server used for local code/config storage.
-
----
-
-### 🚀 Start the Entire Stack
+### 🚀 Start the standard service set
 
 ```bash
 make start-all
 ```
 
-Starts everything:
+Starts:
 
 - Core infra
-- JupyterHub  
-- MedCAT NLP services  
-- OCR services  
+- MedCAT inference service
+- OCR and text-only OCR services
 
-Use for complete deployments, demos, or full-stack development.
+It does not start every optional service. Start Gitea, the production database,
+MedCAT de-identification, MedCAT Trainer, or other optional services with their
+individual targets when needed.
diff --git a/docs/deploy/main.md b/docs/deploy/main.md
index da3d7376..38a87f7d 100755
--- a/docs/deploy/main.md
+++ b/docs/deploy/main.md
@@ -6,7 +6,7 @@ Please read carefully as there can be many points of failure when installing/dep
 
 Please note that the OSes mentioned below are the versions we support, whatever is not listed here is not supported, and we will not provide support for.
 
-- Linux OS (Ubuntu 24.04 LTS +, Debian 10+ are preffered, RHEL 9+).
+- Linux OS (Ubuntu 24.04 LTS+, Debian 10+ preferred, or RHEL 9+).
 - Windows 11+/Windows Server 2022+ (Requires [WSL 2.0](https://learn.microsoft.com/en-us/windows/wsl/about) installation and the installation of an Ubuntu image, for a working setup, follow [this guide](https://documentation.ubuntu.com/wsl/latest/howto/install-ubuntu-wsl2/) to get going, and get back here when things are working).
 - macOS 15+ (Sequoia).
 
@@ -22,7 +22,7 @@ Software required on machine (the minimum/basics to get demos running):
 ## 🔐 Other requirements (User Permissions/Firewall)
 
     - a Linux account with 'admin' rights, if possible, if not, you will need to get your IT team to take a look at this README and install the packages for you using the steps below (make sure they look at [Docker rootless installation steps](https://docs.docker.com/engine/security/rootless))
-    - firewall whitelisting of the following addreses:
+    - firewall allow-listing of the following addresses:
         - https://github.com/
         - https://hub.docker.com/
         - https://docker.io 
diff --git a/docs/deploy/services.md b/docs/deploy/services.md
index 2b7d8e70..b490761d 100644
--- a/docs/deploy/services.md
+++ b/docs/deploy/services.md
@@ -1,7 +1,8 @@
 # 🧩 Services
 
-This section provides a complete overview of all services included in the CogStack-NiFi deployment.  
-All services run in Docker and interact within a shared internal Docker network.
+This section provides an overview of the services included in the CogStack-NiFi
+deployment. Services run in Docker and are attached to the relevant Compose
+network when they need to communicate.
 
 ---
 
@@ -20,18 +21,18 @@ The core services defined in `services.yml` include:
 - **samples-db** — PostgreSQL database populated with demo datasets.
 - **cogstack-databank-db / cogstack-databank-db-mssql** — Production-grade PostgreSQL and optional MSSQL instances.
 - **elasticsearch-1 / elasticsearch-2 / elasticsearch-3** — Multi-node Elasticsearch or OpenSearch cluster.
-- **metricbeat / filebeat** — Elastic monitoring and log forwarder services.
+- **metricbeat / filebeat** — Elastic monitoring and log-forwarding services.
 - **nifi** — Apache NiFi single-node instance with embedded ZooKeeper.
 - **nifi-nginx** — Reverse proxy providing secure access to NiFi.
-- **ocr-service / ocr-service-text-only** — High-performance Python OCR and text extraction services.
-- **nlp-medcat-service-production** — MedCAT NLP model service with REST API.
-- **medcat-trainer-ui / medcat-trainer-nginx** — Web UI and reverse proxy for model training and refinement.
+- **kibana** — Kibana or OpenSearch Dashboards, depending on the backend.
+- **gitea** — Self-hosted Git service.
+- **cogstack-cohort** — Cohort discovery interface.
 
-- **kibana** — OpenSearch Dashboards UI.
-- **jupyter-hub** — Fully featured data science interface.
-- **git-ea** — Self‑hosted Git service (Gitea).
+Additional services, including MedCAT, MedCAT Trainer, OCR, and JupyterHub, are
+started from Compose files in their respective `services/` directories.
 
-> 🔐 **Note:** Important configuration options and environment variables for these services are managed in `services.yml` and the associated `.env` files under `deploy/` and `security/`.
+> 🔐 **Note:** Core-service configuration is managed in `services.yml` and
+> the associated environment files under `deploy/` and `security/env/`.
 
 ## 🗂️ Service Definitions
 
@@ -48,7 +49,7 @@ Some services expose ports to the host for convenience.
 
 ## 🗣️ NLP/OCR and other services API Endpoints
 
-Most web ETL & data-enrichment API services that we use will offer thw following endpoints for querying.
+Most web ETL and data-enrichment APIs provide some or all of these endpoints:
 
 - **GET** `/api/info`
 - **POST** `/api/process`
@@ -60,17 +61,20 @@ Useful for NiFi workflows (see [workflows](./workflows.md)).
 
 ## 🧬 MedCAT Service
 
-Runs a REST API for model inference uses the [MedCAT library](https://github.com/CogStack/cogstack-nlp/tree/main/medcat-v2) which performss clinical concept extraction and linking. 
+This REST API uses the
+[MedCAT library](https://github.com/CogStack/cogstack-nlp/tree/main/medcat-v2)
+for clinical concept extraction and linking.
 
 The service has two operation modes:
 
-- concept detection: exctracts medical concepts: outputs original text + annotations list.
-- de-id mode aka. AnonCAT mode, for de-identifying documents: outputs de-identified text + (will output annotations that represent what was de-id in a future version).
+- **Concept detection** extracts medical concepts and returns the original text
+  with annotations.
+- **De-identification (AnonCAT)** returns de-identified text.
 
 ### Access
 
-- `https://localhost:5555/api/info` - NER container, check if model loads successfully
-- `https://localhost:5556/api/info` - DE-ID/AnonCAT container
+- `http://localhost:5555/api/info` — concept extraction service
+- `http://localhost:5556/api/info` — de-identification service
 
 ### Containers
 
@@ -79,16 +83,14 @@ The service has two operation modes:
 
 ### Service location & files
 
-- dir: `/services/cogstack-nlp/medcat-service/`
-- docker compose file: `/services/cogstack-nlp/medcat-service/docker/docker-compose.yml`
+- directory: `services/cogstack-nlp/medcat-service/`
+- Docker Compose file: `services/cogstack-nlp/medcat-service/docker/docker-compose.yml`
 - env: located in `services/cogstack-nlp/medcat-service/env/`
 
-    ```bash
-        app.env - controls APP settings (number of cpus used, log level, etc) used by the NER container cogstack-medcat-service-production
-        medcat.env - used by the NER container, controls MedCAT settings directly.
-        app_deid.env - used by the DE-ID container, same app setting control, the main difference being the `APP_DEID_MODE`.
-        medcat_deid.env  - used by the DE-ID container, controls MedCAT settings directly
-    ```
+  - `app.env` — application settings for the concept extraction service
+  - `medcat.env` — MedCAT settings for concept extraction
+  - `app_deid.env` — application settings for de-identification
+  - `medcat_deid.env` — MedCAT settings for de-identification
 
 ### Ports
 
@@ -100,8 +102,8 @@ The service has two operation modes:
 ### Models
 
 - A default MedMentions `MedMen` NER+L model (includes MetaCAT models) is available for public use but needs to be downloaded.
-- To download a model head to the directory of the service `services/cogstack-nlp/medcat-service/scripts`
-- Execute: `bash download_medmen.sh`, wait for download to complete.
+- To download it, go to `services/cogstack-nlp/medcat-service/scripts` and run
+  `bash download_medmen.sh`.
 
 ### README
 
@@ -115,7 +117,7 @@ Provides UI workflows for annotation, correction, and iterative model training.
 
 ### Access
 
-- `https://localhost:8001`
+- `http://localhost:8001`
 
 ### Containers
 
@@ -188,13 +190,13 @@ Demo dataset with:
 - cleaned reports  
 - annotation tables  
 
-### Acess
+### Access
 
-- `localhost:5555`
+- `localhost:5554`
 
 ### Ports
 
-- external: `5432`
+- external: `5554`
 - internal: `5432`
 
 ### Credentials
@@ -214,8 +216,7 @@ Place schema files inside and they will be loaded instantly on container startup
 services/cogstack-db/<DB_PROVIDER>/schemas/
 ```
 
-Where `<DB_PROVIDER>` can be: `mssql`,`pgsql`.
-
+Where `<DB_PROVIDER>` can be `mssql` or `pgsql`.
 ### Credentials
 
 PostgreSQL and MSSQL development credentials are defined in `security/env/users_database.env`. Production credentials must be injected through deployment-local configuration or a secret manager.
@@ -223,12 +224,12 @@ PostgreSQL and MSSQL development credentials are defined in `security/env/users_
 ### Access
 
 - PgSQL: `localhost:5558` → container `5432`  
-- MSSQL: `localhost:1443` → container `1433`  
+- MSSQL: `localhost:1433` → container `1433`
 
 ### Containers
 
-- PgSQL: `cogstack-databank-db`
-- MSSQL: `cogstack-databank-db-mssql`
+- PgSQL: `cogstack-production-databank-db`
+- MSSQL: `cogstack-production-databank-db-mssql`
 
 ### Service location & files
 
@@ -264,6 +265,7 @@ Single-user development credentials are defined in `security/env/users_nifi.env`
 ### Containers
 
 - NiFi: `cogstack-nifi`
+- nginx: `cogstack-nifi-nginx`
 
 ### Service location & files
 
@@ -276,9 +278,14 @@ Single-user development credentials are defined in `security/env/users_nifi.env`
 
 ### Ports
 
-| Component            | External Port | Internal Port |
-|---------------------|---------------|----------------|
-| NiFi                | `8443`        | `8082`, `10000` |
+| Component | External port | Internal port |
+|-----------|---------------|---------------|
+| NiFi via nginx | `8443` | `8443` |
+| NiFi direct access | `8082` | `8443` |
+
+The NiFi container also publishes its input socket on container port `10000`
+without assigning a fixed host port. Use `docker compose port nifi 10000` from
+the `deploy/` directory to see the assigned host port.
 
 ---
 
@@ -290,7 +297,7 @@ This service is fully described in the Elasticsearch section of the documentatio
 
 The repo supports both:
 
-- ElasticSearch (native)
+- Elasticsearch (native)
 - OpenSearch (Amazon fork)
 
 Switch between modes via environment variables in `deploy/elasticsearch.env`.
@@ -315,9 +322,10 @@ Development credentials for both backends are defined in `security/env/users_ela
 
 #### Ports
 
-- all ports need to be exposed via firewall to allow for intercluster communication, we assume 1 different port per node if hosted on the same machine/VM, in production mode all machines can have and use the following ports (if they live on separarate VMs/machines ): `9200`, `9300`, `9600`
-- internal: `9300`, `9301`, `9302`, `9600`, `9601`, `9602`, `9200`, `9201`, `9202`
-- external: `9300`, `9301`, `9302`, `9600`, `9601`, `9602`, `9200`, `9201`, `9202`
+The example Compose deployment assigns distinct host ports to nodes running on
+the same machine. Do not expose transport or analyzer ports through a public
+firewall. For a production multi-host cluster, allow only the required traffic
+between trusted cluster nodes.
 
 | Node | HTTP | Transport | Analyzer |
 |------|------|-----------|----------|
@@ -362,15 +370,15 @@ Both run as independent containers in the deployment.
 
 Metricbeat:
 
-- `metricbeat-1`
-- `metricbeat-2`
-- `metricbeat-3`
+- `cogstack-metricbeat-1`
+- `cogstack-metricbeat-2`
+- `cogstack-metricbeat-3`
 
 Filebeat:
 
-- `filebeat-1`
-- `filebeat-2`
-- `filebeat-3`
+- `cogstack-filebeat-1`
+- `cogstack-filebeat-2`
+- `cogstack-filebeat-3`
 
 #### **Service Location & Files**
 
@@ -385,7 +393,7 @@ Filebeat:
 #### **Ports**
 
 No external ports exposed.  
-All communication occurs internally within the `cogstack-net` Docker network.
+All communication occurs internally within the `cognet` Docker network.
 
 #### **Notes**
 
@@ -461,8 +469,7 @@ Image selection controlled by:
 
 ## 🤖 OCR Service
 
-High-performance document text extraction engine replacing legacy Tika for OCR + text processing.
-In the near future it will be possible to use LLMs/custom models for ocr-ing (pending v2 release, ETA 2026).
+High-performance document text-extraction engine for OCR and text processing.
 
 The service comes in **two variants**:
 
@@ -481,7 +488,7 @@ Both expose a simple REST API.
 ### Access
 
 - ocr-service: `http://localhost:8090/api/process`
-- ocr-seervice-text-only: `http://localhost:8091/api/process`
+- ocr-service-text-only: `http://localhost:8091/api/process`
 
 ### Containers
 
@@ -496,7 +503,8 @@ cogstacksystems/cogstack-ocr-service:<release>
 
 ### Service Location & Files
 
-- docker compose file: `services/ocr-service/docker/docker-compose.yml`
+- Docker Compose files: `services/ocr-service/docker/docker-compose.base.yml`
+  and `services/ocr-service/docker/docker-compose.prod.yml`
 - service directory: `services/ocr-service/`
 - logs:  
   - Host: `services/ocr-service/log/`  
@@ -520,7 +528,7 @@ Please check the service's own [README.md](https://github.com/CogStack/ocr-servi
 
 ---
 
-## 🗂️ Git-ea
+## 🗂️ Gitea
 
 Self-hosted Git instance (Gitea).
 Lightweight GitHub/GitLab-style service used for hosting repositories inside secure or offline environments.
@@ -537,7 +545,7 @@ Lightweight GitHub/GitLab-style service used for hosting repositories inside sec
 
 ### Containers
 
-- `gitea`
+- `cogstack-gitea`
 
 ### Service Location & Files
 
@@ -551,7 +559,7 @@ Persistent repository data is stored in the volume defined in `services.yml`.
 
 | Service | External | Internal |
 |---------|----------|----------|
-| Git-ea  | `3000`   | `3000`   |
+| Gitea   | `3000`   | `3000`   |
 
 ### Notes
 
@@ -563,51 +571,37 @@ Persistent repository data is stored in the volume defined in `services.yml`.
 
 ## 🧱 NGINX
 
-*Note: this component may eventually be replaced by **Traefik** as the preferred reverse‑proxy and ingress layer for CogStack deployments.*
-
-NGINX is used as a lightweight reverse proxy to provide secure, unified access to internal CogStack services.  
-It handles HTTPS, routing, and access control for NiFi, MedCAT Trainer, and other components.
-
-MedCAT-Trainer has its own nginx instance that runs independently.
+NGINX is used as a lightweight reverse proxy to provide HTTPS access to NiFi.
+MedCAT Trainer has a separate nginx instance in its own Compose project.
 
 **Purpose:**
 
-- Secure external access to internal services  
-- Reverse proxy for NiFi, MedCAT Trainer, and service UIs  
-- TLS termination (optional)  
-- Basic auth / access control where required  
-
-Two variants are included:
-
-- **nginx-nifi** — main proxy for NiFi and related services  
-- **nginx-medcat-trainer** — specialized proxy for MedCAT Trainer
-
-Two variants:
-
-- **nginx-nifi** — main proxy for services
-- **nginx-medcat-trainer** — dedicated trainer proxy
+- **nifi-nginx** terminates TLS and proxies NiFi UI and API traffic.
+- **medcattrainer_nginx** proxies the MedCAT Trainer application separately.
 
 ### Access
 
 Examples (actual paths depend on config):
 
 - NiFi: `https://localhost:8443`  
-- MedCAT Trainer: `https://localhost:8001`
+- MedCAT Trainer: `http://localhost:8001`
 
 Routing rules are defined in the NGINX configuration files.
 
 ### Containers
 
-- `nifi-nginx` — main proxy for NiFi
-- `medcat-trainer-nginx` — proxy dedicated to MedCAT Trainer
+- `cogstack-nifi-nginx` — proxy for NiFi
+- `medcattrainer_nginx` — proxy for MedCAT Trainer
 
 ### Service Location & Files
 
-- docker compose file: `deploy/services.yml`, trainer - `deploy/cogstack-nlp/medcat-trainer`
+- NiFi Compose file: `deploy/services.yml`
+- MedCAT Trainer Compose file:
+  `services/cogstack-nlp/medcat-trainer/docker-compose-prod.yml`
 - config files:
-  - `services/nginx/config/nifi.conf`
-  - `services/nginx/config/medcat-trainer.conf`
-  - additional templates under `services/nginx/config/`
+  - `services/nginx/config/nginx.conf.template`
+  - `services/cogstack-nlp/medcat-trainer/nginx/nginx.conf`
+  - `services/cogstack-nlp/medcat-trainer/nginx/sites-enabled/`
 - env / certificates:
   - `security/env/certificates_general.env`
   - `security/env/certificates_nifi.env`
diff --git a/docs/deploy/troubleshooting.md b/docs/deploy/troubleshooting.md
index 33bbe417..66da5f5a 100644
--- a/docs/deploy/troubleshooting.md
+++ b/docs/deploy/troubleshooting.md
@@ -1,127 +1,193 @@
 # 📛 Troubleshooting
 
-Always start with fresh containers and volumes, to make sure that there are no volumes from previous experimentations, make sure to always delete all/any cogstack running containers by executing:
+Start with logs and container state. Do not delete containers or volumes as a
+routine first step: volumes may contain NiFi state, database data, or search
+indices.
 
-`docker container rm samples-db elasticsearch-1 kibana nifi  nlp-medcat-service-production tika-service nlp-gate-drugapp nlp-medcat-snomed nlp-gate-bioyodie medcat-trainer-ui medcat-trainer-nginx jupyter-hub -f`
+From the repository root, list containers and their state:
 
-followed by a cleanup or dangling volumes (careful as this will remove all volumes which are NOT being used by a container, if you want to remove specific volumes you will have to manually specifiy the volume names), otherwise, you can specify :
+```bash
+docker ps -a --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'
+```
+
+Inspect the affected service before restarting it:
+
+```bash
+docker logs --tail 300 cogstack-nifi
+docker inspect cogstack-nifi
+```
 
-`docker volume prune -f` <strong> WARNING THIS WILL DELETE ALL UNUSED VOLUMES ON YOUR MACHINE!</strong>. Check the volume names used in services.yml file and delete them as necessary `dockr volume rm volume_name`
+Use the scoped Make targets where possible. For example:
 
-## 🐞 Known Issues/errors
+```bash
+make -C deploy stop-nifi
+make -C deploy start-nifi
+```
 
-Common issues that can be encountered across services.
+To recreate only the NiFi containers while retaining named volumes:
 
-### 🍎 **Apple Silicon**
+```bash
+make -C deploy delete-nifi-containers
+make -C deploy start-nifi
+```
 
-Many services cannot run natively on Apple Silicon (such as M1 and M2 architectures). Common error messages related to Apple silicon follow patterns similar to:
-    <br /><br/>
-    - `no match for platform in manifest`
-    <br /><br/>
-    <br /><br/>
-    - `no matching manifest for linux/arm64/v8 in the manifest list entries`
-    <br /><br/>
-    <br /><br/>
-    - `image with reference cogstacksystems/cogstack-ocr-service:1.0.2 was found but does not match the specified platform: wanted linux/arm64, actual: linux/amd64`
-    <br /><br/>
-To solve these issues; Rosetta is required and enabled in Docker Desktop. Finally an environment variable is required to be set.
+!!! danger "Deleting persistent data"
 
-Rosetta can which can be installed via the following command:
+    `make -C deploy cleanup` runs `docker compose down -v` for the core
+    deployment and deletes its named volumes. Use it only when you deliberately
+    want to reset all core service data. Do not use a global
+    `docker volume prune` command as a CogStack troubleshooting step because it
+    can delete unrelated Docker volumes on the host.
+
+## 🐞 Known issues
+
+### 🍎 Apple Silicon
+
+Some images may only publish an `linux/amd64` variant. Typical errors include:
+
+```text
+no match for platform in manifest
+no matching manifest for linux/arm64/v8 in the manifest list entries
+image ... does not match the specified platform: wanted linux/arm64, actual: linux/amd64
+```
+
+On macOS, install Rosetta if it is not already available:
 
 ```bash
 softwareupdate --install-rosetta
 ```
 
-When Rosetta and Docker Desktop are installed, Rosetta must be enabled. This done by going to Docker Desktop -> Setting -> General and enabling "Use Virtualization framework". After in the same settings go to "features in development" -> "Use Rosetta for x86/amd64 emulation on Apple Silicon". Finally execute the following command:
+Enable Docker Desktop's Apple Virtualization framework and Rosetta-based
+`x86_64` emulation. If an image still needs an explicit platform, set it for the
+current shell before starting the service:
 
 ```bash
 export DOCKER_DEFAULT_PLATFORM=linux/amd64
 ```
 
-to set the environment variable. These issues are known to occur on the "cogstack-nifi", "cogstack-ocr-services" and "jupyter-hub" services and may occur on others.
+Emulation is slower than running a native ARM64 image and may require higher
+Docker Desktop memory limits.
+
+### 🔧 NiFi
 
-### 🔧 **NiFi**
+#### NiFi reports mixed HTTP and HTTPS configuration
 
-When dealing with contaminated deployments ( containers using volumes from previous instances ) :
-    <br /><br/>
-    - `NiFi only supports one mode of HTTP or HTTPS operation...` deleting the volumes should usually solve this issue, if not, please check the `nifi.properties` if there have been modifications done by yourself or a developer on it.
-    <br /><br/>
-    - building the NiFi image manually on a restricted system, this is usually not necessary, but if for some reason this needs to be done then some settings such as proxy configs might need to be set up in the `nifi/Dockerfile` epecially ones related to the `grape` application and dealing with external downloads.
-    <br /><br/>
-    - `keystore.jks`/`truststore.jks` related errors, remove the nifi container & related volumes then restart the nifi instance. 
-     <br /><br/>
-    - `HTTP 421 INVALID PROXY` or `System Error: Invalid host header` occurs when NiFi does not trust the public host/port forwarded by nginx. For remote deployments, set `NIFI_WEB_PROXY_HOST` in `deploy/nifi.env` to the exact browser-visible host and port, for example `localhost:8443,nifi.example.org:8443`, and keep `NIFI_WEB_PROXY_PORT` aligned with that public port. If the public URL is `https://nifi.example.org/nifi`, use port `443`; if it is `https://nifi.example.org:8443/nifi`, use port `8443`. Restart `nifi` and `nifi-nginx` afterwards. If the generated container config still shows the old value, check `/opt/nifi/nifi-current/conf/nifi.properties` in the NiFi container and ensure `nifi.web.proxy.host` matches.
-    <br /><br/>
-    - Possible error when dealing with non-pgsql databases `due to Incorrect syntax near 'LIMIT'.; routing to failure: com.microsoft.sqlserver.jdbc.SQLServerException: Incorrect syntax near 'LIMIT'`, go to the GenerateTableFetch Process -> right-click -> configure -> change database type from Generic to -> MS SQL 2012 + or 2008 (if an older DB system is used)
-    - Possible error on Linux systems related to `nifi.properties` permission error and/or other files from the `nifi/conf/` folder, please see the [nifi doc](../nifi/main.md#important-note-about-nifi-properties) {nifi.properties} section. 
-    <br /><br/>
-    - `Driver class org.postgresql.Driver is not found` or something similar for other MSSQL/SQL drivers, this is a known issue after NiFi version v1.20+, first, make sure you pull the latest version of the repository, then for the JAR file you are using, please execute the following command in order to verify its integrity `jar -tvf ./nifi/drivers/your_file_version.jar`, if this returns a list of files and NO errors then the files are not corrupted and can be loaded. On the NiFi side make sure to go to the `DBCPConnectionPool` controller service and verify the propertiesit a few times, make sure the file path is correct and in the following format: `file:///opt/nifi/drivers/postgresql-42.7.7.jar` for example. If all this fails stop nifi, delete all the Docker volumes associated with it -> restart NiFi, perform the above steps again. You can try forcefully starting the `GenerateTableFetch` or `QueryDatabaseTable` processors by enabling the `DBCPConnectionPool` even if an error popus up after clicking the verify button.
-    <br /><br/>
-    - `502 Bad Gateway`, NiFi simply not starting, even after waiting more than 2-3 minutes. This can occur due to a wide variety of issues, you can check the NiFi container log : “docker logs -f --tail 1000 cogstack-nifi > my_log_file.txt” to capture the output easily. The most common cause is running out of memory, increase or decrease the limits in `nifi/conf/bootstrap.conf` according to your machine's spec, please read [bootstrap.conf](../nifi/main.md#bootstrapconf)
-    <br /><br/>
-    - `Unable to connect to ElasticSearch` using the `ElasticSearchClientService` NiFi controller, make sure the settings are correct (username,password,certificates, etc.) and then click `Apply`, disregard the errors and click `Enable` on the controller to forcefully reload the controller, stop it and then validate the settings, start it again after and it should work.
+If NiFi reports that it supports only one mode of HTTP or HTTPS operation:
 
-### 🛢️ **Elasticsearch Errors**
+1. Check local changes in `nifi/conf/nifi.properties`.
+2. Confirm the HTTPS settings loaded from `deploy/nifi.env`.
+3. Recreate the NiFi containers without deleting their volumes.
+4. Delete NiFi state volumes only if you have confirmed that stale persisted
+   configuration is the cause and the stored state is disposable.
 
-#### ⚡ **VM memory errors, failed bootstrap check**
+#### Keystore or truststore errors
 
-It is quite a common issue for both opensearch and native-ES to error out when it comes to virtual memory allocation, this error typically comes in the form of :
+Run the security initializer again; it validates complete certificate sets and
+does not overwrite a valid existing set:
 
 ```bash
-ERROR: [1] bootstrap checks failed
-[1]: max virtual memory areas vm.max_map_count [65111] is too low, increase to at least [262144]
+make -C deploy init-security
 ```
 
-To solve this one needs to simply execute :
-    <br>
-    - on Linux/Mac OS X : 
-    `sysctl -w vm.max_map_count=262144` in terminal. 
-    To make the same change systemwide plase add `vm.max_map_count=262144` to /etc/sysctl.conf and restart the dockerservice/machine.
-    An example of this can be found under /services/elasticsearch/sysctl.conf
-    <br>
-    - on Windows you need to enter the following commands in a powershell instance:
-    <br>
-    `wsl -d docker-desktop`
-    <br>
-    `sysctl -w vm.max_map_count=262144`
+If it reports an incomplete certificate set, preserve the existing files and
+repair or explicitly replace the set. Check the
+[certificate guide](../security/certificates.md) before replacing a CA or leaf
+certificate.
 
-For more on this issue please read: https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
+#### HTTP 421 or invalid host header
 
-<br>
+NiFi returns `HTTP 421 INVALID PROXY` or `System Error: Invalid host header`
+when it does not trust the public host and port forwarded by nginx.
 
-#### 📄 **OpenSearch: validating opensearch.yml hosts**
+Set `NIFI_WEB_PROXY_HOST` in `deploy/nifi.env` to the browser-visible host and
+port. For example:
+
+```text
+NIFI_WEB_PROXY_HOST="localhost:8443,nifi.example.org:8443"
+```
+
+Use port `443` for `https://nifi.example.org/nifi`, or `8443` for
+`https://nifi.example.org:8443/nifi`. Restart `nifi` and `nifi-nginx` after the
+change.
+
+#### Database processor errors
+
+- For Microsoft SQL Server errors involving `LIMIT`, configure
+  `GenerateTableFetch` with the appropriate Microsoft SQL Server database type
+  instead of `Generic`.
+- If a JDBC driver class is not found, verify the JAR before configuring its
+  absolute container path:
+
+  ```bash
+  jar -tvf ./nifi/drivers/postgresql-42.7.7.jar
+  ```
+
+  A PostgreSQL driver path in NiFi should look like
+  `file:///opt/nifi/drivers/postgresql-42.7.7.jar`. Apply the controller-service
+  configuration, then disable and re-enable the service to reload it.
+
+#### 502 Bad Gateway
+
+A 502 from nginx usually means that NiFi is not ready or has stopped. Inspect
+the container state and logs:
 
 ```bash
-FATAL  Error: [config validation of [opensearch].hosts]: types that failed validation:
-- [config validation of [opensearch].hosts.0]: expected URI with scheme [http|https].
-- [config validation of [opensearch].hosts.1]: could not parse array value from json input
+docker ps -a --filter name=cogstack-nifi
+docker logs --tail 1000 cogstack-nifi
 ```
 
-This issue may appear after the recent switch to using fully customizable environment variables. Strings and ENV vars may be parsed differently depending on the shell version found on the host system.
+NiFi startup can take several minutes. If the process was killed because of
+memory pressure, review the Docker resource limits in `deploy/nifi.env` and the
+JVM settings documented under
+[bootstrap.conf](../nifi/main.md#bootstrapconf).
+
+### 🛢️ Elasticsearch/OpenSearch
+
+#### `vm.max_map_count` bootstrap failure
 
-To solve this, the easiest way is to make sure to load the `elasticsearch.env` variables before starting the Elastic & Kibana containers by doing the following:
+Search nodes may fail with an error similar to:
+
+```text
+bootstrap checks failed
+max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+```
+
+On Linux, set the value temporarily with:
 
 ```bash
-    cd ./deploy/
-    set -a
-    source elasticsearch.env
-    make start-elastic
+sudo sysctl -w vm.max_map_count=262144
 ```
 
-Alternatively (if the script executes without issues):
+For a persistent Linux configuration, add `vm.max_map_count=262144` to
+`/etc/sysctl.conf` or an appropriate file under `/etc/sysctl.d/`, then apply the
+configuration. On Docker Desktop, set the value in the Linux VM used by Docker.
+
+#### OpenSearch host configuration fails validation
+
+Errors saying that `opensearch.hosts` expects a URI usually indicate that the
+environment was not loaded or that the value is not valid JSON/URI syntax.
+Prefer the Make target, which loads the environment automatically:
 
 ```bash
-    cd ./deploy/
-    source export_env_vars.sh
-    make start-elastic
+make -C deploy start-elastic
 ```
 
-### 🗃️ DB-samples issues
+If you invoke Compose directly, load the deployment environment first:
 
 ```bash
-No table data for samples_db
+cd deploy
+source ./export_env_vars.sh
+docker compose -f services.yml up -d elasticsearch-1 elasticsearch-2 kibana
 ```
 
-It is possible that you may have forgotten to pull the large files from the repo, please do : `git lfs pull`.
+### 🗃️ Sample database has no data
+
+If `samples_db` contains no sample tables, make sure Git LFS assets were pulled:
+
+```bash
+git lfs pull
+```
 
-Delete the samples-db container and it's volumes and restart it, you should now see the data in the tables.
+Then recreate the sample database only if its existing data is disposable. Stop
+the service, inspect its volume with `docker volume ls`, and remove only that
+deployment's sample database volume before starting `samples-db` again.
diff --git a/docs/nifi/development_guide.md b/docs/nifi/development_guide.md
index 8d6af43a..b63d73c0 100644
--- a/docs/nifi/development_guide.md
+++ b/docs/nifi/development_guide.md
@@ -22,7 +22,7 @@ make init-security
 make start-nifi
 ```
 
-Then open NiFi at `https://localhost:8082`.
+Then open NiFi through nginx at `https://localhost:8443`.
 
 ## Repository map
 
diff --git a/docs/nifi/main.md b/docs/nifi/main.md
index 6ddc4f29..092a034c 100644
--- a/docs/nifi/main.md
+++ b/docs/nifi/main.md
@@ -280,7 +280,9 @@ To access the scheduling menu, right click on any processor > click configure >
 <strong> <span style="color: red">IMPORTANT:</span></strong> In general, if a process is left on the default `0 sec` run schedule, it will attempt to start to work as soon as flowfiles are in the queue, this can be a problem as, some workflows contain large amounts of flowfiles that may also contain a huge number of records, it can strain the system at times by constantly performing I/O operations (remember that flowfiles are stored both in memory and on disk), it is best to delay the running of tasks by setting the `Run Schedule` to anything but the default, this is especially useful for DB query processors (ExecuteSQLRecord or QueryDatabaseTable), in which the schedule should be set to an interval of minutes as queries may be very slow in executing. This setting will have a direct impact on CPU and memory usage.
 
 `Concurrent tasks` are a good way to keep things running in parallel if you feel that data is not being processed fast enough, just be aware that as stated above, one task represents one thread, and as mentioned in the `maximum thread count` section, it should not exceed the max number.
-It should also be noted that a `processor` cannot have more `tasks` in current execution than the declared number `concurrent tasks`, if a task finishes its work you will notice that another will immidiately take it's place, of course, a new task may not be started if the current task has finished work and the timer is set on `run schedule` to be different than 0.
+A processor cannot run more tasks concurrently than its configured
+`Concurrent Tasks` value. When one task finishes, another can start immediately
+if work is available and the run schedule permits it.
 
 ### Benchmarking
 
diff --git a/mkdocs.yml b/mkdocs.yml
index 75965af9..3e62745c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -42,6 +42,7 @@ nav:
   - 📰 News: news.md
   - 📋 Prerequisites: deploy/main.md
   - 📦 Deployment: deploy/deployment.md
+  - ⚙️ Configuration: deploy/configuration.md
   - 🧩 Services: deploy/services.md
   - 📛 Troubleshooting: deploy/troubleshooting.md
   - 🔀 Workflows:
@@ -58,7 +59,6 @@ nav:
       - Processor Scripting: nifi/processor_scripting.md
       - User Scripts: nifi/user_scripts.md
       - Python Extensions: nifi/user_python_extensions.md
-      - Configuration: deploy/configuration.md
   - 🛡️ Security:
       - security/main.md
       - 🏛️ Certificates and Root CA: security/certificates.md

From f72d931e7057f4112ebdde2db3d91ff2494bcca7 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Thu, 3 Sep 2026 17:02:42 +0100
Subject: [PATCH 03/12] Docs: added j-hub to makefile start docs.

---
 docs/deploy/deployment.md | 108 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/docs/deploy/deployment.md b/docs/deploy/deployment.md
index cc12302f..6875c637 100644
--- a/docs/deploy/deployment.md
+++ b/docs/deploy/deployment.md
@@ -397,6 +397,114 @@ Starts:
 
 Ideal for running ingestion pipelines and ETL workflows.
 
+---
+
+### 🛢️ Elasticsearch/OpenSearch and dashboards
+
+To switch between OpenSearch and Elasticsearch, update the backend variables
+described in [Configuration](./configuration.md) before generating certificates
+or starting the services.
+
+```bash
+make start-elastic
+```
+
+Starts Elasticsearch/OpenSearch nodes 1 and 2 plus Kibana or OpenSearch
+Dashboards.
+
+```bash
+make start-elastic-cluster
+```
+
+Starts all three search nodes. Start the dashboard separately if needed:
+
+```bash
+make start-kibana
+```
+
+For debugging or failure testing, start an individual node with
+`make start-elastic-1`, `make start-elastic-2`, or `make start-elastic-3`.
+
+---
+
+### 🗄️ Databases
+
+```bash
+make start-samples
+```
+
+Starts `samples-db`, which contains the datasets used by the example database
+workflows.
+
+```bash
+make start-production-db
+```
+
+Starts the PostgreSQL `cogstack-databank-db` service for deployments using a
+separate ingestion database.
+
+---
+
+### 📚 JupyterHub
+
+```bash
+make start-jupyter
+```
+
+Starts the JupyterHub service at `https://localhost:8888`. Its production
+Compose configuration joins the external `cogstack-net` network, so start the
+core data infrastructure first unless that network already exists.
+
+JupyterHub is not included in `start-all`; start it explicitly when notebooks
+or interactive analysis are required.
+
+---
+
+### 🧠 MedCAT services
+
+```bash
+make start-medcat-service
+```
+
+Starts the MedCAT concept-extraction API used by the annotation workflow.
+
+```bash
+make start-medcat-service-deid
+```
+
+Starts the MedCAT de-identification API.
+
+```bash
+make start-medcat-trainer
+```
+
+Starts MedCAT Trainer, its nginx proxy, and Solr for annotation and supervised
+training.
+
+---
+
+### 📝 OCR services
+
+```bash
+make start-ocr-services
+```
+
+Starts both `ocr-service` and `ocr-service-text-only`. Use these services for
+document extraction, OCR workflows, and pipeline testing.
+
+---
+
+### 🗂️ Gitea
+
+```bash
+make start-git-ea
+```
+
+Starts the local Gitea service for internal repository and configuration
+hosting.
+
+---
+
 ### 🚀 Start the standard service set
 
 ```bash

From cfaa9e0f424c92e831b20545e82e7b5b2ed6e771 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Thu, 3 Sep 2026 17:24:18 +0100
Subject: [PATCH 04/12] Services: updated OpenSearch[Dashboards] settings for
 ML.

---
 services/elasticsearch/config/opensearch.yml | 13 +++++++++++++
 services/kibana/config/opensearch.yml        |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml
index bbc9fd3c..c1b99618 100644
--- a/services/elasticsearch/config/opensearch.yml
+++ b/services/elasticsearch/config/opensearch.yml
@@ -182,6 +182,19 @@ plugins.security.audit.config.enable_ssl_client_auth: true
 plugins.security.system_indices.enabled: true
 plugins.security.system_indices.permission.enabled: true
 
+# ================= ML Commons =================
+
+plugins.ml_commons.local_model.enabled: true
+plugins.ml_commons.remote_inference.enabled: true
+plugins.ml_commons.only_run_on_ml_node: false
+plugins.ml_commons.allow_registering_model_via_url: true
+plugins.ml_commons.agent_framework_enabled: true
+plugins.ml_commons.rag_pipeline_feature_enabled: true
+
+# Allow remote models to execute from data nodes
+plugins.ml_commons.task_dispatcher.eligible_node_role.remote_model:
+  - data
+
 
 # Enable SQL-like queries, allows you to join across indices etc.
 plugins.sql.enabled: true
diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml
index 72a3103d..922bf789 100644
--- a/services/kibana/config/opensearch.yml
+++ b/services/kibana/config/opensearch.yml
@@ -133,6 +133,8 @@ opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"]
 # map.showRegionBlockedWarning: false
 
 ml_commons_dashboards.enabled: true
+assistant.chat.enabled: true
+observability.query_assist.enabled: true
 
 savedObjects.permission.enabled: true
 

From 7fe38d138e92ba055dc06527661f66b21158f71f Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Thu, 3 Sep 2026 17:50:47 +0100
Subject: [PATCH 05/12] Services: added more settings + roles to OS for ML
 models.

---
 deploy/services.yml                           |  3 ++
 security/es_roles/opensearch/roles.yml        | 19 ++++++++++
 .../es_roles/opensearch/roles_mapping.yml     | 37 +++++++++++++++++++
 services/elasticsearch/config/opensearch.yml  | 12 +++++-
 services/elasticsearch/models/.gitignore      |  3 ++
 services/elasticsearch/models/README.md       | 18 +++++++++
 services/kibana/config/opensearch.yml         |  2 +
 7 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 services/elasticsearch/models/.gitignore
 create mode 100644 services/elasticsearch/models/README.md

diff --git a/deploy/services.yml b/deploy/services.yml
index 41bfe036..1fc01c4d 100644
--- a/deploy/services.yml
+++ b/deploy/services.yml
@@ -139,6 +139,9 @@ x-es-common-volumes: &es-common-volumes
   - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml:ro
   - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config/opensearch-security/roles_mapping.yml:ro
   - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config/opensearch-security/roles.yml:ro
+  # Shared local ML model repository. Register files with a
+  # file:///usr/share/opensearch/models/<model>.zip URL.
+  - ../services/elasticsearch/models:/usr/share/opensearch/models:ro
   # Shared system settings
   - ../services/elasticsearch/sysctl.conf:/etc/sysctl.conf:ro
   # Shared snapshot mounts
diff --git a/security/es_roles/opensearch/roles.yml b/security/es_roles/opensearch/roles.yml
index 77693806..f0e4c6c1 100644
--- a/security/es_roles/opensearch/roles.yml
+++ b/security/es_roles/opensearch/roles.yml
@@ -328,6 +328,25 @@ ml_full_access:
       allowed_actions:
         - 'indices_monitor'
 
+# OpenSearch 3.x protects ML Commons' backing indices as system indices. The
+# standard ml_full_access role grants the ML API actions but not the explicit
+# system-index permission needed when system-index permissions are enabled.
+cogstack_ml_full_access:
+  reserved: false
+  hidden: false
+  cluster_permissions:
+    - 'cluster_monitor'
+    - 'cluster:admin/opensearch/ml/*'
+  index_permissions:
+    - index_patterns:
+        - '*'
+      allowed_actions:
+        - 'indices_monitor'
+    - index_patterns:
+        - '.plugins-ml-*'
+      allowed_actions:
+        - 'system:admin/system_index'
+
 # Allows users to use all Notifications functionality
 notifications_full_access:
   reserved: true
diff --git a/security/es_roles/opensearch/roles_mapping.yml b/security/es_roles/opensearch/roles_mapping.yml
index ba604379..76592a84 100644
--- a/security/es_roles/opensearch/roles_mapping.yml
+++ b/security/es_roles/opensearch/roles_mapping.yml
@@ -31,6 +31,43 @@ all_access:
   and_backend_roles: [""]
   description: "Maps admin to all_access"
 
+# Grant the current administrator ML Commons access and provide a backend role
+# that can be assigned to dedicated ML API/UI users.
+ml_full_access:
+  reserved: false
+  hidden: false
+  backend_roles:
+  - "ml_full_access"
+  hosts: []
+  users:
+  - "admin"
+  and_backend_roles: []
+  description: "Maps ML administrators to ml_full_access"
+
+# Companion role required by OpenSearch 3.x when system-index permissions are
+# enabled. Keep its users/backend roles aligned with ml_full_access above.
+cogstack_ml_full_access:
+  reserved: false
+  hidden: false
+  backend_roles:
+  - "ml_full_access"
+  hosts: []
+  users:
+  - "admin"
+  and_backend_roles: []
+  description: "Allows ML Commons APIs to access their protected system indices"
+
+# Read-only ML users can inspect models, connectors, tasks, and statistics.
+ml_read_access:
+  reserved: false
+  hidden: false
+  backend_roles:
+  - "ml_read_access"
+  hosts: []
+  users: []
+  and_backend_roles: []
+  description: "Maps ML observers to ml_read_access"
+
 own_index:
   reserved: false
   hidden: false
diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml
index c1b99618..2ba4bf1f 100644
--- a/services/elasticsearch/config/opensearch.yml
+++ b/services/elasticsearch/config/opensearch.yml
@@ -188,10 +188,20 @@ plugins.ml_commons.local_model.enabled: true
 plugins.ml_commons.remote_inference.enabled: true
 plugins.ml_commons.only_run_on_ml_node: false
 plugins.ml_commons.allow_registering_model_via_url: true
+plugins.ml_commons.allow_registering_model_via_local_file: true
+# Permit HTTPS model repositories and only the dedicated read-only local model
+# directory; do not allow arbitrary HTTP, FTP, or file-system paths.
+plugins.ml_commons.trusted_url_regex: '^https://.*$|^file:///usr/share/opensearch/models/[^?#]+\.zip$'
+plugins.ml_commons.model_access_control_enabled: true
+plugins.ml_commons.connector_access_control_enabled: true
+plugins.ml_commons.safe_delete_model: true
 plugins.ml_commons.agent_framework_enabled: true
 plugins.ml_commons.rag_pipeline_feature_enabled: true
 
-# Allow remote models to execute from data nodes
+# This cluster has no dedicated ML nodes, so allow both local and remote models
+# to execute on data nodes.
+plugins.ml_commons.task_dispatcher.eligible_node_role.local_model:
+  - data
 plugins.ml_commons.task_dispatcher.eligible_node_role.remote_model:
   - data
 
diff --git a/services/elasticsearch/models/.gitignore b/services/elasticsearch/models/.gitignore
new file mode 100644
index 00000000..7c9d611b
--- /dev/null
+++ b/services/elasticsearch/models/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.md
diff --git a/services/elasticsearch/models/README.md b/services/elasticsearch/models/README.md
new file mode 100644
index 00000000..1683d704
--- /dev/null
+++ b/services/elasticsearch/models/README.md
@@ -0,0 +1,18 @@
+# Local OpenSearch ML models
+
+Place custom TorchScript or ONNX model ZIP files in this directory before
+starting the OpenSearch containers. The directory is mounted read-only at the
+same path on every OpenSearch node:
+
+```text
+/usr/share/opensearch/models
+```
+
+Register a model through the ML Commons REST API or OpenSearch Dashboards Dev
+Tools using a URL such as:
+
+```text
+file:///usr/share/opensearch/models/my-model.zip
+```
+
+Do not commit model binaries to the repository.
diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml
index 922bf789..a01588c8 100644
--- a/services/kibana/config/opensearch.yml
+++ b/services/kibana/config/opensearch.yml
@@ -132,6 +132,8 @@ opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"]
 # for your map rendering services.
 # map.showRegionBlockedWarning: false
 
+# Enable ML Commons model monitoring and management. Model registration and
+# deployment are performed through the REST API or the Dashboards Dev Tools UI.
 ml_commons_dashboards.enabled: true
 assistant.chat.enabled: true
 observability.query_assist.enabled: true

From 1797e62f3e28a2e979e89a7f9af128c49654edb5 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Fri, 4 Sep 2026 11:55:20 +0100
Subject: [PATCH 06/12] Deploy/services: added extra services for ollama and
 openwebui.

---
 .gitignore                                   |  2 +
 deploy/Makefile                              | 36 +++++++++-
 deploy/elasticsearch.env                     | 12 ++--
 deploy/export_env_vars.sh                    |  2 +
 deploy/ollama.env                            | 36 ++++++++++
 deploy/openwebui.env                         | 39 +++++++++++
 deploy/services.yml                          | 74 ++++++++++++++++++++
 docs/deploy/configuration.md                 |  1 +
 services/elasticsearch/config/opensearch.yml |  9 +++
 9 files changed, 203 insertions(+), 8 deletions(-)
 create mode 100644 deploy/ollama.env
 create mode 100644 deploy/openwebui.env

diff --git a/.gitignore b/.gitignore
index c781ab91..f5fb47b9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,3 +52,5 @@ site/
 **/*.env
 !*.env.template
 !**/*.env.template
+!deploy/ollama.env
+!deploy/openwebui.env
diff --git a/deploy/Makefile b/deploy/Makefile
index bcbf31be..8da3f56e 100644
--- a/deploy/Makefile
+++ b/deploy/Makefile
@@ -351,6 +351,26 @@ start-filebeat-3:
 start-kibana:
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) kibana
 
+start-ollama: ## Start the Ollama inference service
+	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) ollama
+
+pull-ollama-model: start-ollama ## Pull the model configured by OLLAMA_MODEL
+	$(WITH_ENV) docker compose -f services.yml exec -T ollama ollama pull "$${OLLAMA_MODEL}"
+
+setup-ollama: pull-ollama-model ## Start Ollama and pull its configured model
+
+start-openwebui: ## Start Open WebUI and its Ollama dependency
+	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) open-webui
+
+restart-openwebui: ## Restart the Open WebUI container
+	$(WITH_ENV) docker compose -f services.yml restart open-webui
+
+status-openwebui: ## Show Open WebUI and Ollama container status
+	$(WITH_ENV) docker compose -f services.yml ps open-webui ollama
+
+logs-openwebui: ## Follow the Open WebUI logs
+	$(WITH_ENV) docker compose -f services.yml logs --tail=100 -f open-webui
+
 start-samples: ## Start the sample database service
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) samples-db
 
@@ -379,7 +399,7 @@ start-data-infra: start-nifi start-elastic start-samples ## Start core data infr
 
 start-all: start-data-infra start-medcat-service start-ocr-services ## Start all major services
 
-.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-samples start-jupyter
+.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama pull-ollama-model setup-ollama start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-jupyter
 
 
 # stop services
@@ -423,6 +443,12 @@ delete-elastic: ## Remove Elasticsearch and Kibana containers
 delete-elastic-volumes: ## Delete Elasticsearch and Kibana volumes
 	$(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana
 
+delete-ollama: ## Remove the Ollama container but preserve downloaded models
+	$(WITH_ENV) docker compose -f services.yml rm -f -s ollama
+
+delete-openwebui: ## Remove the Open WebUI container but preserve its data
+	$(WITH_ENV) docker compose -f services.yml rm -f -s open-webui
+
 delete-databank:
 	$(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql
 
@@ -494,6 +520,12 @@ stop-filebeat-3:
 stop-kibana:
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) kibana
 
+stop-ollama: ## Stop the Ollama inference service
+	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) ollama
+
+stop-openwebui: ## Stop the Open WebUI service
+	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) open-webui
+
 stop-samples: ## Stop the sample database service
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) samples-db
 
@@ -522,7 +554,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples ## Stop core data infrastru
 
 stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services ## Stop all major services
 
-.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes delete-medcat-trainer delete-medcat-trainer-volumes delete-jupyter delete-medcat-service delete-medcat-service-deid delete-ocr-services make-delete-jupyter make-delete-medcat-service make-delete-medcat-service-deid make-delete-ocr-services stop-elastic stop-samples stop-jupyter
+.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-ollama delete-openwebui delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes delete-medcat-trainer delete-medcat-trainer-volumes delete-jupyter delete-medcat-service delete-medcat-service-deid delete-ocr-services make-delete-jupyter make-delete-medcat-service make-delete-medcat-service-deid make-delete-ocr-services stop-elastic stop-ollama stop-openwebui stop-samples stop-jupyter
 
 
 # cleanup
diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env
index 56fa8c61..0172893a 100644
--- a/deploy/elasticsearch.env
+++ b/deploy/elasticsearch.env
@@ -16,7 +16,7 @@ ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.8.0
 
 ELASTICSEARCH_LOG_LEVEL=INFO
 
-ELASTICSEARCH_SEED_HOSTS=localhost,elasticsearch-2,elasticsearch-1
+ELASTICSEARCH_SEED_HOSTS=localhost,elasticsearch-1
 ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES=es01
 
 
@@ -93,15 +93,15 @@ ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
 ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
 
 # MEMORY CONFIG
-ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
+ELASTICSEARCH_JAVA_OPTS="-Xms24g -Xmx24g -Des.failure_store_feature_flag_enabled=true"
 ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
 OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
 
 ELASTICSEARCH_DOCKER_CPU_MIN=1
-ELASTICSEARCH_DOCKER_CPU_MAX=1
-ELASTICSEARCH_DOCKER_RAM=1g
+ELASTICSEARCH_DOCKER_CPU_MAX=4
+ELASTICSEARCH_DOCKER_RAM=32g
 
-ELASTICSEARCH_DOCKER_SHM_SIZE=512m
+ELASTICSEARCH_DOCKER_SHM_SIZE=1g
 
 ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
 ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
@@ -155,7 +155,7 @@ ELASTICSEARCH_INDICES_MEMORY_INDEX_BUFFER_SIZE="25%"
 ############################################################ Shared Env vars for MetricBeats/Kibana etc ##############################################################
 
 # please respect the double quotes "" on each address or you may get type errors from Kibana
-ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200","https://elasticsearch-2:9200"]'
+ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200"]'
 
 KIBANA_HOST="https://kibana:5601"
 
diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh
index 1d05060b..8528034b 100755
--- a/deploy/export_env_vars.sh
+++ b/deploy/export_env_vars.sh
@@ -29,6 +29,8 @@ env_files=(
   "$DEPLOY_DIR/general.env"
   "$DEPLOY_DIR/nifi.env"
   "$DEPLOY_DIR/elasticsearch.env"
+  "$DEPLOY_DIR/ollama.env"
+  "$DEPLOY_DIR/openwebui.env"
   "$DEPLOY_DIR/database.env"
   "$DEPLOY_DIR/network_settings.env"
   "$DEPLOY_DIR/project.env"
diff --git a/deploy/ollama.env b/deploy/ollama.env
new file mode 100644
index 00000000..1a6b70a7
--- /dev/null
+++ b/deploy/ollama.env
@@ -0,0 +1,36 @@
+####################################################################### Ollama Env vars #######################################################################
+
+# Official Ollama server image.
+OLLAMA_DOCKER_IMAGE=ollama/ollama:latest
+OLLAMA_CONTAINER_NAME=cogstack-ollama
+
+# Use the native image architecture on Apple Silicon instead of inheriting
+# the repository-wide linux/amd64 default used by OpenSearch.
+# Change to linux/amd64 when deploying on an x86-64 host.
+OLLAMA_DOCKER_PLATFORM=linux/arm64
+
+# Ollama listens on every container interface so OpenSearch can reach it over
+# cogstack-net. The published host port remains bound to loopback in services.yml.
+OLLAMA_HOST=0.0.0.0:11434
+OLLAMA_OUTPUT_HOST=127.0.0.1
+OLLAMA_OUTPUT_PORT=11434
+
+# Default model for CogStack Q&A setup. This variable documents the selected
+# model; pull it once after starting Ollama (see the command below).
+OLLAMA_MODEL=qwen3.5:9b-q4_K_M
+
+# A 16K context provides room for OpenSearch agent instructions, mappings,
+# tool definitions, and retrieved documents without paying for the full 256K
+# context window in memory.
+OLLAMA_CONTEXT_LENGTH=16384
+
+# Keep one model resident and avoid excessive memory use alongside OpenSearch.
+OLLAMA_KEEP_ALIVE=5m
+OLLAMA_MAX_LOADED_MODELS=1
+OLLAMA_NUM_PARALLEL=1
+
+# Docker resource allocation.
+OLLAMA_DOCKER_CPU_MIN=1
+OLLAMA_DOCKER_CPU_MAX=4
+OLLAMA_DOCKER_RAM_MIN=2g
+OLLAMA_DOCKER_RAM=12g
diff --git a/deploy/openwebui.env b/deploy/openwebui.env
new file mode 100644
index 00000000..48f01ccb
--- /dev/null
+++ b/deploy/openwebui.env
@@ -0,0 +1,39 @@
+###################################################################### Open WebUI Env vars ######################################################################
+
+# Official Open WebUI image. Pin this tag to a numbered release when a
+# reproducible production deployment is required.
+OPENWEBUI_DOCKER_IMAGE=ghcr.io/open-webui/open-webui:main
+OPENWEBUI_CONTAINER_NAME=cogstack-open-webui
+
+# Open WebUI reaches Ollama through the private Compose network. Do not use
+# localhost here because localhost inside this container is Open WebUI itself.
+OLLAMA_BASE_URL=http://ollama:11434
+
+# The first account registered becomes the administrator. After creating it,
+# signups can be disabled from the Open WebUI Admin Panel.
+WEBUI_AUTH=true
+ENABLE_SIGNUP=true
+DEFAULT_USER_ROLE=user
+WEBUI_NAME=CogStack
+
+# Leave this empty when using the official image entrypoint. It generates a
+# random key and persists it in the openwebui-data volume on first startup.
+WEBUI_SECRET_KEY=
+
+# Browser-facing address. Port 3001 avoids the existing Gitea port 3000.
+WEBUI_URL=http://localhost:3001
+OPENWEBUI_OUTPUT_HOST=127.0.0.1
+OPENWEBUI_OUTPUT_PORT=3001
+
+# Open WebUI is lightweight compared with model inference, which stays in the
+# separate Ollama container.
+OPENWEBUI_DOCKER_CPU_MIN=0.25
+OPENWEBUI_DOCKER_CPU_MAX=1
+OPENWEBUI_DOCKER_RAM_MIN=256m
+OPENWEBUI_DOCKER_RAM=2g
+
+# Disable optional usage reporting and update checks in this local deployment.
+ANONYMIZED_TELEMETRY=false
+DO_NOT_TRACK=true
+SCARF_NO_ANALYTICS=true
+ENABLE_VERSION_UPDATE_CHECK=false
diff --git a/deploy/services.yml b/deploy/services.yml
index 1fc01c4d..88166a32 100644
--- a/deploy/services.yml
+++ b/deploy/services.yml
@@ -478,6 +478,72 @@ services:
     extra_hosts: *common-hosts
     logging: *es-logging-common
 
+#---------------------------------------------------------------------------#
+# Ollama local model inference                                              #
+#---------------------------------------------------------------------------#
+  ollama:
+    image: ${OLLAMA_DOCKER_IMAGE:-ollama/ollama:latest}
+    container_name: ${OLLAMA_CONTAINER_NAME:-cogstack-ollama}
+    platform: ${OLLAMA_DOCKER_PLATFORM:-linux/arm64}
+    restart: unless-stopped
+    env_file:
+      - ./ollama.env
+    deploy:
+      resources:
+        limits:
+          cpus: "${OLLAMA_DOCKER_CPU_MAX:-4}"
+          memory: "${OLLAMA_DOCKER_RAM:-12g}"
+        reservations:
+          cpus: "${OLLAMA_DOCKER_CPU_MIN:-1}"
+          memory: "${OLLAMA_DOCKER_RAM_MIN:-2g}"
+    volumes:
+      - ollama-models:/root/.ollama
+    ports:
+      # Keep the unauthenticated Ollama API local to the Docker host.
+      - "${OLLAMA_OUTPUT_HOST:-127.0.0.1}:${OLLAMA_OUTPUT_PORT:-11434}:11434"
+    expose:
+      - "11434"
+    networks:
+      - cognet
+    logging: *logging-common
+    healthcheck:
+      test: ["CMD", "ollama", "list"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 20s
+
+#---------------------------------------------------------------------------#
+# Open WebUI for local Ollama chat and model management                      #
+#---------------------------------------------------------------------------#
+  open-webui:
+    image: ${OPENWEBUI_DOCKER_IMAGE:-ghcr.io/open-webui/open-webui:main}
+    container_name: ${OPENWEBUI_CONTAINER_NAME:-cogstack-open-webui}
+    restart: unless-stopped
+    env_file:
+      - ./openwebui.env
+    depends_on:
+      ollama:
+        condition: service_healthy
+    deploy:
+      resources:
+        limits:
+          cpus: "${OPENWEBUI_DOCKER_CPU_MAX:-1}"
+          memory: "${OPENWEBUI_DOCKER_RAM:-2g}"
+        reservations:
+          cpus: "${OPENWEBUI_DOCKER_CPU_MIN:-0.25}"
+          memory: "${OPENWEBUI_DOCKER_RAM_MIN:-256m}"
+    volumes:
+      - openwebui-data:/app/backend/data
+    ports:
+      # Port 3000 is already used by Gitea in this Compose project.
+      - "${OPENWEBUI_OUTPUT_HOST:-127.0.0.1}:${OPENWEBUI_OUTPUT_PORT:-3001}:8080"
+    expose:
+      - "8080"
+    networks:
+      - cognet
+    logging: *logging-common
+
 #---------------------------------------------------------------------------#
 # NiFi webapp                                                               #
 #---------------------------------------------------------------------------#
@@ -635,6 +701,14 @@ volumes:
   filebeat-data-3:
     driver: local
 
+  # Ollama model cache
+  ollama-models:
+    driver: local
+
+  # Open WebUI database, uploads, configuration, and generated secret key
+  openwebui-data:
+    driver: local
+
   # NiFi related
   nifi-vol-logs:
     driver: local
diff --git a/docs/deploy/configuration.md b/docs/deploy/configuration.md
index c4240be5..0c63b0e0 100644
--- a/docs/deploy/configuration.md
+++ b/docs/deploy/configuration.md
@@ -24,6 +24,7 @@ The core Compose file, `deploy/services.yml`, loads these files:
 | `deploy/general.env` | General Docker logging and shared defaults |
 | `deploy/nifi.env` | NiFi image, ports, resources, proxy settings, and paths |
 | `deploy/elasticsearch.env` | Elasticsearch/OpenSearch, Dashboards/Kibana, and Beats |
+| `deploy/ollama.env` | Ollama image, model selection, port, and resource limits |
 | `deploy/database.env` | Sample and production database resources |
 | `deploy/gitea.env` | Gitea image, ports, and application settings |
 | `deploy/nginx.env` | NiFi nginx proxy settings |
diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml
index 2ba4bf1f..9383c648 100644
--- a/services/elasticsearch/config/opensearch.yml
+++ b/services/elasticsearch/config/opensearch.yml
@@ -192,6 +192,15 @@ plugins.ml_commons.allow_registering_model_via_local_file: true
 # Permit HTTPS model repositories and only the dedicated read-only local model
 # directory; do not allow arbitrary HTTP, FTP, or file-system paths.
 plugins.ml_commons.trusted_url_regex: '^https://.*$|^file:///usr/share/opensearch/models/[^?#]+\.zip$'
+# Permit ML Commons connectors to call only the Ollama service on the private
+# CogStack Docker network. Docker resolves this stable service name to a
+# private container IP, so both the general and private endpoint allowlists
+# are required by OpenSearch 3.7 and later.
+plugins.ml_commons.connector.private_ip_enabled: true
+plugins.ml_commons.trusted_connector_endpoints_regex:
+  - '^http://ollama:11434/.*$'
+plugins.ml_commons.trusted_connector_private_endpoints_regex:
+  - '^http://ollama:11434/.*$'
 plugins.ml_commons.model_access_control_enabled: true
 plugins.ml_commons.connector_access_control_enabled: true
 plugins.ml_commons.safe_delete_model: true

From 65228d265c7f52b9197385b6d07a5b3734f6bc8b Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Fri, 4 Sep 2026 11:57:21 +0100
Subject: [PATCH 07/12] Deploy: reverted elastic env vars to default.

---
 deploy/elasticsearch.env | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env
index 0172893a..d23658d6 100644
--- a/deploy/elasticsearch.env
+++ b/deploy/elasticsearch.env
@@ -17,7 +17,33 @@ ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.8.0
 ELASTICSEARCH_LOG_LEVEL=INFO
 
 ELASTICSEARCH_SEED_HOSTS=localhost,elasticsearch-1
-ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES=es01
+ELASTICSEARCHhot backups of configs
+ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
+
+# Security certificates
+ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
+
+# MEMORY CONFIG
+ELASTICSEARCH_JAVA_OPTS="-Xms24g -Xmx24g -Des.failure_store_feature_flag_enabled=true"
+ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
+OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
+
+ELASTICSEARCH_DOCKER_CPU_MIN=1
+ELASTICSEARCH_DOCKER_CPU_MAX=4
+ELASTICSEARCH_DOCKER_RAM=32g
+
+ELASTICSEARCH_DOCKER_SHM_SIZE=1g
+
+ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
+ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
+
+# disables swapping, imporving r/w performance (at the expense of RAM)
+ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK=true
+
+#  system_call_filter | https://en.wikipedia.org/wiki/Seccomp
+ELASTICSEARCH_SYSTEM_CALL_FILTER=false
+
+#_INITIAL_CLUSTER_MANAGER_NODES=es01
 
 
 ELASTICSEARCH_NETWORK_HOST="0.0.0.0"

From 13cdb5d0e07458756c0afae6d47158984333fe26 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Fri, 4 Sep 2026 12:12:36 +0100
Subject: [PATCH 08/12] Deploy: fixed es.env typo.

---
 deploy/elasticsearch.env | 40 +++++++---------------------------------
 1 file changed, 7 insertions(+), 33 deletions(-)

diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env
index d23658d6..56fa8c61 100644
--- a/deploy/elasticsearch.env
+++ b/deploy/elasticsearch.env
@@ -16,34 +16,8 @@ ELASTICSEARCH_DOCKER_IMAGE=opensearchproject/opensearch:3.8.0
 
 ELASTICSEARCH_LOG_LEVEL=INFO
 
-ELASTICSEARCH_SEED_HOSTS=localhost,elasticsearch-1
-ELASTICSEARCHhot backups of configs
-ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
-
-# Security certificates
-ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
-
-# MEMORY CONFIG
-ELASTICSEARCH_JAVA_OPTS="-Xms24g -Xmx24g -Des.failure_store_feature_flag_enabled=true"
-ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
-OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
-
-ELASTICSEARCH_DOCKER_CPU_MIN=1
-ELASTICSEARCH_DOCKER_CPU_MAX=4
-ELASTICSEARCH_DOCKER_RAM=32g
-
-ELASTICSEARCH_DOCKER_SHM_SIZE=1g
-
-ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
-ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
-
-# disables swapping, imporving r/w performance (at the expense of RAM)
-ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK=true
-
-#  system_call_filter | https://en.wikipedia.org/wiki/Seccomp
-ELASTICSEARCH_SYSTEM_CALL_FILTER=false
-
-#_INITIAL_CLUSTER_MANAGER_NODES=es01
+ELASTICSEARCH_SEED_HOSTS=localhost,elasticsearch-2,elasticsearch-1
+ELASTICSEARCH_INITIAL_CLUSTER_MANAGER_NODES=es01
 
 
 ELASTICSEARCH_NETWORK_HOST="0.0.0.0"
@@ -119,15 +93,15 @@ ELASTICSEARCH_BACKUP_PARTITION_CONFIG=../data/es_snapshot_backups/config_backup
 ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
 
 # MEMORY CONFIG
-ELASTICSEARCH_JAVA_OPTS="-Xms24g -Xmx24g -Des.failure_store_feature_flag_enabled=true"
+ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
 ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
 OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
 
 ELASTICSEARCH_DOCKER_CPU_MIN=1
-ELASTICSEARCH_DOCKER_CPU_MAX=4
-ELASTICSEARCH_DOCKER_RAM=32g
+ELASTICSEARCH_DOCKER_CPU_MAX=1
+ELASTICSEARCH_DOCKER_RAM=1g
 
-ELASTICSEARCH_DOCKER_SHM_SIZE=1g
+ELASTICSEARCH_DOCKER_SHM_SIZE=512m
 
 ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
 ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
@@ -181,7 +155,7 @@ ELASTICSEARCH_INDICES_MEMORY_INDEX_BUFFER_SIZE="25%"
 ############################################################ Shared Env vars for MetricBeats/Kibana etc ##############################################################
 
 # please respect the double quotes "" on each address or you may get type errors from Kibana
-ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200"]'
+ELASTICSEARCH_HOSTS='["https://elasticsearch-1:9200","https://elasticsearch-2:9200"]'
 
 KIBANA_HOST="https://kibana:5601"
 

From b56d75f498458f2e9bbfed746c3a40d12a47aad9 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Sat, 5 Sep 2026 09:55:14 +0100
Subject: [PATCH 09/12] Services: updated OpenSearch ml settings.

---
 services/elasticsearch/config/opensearch.yml | 3 +++
 services/kibana/config/opensearch.yml        | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml
index 9383c648..274b4ee9 100644
--- a/services/elasticsearch/config/opensearch.yml
+++ b/services/elasticsearch/config/opensearch.yml
@@ -206,6 +206,9 @@ plugins.ml_commons.connector_access_control_enabled: true
 plugins.ml_commons.safe_delete_model: true
 plugins.ml_commons.agent_framework_enabled: true
 plugins.ml_commons.rag_pipeline_feature_enabled: true
+# Upper bound for asynchronous ML Commons tasks. Remote connector connection
+# and response timeouts are configured per connector by opensearch-ai-init.sh.
+plugins.ml_commons.ml_task_timeout_in_seconds: 600
 
 # This cluster has no dedicated ML nodes, so allow both local and remote models
 # to execute on data nodes.
diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml
index a01588c8..f37d5131 100644
--- a/services/kibana/config/opensearch.yml
+++ b/services/kibana/config/opensearch.yml
@@ -85,7 +85,7 @@ opensearch.pingTimeout: 10000
 
 # Time in milliseconds to wait for responses from the back end or OpenSearch. This value
 # must be a positive integer.
-opensearch.requestTimeout: 200000
+opensearch.requestTimeout: 420000
 
 # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side
 # headers, set this value to [] (an empty list).
@@ -141,7 +141,7 @@ observability.query_assist.enabled: true
 savedObjects.permission.enabled: true
 
 # Set the value to true to enable multiple data source feature
-data_source.enabled: true
+data_source.enabled: false
 # Set the value to true to enable workspace feature
 workspace.enabled: true
 # Set the value to true to enable explore feature

From ad0fc053812fecab7bfcf102db02305b972d7006 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Sat, 5 Sep 2026 11:17:10 +0100
Subject: [PATCH 10/12] Deploy: added OpenSearch settings + added script for
 llm setup.

---
 deploy/Makefile               |   5 +-
 deploy/elasticsearch.env      |  23 ++
 deploy/general.env            |   2 +-
 deploy/ollama.env             |   7 +-
 deploy/services.yml           |   3 +-
 scripts/opensearch-ai-init.sh | 416 ++++++++++++++++++++++++++++++++++
 6 files changed, 447 insertions(+), 9 deletions(-)
 create mode 100755 scripts/opensearch-ai-init.sh

diff --git a/deploy/Makefile b/deploy/Makefile
index 8da3f56e..3f618642 100644
--- a/deploy/Makefile
+++ b/deploy/Makefile
@@ -359,6 +359,9 @@ pull-ollama-model: start-ollama ## Pull the model configured by OLLAMA_MODEL
 
 setup-ollama: pull-ollama-model ## Start Ollama and pull its configured model
 
+opensearch-ai-init: ## Create or update Ollama-backed OpenSearch chat and PPL assistants
+	$(WITH_ENV) ../scripts/opensearch-ai-init.sh
+
 start-openwebui: ## Start Open WebUI and its Ollama dependency
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) open-webui
 
@@ -399,7 +402,7 @@ start-data-infra: start-nifi start-elastic start-samples ## Start core data infr
 
 start-all: start-data-infra start-medcat-service start-ocr-services ## Start all major services
 
-.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama pull-ollama-model setup-ollama start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-jupyter
+.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama pull-ollama-model setup-ollama opensearch-ai-init start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-jupyter
 
 
 # stop services
diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env
index 56fa8c61..edad8df1 100644
--- a/deploy/elasticsearch.env
+++ b/deploy/elasticsearch.env
@@ -106,6 +106,29 @@ ELASTICSEARCH_DOCKER_SHM_SIZE=512m
 ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m"
 ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10
 
+# OpenSearch ML Commons bootstrap settings. Connector timeouts are seconds;
+# the Dashboards opensearch.requestTimeout setting is milliseconds.
+OPENSEARCH_AI_OPENSEARCH_CONTAINER=elasticsearch-1
+# Local development credentials; update these when the OpenSearch admin login
+# is rotated or override them when invoking the bootstrap script.
+OPENSEARCH_AI_OPENSEARCH_USER=admin
+OPENSEARCH_AI_OPENSEARCH_PASSWORD=admin
+OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS=120
+OPENSEARCH_AI_READ_TIMEOUT_SECONDS=360
+OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS=300
+OPENSEARCH_AI_MAX_TOKENS=256
+OPENSEARCH_AI_REASONING_EFFORT=none
+
+# Stable resource names let scripts/opensearch-ai-init.sh safely find and
+# update resources instead of creating duplicates on every run.
+OPENSEARCH_AI_CHAT_CONNECTOR_NAME="Ollama Qwen Agent Connector"
+OPENSEARCH_AI_CHAT_MODEL_NAME="Qwen 3.5 Ollama Agent Model"
+OPENSEARCH_AI_CHAT_AGENT_NAME="Qwen 3.5 Conversational Agent v2"
+OPENSEARCH_AI_ROOT_AGENT_NAME="Qwen Dashboards Root Agent"
+OPENSEARCH_AI_PPL_CONNECTOR_NAME="Ollama Qwen PPL Connector"
+OPENSEARCH_AI_PPL_MODEL_NAME="Qwen 3.5 Ollama PPL Model"
+OPENSEARCH_AI_PPL_AGENT_NAME="Qwen PPL Query Assist Agent"
+
 # disables swapping, imporving r/w performance (at the expense of RAM)
 ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK=true
 
diff --git a/deploy/general.env b/deploy/general.env
index fd51869a..768710c2 100644
--- a/deploy/general.env
+++ b/deploy/general.env
@@ -21,6 +21,6 @@ VOLUME_PREFIX=deploy_
 DEPLOYMENT_ENV_FILE_PATH_GENERAL=../../../deploy/general.env
 
 # this is the default, for containers which dont rely on individual settings
-DOCKER_SHM_SIZE=1g
+DOCKER_SHM_SIZE=512m
 DOCKER_LOG_SIZE_PER_FILE="100m"
 DOCKER_LOG_NUM_FILES=10
diff --git a/deploy/ollama.env b/deploy/ollama.env
index 1a6b70a7..48fa3df5 100644
--- a/deploy/ollama.env
+++ b/deploy/ollama.env
@@ -4,11 +4,6 @@
 OLLAMA_DOCKER_IMAGE=ollama/ollama:latest
 OLLAMA_CONTAINER_NAME=cogstack-ollama
 
-# Use the native image architecture on Apple Silicon instead of inheriting
-# the repository-wide linux/amd64 default used by OpenSearch.
-# Change to linux/amd64 when deploying on an x86-64 host.
-OLLAMA_DOCKER_PLATFORM=linux/arm64
-
 # Ollama listens on every container interface so OpenSearch can reach it over
 # cogstack-net. The published host port remains bound to loopback in services.yml.
 OLLAMA_HOST=0.0.0.0:11434
@@ -33,4 +28,4 @@ OLLAMA_NUM_PARALLEL=1
 OLLAMA_DOCKER_CPU_MIN=1
 OLLAMA_DOCKER_CPU_MAX=4
 OLLAMA_DOCKER_RAM_MIN=2g
-OLLAMA_DOCKER_RAM=12g
+OLLAMA_DOCKER_RAM=16g
diff --git a/deploy/services.yml b/deploy/services.yml
index 88166a32..ce0f42ec 100644
--- a/deploy/services.yml
+++ b/deploy/services.yml
@@ -484,7 +484,7 @@ services:
   ollama:
     image: ${OLLAMA_DOCKER_IMAGE:-ollama/ollama:latest}
     container_name: ${OLLAMA_CONTAINER_NAME:-cogstack-ollama}
-    platform: ${OLLAMA_DOCKER_PLATFORM:-linux/arm64}
+    platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64}
     restart: unless-stopped
     env_file:
       - ./ollama.env
@@ -519,6 +519,7 @@ services:
   open-webui:
     image: ${OPENWEBUI_DOCKER_IMAGE:-ghcr.io/open-webui/open-webui:main}
     container_name: ${OPENWEBUI_CONTAINER_NAME:-cogstack-open-webui}
+    platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64}
     restart: unless-stopped
     env_file:
       - ./openwebui.env
diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh
new file mode 100755
index 00000000..f7890214
--- /dev/null
+++ b/scripts/opensearch-ai-init.sh
@@ -0,0 +1,416 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+log() {
+  printf '[opensearch-ai-init] %s\n' "$*" >&2
+}
+
+die() {
+  printf '[opensearch-ai-init] ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+load_env_file() {
+  local env_file="$1"
+  if [[ -f "$env_file" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$env_file"
+    set +a
+  fi
+}
+
+# Make exports these variables already. Loading the files here also makes the
+# script convenient to run directly from the repository root. Preserve values
+# supplied by the caller so one-off command-line overrides take precedence.
+ENV_OVERRIDE_NAMES=(
+  ELASTIC_USER
+  ELASTIC_PASSWORD
+  OLLAMA_CONTAINER_NAME
+  OLLAMA_MODEL
+  OPENSEARCH_AI_OPENSEARCH_CONTAINER
+  OPENSEARCH_AI_OPENSEARCH_USER
+  OPENSEARCH_AI_OPENSEARCH_PASSWORD
+  OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS
+  OPENSEARCH_AI_READ_TIMEOUT_SECONDS
+  OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS
+  OPENSEARCH_AI_MAX_TOKENS
+  OPENSEARCH_AI_REASONING_EFFORT
+  OPENSEARCH_AI_CHAT_CONNECTOR_NAME
+  OPENSEARCH_AI_CHAT_MODEL_NAME
+  OPENSEARCH_AI_CHAT_AGENT_NAME
+  OPENSEARCH_AI_ROOT_AGENT_NAME
+  OPENSEARCH_AI_PPL_CONNECTOR_NAME
+  OPENSEARCH_AI_PPL_MODEL_NAME
+  OPENSEARCH_AI_PPL_AGENT_NAME
+)
+ENV_OVERRIDES=()
+for variable_name in "${ENV_OVERRIDE_NAMES[@]}"; do
+  if [[ -n "${!variable_name+x}" ]]; then
+    ENV_OVERRIDES+=("$variable_name=${!variable_name}")
+  fi
+done
+load_env_file "$PROJECT_DIR/security/env/users_elasticsearch.env"
+load_env_file "$PROJECT_DIR/deploy/elasticsearch.env"
+load_env_file "$PROJECT_DIR/deploy/ollama.env"
+for variable_assignment in "${ENV_OVERRIDES[@]}"; do
+  variable_name="${variable_assignment%%=*}"
+  printf -v "$variable_name" '%s' "${variable_assignment#*=}"
+  export "${variable_name?}"
+done
+
+OPENSEARCH_CONTAINER="${OPENSEARCH_AI_OPENSEARCH_CONTAINER:-elasticsearch-1}"
+OPENSEARCH_USER="${OPENSEARCH_AI_OPENSEARCH_USER:-${ELASTIC_USER:-admin}}"
+OPENSEARCH_PASSWORD="${OPENSEARCH_AI_OPENSEARCH_PASSWORD:-${ELASTIC_PASSWORD:-}}"
+OLLAMA_CONTAINER="${OLLAMA_CONTAINER_NAME:-cogstack-ollama}"
+OLLAMA_MODEL_NAME="${OLLAMA_MODEL:-qwen3.5:9b-q4_K_M}"
+
+CONNECTION_TIMEOUT="${OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS:-120}"
+READ_TIMEOUT="${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360}"
+DEPLOY_TIMEOUT="${OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS:-300}"
+MAX_TOKENS="${OPENSEARCH_AI_MAX_TOKENS:-256}"
+REASONING_EFFORT="${OPENSEARCH_AI_REASONING_EFFORT:-none}"
+
+CHAT_CONNECTOR_NAME="${OPENSEARCH_AI_CHAT_CONNECTOR_NAME:-Ollama Qwen Agent Connector}"
+CHAT_MODEL_NAME="${OPENSEARCH_AI_CHAT_MODEL_NAME:-Qwen 3.5 Ollama Agent Model}"
+CHAT_AGENT_NAME="${OPENSEARCH_AI_CHAT_AGENT_NAME:-Qwen 3.5 Conversational Agent v2}"
+ROOT_AGENT_NAME="${OPENSEARCH_AI_ROOT_AGENT_NAME:-Qwen Dashboards Root Agent}"
+PPL_CONNECTOR_NAME="${OPENSEARCH_AI_PPL_CONNECTOR_NAME:-Ollama Qwen PPL Connector}"
+PPL_MODEL_NAME="${OPENSEARCH_AI_PPL_MODEL_NAME:-Qwen 3.5 Ollama PPL Model}"
+PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-Qwen PPL Query Assist Agent}"
+
+OPENSEARCH_URL="https://localhost:9200"
+OPENSEARCH_CA="/usr/share/opensearch/config/root-ca.crt"
+
+[[ -n "$OPENSEARCH_PASSWORD" ]] \
+  || die "set OPENSEARCH_AI_OPENSEARCH_PASSWORD or ELASTIC_PASSWORD"
+
+for command_name in docker jq; do
+  command -v "$command_name" >/dev/null 2>&1 || die "$command_name is required"
+done
+
+for integer_value in "$CONNECTION_TIMEOUT" "$READ_TIMEOUT" "$DEPLOY_TIMEOUT" "$MAX_TOKENS"; do
+  [[ "$integer_value" =~ ^[1-9][0-9]*$ ]] || die "timeout and token settings must be positive integers"
+done
+
+[[ "$(docker inspect --format '{{.State.Running}}' "$OPENSEARCH_CONTAINER" 2>/dev/null)" == "true" ]] \
+  || die "OpenSearch container '$OPENSEARCH_CONTAINER' is not running"
+[[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_CONTAINER" 2>/dev/null)" == "true" ]] \
+  || die "Ollama container '$OLLAMA_CONTAINER' is not running"
+
+os_request() {
+  local method="$1"
+  local path="$2"
+  local payload="${3-}"
+  local response body status
+  local curl_args=(
+    --silent
+    --show-error
+    --cacert "$OPENSEARCH_CA"
+    --user "$OPENSEARCH_USER:$OPENSEARCH_PASSWORD"
+    --request "$method"
+    --header 'Content-Type: application/json'
+    --write-out $'\n%{http_code}'
+  )
+
+  if [[ -n "$payload" ]]; then
+    response="$(printf '%s' "$payload" | docker exec -i "$OPENSEARCH_CONTAINER" \
+      curl "${curl_args[@]}" --data-binary @- "$OPENSEARCH_URL$path")" || return
+  else
+    response="$(docker exec "$OPENSEARCH_CONTAINER" \
+      curl "${curl_args[@]}" "$OPENSEARCH_URL$path")" || return
+  fi
+
+  status="${response##*$'\n'}"
+  body="${response%$'\n'*}"
+  if [[ ! "$status" =~ ^2[0-9][0-9]$ ]]; then
+    log "$method $path failed with HTTP $status"
+    [[ -n "$body" ]] && printf '%s\n' "$body" >&2
+    return 1
+  fi
+  printf '%s' "$body"
+}
+
+wait_for_opensearch() {
+  local deadline=$((SECONDS + DEPLOY_TIMEOUT))
+  local response
+  log "Waiting for OpenSearch to reach yellow health"
+  while (( SECONDS < deadline )); do
+    if response="$(os_request GET '/_cluster/health?wait_for_status=yellow&timeout=5s' 2>/dev/null)" \
+      && [[ "$(jq -r '.status // empty' <<<"$response")" =~ ^(yellow|green)$ ]]; then
+      return 0
+    fi
+    sleep 5
+  done
+  die "OpenSearch did not reach yellow health within ${DEPLOY_TIMEOUT}s"
+}
+
+wait_for_ollama() {
+  local deadline=$((SECONDS + DEPLOY_TIMEOUT))
+  log "Waiting for Ollama and ensuring model '$OLLAMA_MODEL_NAME' is available"
+  while (( SECONDS < deadline )); do
+    if docker exec "$OLLAMA_CONTAINER" ollama list >/dev/null 2>&1; then
+      if ! docker exec "$OLLAMA_CONTAINER" ollama show "$OLLAMA_MODEL_NAME" >/dev/null 2>&1; then
+        docker exec "$OLLAMA_CONTAINER" ollama pull "$OLLAMA_MODEL_NAME"
+      fi
+      return 0
+    fi
+    sleep 5
+  done
+  die "Ollama did not become ready within ${DEPLOY_TIMEOUT}s"
+}
+
+search_resource_id() {
+  local resource="$1"
+  local name="$2"
+  local response
+  response="$(os_request POST "/_plugins/_ml/${resource}/_search" '{"query":{"match_all":{}},"size":1000}')"
+  jq -r --arg name "$name" '.hits.hits[]? | select(._source.name == $name) | ._id' \
+    <<<"$response" | head -n 1
+}
+
+models_for_connector() {
+  local connector_id="$1"
+  local response
+  response="$(os_request POST '/_plugins/_ml/models/_search' '{"query":{"match_all":{}},"size":1000}')"
+  jq -r --arg connector_id "$connector_id" \
+    '.hits.hits[]? | select(._source.connector_id == $connector_id) | [._id, (._source.model_state // "")] | @tsv' \
+    <<<"$response"
+}
+
+wait_for_model_state() {
+  local model_id="$1"
+  local deadline=$((SECONDS + DEPLOY_TIMEOUT))
+  local response state
+  while (( SECONDS < deadline )); do
+    response="$(os_request GET "/_plugins/_ml/models/$model_id")"
+    state="$(jq -r '.model_state // empty' <<<"$response")"
+    case "$state" in
+      DEPLOYED)
+        return 0
+        ;;
+      DEPLOY_FAILED)
+        die "model '$model_id' failed to deploy"
+        ;;
+    esac
+    sleep 3
+  done
+  die "model '$model_id' did not deploy within ${DEPLOY_TIMEOUT}s"
+}
+
+wait_for_model_undeployed() {
+  local model_id="$1"
+  local deadline=$((SECONDS + DEPLOY_TIMEOUT))
+  local response state
+  while (( SECONDS < deadline )); do
+    response="$(os_request GET "/_plugins/_ml/models/$model_id")"
+    state="$(jq -r '.model_state // empty' <<<"$response")"
+    if [[ "$state" != "DEPLOYED" && "$state" != "DEPLOYING" ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+  die "model '$model_id' did not undeploy within ${DEPLOY_TIMEOUT}s"
+}
+
+deploy_model() {
+  local model_id="$1"
+  local state
+  state="$(os_request GET "/_plugins/_ml/models/$model_id" | jq -r '.model_state // empty')"
+  if [[ "$state" != "DEPLOYED" ]]; then
+    log "Deploying model $model_id"
+    os_request POST "/_plugins/_ml/models/$model_id/_deploy" '{}' >/dev/null
+  fi
+  wait_for_model_state "$model_id"
+}
+
+undeploy_connector_models() {
+  local connector_id="$1"
+  local model_id state
+  while IFS=$'\t' read -r model_id state; do
+    [[ -n "$model_id" ]] || continue
+    if [[ "$state" == "DEPLOYED" || "$state" == "DEPLOYING" ]]; then
+      log "Undeploying model $model_id before updating connector $connector_id"
+      os_request POST "/_plugins/_ml/models/$model_id/_undeploy" '{}' >/dev/null
+      wait_for_model_undeployed "$model_id"
+    fi
+  done < <(models_for_connector "$connector_id")
+}
+
+redeploy_connector_models() {
+  local connector_id="$1"
+  local model_id state
+  while IFS=$'\t' read -r model_id state; do
+    [[ -n "$model_id" ]] || continue
+    deploy_model "$model_id"
+  done < <(models_for_connector "$connector_id")
+}
+
+upsert_connector() {
+  local name="$1"
+  local payload="$2"
+  local connector_id
+  connector_id="$(search_resource_id connectors "$name")"
+  if [[ -n "$connector_id" ]]; then
+    undeploy_connector_models "$connector_id"
+    log "Updating connector '$name' ($connector_id)"
+    os_request PUT "/_plugins/_ml/connectors/$connector_id" "$payload" >/dev/null
+    redeploy_connector_models "$connector_id"
+  else
+    log "Creating connector '$name'"
+    connector_id="$(os_request POST '/_plugins/_ml/connectors/_create' "$payload" | jq -r '.connector_id // empty')"
+    [[ -n "$connector_id" ]] || die "connector '$name' was created without a connector_id"
+  fi
+  printf '%s\n' "$connector_id"
+}
+
+register_model() {
+  local name="$1"
+  local connector_id="$2"
+  local payload response model_id task_id task_response task_state
+  payload="$(jq -nc \
+    --arg name "$name" \
+    --arg connector_id "$connector_id" \
+    '{name:$name, description:("Ollama remote model for " + $name), function_name:"remote", connector_id:$connector_id}')"
+  response="$(os_request POST '/_plugins/_ml/models/_register' "$payload")"
+  model_id="$(jq -r '.model_id // empty' <<<"$response")"
+  task_id="$(jq -r '.task_id // empty' <<<"$response")"
+
+  if [[ -z "$model_id" && -n "$task_id" ]]; then
+    local deadline=$((SECONDS + DEPLOY_TIMEOUT))
+    while (( SECONDS < deadline )); do
+      task_response="$(os_request GET "/_plugins/_ml/tasks/$task_id")"
+      task_state="$(jq -r '.state // empty' <<<"$task_response")"
+      model_id="$(jq -r '.model_id // empty' <<<"$task_response")"
+      [[ "$task_state" == "FAILED" ]] && die "model registration task '$task_id' failed"
+      [[ "$task_state" == "COMPLETED" && -n "$model_id" ]] && break
+      sleep 2
+    done
+  fi
+
+  [[ -n "$model_id" ]] || die "model '$name' was registered without a model_id"
+  printf '%s\n' "$model_id"
+}
+
+ensure_model() {
+  local name="$1"
+  local connector_id="$2"
+  local model_id current_connector_id
+  model_id="$(search_resource_id models "$name")"
+  if [[ -n "$model_id" ]]; then
+    current_connector_id="$(os_request GET "/_plugins/_ml/models/$model_id" | jq -r '.connector_id // empty')"
+    if [[ "$current_connector_id" != "$connector_id" ]]; then
+      die "model '$name' already exists with connector '$current_connector_id', expected '$connector_id'"
+    fi
+    log "Reusing model '$name' ($model_id)"
+  else
+    log "Registering model '$name'"
+    model_id="$(register_model "$name" "$connector_id")"
+  fi
+  deploy_model "$model_id"
+  printf '%s\n' "$model_id"
+}
+
+upsert_agent() {
+  local name="$1"
+  local payload="$2"
+  local agent_id
+  agent_id="$(search_resource_id agents "$name")"
+  if [[ -n "$agent_id" ]]; then
+    log "Updating agent '$name' ($agent_id)"
+    # Agent type and top-level interface parameters are creation-time fields.
+    # Existing agents retain them while supported mutable fields are refreshed.
+    os_request PUT "/_plugins/_ml/agents/$agent_id" \
+      "$(jq -c 'del(.type, .parameters)' <<<"$payload")" >/dev/null
+  else
+    log "Registering agent '$name'"
+    agent_id="$(os_request POST '/_plugins/_ml/agents/_register' "$payload" | jq -r '.agent_id // empty')"
+    [[ -n "$agent_id" ]] || die "agent '$name' was registered without an agent_id"
+  fi
+  printf '%s\n' "$agent_id"
+}
+
+write_ml_config() {
+  local config_id="$1"
+  local config_type="$2"
+  local agent_id="$3"
+  local payload
+  payload="$(jq -nc --arg type "$config_type" --arg agent_id "$agent_id" \
+    '{type:$type, configuration:{agent_id:$agent_id}}')"
+  log "Writing ML config '$config_id' -> $agent_id"
+  os_request PUT "/.plugins-ml-config/_doc/$config_id?refresh=true" "$payload" >/dev/null
+}
+
+wait_for_opensearch
+wait_for_ollama
+
+# The single quotes deliberately preserve ML Commons ${parameters.*}
+# placeholders until connector inference time.
+# shellcheck disable=SC2016
+printf -v CHAT_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"${parameters.system_prompt}"},${parameters._chat_history:-}{"role":"user","content":"${parameters.prompt}"}${parameters._interactions:-}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s"${parameters.tool_configs:-} }' \
+  "$MAX_TOKENS" "$REASONING_EFFORT"
+
+# shellcheck disable=SC2016
+printf -v PPL_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Translate the request into a valid OpenSearch PPL query. Return only the PPL query."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
+  "$MAX_TOKENS" "$REASONING_EFFORT"
+
+CLIENT_CONFIG="$(jq -nc \
+  --argjson connection_timeout "$CONNECTION_TIMEOUT" \
+  --argjson read_timeout "$READ_TIMEOUT" \
+  '{max_connection:10, connection_timeout:$connection_timeout, read_timeout:$read_timeout, max_retry_times:0}')"
+
+CHAT_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$CHAT_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$CHAT_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"OpenAI-compatible Ollama connector for OpenSearch conversational agents", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
+PPL_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$PPL_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$PPL_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"Prompt-compatible Ollama connector for OpenSearch PPL query assist", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
+CHAT_CONNECTOR_ID="$(upsert_connector "$CHAT_CONNECTOR_NAME" "$CHAT_CONNECTOR_PAYLOAD")"
+CHAT_MODEL_ID="$(ensure_model "$CHAT_MODEL_NAME" "$CHAT_CONNECTOR_ID")"
+
+CHAT_AGENT_PAYLOAD="$(jq -nc \
+  --arg name "$CHAT_AGENT_NAME" \
+  --arg model_id "$CHAT_MODEL_ID" \
+  '{name:$name, description:"Local Qwen assistant served through Ollama", type:"conversational", app_type:"os_chat", memory:{type:"conversation_index"}, llm:{model_id:$model_id, parameters:{max_iteration:"3", response_filter:"$.response", system_prompt:"You are a helpful OpenSearch assistant. Keep answers concise unless the user requests detail.", prompt:"${parameters.question}", message_history_limit:"5"}}, parameters:{_llm_interface:"openai/v1/chat/completions"}}')"
+CHAT_AGENT_ID="$(upsert_agent "$CHAT_AGENT_NAME" "$CHAT_AGENT_PAYLOAD")"
+
+ROOT_AGENT_PAYLOAD="$(jq -nc \
+  --arg name "$ROOT_AGENT_NAME" \
+  --arg agent_id "$CHAT_AGENT_ID" \
+  '{name:$name, description:"Root agent for OpenSearch Assistant", type:"flow", app_type:"os_chat", tools:[{type:"AgentTool", name:"LLMResponseGenerator", include_output_in_agent_response:true, parameters:{agent_id:$agent_id}}]}')"
+ROOT_AGENT_ID="$(upsert_agent "$ROOT_AGENT_NAME" "$ROOT_AGENT_PAYLOAD")"
+
+PPL_CONNECTOR_ID="$(upsert_connector "$PPL_CONNECTOR_NAME" "$PPL_CONNECTOR_PAYLOAD")"
+PPL_MODEL_ID="$(ensure_model "$PPL_MODEL_NAME" "$PPL_CONNECTOR_ID")"
+
+PPL_AGENT_PAYLOAD="$(jq -nc \
+  --arg name "$PPL_AGENT_NAME" \
+  --arg model_id "$PPL_MODEL_ID" \
+  '{name:$name, description:"Generate PPL queries from natural-language questions", type:"flow", app_type:"query_assist", tools:[{type:"PPLTool", name:"TransferQuestionToPPLAndExecuteTool", description:"Translate a natural-language question into an OpenSearch PPL query for the supplied index. Inputs: {index:IndexName, question:UserQuestion}.", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content", execute:false}}]}')"
+PPL_AGENT_ID="$(upsert_agent "$PPL_AGENT_NAME" "$PPL_AGENT_PAYLOAD")"
+
+write_ml_config os_chat os_chat_root_agent "$ROOT_AGENT_ID"
+write_ml_config os_query_assist_ppl os_query_assist_ppl_agent "$PPL_AGENT_ID"
+
+log "Bootstrap complete"
+printf '  chat_connector_id=%s\n' "$CHAT_CONNECTOR_ID"
+printf '  chat_model_id=%s\n' "$CHAT_MODEL_ID"
+printf '  chat_agent_id=%s\n' "$CHAT_AGENT_ID"
+printf '  root_agent_id=%s\n' "$ROOT_AGENT_ID"
+printf '  ppl_connector_id=%s\n' "$PPL_CONNECTOR_ID"
+printf '  ppl_model_id=%s\n' "$PPL_MODEL_ID"
+printf '  ppl_agent_id=%s\n' "$PPL_AGENT_ID"

From 12a7c674b2737737935c05ac771048947538e804 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Mon, 7 Sep 2026 09:56:34 +0100
Subject: [PATCH 11/12] Deploy: final adjustments for OpenSearch LLM
 integration.

---
 deploy/Makefile                       |  2 +-
 deploy/elasticsearch.env              | 10 ++++++
 scripts/opensearch-ai-init.sh         | 46 +++++++++++++++++++++++++++
 services/kibana/config/opensearch.yml |  1 +
 4 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/deploy/Makefile b/deploy/Makefile
index 3f618642..5bbb7daa 100644
--- a/deploy/Makefile
+++ b/deploy/Makefile
@@ -359,7 +359,7 @@ pull-ollama-model: start-ollama ## Pull the model configured by OLLAMA_MODEL
 
 setup-ollama: pull-ollama-model ## Start Ollama and pull its configured model
 
-opensearch-ai-init: ## Create or update Ollama-backed OpenSearch chat and PPL assistants
+opensearch-ai-init: ## Create or update all Ollama-backed OpenSearch Dashboards AI agents and configs
 	$(WITH_ENV) ../scripts/opensearch-ai-init.sh
 
 start-openwebui: ## Start Open WebUI and its Ollama dependency
diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env
index edad8df1..ba2d1fae 100644
--- a/deploy/elasticsearch.env
+++ b/deploy/elasticsearch.env
@@ -117,6 +117,7 @@ OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS=120
 OPENSEARCH_AI_READ_TIMEOUT_SECONDS=360
 OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS=300
 OPENSEARCH_AI_MAX_TOKENS=256
+OPENSEARCH_AI_UTILITY_MAX_TOKENS=1024
 OPENSEARCH_AI_REASONING_EFFORT=none
 
 # Stable resource names let scripts/opensearch-ai-init.sh safely find and
@@ -128,6 +129,15 @@ OPENSEARCH_AI_ROOT_AGENT_NAME="Qwen Dashboards Root Agent"
 OPENSEARCH_AI_PPL_CONNECTOR_NAME="Ollama Qwen PPL Connector"
 OPENSEARCH_AI_PPL_MODEL_NAME="Qwen 3.5 Ollama PPL Model"
 OPENSEARCH_AI_PPL_AGENT_NAME="Qwen PPL Query Assist Agent"
+OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME="Ollama Qwen Time Range Connector"
+OPENSEARCH_AI_TIME_RANGE_MODEL_NAME="Qwen 3.5 Ollama Time Range Model"
+OPENSEARCH_AI_TIME_RANGE_AGENT_NAME="Qwen Query Time Range Parser Agent"
+OPENSEARCH_AI_UTILITY_CONNECTOR_NAME="Ollama Qwen Dashboards Utility Connector"
+OPENSEARCH_AI_UTILITY_MODEL_NAME="Qwen 3.5 Dashboards Utility Model"
+OPENSEARCH_AI_AD_CONNECTOR_NAME="Ollama Qwen Anomaly Suggestion Connector"
+OPENSEARCH_AI_AD_MODEL_NAME="Qwen 3.5 Anomaly Suggestion Model"
+OPENSEARCH_AI_VISUAL_CONNECTOR_NAME="Ollama Qwen Visualization Connector"
+OPENSEARCH_AI_VISUAL_MODEL_NAME="Qwen 3.5 Visualization Model"
 
 # disables swapping, imporving r/w performance (at the expense of RAM)
 ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK=true
diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh
index f7890214..1b4ef520 100755
--- a/scripts/opensearch-ai-init.sh
+++ b/scripts/opensearch-ai-init.sh
@@ -47,6 +47,9 @@ ENV_OVERRIDE_NAMES=(
   OPENSEARCH_AI_PPL_CONNECTOR_NAME
   OPENSEARCH_AI_PPL_MODEL_NAME
   OPENSEARCH_AI_PPL_AGENT_NAME
+  OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME
+  OPENSEARCH_AI_TIME_RANGE_MODEL_NAME
+  OPENSEARCH_AI_TIME_RANGE_AGENT_NAME
 )
 ENV_OVERRIDES=()
 for variable_name in "${ENV_OVERRIDE_NAMES[@]}"; do
@@ -82,6 +85,9 @@ ROOT_AGENT_NAME="${OPENSEARCH_AI_ROOT_AGENT_NAME:-Qwen Dashboards Root Agent}"
 PPL_CONNECTOR_NAME="${OPENSEARCH_AI_PPL_CONNECTOR_NAME:-Ollama Qwen PPL Connector}"
 PPL_MODEL_NAME="${OPENSEARCH_AI_PPL_MODEL_NAME:-Qwen 3.5 Ollama PPL Model}"
 PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-Qwen PPL Query Assist Agent}"
+TIME_RANGE_CONNECTOR_NAME="${OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME:-Ollama Qwen Time Range Connector}"
+TIME_RANGE_MODEL_NAME="${OPENSEARCH_AI_TIME_RANGE_MODEL_NAME:-Qwen 3.5 Ollama Time Range Model}"
+TIME_RANGE_AGENT_NAME="${OPENSEARCH_AI_TIME_RANGE_AGENT_NAME:-Qwen Query Time Range Parser Agent}"
 
 OPENSEARCH_URL="https://localhost:9200"
 OPENSEARCH_CA="/usr/share/opensearch/config/root-ca.crt"
@@ -360,6 +366,11 @@ printf -v PPL_REQUEST_BODY \
   '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Translate the request into a valid OpenSearch PPL query. Return only the PPL query."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
   "$MAX_TOKENS" "$REASONING_EFFORT"
 
+# shellcheck disable=SC2016
+printf -v TIME_RANGE_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Parse time constraints for an OpenSearch date picker. Follow the requested XML output format exactly and return no explanation."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
+  "$MAX_TOKENS" "$REASONING_EFFORT"
+
 CLIENT_CONFIG="$(jq -nc \
   --argjson connection_timeout "$CONNECTION_TIMEOUT" \
   --argjson read_timeout "$READ_TIMEOUT" \
@@ -379,6 +390,13 @@ PPL_CONNECTOR_PAYLOAD="$(jq -nc \
   --argjson client_config "$CLIENT_CONFIG" \
   '{name:$name, description:"Prompt-compatible Ollama connector for OpenSearch PPL query assist", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
 
+TIME_RANGE_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$TIME_RANGE_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$TIME_RANGE_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"Ollama connector for OpenSearch query time-range parsing", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
 CHAT_CONNECTOR_ID="$(upsert_connector "$CHAT_CONNECTOR_NAME" "$CHAT_CONNECTOR_PAYLOAD")"
 CHAT_MODEL_ID="$(ensure_model "$CHAT_MODEL_NAME" "$CHAT_CONNECTOR_ID")"
 
@@ -403,8 +421,33 @@ PPL_AGENT_PAYLOAD="$(jq -nc \
   '{name:$name, description:"Generate PPL queries from natural-language questions", type:"flow", app_type:"query_assist", tools:[{type:"PPLTool", name:"TransferQuestionToPPLAndExecuteTool", description:"Translate a natural-language question into an OpenSearch PPL query for the supplied index. Inputs: {index:IndexName, question:UserQuestion}.", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content", execute:false}}]}')"
 PPL_AGENT_ID="$(upsert_agent "$PPL_AGENT_NAME" "$PPL_AGENT_PAYLOAD")"
 
+TIME_RANGE_CONNECTOR_ID="$(upsert_connector "$TIME_RANGE_CONNECTOR_NAME" "$TIME_RANGE_CONNECTOR_PAYLOAD")"
+TIME_RANGE_MODEL_ID="$(ensure_model "$TIME_RANGE_MODEL_NAME" "$TIME_RANGE_CONNECTOR_ID")"
+
+# OpenSearch Dashboards accepts these timestamp formats without a timezone
+# suffix and leaves the existing date picker unchanged if no tags are returned.
+# shellcheck disable=SC2016
+TIME_RANGE_TOOL_PROMPT='Analyze whether the question contains a time constraint that applies to the selected time field.
+Current time (ISO 8601): ${parameters.current_time_iso}
+Selected time field: ${parameters.time_field}
+Other time fields in the index: ${parameters.other_time_fields}
+Question: ${parameters.question}
+
+Resolve relative expressions such as "last 24 hours" using the current time. If a time constraint applies to the selected time field, return exactly:
+<start>YYYY-MM-DD HH:mm:ss</start>
+<end>YYYY-MM-DD HH:mm:ss</end>
+The start must not be after the end. Do not include a timezone suffix, Markdown, or explanation. If the question has no applicable time constraint, or explicitly applies it to one of the other time fields, return an empty response.'
+
+TIME_RANGE_AGENT_PAYLOAD="$(jq -nc \
+  --arg name "$TIME_RANGE_AGENT_NAME" \
+  --arg model_id "$TIME_RANGE_MODEL_ID" \
+  --arg prompt "$TIME_RANGE_TOOL_PROMPT" \
+  '{name:$name, description:"Parse natural-language time constraints for the Discover date picker", type:"flow", app_type:"query_assist", tools:[{type:"MLModelTool", name:"QueryTimeRangeParserTool", description:"Extract an absolute start and end time for the selected OpenSearch time field.", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')"
+TIME_RANGE_AGENT_ID="$(upsert_agent "$TIME_RANGE_AGENT_NAME" "$TIME_RANGE_AGENT_PAYLOAD")"
+
 write_ml_config os_chat os_chat_root_agent "$ROOT_AGENT_ID"
 write_ml_config os_query_assist_ppl os_query_assist_ppl_agent "$PPL_AGENT_ID"
+write_ml_config os_query_time_range_parser os_query_time_range_parser_agent "$TIME_RANGE_AGENT_ID"
 
 log "Bootstrap complete"
 printf '  chat_connector_id=%s\n' "$CHAT_CONNECTOR_ID"
@@ -414,3 +457,6 @@ printf '  root_agent_id=%s\n' "$ROOT_AGENT_ID"
 printf '  ppl_connector_id=%s\n' "$PPL_CONNECTOR_ID"
 printf '  ppl_model_id=%s\n' "$PPL_MODEL_ID"
 printf '  ppl_agent_id=%s\n' "$PPL_AGENT_ID"
+printf '  time_range_connector_id=%s\n' "$TIME_RANGE_CONNECTOR_ID"
+printf '  time_range_model_id=%s\n' "$TIME_RANGE_MODEL_ID"
+printf '  time_range_agent_id=%s\n' "$TIME_RANGE_AGENT_ID"
diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml
index f37d5131..f522023e 100644
--- a/services/kibana/config/opensearch.yml
+++ b/services/kibana/config/opensearch.yml
@@ -136,6 +136,7 @@ opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"]
 # deployment are performed through the REST API or the Dashboards Dev Tools UI.
 ml_commons_dashboards.enabled: true
 assistant.chat.enabled: true
+assistant.incontextInsight.enabled: true
 observability.query_assist.enabled: true
 
 savedObjects.permission.enabled: true

From 24e233dba40436761e0199bb8f51f9ac7f125368 Mon Sep 17 00:00:00 2001
From: vladd-bit <vlad.a.dinu@gmail.com>
Date: Mon, 7 Sep 2026 10:08:47 +0100
Subject: [PATCH 12/12] Scripts: OpenSearch update.

---
 scripts/opensearch-ai-init.sh | 206 +++++++++++++++++++++++++++++++++-
 1 file changed, 205 insertions(+), 1 deletion(-)

diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh
index 1b4ef520..0b80a83f 100755
--- a/scripts/opensearch-ai-init.sh
+++ b/scripts/opensearch-ai-init.sh
@@ -39,6 +39,7 @@ ENV_OVERRIDE_NAMES=(
   OPENSEARCH_AI_READ_TIMEOUT_SECONDS
   OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS
   OPENSEARCH_AI_MAX_TOKENS
+  OPENSEARCH_AI_UTILITY_MAX_TOKENS
   OPENSEARCH_AI_REASONING_EFFORT
   OPENSEARCH_AI_CHAT_CONNECTOR_NAME
   OPENSEARCH_AI_CHAT_MODEL_NAME
@@ -50,6 +51,12 @@ ENV_OVERRIDE_NAMES=(
   OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME
   OPENSEARCH_AI_TIME_RANGE_MODEL_NAME
   OPENSEARCH_AI_TIME_RANGE_AGENT_NAME
+  OPENSEARCH_AI_UTILITY_CONNECTOR_NAME
+  OPENSEARCH_AI_UTILITY_MODEL_NAME
+  OPENSEARCH_AI_AD_CONNECTOR_NAME
+  OPENSEARCH_AI_AD_MODEL_NAME
+  OPENSEARCH_AI_VISUAL_CONNECTOR_NAME
+  OPENSEARCH_AI_VISUAL_MODEL_NAME
 )
 ENV_OVERRIDES=()
 for variable_name in "${ENV_OVERRIDE_NAMES[@]}"; do
@@ -76,6 +83,7 @@ CONNECTION_TIMEOUT="${OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS:-120}"
 READ_TIMEOUT="${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360}"
 DEPLOY_TIMEOUT="${OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS:-300}"
 MAX_TOKENS="${OPENSEARCH_AI_MAX_TOKENS:-256}"
+UTILITY_MAX_TOKENS="${OPENSEARCH_AI_UTILITY_MAX_TOKENS:-1024}"
 REASONING_EFFORT="${OPENSEARCH_AI_REASONING_EFFORT:-none}"
 
 CHAT_CONNECTOR_NAME="${OPENSEARCH_AI_CHAT_CONNECTOR_NAME:-Ollama Qwen Agent Connector}"
@@ -88,6 +96,12 @@ PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-Qwen PPL Query Assist Agent}"
 TIME_RANGE_CONNECTOR_NAME="${OPENSEARCH_AI_TIME_RANGE_CONNECTOR_NAME:-Ollama Qwen Time Range Connector}"
 TIME_RANGE_MODEL_NAME="${OPENSEARCH_AI_TIME_RANGE_MODEL_NAME:-Qwen 3.5 Ollama Time Range Model}"
 TIME_RANGE_AGENT_NAME="${OPENSEARCH_AI_TIME_RANGE_AGENT_NAME:-Qwen Query Time Range Parser Agent}"
+UTILITY_CONNECTOR_NAME="${OPENSEARCH_AI_UTILITY_CONNECTOR_NAME:-Ollama Qwen Dashboards Utility Connector}"
+UTILITY_MODEL_NAME="${OPENSEARCH_AI_UTILITY_MODEL_NAME:-Qwen 3.5 Dashboards Utility Model}"
+AD_CONNECTOR_NAME="${OPENSEARCH_AI_AD_CONNECTOR_NAME:-Ollama Qwen Anomaly Suggestion Connector}"
+AD_MODEL_NAME="${OPENSEARCH_AI_AD_MODEL_NAME:-Qwen 3.5 Anomaly Suggestion Model}"
+VISUAL_CONNECTOR_NAME="${OPENSEARCH_AI_VISUAL_CONNECTOR_NAME:-Ollama Qwen Visualization Connector}"
+VISUAL_MODEL_NAME="${OPENSEARCH_AI_VISUAL_MODEL_NAME:-Qwen 3.5 Visualization Model}"
 
 OPENSEARCH_URL="https://localhost:9200"
 OPENSEARCH_CA="/usr/share/opensearch/config/root-ca.crt"
@@ -99,7 +113,7 @@ for command_name in docker jq; do
   command -v "$command_name" >/dev/null 2>&1 || die "$command_name is required"
 done
 
-for integer_value in "$CONNECTION_TIMEOUT" "$READ_TIMEOUT" "$DEPLOY_TIMEOUT" "$MAX_TOKENS"; do
+for integer_value in "$CONNECTION_TIMEOUT" "$READ_TIMEOUT" "$DEPLOY_TIMEOUT" "$MAX_TOKENS" "$UTILITY_MAX_TOKENS"; do
   [[ "$integer_value" =~ ^[1-9][0-9]*$ ]] || die "timeout and token settings must be positive integers"
 done
 
@@ -351,6 +365,22 @@ write_ml_config() {
   os_request PUT "/.plugins-ml-config/_doc/$config_id?refresh=true" "$payload" >/dev/null
 }
 
+ensure_utility_agent() {
+  local name="$1"
+  local description="$2"
+  local app_type="$3"
+  local prompt="$4"
+  local payload
+  payload="$(jq -nc \
+    --arg name "$name" \
+    --arg description "$description" \
+    --arg app_type "$app_type" \
+    --arg model_id "$UTILITY_MODEL_ID" \
+    --arg prompt "$prompt" \
+    '{name:$name, description:$description, type:"flow", app_type:$app_type, tools:[{type:"MLModelTool", name:"LLMResponseGenerator", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')"
+  upsert_agent "$name" "$payload"
+}
+
 wait_for_opensearch
 wait_for_ollama
 
@@ -371,6 +401,29 @@ printf -v TIME_RANGE_REQUEST_BODY \
   '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Parse time constraints for an OpenSearch date picker. Follow the requested XML output format exactly and return no explanation."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
   "$MAX_TOKENS" "$REASONING_EFFORT"
 
+# General-purpose strict-output connector used by the feature-specific agents
+# below. Their MLModelTool prompts define the exact response contract expected
+# by each OpenSearch Dashboards route.
+# shellcheck disable=SC2016
+printf -v UTILITY_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Follow the requested output format exactly. Do not add Markdown fences or commentary outside that format."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
+  "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT"
+
+# The anomaly detector UI calls String.split on every multi-value property.
+# Enforce strings at the Ollama decoder so the UI never receives JSON arrays.
+# shellcheck disable=SC2016
+printf -v AD_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Suggest anomaly detector parameters from the supplied index mapping. Follow the JSON schema exactly."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s", "response_format":{"type":"json_schema","json_schema":{"name":"anomaly_detector_parameters","strict":true,"schema":{"type":"object","properties":{"categoryField":{"type":"string"},"aggregationField":{"type":"string"},"aggregationMethod":{"type":"string"},"dateFields":{"type":"string"}},"required":["categoryField","aggregationField","aggregationMethod","dateFields"],"additionalProperties":false}}} }' \
+  "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT"
+
+# Qwen 3.5 is multimodal. Send notebook visualization screenshots through the
+# OpenAI-compatible image_url message format and make the model emit the nested
+# JSON shape expected by the Dashboards investigation plugin.
+# shellcheck disable=SC2016
+printf -v VISUAL_REQUEST_BODY \
+  '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Summarize the supplied OpenSearch visualization. Return only JSON shaped as {\\"output\\":{\\"message\\":{\\"content\\":[{\\"text\\":\\"concise summary\\"}]}}}."},{"role":"user","content":[{"type":"text","text":"Explain the notable trends, outliers, and operational meaning. Local timezone offset in minutes: ${parameters.local_time_offset}"},{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,${parameters.image_base64}"}}]}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \
+  "$UTILITY_MAX_TOKENS" "$REASONING_EFFORT"
+
 CLIENT_CONFIG="$(jq -nc \
   --argjson connection_timeout "$CONNECTION_TIMEOUT" \
   --argjson read_timeout "$READ_TIMEOUT" \
@@ -397,6 +450,27 @@ TIME_RANGE_CONNECTOR_PAYLOAD="$(jq -nc \
   --argjson client_config "$CLIENT_CONFIG" \
   '{name:$name, description:"Ollama connector for OpenSearch query time-range parsing", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
 
+UTILITY_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$UTILITY_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$UTILITY_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"Ollama connector for OpenSearch Dashboards feature agents", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
+AD_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$AD_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$AD_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"Structured-output Ollama connector for anomaly detector suggestions", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
+VISUAL_CONNECTOR_PAYLOAD="$(jq -nc \
+  --arg name "$VISUAL_CONNECTOR_NAME" \
+  --arg model "$OLLAMA_MODEL_NAME" \
+  --arg request_body "$VISUAL_REQUEST_BODY" \
+  --argjson client_config "$CLIENT_CONFIG" \
+  '{name:$name, description:"OpenAI-compatible Ollama connector for visualization screenshots", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')"
+
 CHAT_CONNECTOR_ID="$(upsert_connector "$CHAT_CONNECTOR_NAME" "$CHAT_CONNECTOR_PAYLOAD")"
 CHAT_MODEL_ID="$(ensure_model "$CHAT_MODEL_NAME" "$CHAT_CONNECTOR_ID")"
 
@@ -445,9 +519,136 @@ TIME_RANGE_AGENT_PAYLOAD="$(jq -nc \
   '{name:$name, description:"Parse natural-language time constraints for the Discover date picker", type:"flow", app_type:"query_assist", tools:[{type:"MLModelTool", name:"QueryTimeRangeParserTool", description:"Extract an absolute start and end time for the selected OpenSearch time field.", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')"
 TIME_RANGE_AGENT_ID="$(upsert_agent "$TIME_RANGE_AGENT_NAME" "$TIME_RANGE_AGENT_PAYLOAD")"
 
+UTILITY_CONNECTOR_ID="$(upsert_connector "$UTILITY_CONNECTOR_NAME" "$UTILITY_CONNECTOR_PAYLOAD")"
+UTILITY_MODEL_ID="$(ensure_model "$UTILITY_MODEL_NAME" "$UTILITY_CONNECTOR_ID")"
+
+# Keep the placeholders literal so ML Commons substitutes request parameters
+# when each agent executes.
+# shellcheck disable=SC2016
+DATA2SUMMARY_PROMPT='Summarize these OpenSearch results for the user.
+Question: ${parameters.question}
+PPL query: ${parameters.ppl}
+Sample count: ${parameters.sample_count}
+Total count: ${parameters.total_count}
+Sample data: ${parameters.sample_data}
+Highlight the most useful findings and uncertainty. Return plain text.'
+
+# shellcheck disable=SC2016
+LOG_DATA2SUMMARY_PROMPT='Summarize these OpenSearch log results.
+Question: ${parameters.question}
+PPL query: ${parameters.ppl}
+Sample count: ${parameters.sample_count}
+Total count: ${parameters.total_count}
+Sample data: ${parameters.sample_data}
+Emphasize recurring log patterns, error signals, unusual values, and likely operational impact. Return plain text.'
+
+# shellcheck disable=SC2016
+INDEX_TYPE_PROMPT='Decide whether the index contains logs, events, traces, audit records, or other timestamped operational records.
+Mapping: ${parameters.schema}
+Sample documents: ${parameters.sampleData}
+Return only valid JSON in this exact shape: {"isRelated":true,"reason":"short reason"}. Use false when the data is primarily business entities or reference data.'
+
+# shellcheck disable=SC2016
+QUERY_RESPONSE_SUMMARY_PROMPT='Explain the result of an OpenSearch PPL query.
+Index: ${parameters.index}
+Question: ${parameters.question}
+PPL: ${parameters.query}
+Response: ${parameters.response}
+Give a concise factual answer, mention important totals or trends, and do not invent missing values.'
+
+# shellcheck disable=SC2016
+QUERY_ERROR_SUMMARY_PROMPT='Explain why this OpenSearch PPL query failed and suggest a corrected query.
+Index: ${parameters.index}
+Question: ${parameters.question}
+PPL: ${parameters.query}
+Error response: ${parameters.response}
+Available fields: ${parameters.fields}
+Return a concise explanation followed by one corrected PPL query.'
+
+# shellcheck disable=SC2016
+ALERT_SUMMARY_PROMPT='Summarize this OpenSearch alert context.
+Question: ${parameters.question}
+Index: ${parameters.index}
+Query input: ${parameters.input}
+Context: ${parameters.context}
+Return the response as <summarization>concise evidence-based summary</summarization><final insights>recommended next checks</final insights>.'
+
+# shellcheck disable=SC2016
+LOG_ALERT_SUMMARY_PROMPT='Summarize this OpenSearch log alert context.
+Question: ${parameters.question}
+Index: ${parameters.index}
+Query input: ${parameters.input}
+Context: ${parameters.context}
+Top log patterns: ${parameters.topNLogPatternData}
+Return the response as <summarization>concise evidence-based summary</summarization><final insights>notable patterns and recommended next checks</final insights>.'
+
+# shellcheck disable=SC2016
+TEXT2VEGA_PROMPT='Create a Vega-Lite v5 specification for these OpenSearch query results.
+Question: ${parameters.input_question}
+PPL: ${parameters.ppl}
+Data schema: ${parameters.dataSchema}
+Sample data: ${parameters.sampleData}
+Use field names that exist in the supplied data. Return only one valid JSON object. Do not include width, height, or inline data.'
+
+# shellcheck disable=SC2016
+TEXT2VEGA_INSTRUCTIONS_PROMPT='Modify or create a Vega-Lite v5 specification for these OpenSearch query results.
+Question: ${parameters.input_question}
+Requested visualization change: ${parameters.input_instruction}
+PPL: ${parameters.ppl}
+Data schema: ${parameters.dataSchema}
+Sample data: ${parameters.sampleData}
+Use field names that exist in the supplied data. Return only one valid JSON object. Do not include width, height, or inline data.'
+
+DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Data Summary Agent" "Summarize Discover result samples" "os_data2summary" "$DATA2SUMMARY_PROMPT")"
+LOG_DATA2SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Log Data Summary Agent" "Summarize Discover log result samples" "os_data2summary" "$LOG_DATA2SUMMARY_PROMPT")"
+INDEX_TYPE_AGENT_ID="$(ensure_utility_agent "Qwen Index Type Detection Agent" "Classify whether an index contains operational log data" "os_index_type_detect" "$INDEX_TYPE_PROMPT")"
+QUERY_RESPONSE_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Query Response Summary Agent" "Summarize successful PPL query results" "query_assist" "$QUERY_RESPONSE_SUMMARY_PROMPT")"
+QUERY_ERROR_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Query Error Summary Agent" "Explain failed PPL queries" "query_assist" "$QUERY_ERROR_SUMMARY_PROMPT")"
+ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Alert Summary Agent" "Summarize alert context" "os_summary" "$ALERT_SUMMARY_PROMPT")"
+LOG_ALERT_SUMMARY_AGENT_ID="$(ensure_utility_agent "Qwen Log Alert Summary Agent" "Summarize log alert context" "os_summary" "$LOG_ALERT_SUMMARY_PROMPT")"
+TEXT2VEGA_AGENT_ID="$(ensure_utility_agent "Qwen Text to Vega Agent" "Create Vega-Lite specifications from query results" "os_text2vega" "$TEXT2VEGA_PROMPT")"
+TEXT2VEGA_INSTRUCTIONS_AGENT_ID="$(ensure_utility_agent "Qwen Text to Vega Instructions Agent" "Modify Vega-Lite specifications using user instructions" "os_text2vega" "$TEXT2VEGA_INSTRUCTIONS_PROMPT")"
+
+AD_CONNECTOR_ID="$(upsert_connector "$AD_CONNECTOR_NAME" "$AD_CONNECTOR_PAYLOAD")"
+AD_MODEL_ID="$(ensure_model "$AD_MODEL_NAME" "$AD_CONNECTOR_ID")"
+
+# The anomaly detector UI supplies only an index name. Fetch its mapping first,
+# then ask the model for the exact comma-delimited JSON fields consumed by the
+# UI form.
+# shellcheck disable=SC2016
+SUGGEST_AD_PROMPT='Suggest OpenSearch anomaly detector fields using this index mapping.
+Index: ${parameters.index}
+Mapping and settings: ${parameters.IndexMappingTool.output}
+Choose one to three suitable numeric or countable aggregation fields, matching aggregation methods such as avg, sum, min, max, or count, all date fields, and at most one keyword category field.
+Return only valid JSON in this exact shape: {"categoryField":"service.keyword","aggregationField":"latency,error_count","aggregationMethod":"avg,sum","dateFields":"@timestamp"}. Every property value must be a JSON string, never an array. Represent multiple fields or methods inside one comma-delimited string. Use an empty categoryField when none is suitable. The aggregationField and aggregationMethod comma-delimited lists must have equal lengths.'
+
+SUGGEST_AD_AGENT_PAYLOAD="$(jq -nc \
+  --arg model_id "$AD_MODEL_ID" \
+  --arg prompt "$SUGGEST_AD_PROMPT" \
+  '{name:"Qwen Anomaly Detector Suggestion Agent", description:"Suggest anomaly detector fields from an index mapping", type:"flow", app_type:"os_suggest_ad", tools:[{type:"IndexMappingTool", name:"IndexMappingTool", include_output_in_agent_response:false, parameters:{index:"${parameters.index}"}},{type:"MLModelTool", name:"SuggestAnomalyDetectorTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", prompt:$prompt, response_filter:"$.choices[0].message.content"}}]}')"
+SUGGEST_AD_AGENT_ID="$(upsert_agent "Qwen Anomaly Detector Suggestion Agent" "$SUGGEST_AD_AGENT_PAYLOAD")"
+
+VISUAL_CONNECTOR_ID="$(upsert_connector "$VISUAL_CONNECTOR_NAME" "$VISUAL_CONNECTOR_PAYLOAD")"
+VISUAL_MODEL_ID="$(ensure_model "$VISUAL_MODEL_NAME" "$VISUAL_CONNECTOR_ID")"
+VISUAL_SUMMARY_AGENT_PAYLOAD="$(jq -nc \
+  --arg model_id "$VISUAL_MODEL_ID" \
+  '{name:"Qwen Visualization Summary Agent", description:"Summarize OpenSearch notebook visualization screenshots", type:"flow", app_type:"os_visualization_summary", tools:[{type:"MLModelTool", name:"VisualizationSummaryTool", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content"}}]}')"
+VISUAL_SUMMARY_AGENT_ID="$(upsert_agent "Qwen Visualization Summary Agent" "$VISUAL_SUMMARY_AGENT_PAYLOAD")"
+
 write_ml_config os_chat os_chat_root_agent "$ROOT_AGENT_ID"
 write_ml_config os_query_assist_ppl os_query_assist_ppl_agent "$PPL_AGENT_ID"
 write_ml_config os_query_time_range_parser os_query_time_range_parser_agent "$TIME_RANGE_AGENT_ID"
+write_ml_config os_data2summary os_data2summary_agent "$DATA2SUMMARY_AGENT_ID"
+write_ml_config os_data2summary_with_log_pattern os_data2summary_agent "$LOG_DATA2SUMMARY_AGENT_ID"
+write_ml_config os_index_type_detect os_index_type_detect_agent "$INDEX_TYPE_AGENT_ID"
+write_ml_config os_query_assist_response_summary os_query_assist_response_summary_agent "$QUERY_RESPONSE_SUMMARY_AGENT_ID"
+write_ml_config os_query_assist_error_summary os_query_assist_error_summary_agent "$QUERY_ERROR_SUMMARY_AGENT_ID"
+write_ml_config os_summary os_summary_agent "$ALERT_SUMMARY_AGENT_ID"
+write_ml_config os_summary_with_log_pattern os_summary_agent "$LOG_ALERT_SUMMARY_AGENT_ID"
+write_ml_config os_text2vega os_text2vega_agent "$TEXT2VEGA_AGENT_ID"
+write_ml_config os_text2vega_with_instructions os_text2vega_agent "$TEXT2VEGA_INSTRUCTIONS_AGENT_ID"
+write_ml_config os_suggest_ad os_suggest_ad_agent "$SUGGEST_AD_AGENT_ID"
+write_ml_config os_visualization_summary os_visualization_summary_agent "$VISUAL_SUMMARY_AGENT_ID"
 
 log "Bootstrap complete"
 printf '  chat_connector_id=%s\n' "$CHAT_CONNECTOR_ID"
@@ -460,3 +661,6 @@ printf '  ppl_agent_id=%s\n' "$PPL_AGENT_ID"
 printf '  time_range_connector_id=%s\n' "$TIME_RANGE_CONNECTOR_ID"
 printf '  time_range_model_id=%s\n' "$TIME_RANGE_MODEL_ID"
 printf '  time_range_agent_id=%s\n' "$TIME_RANGE_AGENT_ID"
+printf '  utility_model_id=%s\n' "$UTILITY_MODEL_ID"
+printf '  anomaly_suggestion_model_id=%s\n' "$AD_MODEL_ID"
+printf '  visualization_model_id=%s\n' "$VISUAL_MODEL_ID"