LCORE-4313: in-process PII redaction

README.md

@@ -193,6 +193,13 @@ To quickly get hands on LCS, we can run it using the default configurations prov
    ```bash
    uv sync --group dev --group ogxlibdev
    ```
+
+   If you plan to enable OpenTelemetry tracing with in-process PII redaction
+   (OTEL_TRACES_EXPORTER=redacting_otlp), also download the spaCy model:
+   ```bash
+   uv run python -m spacy download en_core_web_lg
+   ```
+
 1. create OGX `run.yaml`. you can do this by running the local run generation script
    ```bash
    ./scripts/generate_local_run.sh

deploy/lightspeed-stack/Containerfile

@@ -39,7 +39,9 @@ COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR
 COPY ${LSC_SOURCE_DIR}/providers/lightspeed_stack_providers /app-root/lightspeed_stack_providers
 COPY ${LSC_SOURCE_DIR}/providers/resources/external_providers /app-root/providers.d
 
-# Bundle additional dependencies for library mode.
+# Bundle additional dependencies for library mode and install the spaCy model for PII
+# redaction. The model is baked into the image so it is available in disconnected
+# environments where runtime downloads are not possible.
 # Source cachi2 environment for hermetic builds if available, otherwise use normal installation
 # cachi2.env has these env vars:
 # PIP_FIND_LINKS=/cachi2/output/deps/pip
@@ -51,7 +53,8 @@ RUN if [ -f /cachi2/cachi2.env ]; then \
     pip install --no-cache-dir --ignore-installed --no-index --find-links ${PIP_FIND_LINKS} --no-deps -r requirements.hashes.wheel.txt -r requirements.hashes.wheel.pypi.txt -r requirements.hashes.source.txt && \
     pip check; \
     else \
-    uv sync --locked --no-dev --group ogxlibdev; \
+    uv sync --locked --no-dev --group ogxlibdev && \
+    uv run --with pip python -m spacy download en_core_web_lg; \
     fi
 
 # Explicitly remove some packages to mitigate some CVEs

docker-compose-library.yaml

@@ -80,6 +80,10 @@ services:
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-lightspeed-stack-e2e}
       - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-false}
+      # In-process PII redaction is enabled: OTEL_TRACES_EXPORTER=redacting_otlp
+      # routes spans through the RedactingSpanExporter before export. The spaCy
+      # model (en_core_web_lg) is baked into the image at build time.
+      - OTEL_TRACES_EXPORTER=${OTEL_TRACES_EXPORTER:-redacting_otlp}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]
       interval: 10s   # how often to run the check

docker-compose.yaml

@@ -123,6 +123,10 @@ services:
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-lightspeed-stack-e2e}
       - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-false}
+      # In-process PII redaction is enabled: OTEL_TRACES_EXPORTER=redacting_otlp
+      # routes spans through the RedactingSpanExporter before export. The spaCy
+      # model (en_core_web_lg) is baked into the image at build time.
+      - OTEL_TRACES_EXPORTER=${OTEL_TRACES_EXPORTER:-redacting_otlp}
     depends_on:
         ogx:
           condition: service_healthy
@@ -229,7 +233,6 @@ services:
       retries: 3
       start_period: 5s
 
-
 volumes:
   ogx-storage:
   mock-tls-certs:

docs/devel_doc/installation_linux.md

@@ -15,6 +15,8 @@
 1. `cd service`
 1. `uv info` -- should return no error
 1. `uv install`
+1. (Optional) If you plan to use OpenTelemetry tracing with in-process PII redaction:
+   `uv run python -m spacy download en_core_web_lg`
 
 Note: `pip` installs the `uv` binary into `~/.local/bin`, which is often missing
 from `$PATH` on fresh distributions. If the `uv` command is not found, please

docs/devel_doc/installation_macos.md

@@ -15,3 +15,5 @@
 `git clone https://github.com/lightspeed-core/lightspeed-stack`
 1. `cd lightspeed-stack`
 1. `uv sync`
+1. (Optional) If you plan to use OpenTelemetry tracing with in-process PII redaction:
+   `uv run python -m spacy download en_core_web_lg`

pyproject.toml

@@ -94,6 +94,7 @@ dependencies = [
     "presidio-analyzer>=2.2.364",
     "presidio-anonymizer>=2.2.364",
     "boto3",
+    # Used by the redacting span exporter for safe HTML redaction
     "nh3>=0.3.6",
 ]
 
@@ -116,6 +117,11 @@ paths = ["tests/e2e/features"]
 [project.scripts]
 lightspeed-stack = "lightspeed_stack:main"
 
+# In-process PII redaction span exporter. Enable by setting
+# OTEL_TRACES_EXPORTER=redacting_otlp (auto-instrumentation picks it up).
+[project.entry-points.opentelemetry_traces_exporter]
+redacting_otlp = "observability.redacting_span_exporter:RedactingSpanExporter"
+
 [project.urls]
 Homepage = "https://github.com/lightspeed-core/lightspeed-stack"
 Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"

src/observability/redacting_span_exporter.py

@@ -0,0 +1,182 @@
+"""Span exporter that redacts PII from span content before delegating to OTLP.
+
+In-process (producer-side) sanitization: redaction happens inside lightspeed-core,
+before spans leave the process, so no unredacted span is ever put on the wire.
+
+It is wired via the ``opentelemetry_traces_exporter`` entry point, selected with
+``OTEL_TRACES_EXPORTER=redacting_otlp``. Auto-instrumentation
+(``opentelemetry-instrument``) instantiates it and wraps it in the
+``BatchSpanProcessor``, so ``export()`` runs on the processor's background thread,
+off the request path. The request/handler code is unchanged.
+
+Redaction is delegated to the vendored ``pii_redaction_engine`` (Presidio + spaCy).
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Sequence
+
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+    OTLPSpanExporter as GRPCSpanExporter,
+)
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+    OTLPSpanExporter as HTTPSpanExporter,
+)
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+
+from log import get_logger
+from pii_redaction_engine.anonymizer import PIIAnonymizer
+
+logger = get_logger(__name__)
+
+# Span attributes whose string values may carry PII (see utils.otel_tracing).
+# Note: request.safety_identifier is intentionally excluded (it is a caller-supplied
+# identifier for tracking, not user content).
+CONTENT_ATTRIBUTES: tuple[str, ...] = (
+    "request.input",
+    "response.output",
+    "rag.input",
+    "tool_results",
+    "feedback.comment",
+)
+
+
+def _default_otlp_exporter() -> SpanExporter:
+    """Return an OTLP span exporter matching the configured protocol.
+
+    Matches the stock SDK's supported values exactly: ``grpc`` (default) or
+    ``http/protobuf``. Both read the standard ``OTEL_EXPORTER_OTLP_*`` environment
+    for endpoint, headers, and TLS. Any other value (e.g. ``http/json``, which
+    neither OTLP span exporter supports) is rejected rather than silently sending
+    protobuf.
+
+    Returns:
+        A gRPC or HTTP OTLP span exporter.
+
+    Raises:
+        ValueError: if the configured protocol is not ``grpc`` or ``http/protobuf``.
+
+    """
+    protocol = (
+        (
+            os.getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")
+            or os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL")
+            or "grpc"
+        )
+        .strip()
+        .lower()
+    )
+    if protocol == "grpc":
+        return GRPCSpanExporter()
+    if protocol == "http/protobuf":
+        return HTTPSpanExporter()
+    raise ValueError(
+        f"unsupported OTEL_EXPORTER_OTLP_PROTOCOL {protocol!r}; "
+        "supported values are 'grpc' and 'http/protobuf'"
+    )
+
+
+class RedactingSpanExporter(SpanExporter):
+    """Wraps an OTLP span exporter, redacting content attributes before export.
+
+    Attributes:
+        delegate: The downstream exporter that actually ships the (redacted) spans.
+
+    """
+
+    def __init__(
+        self,
+        delegate: SpanExporter | None = None,
+        anonymizer: PIIAnonymizer | None = None,
+    ) -> None:
+        """Initialize the exporter.
+
+        Args:
+            delegate: Downstream exporter. Defaults to an OTLP exporter matching
+                ``OTEL_EXPORTER_OTLP_PROTOCOL`` (gRPC or HTTP), which reads the
+                standard ``OTEL_EXPORTER_OTLP_*`` environment.
+            anonymizer: Redaction engine. Defaults to a ``PIIAnonymizer``, which
+                loads the spaCy model once at construction (off the request path).
+                Injectable for testing.
+
+        """
+        self._delegate = delegate or _default_otlp_exporter()
+        self._anonymizer = anonymizer or PIIAnonymizer()
+        logger.info(
+            "RedactingSpanExporter active; content attributes redacted in-process"
+        )
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        """Redact content attributes, then delegate the export.
+
+        Fail closed: if redaction of a span raises, that span is dropped rather
+        than exported unredacted, so a redaction failure can never leak PII. Other
+        spans in the batch are still exported.
+
+        Args:
+            spans: The spans to redact and export.
+
+        Returns:
+            The delegate's export result, or ``SUCCESS`` if every span was dropped
+            (reporting failure would only make the batch processor retry the same
+            failing spans).
+
+        """
+        safe_spans: list[ReadableSpan] = []
+        for span in spans:
+            try:
+                self._redact_span(span)
+            except Exception:  # pylint: disable=broad-except  # noqa: BLE001
+                # Log the span name only, never the content being redacted.
+                logger.error(
+                    "in-process redaction failed for span %r; dropping it to avoid "
+                    "exporting unredacted PII",
+                    span.name,
+                )
+                continue
+            safe_spans.append(span)
+
+        if not safe_spans:
+            return SpanExportResult.SUCCESS
+        return self._delegate.export(safe_spans)
+
+    def _redact_span(self, span: ReadableSpan) -> None:
+        """Rewrite the content attributes of a span in place with redacted text.
+
+        Args:
+            span: The span whose content attributes are redacted in place.
+
+        Raises:
+            Exception: if the redaction engine fails. The caller drops the span
+                (fail closed) rather than exporting unredacted content.
+
+        """
+        attributes = span.attributes
+        if not attributes:
+            return
+
+        new_attributes = dict(attributes)
+        changed = False
+        for key in CONTENT_ATTRIBUTES:
+            value = new_attributes.get(key)
+            if not isinstance(value, str) or not value:
+                continue
+            redacted = self._anonymizer.anonymize_text(value)
+            if redacted != value:
+                new_attributes[key] = redacted
+                changed = True
+
+        if changed:
+            # ReadableSpan exposes attributes via self._attributes; replacing it
+            # with a plain dict is what the OTLP encoder reads at export time.
+            span._attributes = new_attributes  # pylint: disable=protected-access
+
+    def shutdown(self) -> None:
+        """Shut down the delegate exporter."""
+        self._delegate.shutdown()
+
+    def force_flush(self, timeout_millis: int = 30_000) -> bool:
+        """Force-flush the delegate exporter."""
+        return self._delegate.force_flush(timeout_millis)

tests/e2e/features/otel_redaction.feature

@@ -0,0 +1,63 @@
+# In-process PII redaction e2e.
+#
+# The e2e stack runs with in-process redaction on by default: lightspeed-stack is
+# built with the spaCy model (en_core_web_lg, baked into the image) and runs the
+# RedactingSpanExporter (OTEL_TRACES_EXPORTER=redacting_otlp, set in docker-compose).
+# Spans are redacted in-process before being exported over OTLP to the mock
+# collector (mock-otel), which the test uses as a telemetry sink. This scenario
+# asserts a known PII value sent in a query never reaches the collector, while a
+# non-PII marker still arrives.
+@cfg_authorized @OTel @OTel-redaction @skip-in-prow
+Feature: In-process PII redaction of exported telemetry
+
+  Background:
+    Given The service is started locally
+      And The system is in default state
+      And An OpenTelemetry service is running and listening for OTLP data
+      And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
+      And REST API service prefix is /v1
+      And the Lightspeed stack configuration directory is "tests/e2e/configuration"
+      And The service uses the lightspeed-stack-authorized.yaml configuration
+      And The service is restarted
+
+
+  Scenario: PII in a responses endpoint query is redacted in-process before telemetry is exported
+    Given The system is in default state
+    When I use "responses" to ask question with authorization header
+      """
+      {
+        "input": "Reach me at sam@redhat.com regarding e2e-redaction-marker",
+        "model": "{PROVIDER}/{MODEL}",
+        "stream": false,
+        "safety_identifier": "e2e-redaction-marker"
+      }
+      """
+    Then The status code of the response is 200
+     And The OpenTelemetry service received data containing e2e-redaction-marker
+     And The OpenTelemetry service did not receive data containing sam@redhat.com
+
+
+  Scenario: PII in a query endpoint request is redacted in-process before telemetry is exported
+    Given The system is in default state
+    When I use "query" to ask question with authorization header
+      """
+      {
+        "query": "My email is john.doe@redhat.com for e2e-query-redaction-marker"
+      }
+      """
+    Then The status code of the response is 200
+     And The OpenTelemetry service received data containing e2e-query-redaction-marker
+     And The OpenTelemetry service did not receive data containing john.doe@redhat.com
+
+
+  Scenario: PII in a streaming_query endpoint request is redacted in-process before telemetry is exported
+    Given The system is in default state
+    When I use "streaming_query" to ask question with authorization header
+      """
+      {
+        "query": "Contact me at alice.smith@redhat.com for e2e-streaming-redaction-marker"
+      }
+      """
+    Then The status code of the response is 200
+     And The OpenTelemetry service received data containing e2e-streaming-redaction-marker
+     And The OpenTelemetry service did not receive data containing alice.smith@redhat.com

tests/e2e/features/steps/opentelemetry.py

@@ -85,3 +85,29 @@ def collector_received_data(context: Context, marker: str) -> None:
         f"Mock OTEL collector did not receive data containing {marker!r} "
         f"within {_DELIVERY_TIMEOUT_S:.0f}s"
     )
+
+
+@then("The OpenTelemetry service did not receive data containing {marker}")
+def collector_did_not_receive_data(context: Context, marker: str) -> None:
+    """Assert the mock collector buffered telemetry that does NOT contain ``marker``.
+
+    Used to verify in-process redaction: a known PII value sent in a query must be
+    absent from the exported telemetry. Ordering matters — a preceding "received
+    data containing ..." step should already have confirmed the span was delivered,
+    so a single check proves the value is absent (redacted) rather than merely
+    not-yet-delivered.
+    """
+    assert getattr(
+        context, "otel_collector_ready", False
+    ), "The OpenTelemetry service must be started before asserting on delivery"
+    marker = marker.strip()
+    response = requests.get(
+        f"{MOCK_OTEL_CONTROL_BASE}/received", params={"contains": marker}, timeout=5
+    )
+    assert (
+        response.status_code == 200
+    ), f"Failed to query mock OTEL collector: HTTP {response.status_code}"
+    assert not response.json().get("found"), (
+        f"Mock OTEL collector unexpectedly received data containing {marker!r} "
+        "(expected it to be redacted in-process)"
+    )

tests/e2e/test_list.txt

@@ -16,6 +16,7 @@ features/authorized_noop_token.feature
 features/conversations.feature
 features/faiss.feature
 features/opentelemetry.feature
+features/otel_redaction.feature
 features/prompts.feature
 features/query.feature
 features/responses.feature