feat(observability): expose translation diagnostics

crates/libsy-llm-client/src/client.rs

@@ -19,8 +19,9 @@ use switchyard_protocol::{
     Response, RoutedLlmClient,
 };
 use switchyard_translation::{
-    WireFormat, decode_aggregated_response, decode_request, decode_stream,
-    encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,
+    WireFormat, decode_aggregated_response_with_diagnostics, decode_request_with_diagnostics,
+    decode_stream, encode_aggregated_response_with_extensions_and_diagnostics,
+    encode_request_with_diagnostics, encode_stream_with_extensions,
 };
 use tracing::Instrument;
 
@@ -240,8 +241,14 @@ impl TranslatingLlmClient {
         model: &ModelId,
         endpoint: UpstreamEndpoint,
     ) -> Result<EncodedResponse> {
-        let mut body = encode_request(&llm_request, wire_format)
+        let encoded = encode_request_with_diagnostics(&llm_request, wire_format)
             .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
+        metrics::record_translation_diagnostics(
+            &encoded.diagnostics,
+            metrics::TranslationOperation::RequestEncode,
+            wire_format,
+        );
+        let mut body = encoded.body;
         // `encode_request` round-trips a preserved same-format body verbatim,
         // which keeps the caller's original `model`; force the resolved model so
         // the upstream always sees the target id.
@@ -498,9 +505,14 @@ impl TranslatingLlmClient {
                 let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
                     LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
                 })?;
-                let agg = decode_aggregated_response(&body, wire_format)
+                let decoded = decode_aggregated_response_with_diagnostics(&body, wire_format)
                     .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
-                LlmResponse::Agg(agg)
+                metrics::record_translation_diagnostics(
+                    &decoded.diagnostics,
+                    metrics::TranslationOperation::ResponseDecode,
+                    wire_format,
+                );
+                LlmResponse::Agg(decoded.response)
             }
         };
 
@@ -531,8 +543,14 @@ impl TranslatingLlmClient {
         model: Option<&ModelId>,
         wire_format: WireFormat,
     ) -> Result<RawResponse> {
-        let llm_request = decode_request(wire_format, &raw_http_request)
+        let decoded = decode_request_with_diagnostics(wire_format, &raw_http_request)
             .map_err(|error| LlmClientError::RequestTranslation(error.to_string()))?;
+        metrics::record_translation_diagnostics(
+            &decoded.diagnostics,
+            metrics::TranslationOperation::RequestDecode,
+            wire_format,
+        );
+        let llm_request = decoded.request;
         let request_extensions = llm_request.extensions.clone();
         // The model that serves the call — the rewrite target when the caller pinned
         // one, else the request's own model. Mirrors `call_rewrite_model`'s own
@@ -559,14 +577,19 @@ impl TranslatingLlmClient {
 
         match response.llm_response {
             LlmResponse::Agg(agg) => {
-                let body = encode_aggregated_response_with_extensions(
+                let encoded = encode_aggregated_response_with_extensions_and_diagnostics(
                     &agg,
                     wire_format,
                     served_model.as_deref(),
                     &request_extensions,
                 )
                 .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
-                Ok(RawResponse::Buffered(body))
+                metrics::record_translation_diagnostics(
+                    &encoded.diagnostics,
+                    metrics::TranslationOperation::ResponseEncode,
+                    wire_format,
+                );
+                Ok(RawResponse::Buffered(encoded.body))
             }
             LlmResponse::Stream(chunks) => {
                 let events = encode_stream_with_extensions(

crates/libsy-llm-client/src/metrics.rs

@@ -12,12 +12,34 @@ use std::{
 use opentelemetry::metrics::ObservableGauge;
 use opentelemetry::{KeyValue, global};
 use switchyard_libsy::Result;
-use switchyard_protocol::{ModelId, Response};
+use switchyard_protocol::{ModelId, Response, WireFormat};
+use switchyard_translation::{DiagnosticSeverity, TranslationDiagnostic};
 
 static TOTAL_REQUESTS: AtomicU64 = AtomicU64::new(0);
 static TOTAL_ERRORS: AtomicU64 = AtomicU64::new(0);
 static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = OnceLock::new();
 
+/// Runtime boundary at which a buffered translation diagnostic was emitted.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum TranslationOperation {
+    RequestDecode,
+    RequestEncode,
+    ResponseDecode,
+    ResponseEncode,
+}
+
+impl TranslationOperation {
+    /// Returns the bounded metric-label value for this operation.
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::RequestDecode => "request_decode",
+            Self::RequestEncode => "request_encode",
+            Self::ResponseDecode => "response_decode",
+            Self::ResponseEncode => "response_encode",
+        }
+    }
+}
+
 /// Registers process-wide compatibility gauges with the installed global meter provider.
 pub fn initialize() {
     TOTAL_GAUGES.get_or_init(|| {
@@ -100,6 +122,47 @@ pub(crate) fn record_retry_recovered() {
         .add(1, &[]);
 }
 
+/// Records translation diagnostics without putting request-derived values in metric labels.
+pub fn record_translation_diagnostics(
+    diagnostics: &[TranslationDiagnostic],
+    operation: TranslationOperation,
+    format: WireFormat,
+) {
+    for diagnostic in diagnostics {
+        let severity = diagnostic_severity_label(&diagnostic.severity);
+        global::meter("switchyard")
+            .u64_counter("switchyard.translation_diagnostics")
+            .build()
+            .add(
+                1,
+                &[
+                    KeyValue::new("code", diagnostic.code.clone()),
+                    KeyValue::new("format", format.as_str()),
+                    KeyValue::new("operation", operation.as_str()),
+                    KeyValue::new("severity", severity),
+                ],
+            );
+        tracing::warn!(
+            target: "libsy",
+            code = %diagnostic.code,
+            format = format.as_str(),
+            operation = operation.as_str(),
+            severity,
+            diagnostic = %diagnostic.message,
+            path = diagnostic.path.as_deref().unwrap_or(""),
+            "LLM protocol translation emitted a diagnostic"
+        );
+    }
+}
+
+const fn diagnostic_severity_label(severity: &DiagnosticSeverity) -> &'static str {
+    match severity {
+        DiagnosticSeverity::Info => "info",
+        DiagnosticSeverity::Warning => "warning",
+        DiagnosticSeverity::Error => "error",
+    }
+}
+
 /// Records the time needed to produce the routing outcome, including classifier calls,
 /// target resolution, request rewrites, and decision publishing.
 pub(crate) fn record_routing_overhead(algorithm: &str, overhead: Duration) {

crates/libsy-llm-client/tests/observability.rs

@@ -38,6 +38,7 @@ use switchyard_libsy::{
     LlmClassifierConfig, LlmTaskClassifier, PickerMode, RoutingOutcome, StageRouter,
     StageRouterConfig, Step, TaskClassifierConfig,
 };
+use switchyard_llm_client::metrics::{TranslationOperation, record_translation_diagnostics};
 use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
 use switchyard_protocol::ModelId;
 use switchyard_protocol::{
@@ -48,6 +49,7 @@ use switchyard_protocol::{
     LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,
     text_response,
 };
+use switchyard_translation::TranslationDiagnostic;
 
 #[derive(Debug, thiserror::Error)]
 #[error("{0}")]
@@ -644,6 +646,95 @@ fn otel_attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {
         .map(|attribute| &attribute.value)
 }
 
+// Verifies the required metric labels and exactly one structured warning per diagnostic.
+#[tokio::test]
+async fn translation_diagnostics_emit_a_metric_and_structured_warning() {
+    let _guard = serialize_test().lock().await;
+    let (store, exporter, provider, _, _) = telemetry();
+    let attributes = [
+        ("code", "lossy_conversion"),
+        ("format", "anthropic_messages"),
+        ("operation", "request_encode"),
+        ("severity", "warning"),
+    ];
+    let before = u64_counter_value(
+        &flushed_metrics(exporter, provider),
+        "switchyard.translation_diagnostics",
+        &attributes,
+    )
+    .unwrap_or_default();
+    let event_count = store.events().len();
+
+    record_translation_diagnostics(
+        &[TranslationDiagnostic::warning(
+            "lossy_conversion",
+            "Anthropic structured output dropped unsupported JSON Schema constraints",
+        )
+        .at_path("$.response_format")],
+        TranslationOperation::RequestEncode,
+        WireFormat::AnthropicMessages,
+    );
+
+    let snapshots = flushed_metrics(exporter, provider);
+    let after = u64_counter_value(
+        &snapshots,
+        "switchyard.translation_diagnostics",
+        &attributes,
+    );
+    assert_eq!(after, Some(before + 1));
+    let mut metric_attribute_keys = snapshots
+        .iter()
+        .flat_map(|snapshot| snapshot.scope_metrics())
+        .flat_map(|scope| scope.metrics())
+        .filter(|metric| metric.name() == "switchyard.translation_diagnostics")
+        .filter_map(|metric| match metric.data() {
+            AggregatedMetrics::U64(MetricData::Sum(sum)) => sum
+                .data_points()
+                .find(|point| attributes_match(point.attributes(), &attributes))
+                .map(|point| {
+                    point
+                        .attributes()
+                        .map(|attribute| attribute.key.as_str().to_string())
+                        .collect::<Vec<_>>()
+                }),
+            _ => None,
+        })
+        .next()
+        .expect("missing translation diagnostic metric attributes");
+    metric_attribute_keys.sort_unstable();
+    assert_eq!(
+        metric_attribute_keys,
+        ["code", "format", "operation", "severity"]
+    );
+    let events = store.events();
+    let matching_warnings = events[event_count..]
+        .iter()
+        .filter(|event| {
+            event.target == "libsy"
+                && event.level == "WARN"
+                && event
+                    .fields
+                    .get("code")
+                    .is_some_and(|value| value == "lossy_conversion")
+                && event
+                    .fields
+                    .get("format")
+                    .is_some_and(|value| value == "anthropic_messages")
+                && event
+                    .fields
+                    .get("operation")
+                    .is_some_and(|value| value == "request_encode")
+                && event.fields.get("diagnostic").is_some_and(|value| {
+                    value.contains("dropped unsupported JSON Schema constraints")
+                })
+        })
+        .count();
+    assert_eq!(
+        matching_warnings, 1,
+        "expected one structured translation warning"
+    );
+}
+
 #[tokio::test]
 async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard_libsy::Result<()> {
     let _guard = serialize_test().lock().await;

crates/switchyard-server/src/lib.rs

@@ -37,6 +37,7 @@ use libsy::{Algorithm, LibsyError, RoutingOutcome};
 use parking_lot::Mutex;
 use serde::{Deserialize, Serialize};
 use serde_json::{Value, json};
+use switchyard_llm_client::metrics::{TranslationOperation, record_translation_diagnostics};
 use switchyard_llm_client::{AuxiliaryOperation, ClientRouter, RunObservation, RunObserver};
 use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};
 use switchyard_runner::{
@@ -46,7 +47,9 @@ use tokio::net::{TcpListener, TcpSocket};
 use tokio::task;
 use tracing::{Instrument, Level};
 
-use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};
+use switchyard_translation::{
+    WireFormat, decode_request_with_diagnostics, encode_aggregated_response_with_diagnostics,
+};
 
 use crate::response::into_http_response;
 use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};
@@ -720,12 +723,19 @@ async fn decision(
             // The request moved into the decision run, so its namespace mapping
             // is gone by here. A Codex tool call in this preview keeps its
             // qualified name.
-            match encode_aggregated_response(
+            match encode_aggregated_response_with_diagnostics(
                 &aggregate,
                 input_format,
                 outcome.selected_model_id().ok().map(ModelId::as_str),
             ) {
-                Ok(response) => Some(response),
+                Ok(encoded) => {
+                    record_translation_diagnostics(
+                        &encoded.diagnostics,
+                        TranslationOperation::ResponseEncode,
+                        input_format,
+                    );
+                    Some(encoded.body)
+                }
                 Err(error) => return server_error(error.to_string()),
             }
         }
@@ -941,8 +951,14 @@ fn resolve_route(
     body: Value,
     wire_format: WireFormat,
 ) -> std::result::Result<(&Route, Request), Response> {
-    let llm_request = decode_request(wire_format, &body)
+    let decoded = decode_request_with_diagnostics(wire_format, &body)
         .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
+    record_translation_diagnostics(
+        &decoded.diagnostics,
+        TranslationOperation::RequestDecode,
+        wire_format,
+    );
+    let llm_request = decoded.request;
     let requested_model = llm_request
         .model
         .clone()

crates/switchyard-server/src/response.rs

@@ -7,9 +7,11 @@ use std::error::Error;
 
 use axum::Json;
 use axum::response::{IntoResponse, Response as HttpResponse};
+use switchyard_llm_client::metrics::{TranslationOperation, record_translation_diagnostics};
 use switchyard_protocol::{LlmResponse, ProviderExtensions, Response as AlgorithmResponse};
 use switchyard_translation::{
-    WireFormat, encode_aggregated_response_with_extensions, encode_stream_with_extensions,
+    WireFormat, encode_aggregated_response_with_extensions_and_diagnostics,
+    encode_stream_with_extensions,
 };
 
 use crate::sse::frame_stream;
@@ -27,13 +29,18 @@ pub(crate) fn into_http_response(
 ) -> Result<HttpResponse, BoxError> {
     match response.llm_response {
         LlmResponse::Agg(response) => {
-            let body = encode_aggregated_response_with_extensions(
+            let encoded = encode_aggregated_response_with_extensions_and_diagnostics(
                 &response,
                 target_format,
                 served_model.as_deref(),
                 &request_extensions,
             )?;
-            Ok(Json(body).into_response())
+            record_translation_diagnostics(
+                &encoded.diagnostics,
+                TranslationOperation::ResponseEncode,
+                target_format,
+            );
+            Ok(Json(encoded.body).into_response())
         }
         LlmResponse::Stream(stream) => {
             let events = encode_stream_with_extensions(