databricks · MarioCadenas · Aug 14, 2026 · Aug 21, 2026 · Aug 21, 2026 · Aug 21, 2026
@@ -83,8 +83,8 @@
     "@opentelemetry/resources": "2.8.0",
     "@opentelemetry/sdk-logs": "0.219.0",
     "@opentelemetry/sdk-metrics": "2.8.0",
-    "@opentelemetry/sdk-node": "0.219.0",
     "@opentelemetry/sdk-trace-base": "2.8.0",
+    "@opentelemetry/sdk-trace-node": "2.8.0",
     "@opentelemetry/semantic-conventions": "1.38.0",
     "@types/semver": "7.7.1",
     "apache-arrow": "21.1.0",

@@ -223,6 +223,12 @@ export class AppKit<TPlugins extends InputPluginMap> {
     const instance = new AppKit(mergedConfig);
 
     await Promise.all(instance.#setupPromises);
+
+    // Build the global tracer provider now that every plugin's setup() has run
+    // and contributed any span processors. Deferred to here so a single provider
+    // carries all processors (OTLP + plugin-contributed); see TelemetryManager.
+    TelemetryManager.start();
+
     await instance.#context.emitLifecycle("setup:complete");
 
     const handle = instance as unknown as PluginMap<T>;

@@ -45,6 +45,8 @@ vi.mock("../../telemetry", async () => {
     ...actual,
     TelemetryManager: {
       initialize: vi.fn(),
+      start: vi.fn(),
+      registerSpanProcessor: vi.fn(),
       getProvider: () => ({
         getTracer: () => ({
           startActiveSpan: vi.fn((_name: string, fn: (span: any) => any) =>

@@ -67,6 +67,7 @@ import {
   currentTraceId,
   initAgentTracing,
   linkTraceToRun,
+  startAgentTracing,
   traceAgent,
   traceTool,
 } from "./mlflow";
@@ -297,6 +298,12 @@ export class AgentsPlugin extends Plugin implements ToolProvider {
 
   async setup() {
     await initAgentTracing();
+    // Seed mlflow's config right after TelemetryManager.start() (before the
+    // server serves), so the first turn's request-root span is forwarded and
+    // that turn assembles into a trace — not dropped as a cold-start artifact.
+    this.context?.onLifecycle("setup:complete", () => {
+      startAgentTracing();
+    });
     const { agents, defaultAgentName } = await this.buildAgentRegistry();
     this.agents = agents;
     this.defaultAgentName = defaultAgentName;

@@ -1,12 +1,120 @@
+import { SpanKind } from "@opentelemetry/api";
+import type { SpanProcessor } from "@opentelemetry/sdk-trace-base";
+
 import { createLogger } from "../../logging/logger";
+import { TelemetryManager } from "../../telemetry";
 
 const logger = createLogger("agents");
 
 type MlflowModule = typeof import("mlflow-tracing");
 
+interface MlflowInitConfig {
+  trackingUri: string;
+  experimentId: string;
+  host?: string;
+}
+
 let mlflow: MlflowModule | undefined;
 let enabled = false;
 let initStarted = false;
+let configured = false;
+let initConfig: MlflowInitConfig | undefined;
+let gatedProcessor: GatedMlflowSpanProcessor | undefined;
+
+/**
+ * Wraps mlflow's OTel `SpanProcessor` so it stays inert until mlflow's global
+ * config is seeded. mlflow's `onStart` calls its own `getConfig()`, which THROWS
+ * before `init()` runs — and that throw propagates out of `tracer.startSpan()`,
+ * so it would break unrelated AppKit spans (HTTP, analytics) created between
+ * `TelemetryManager.start()` and the first agent turn. We contribute this to
+ * AppKit's single tracer provider during `setup()`, but only start forwarding
+ * once `ready()` is called — right after the lazy `init()` in {@link ensureConfigured}.
+ *
+ * It also drops the exporters' own outbound spans (parentless CLIENT spans —
+ * outgoing requests made outside any agent turn, e.g. mlflow/OTLP shipping a
+ * trace). Forwarding those would loop: each upload is an HTTP call that
+ * auto-instrumentation turns into a new span to trace and upload.
+ */
+export class GatedMlflowSpanProcessor implements SpanProcessor {
+  #inner: SpanProcessor;
+  #ready = false;
+  // Spans we forwarded `onStart` for, so `onEnd` stays balanced — mlflow never
+  // sees an end without a matching start.
+  #forwarded = new WeakSet<object>();
+
+  constructor(inner: SpanProcessor) {
+    this.#inner = inner;
+  }
+
+  ready(): void {
+    this.#ready = true;
+  }
+
+  onStart(
+    span: Parameters<SpanProcessor["onStart"]>[0],
+    parentContext: Parameters<SpanProcessor["onStart"]>[1],
+  ): void {
+    if (!this.#ready) return;
+    // Drop the exporters' own outbound calls. A parentless (root) CLIENT span is
+    // an outgoing request made outside any agent turn — e.g. mlflow or OTLP
+    // shipping a trace. Forwarding those would loop: each upload is itself an
+    // HTTP call that auto-instrumentation turns into a new span to trace and
+    // upload. Spans inside a real request tree keep their parent (or are the
+    // incoming SERVER root), so they still flow through.
+    if (span.kind === SpanKind.CLIENT && !span.parentSpanContext?.spanId) {
+      return;
+    }
+    this.#forwarded.add(span);
+    this.#inner.onStart(span, parentContext);
+  }
+
+  onEnd(span: Parameters<SpanProcessor["onEnd"]>[0]): void {
+    if (!this.#forwarded.has(span)) return;
+    this.#inner.onEnd(span);
+  }
+
+  forceFlush(): Promise<void> {
+    return this.#inner.forceFlush();
+  }
+
+  shutdown(): Promise<void> {
+    return this.#inner.shutdown();
+  }
+}
+
+/**
+ * Build mlflow's OTel `SpanProcessor` ourselves rather than letting `init()`
+ * build and globally register its own tracer provider. This lets AppKit own the
+ * single global provider (OTLP + this processor), so agent spans reach both
+ * MLflow and any OTLP endpoint without two SDKs racing for the global slot.
+ *
+ * Deep-imports `mlflow-tracing` internals that aren't on its public entrypoint —
+ * pinned to the exact version in package.json and guarded by a test that fails
+ * loudly if a version bump renames them.
+ */
+async function buildMlflowSpanProcessor(
+  config: MlflowInitConfig,
+): Promise<SpanProcessor> {
+  const { createAuthProvider } = await import("mlflow-tracing/dist/auth");
+  const { MlflowClient } = await import("mlflow-tracing");
+  const { MlflowSpanExporter, MlflowSpanProcessor } =
+    await import("mlflow-tracing/dist/exporters/mlflow");
+
+  const authProvider = createAuthProvider({
+    trackingUri: config.trackingUri,
+    ...(config.host ? { host: config.host } : {}),
+  });
+  const client = new MlflowClient({
+    trackingUri: config.trackingUri,
+    authProvider,
+  });
+  // mlflow builds against a different @opentelemetry/sdk-trace-base major than
+  // AppKit; the SpanProcessor contract (onStart/onEnd/forceFlush/shutdown) is
+  // stable across them, so bridge the nominal type mismatch with one cast.
+  return new MlflowSpanProcessor(
+    new MlflowSpanExporter(client),
+  ) as unknown as SpanProcessor;
+}
 
 /** The bound MLflow experiment id, from the optional `experiment` resource. */
 function experimentId(): string | undefined {
@@ -30,6 +138,13 @@ function normalizedDatabricksHost(): string | undefined {
 /**
  * Initialize MLflow agent tracing once, when an experiment is bound — i.e. the
  * agents plugin's optional `experiment` resource is set (`MLFLOW_EXPERIMENT_ID`).
+ * Called from the agents plugin's `setup()`, before `TelemetryManager.start()`.
+ *
+ * Rather than let `mlflow.init()` stand up and globally register its own tracer
+ * provider (which would race AppKit's and drop one exporter), we build mlflow's
+ * span processor ourselves and contribute it to AppKit's single provider via
+ * {@link TelemetryManager.registerSpanProcessor}. mlflow's global config is
+ * seeded lazily in {@link ensureConfigured} on first trace, after `start()`.
  *
  * Auth is resolved by the `mlflow-tracing` SDK from the app's own Databricks
  * credentials — `DATABRICKS_HOST`/`DATABRICKS_TOKEN` or a `~/.databrickscfg`
@@ -49,11 +164,14 @@ export async function initAgentTracing(): Promise<void> {
   try {
     mlflow = await import("mlflow-tracing");
     const host = normalizedDatabricksHost();
-    mlflow.init({
+    initConfig = {
       trackingUri: process.env.MLFLOW_TRACKING_URI?.trim() || "databricks",
       experimentId: id,
       ...(host ? { host } : {}),
-    });
+    };
+    const processor = await buildMlflowSpanProcessor(initConfig);
+    gatedProcessor = new GatedMlflowSpanProcessor(processor);
+    TelemetryManager.registerSpanProcessor(gatedProcessor);
     enabled = true;
     logger.info("MLflow agent tracing enabled (experiment %s)", id);
   } catch (err) {
@@ -71,6 +189,41 @@ export interface SpanRecorder {
 
 const noopRecorder: SpanRecorder = { setOutputs() {} };
 
+/**
+ * Seed mlflow's global config on first use — AFTER `TelemetryManager.start()`
+ * has registered AppKit's provider. `init()` also stands up its own tracer
+ * provider and tries to register it globally, but that loses to AppKit's
+ * already-registered provider (non-fatal); we call it only for the config
+ * side-effect mlflow's span processor requires, then enable forwarding on the
+ * gated processor. Returns whether tracing is usable.
+ */
+function ensureConfigured(): boolean {
+  if (configured) return enabled;
+  configured = true;
+  if (!mlflow || !initConfig) return false;
+  try {
+    mlflow.init(initConfig);
+    gatedProcessor?.ready();
+    return true;
+  } catch (err) {
+    enabled = false;
+    logger.warn("MLflow agent tracing disabled (init failed): %O", err);
+    return false;
+  }
+}
+
+/**
+ * Seed mlflow's config eagerly, right after `TelemetryManager.start()` — the
+ * agents plugin wires this to the `"setup:complete"` lifecycle event, before the
+ * server serves any request. Doing it here means the request's own root span is
+ * already forwarded when the first turn runs, so that turn assembles into a
+ * trace instead of being dropped (mlflow roots a trace only at the top-level
+ * span). Idempotent, and `trace()` still seeds lazily as a fallback.
+ */
+export function startAgentTracing(): void {
+  ensureConfigured();
+}
+
 /**
  * Run `fn` inside an MLflow span of `spanType` when tracing is enabled,
  * otherwise just run it (zero overhead). Spans auto-nest via the SDK's active
@@ -86,9 +239,10 @@ async function trace<T>(
   fn: (span: SpanRecorder) => Promise<T>,
 ): Promise<T> {
   if (!enabled || !mlflow) return fn(noopRecorder);
-  const type =
-    spanType === "AGENT" ? mlflow.SpanType.AGENT : mlflow.SpanType.TOOL;
-  return await mlflow.withSpan<T>(
+  const m = mlflow;
+  if (!ensureConfigured()) return fn(noopRecorder);
+  const type = spanType === "AGENT" ? m.SpanType.AGENT : m.SpanType.TOOL;
+  return await m.withSpan<T>(
     async (span) => {
       if (inputs !== undefined) span.setInputs(inputs);
       let outputsSet = false;

@@ -32,6 +32,7 @@ interface FakeContext {
   getToolProviders(): Array<{ name: string; provider: ToolProvider }>;
   getPluginNames(): string[];
   addRoute(): void;
+  onLifecycle(): void;
   executeTool: (
     req: unknown,
     pluginName: string,
@@ -48,6 +49,7 @@ function fakeContext(
     getToolProviders: () => providers,
     getPluginNames: () => providers.map((p) => p.name),
     addRoute: vi.fn(),
+    onLifecycle: vi.fn(),
     executeTool: vi.fn(async (_req, p, n, args) => ({
       plugin: p,
       tool: n,