From 0febb045707ee9be014ee8ab71cdf2f0850d1b51 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:18:25 -0500 Subject: [PATCH 1/4] feat(agents): recognize native Codex and Claude Code telemetry Codex and Claude Code export OTLP traces with their own attribute names, so the Agents view either skipped their datasets (Codex) or showed runs without tool names or token usage (Claude Code). Detect both harnesses from the dataset schema and derive the GenAI span fields the view reads: the turn becomes the agent span, model calls and tool calls get their operation, model, tool name and usage. Claude Code input usage includes cache reads and writes. Recorded values are never replaced, and the details panel still shows the fields as recorded. Codex traces a turn with hundreds of internal spans, so the run query keeps only turn, model-call and tool spans from Codex services, and the mapping attaches model and tool calls to their turn. Co-Authored-By: Claude Opus 5.5 --- frontend/e2e/agents.spec.ts | 94 ++++++- .../src/features/agents/AgentRunDetails.tsx | 20 +- .../features/agents/AgentTelemetryGuide.tsx | 15 +- frontend/src/features/agents/AgentsPage.tsx | 11 +- .../features/agents/agentTelemetrySetup.ts | 2 +- frontend/src/lib/agentHarnesses.test.ts | 242 ++++++++++++++++++ frontend/src/lib/agentHarnesses.ts | 195 ++++++++++++++ frontend/src/lib/agents.ts | 16 +- 8 files changed, 575 insertions(+), 20 deletions(-) create mode 100644 frontend/src/lib/agentHarnesses.test.ts create mode 100644 frontend/src/lib/agentHarnesses.ts diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index d8878ac4a..8a65f793a 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -294,7 +294,7 @@ const agentFields = [...new Set(agentRows.flatMap((row) => Object.keys(row)))]; type Captured = { query: string; startTime: string; endTime: string }; async function serveAgents( page: Page, - options: { status?: number; rows?: LogRecord[]; datasets?: string[] } = {}, + options: { status?: number; rows?: LogRecord[]; datasets?: string[]; fields?: string[] } = {}, ) { const captured: Captured[] = []; const state = { status: options.status ?? 200, rows: options.rows ?? agentRows }; @@ -305,7 +305,9 @@ async function serveAgents( json: (options.datasets ?? ['agent_telemetry']).map((name) => ({ name })), }); if (path.endsWith('/schema')) - return route.fulfill({ json: { fields: agentFields.map((name) => ({ name })) } }); + return route.fulfill({ + json: { fields: (options.fields ?? agentFields).map((name) => ({ name })) }, + }); if (path.endsWith('/info')) return route.fulfill({ json: { telemetryType: 'traces', logSource: [{ log_source_format: 'otel-traces' }] }, @@ -723,3 +725,91 @@ for (const trace of ['trace-with-dashes', '']) { await expect(details.getByRole('tab', { name: 'Span tree (1)' })).toBeVisible(); }); } + +test('native Codex traces become runs with mapped model calls, tools and usage', async ({ + page, +}) => { + // Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the + // response handler below each model call, and intermediate spans filtered out. + const trace = 'e5'.repeat(16); + const codex = (span: Omit) => + spanRows({ + ...span, + trace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); + const rows = [ + codex({ + id: 'e500000000000001', + parent: 'e5000000000000aa', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.output_tokens': '38', + }, + }), + codex({ + id: 'e500000000000002', + parent: 'e5000000000000bb', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 1600, + attributes: { model: 'gpt-6-luna' }, + }), + codex({ + id: 'e500000000000003', + parent: 'e500000000000002', + name: 'receiving_stream', + start: base + 1110, + duration: 1580, + }), + codex({ + id: 'e500000000000004', + parent: 'e500000000000003', + name: 'handle_responses', + start: base + 2600, + duration: 40, + attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, + }), + codex({ + id: 'e500000000000005', + parent: 'e5000000000000cc', + name: 'exec', + start: base + 2700, + duration: 137, + attributes: { tool_name: 'exec', call_id: 'call_codex' }, + }), + ].flat(); + const { captured } = await serveAgents(page, { + rows, + datasets: ['codex-traces'], + fields: [...new Set([...rows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens'])], + }); + await page.goto(`/agents?${range}`); + await page.getByLabel('Agents dataset').selectOption('codex-traces'); + const row = runRow(page, 'Codex'); + await expect(cell(row, 'Status')).toHaveText('Successful'); + await expect(cell(row, 'Model')).toHaveText('gpt-6-luna'); + await expect(cell(row, 'Tokens')).toContainText('32,558 in'); + await expect(cell(row, 'Tokens')).toContainText('38 out'); + await expect(cell(row, 'Spans')).toHaveText('3'); + expect(captured[0].query).toContain(`"span_name" = 'session_task.turn'`); + expect(captured[0].query).toContain(`LIKE 'codex%'`); + + const details = await openRun(page, 'Codex'); + const children = details + .getByRole('list', { name: 'Run span tree' }) + .getByRole('list', { name: 'Child spans' }); + await expect(children.getByRole('button')).toHaveCount(2); + await children.getByRole('button', { name: /exec/ }).click(); + const span = details.getByRole('region', { name: 'Selected span' }); + await expect(span).toContainText('Mapped from native Codex telemetry'); + // Facts and raw records show what Codex recorded, not the attached tree position. + await expect(fact(span, 'Parent span ID')).toHaveText('e5000000000000cc'); + await span.getByText('Recorded fields (1 record)').click(); + await expect(span).toContainText('"tool_name": "exec"'); + await expect(span).not.toContainText('gen_ai.tool.name'); +}); diff --git a/frontend/src/features/agents/AgentRunDetails.tsx b/frontend/src/features/agents/AgentRunDetails.tsx index 2d78b0624..9d4e99640 100644 --- a/frontend/src/features/agents/AgentRunDetails.tsx +++ b/frontend/src/features/agents/AgentRunDetails.tsx @@ -1,6 +1,7 @@ import { useState } from 'react'; import { Bot, GitBranch, MessageSquare, Wrench } from 'lucide-react'; import { Badge, Button, Tabs } from '../../components/ui'; +import { AGENT_HARNESSES, mappedHarness, recordedFields } from '../../lib/agentHarnesses'; import type { AgentRun, AgentRunStatus, AgentSpan } from '../../lib/agentRuns'; export function formatAgentDuration(value?: number) { @@ -57,6 +58,11 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { const [tab, setTab] = useState('tree'); const [spanId, setSpanId] = useState(run.roots[0]?.id ?? run.spans[0]?.id); const selected = run.spans.find((span) => span.id === spanId) ?? run.spans[0]; + const selectedHarness = selected?.rows.map(mappedHarness).find(Boolean); + // Harness mapping can attach a span to its run when intermediate spans were not loaded. + const recordedParent = selected?.rows + .map((row) => recordedFields(row).span_parent_span_id ?? recordedFields(row).parent_span_id) + .find((value) => typeof value === 'string' && value.trim() && !/^0+$/.test(value.trim())); const models = run.spans.filter((span) => span.kind === 'model'); const tools = run.spans.filter((span) => span.kind === 'tool'); const errors = run.spans.filter((span) => span.errors.length || span.status === 'failed'); @@ -233,7 +239,11 @@ export function AgentRunDetails({ run }: { run: AgentRun }) {
Parent span ID
- {selected.parentSpanId ?? 'Not recorded'} + + {String( + (selectedHarness ? recordedParent : selected.parentSpanId) ?? 'Not recorded', + )} +
@@ -255,6 +265,12 @@ export function AgentRunDetails({ run }: { run: AgentRun }) {
+ {selectedHarness && ( +

+ Mapped from native {AGENT_HARNESSES[selectedHarness].label} telemetry. Recorded fields + are shown unchanged. +

+ )}
Recorded fields ({selected.rows.length}{' '} @@ -262,7 +278,7 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { {selected.rows.map((row, index) => ( 
-                {JSON.stringify(row, null, 2)}
+                {JSON.stringify(recordedFields(row), null, 2)}
))}
diff --git a/frontend/src/features/agents/AgentTelemetryGuide.tsx b/frontend/src/features/agents/AgentTelemetryGuide.tsx index a292535c4..be1db2445 100644 --- a/frontend/src/features/agents/AgentTelemetryGuide.tsx +++ b/frontend/src/features/agents/AgentTelemetryGuide.tsx @@ -75,24 +75,23 @@ export function AgentTelemetryGuide() {

3. Validate real telemetry

Confirm collector receipt and stored rows first, then run discovery, parent links, models, - tools and usage in Agents. Test successful, failed and interrupted work. Native CLI - compatibility has not been verified end to end; capture Codex’s actual trace schema from - your installed runtime. + tools and usage in Agents. Test successful, failed and interrupted work. Agents reads + native Claude Code and Codex trace fields directly, so their trace datasets need no field + mapping. Runs come from traces; a logs dataset is not listed on its own.

Mapping and validation checklist +

For other agents and frameworks:

  • Map agent and operation to gen_ai.agent.name and{' '} gen_ai.operation.name; models to gen_ai.request.model /{' '} - gen_ai.response.model; tools to gen_ai.tool.name (Claude:{' '} - tool_name). + gen_ai.response.model; tools to gen_ai.tool.name.
  • Map usage to gen_ai.usage.input_tokens and{' '} - gen_ai.usage.output_tokens. Claude input totals include{' '} - input_tokens + cache_read_tokens + cache_creation_tokens. Preserve absent - values as unknown. + gen_ai.usage.output_tokens, including cached input tokens in the input + total. Preserve absent values as unknown.
  • Validate 32-digit hexadecimal trace IDs, 16-digit hexadecimal span IDs, parent IDs, diff --git a/frontend/src/features/agents/AgentsPage.tsx b/frontend/src/features/agents/AgentsPage.tsx index 56bcbfd97..aeeccca93 100644 --- a/frontend/src/features/agents/AgentsPage.tsx +++ b/frontend/src/features/agents/AgentsPage.tsx @@ -24,6 +24,7 @@ import { buildAgentRunQuery, discoverAgentDatasets, } from '../../lib/agents'; +import { detectAgentHarness, mapHarnessRows } from '../../lib/agentHarnesses'; import { buildAgentRuns } from '../../lib/agentRuns'; import { agentValueMatchesSearch } from '../../lib/agentSearch'; import { @@ -101,6 +102,7 @@ export function AgentsPage() { rows: agentDemoRows(Date.parse(bounds.endTime)).filter( (row) => Date.parse(String(row.p_timestamp)) >= Date.parse(bounds.startTime), ), + limitReached: false, timeColumn: 'p_timestamp', traces: false, }; @@ -114,7 +116,12 @@ export function AgentsPage() { buildAgentRunQuery(dataset, fields, bounds, timeColumn), signal, ); - return { rows, timeColumn, traces: info.logSourceFormats.includes('otel-traces') }; + return { + rows: mapHarnessRows(detectAgentHarness(fields), rows), + limitReached: rows.length >= AGENT_RECORD_LIMIT, + timeColumn, + traces: info.logSourceFormats.includes('otel-traces'), + }; }, [client, mode, dataset, bounds], ), @@ -132,7 +139,7 @@ export function AgentsPage() { const runs = useMemo( () => buildAgentRuns(result.data?.rows ?? [], result.data?.timeColumn, { - limitReached: (result.data?.rows.length ?? 0) >= AGENT_RECORD_LIMIT, + limitReached: result.data?.limitReached, bounds, }), [result.data, bounds], diff --git a/frontend/src/features/agents/agentTelemetrySetup.ts b/frontend/src/features/agents/agentTelemetrySetup.ts index 55619e7ae..94dee5d78 100644 --- a/frontend/src/features/agents/agentTelemetrySetup.ts +++ b/frontend/src/features/agents/agentTelemetrySetup.ts @@ -62,7 +62,7 @@ Inspect the installed runtime/version and its actual config location first. Merg ${config} -Use an already configured collector; configure its authenticated Parseable exporter separately with the destination trace dataset and X-P-Stream. Use separate collector logs and traces pipelines/exporters with distinct X-P-Stream values for their destination datasets. Keep credentials in the existing secret mechanism. Inspect real payloads from this installed runtime, map provider fields to the dashboard GenAI span contract, and preserve missing values as unknown. Claude input token totals include input_tokens + cache_read_tokens + cache_creation_tokens, only when those values are present. Map tool_name to gen_ai.tool.name. Codex trace schema must be captured and checked from this installed runtime. Prompt capture does not guarantee a full transcript. +Use an already configured collector; configure its authenticated Parseable exporter separately with the destination trace dataset and X-P-Stream. Use separate collector logs and traces pipelines/exporters with distinct X-P-Stream values for their destination datasets. Keep credentials in the existing secret mechanism. Parseable Agents reads native ${provider === 'claude' ? 'Claude Code' : 'Codex'} trace fields directly, so forward them unchanged rather than renaming them; runs come from the traces dataset. Prompt capture does not guarantee a full transcript. Validate transport first (collector receipt and stored rows), then dashboard compatibility: run discovery, trace/parent relationships, models, tools, usage, and any captured messages. Test successful, failed, and interrupted/incomplete work; missing telemetry remains unknown/incomplete. Do not invent content, tokens, costs, evaluations, or live state. A synthetic fixture alone does not prove native CLI compatibility. Report changed files, restart steps, observed payload mappings, evidence, limitations, and how to restore previous exporter settings.`; } diff --git a/frontend/src/lib/agentHarnesses.test.ts b/frontend/src/lib/agentHarnesses.test.ts new file mode 100644 index 000000000..b69d67d6a --- /dev/null +++ b/frontend/src/lib/agentHarnesses.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from 'vitest'; +import { + detectAgentHarness, + mapHarnessRows, + mappedHarness, + recordedFields, +} from './agentHarnesses'; +import { buildAgentRunQuery, isAgentSchema } from './agents'; +import { buildAgentRuns } from './agentRuns'; +import type { LogRecord } from './types'; + +// Trimmed from spans exported by Codex CLI 0.162.1 (app server) and Claude Code +// 2.1.296 with CLAUDE_CODE_ENHANCED_TELEMETRY_BETA, as stored by Parseable. +function span( + trace: string, + id: string, + parent: string, + name: string, + start: number, + durationMs: number, + fields: LogRecord = {}, +): LogRecord { + const begin = Date.parse('2026-10-13T02:50:34Z') + start; + return { + p_timestamp: new Date(begin + durationMs).toISOString(), + span_trace_id: trace, + span_span_id: id, + span_parent_span_id: parent, + span_name: name, + span_start_time_unix_nano: new Date(begin).toISOString(), + span_end_time_unix_nano: new Date(begin + durationMs).toISOString(), + span_duration_ns: durationMs * 1e6, + span_status_code: 0, + span_status_description: 'UNSET', + ...fields, + }; +} +const codexTrace = '3eead3c4c7897ddb3613163ebdeaa6c0'; +const codex = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(codexTrace, id, parent, name, start, ms, { 'service.name': 'codex-app-server', ...f }); +const codexTurn = [ + codex('a131b912f2251684', '0d2585ff2dc6a775', 'session_task.turn', 0, 4027, { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.cached_input_tokens': '27136', + 'codex.turn.token_usage.output_tokens': '38', + }), + codex('f29e280320d84c8e', '6e055c16cf2ba610', 'try_run_sampling_request', 100, 1648, { + model: 'gpt-6-luna', + }), + codex('845411c03a89e0ff', 'f29e280320d84c8e', 'receiving_stream', 102, 1645), + codex('5e5901e207ece80e', '845411c03a89e0ff', 'handle_responses', 1700, 38, { + 'gen_ai.usage.input_tokens': '16251', + 'gen_ai.usage.output_tokens': '33', + 'gen_ai.usage.cache_read.input_tokens': '11008', + }), + codex('a9a8355c5b8e769e', '174bdda1bf2f0d51', 'exec', 1760, 137, { + tool_name: 'exec', + call_id: 'call_vRR2GrjSnDyX8FWz50ONlZqM', + }), + codex('ff4f6fbba143b1c0', '638f0e6a3947c618', 'try_run_sampling_request', 2900, 967, { + model: 'gpt-6-luna', + }), + codex('761175078dbf3f02', 'ff4f6fbba143b1c0', 'receiving_stream', 2902, 965), + codex('297376cdba4bc3cc', '761175078dbf3f02', 'handle_responses', 3800, 37, { + 'gen_ai.usage.input_tokens': '16307', + 'gen_ai.usage.output_tokens': '5', + }), +]; +const claudeTrace = '0e638bc7089fb020dfd801f6c58a5dc6'; +const claude = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(claudeTrace, id, parent, name, start, ms, { 'service.name': 'claude-code', ...f }); +const claudeRequest = { + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-haiku-4-5', + model: 'claude-haiku-4-5', + 'llm_request.context': 'interaction', +}; +const claudeInteraction = [ + claude('4b1b0ceaf03121e7', '', 'claude_code.interaction', 0, 3162, { + 'interaction.sequence': '1', + user_prompt: '', + }), + claude('06675bb5f58964d7', '4b1b0ceaf03121e7', 'claude_code.llm_request', 48, 1362, { + ...claudeRequest, + input_tokens: '9', + cache_read_tokens: '12598', + cache_creation_tokens: '8595', + output_tokens: '102', + }), + claude('cf906dbe161dcbc5', '4b1b0ceaf03121e7', 'claude_code.tool', 1391, 43, { + tool_name: 'Bash', + tool_use_id: 'toolu_017FNPpeGEpbuRnKoQNMyXgb', + 'gen_ai.tool.call.id': 'toolu_017FNPpeGEpbuRnKoQNMyXgb', + }), + claude('c3b6fd085798feb9', 'cf906dbe161dcbc5', 'claude_code.tool.execution', 1396, 38), + claude('8062decff7ff63cf', '4b1b0ceaf03121e7', 'claude_code.llm_request', 1448, 1707, { + ...claudeRequest, + input_tokens: '8', + cache_read_tokens: '21193', + cache_creation_tokens: '151', + output_tokens: '52', + }), +]; + +describe('coding-agent harness detection', () => { + it('recognizes Codex and Claude Code trace schemas without GenAI agent fields', () => { + const codexFields = ['span_name', 'service.name', 'codex.turn.token_usage.input_tokens']; + expect(detectAgentHarness(codexFields)).toBe('codex'); + expect(isAgentSchema(codexFields)).toBe(true); + expect(detectAgentHarness(['span_name', 'interaction.sequence'])).toBe('claude-code'); + // Logs carry no span names and cannot be grouped into runs. + expect(detectAgentHarness(['codex.turn.token_usage.input_tokens'])).toBeUndefined(); + expect(detectAgentHarness(['span_name', 'model', 'tool_name'])).toBeUndefined(); + }); + it('seeds Codex runs from turns and loads only run spans from Codex services', () => { + const fields = [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'tool_name', + 'gen_ai.usage.input_tokens', + 'codex.turn.token_usage.input_tokens', + ]; + const { sql } = buildAgentRunQuery('codex', fields, { + startTime: '2026-10-13T00:00:00Z', + endTime: '2026-10-14T00:00:00Z', + }); + expect(sql).toContain( + `AND ("span_name" = 'session_task.turn') AND "span_trace_id" IS NOT NULL`, + ); + expect(sql).toContain( + `AND NOT (COALESCE("service.name", '') LIKE 'codex%' AND NOT (COALESCE("span_name", '') IN ('session_task.turn', 'try_run_sampling_request', 'receiving_stream') OR (COALESCE("span_name", '') = 'handle_responses' AND "gen_ai.usage.input_tokens" IS NOT NULL) OR "tool_name" IS NOT NULL)) ORDER BY`, + ); + // Absent columns are never referenced. + const sparse = buildAgentRunQuery( + 'codex', + ['p_timestamp', 'span_trace_id', 'span_name', 'service.name', 'codex.turn.phase'], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ).sql; + expect(sparse).not.toContain('tool_name'); + expect(sparse).not.toContain('gen_ai.usage'); + }); + it('keeps every Claude Code span and seeds runs from interactions or model calls', () => { + const { sql } = buildAgentRunQuery( + 'claude', + [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'gen_ai.request.model', + 'interaction.sequence', + ], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ); + expect(sql).toContain( + `("gen_ai.request.model" IS NOT NULL OR "span_name" = 'claude_code.interaction')`, + ); + expect(sql).not.toContain('COALESCE'); + }); +}); + +describe('coding-agent harness mapping', () => { + it('builds a Codex turn with its model calls, tool call and per-request usage', () => { + const rows = mapHarnessRows('codex', codexTurn); + const [run] = buildAgentRuns(rows); + expect(run.name).toBe('Codex'); + expect(run.status).toBe('success'); + expect(run.warnings).toEqual([]); + expect(run.models).toEqual(['gpt-6-luna']); + // Per-request usage sums to the turn's own total; the rollup is not added twice. + expect([run.inputTokens, run.outputTokens]).toEqual([32558, 38]); + expect(run.roots).toHaveLength(1); + expect(run.roots[0].kind).toBe('agent'); + expect( + run.roots[0].children.map((child) => [child.kind, child.name, child.inputTokens]), + ).toEqual([ + ['model', 'try_run_sampling_request', 16251], + ['tool', 'exec', undefined], + ['model', 'try_run_sampling_request', 16307], + ]); + expect(run.roots[0].children[1].toolName).toBe('exec'); + expect(run.roots[0].children[1].durationMs).toBe(137); + expect(run.spans.some((entry) => entry.name === 'handle_responses')).toBe(false); + }); + it('shows recorded Codex fields unchanged and does not modify the query result', () => { + const original = structuredClone(codexTurn); + const rows = mapHarnessRows('codex', codexTurn); + expect(codexTurn).toEqual(original); + const tool = rows.find((row) => row.span_name === 'exec')!; + expect(tool['gen_ai.tool.name']).toBe('exec'); + expect(tool.span_parent_span_id).toBe('a131b912f2251684'); + expect(mappedHarness(tool)).toBe('codex'); + expect(recordedFields(tool)).toEqual(original[4]); + expect(recordedFields(original[0])).toBe(original[0]); + }); + it('never replaces recorded GenAI values', () => { + const [turn] = mapHarnessRows('codex', [ + { ...codexTurn[0], 'gen_ai.agent.name': 'Reviewer', 'gen_ai.usage.input_tokens': 7 }, + ]); + expect(turn['gen_ai.agent.name']).toBe('Reviewer'); + expect(turn['gen_ai.usage.input_tokens']).toBe(7); + }); + it('does not attach detached spans to one of several turns in a trace', () => { + const second = { ...codexTurn[0], span_span_id: 'b131b912f2251684' }; + const rows = mapHarnessRows('codex', [...codexTurn, second]); + expect(rows.find((row) => row.span_name === 'exec')!.span_parent_span_id).toBe( + '174bdda1bf2f0d51', + ); + expect(buildAgentRuns(rows)[0].warnings).toContain( + 'A parent span is outside the captured sample or missing.', + ); + }); + it('builds a Claude Code interaction with cache-inclusive input usage and tool names', () => { + const [run] = buildAgentRuns(mapHarnessRows('claude-code', claudeInteraction)); + expect(run.name).toBe('Claude Code'); + expect(run.status).toBe('success'); + expect(run.models).toEqual(['claude-haiku-4-5']); + expect([run.inputTokens, run.outputTokens]).toEqual([9 + 12598 + 8595 + 8 + 21193 + 151, 154]); + expect( + run.roots[0].children.map((child) => [child.kind, child.toolName ?? child.model]), + ).toEqual([ + ['model', 'claude-haiku-4-5'], + ['tool', 'Bash'], + ['model', 'claude-haiku-4-5'], + ]); + expect(run.roots[0].children[1].children[0].kind).toBe('span'); + }); + it('keeps absent Claude Code usage unknown', () => { + const [request] = mapHarnessRows('claude-code', [ + claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, claudeRequest), + ]); + expect(request['gen_ai.usage.input_tokens']).toBeUndefined(); + expect(request['gen_ai.usage.output_tokens']).toBeUndefined(); + expect(buildAgentRuns([request])[0].inputTokens).toBeUndefined(); + }); + it('leaves rows untouched when no harness is detected', () => { + expect(mapHarnessRows(undefined, codexTurn)).toBe(codexTurn); + }); +}); diff --git a/frontend/src/lib/agentHarnesses.ts b/frontend/src/lib/agentHarnesses.ts new file mode 100644 index 000000000..4877f1b08 --- /dev/null +++ b/frontend/src/lib/agentHarnesses.ts @@ -0,0 +1,195 @@ +import { quoteIdentifier, quoteLiteral } from './query'; +import type { LogRecord } from './types'; + +// Coding-agent CLIs export OTLP with their own attribute names. A harness profile +// recognizes that telemetry and adds the GenAI span fields the Agents view reads. +// Recorded values are never replaced, and absent values stay absent. +export type AgentHarness = 'codex' | 'claude-code'; + +type HarnessProfile = { + label: string; + // Any of these fields identifies the harness in a dataset schema. + signature: string[]; + // The span that represents one agent run (a user turn). + runSpan: string; +}; + +export const AGENT_HARNESSES: Record = { + codex: { + label: 'Codex', + signature: ['codex.turn.token_usage.input_tokens', 'codex.turn.phase'], + runSpan: 'session_task.turn', + }, + 'claude-code': { + label: 'Claude Code', + signature: ['interaction.sequence', 'llm_request.context'], + runSpan: 'claude_code.interaction', + }, +}; + +// Codex traces a turn with hundreds of internal spans. Only these, plus spans +// carrying a tool name, describe the run; the others would exhaust the record limit. +const CODEX_MODEL_CALL = 'try_run_sampling_request'; +const CODEX_STREAM = 'receiving_stream'; +const CODEX_RESPONSE = 'handle_responses'; +const CODEX_SPANS = [AGENT_HARNESSES.codex.runSpan, CODEX_MODEL_CALL, CODEX_STREAM]; + +export function detectAgentHarness(fields: string[]): AgentHarness | undefined { + if (!fields.includes('span_name')) return; + return (Object.keys(AGENT_HARNESSES) as AgentHarness[]).find((harness) => + AGENT_HARNESSES[harness].signature.some((field) => fields.includes(field)), + ); +} + +/** SQL selecting the spans that start a run of this harness. */ +export function harnessRunPredicate(harness: AgentHarness) { + return `${quoteIdentifier('span_name')} = ${quoteLiteral(AGENT_HARNESSES[harness].runSpan)}`; +} + +/** SQL keeping the run-relevant spans of a harness; other services' rows are untouched. */ +export function harnessRowPredicate(harness: AgentHarness, fields: string[]) { + if (harness !== 'codex' || !fields.includes('service.name')) return; + const name = `COALESCE(${quoteIdentifier('span_name')}, '')`; + const kept = [`${name} IN (${CODEX_SPANS.map(quoteLiteral).join(', ')})`]; + if (fields.includes('gen_ai.usage.input_tokens')) + kept.push( + `(${name} = ${quoteLiteral(CODEX_RESPONSE)} AND ${quoteIdentifier('gen_ai.usage.input_tokens')} IS NOT NULL)`, + ); + if (fields.includes('tool_name')) kept.push(`${quoteIdentifier('tool_name')} IS NOT NULL`); + return `NOT (COALESCE(${quoteIdentifier('service.name')}, '') LIKE 'codex%' AND NOT (${kept.join(' OR ')}))`; +} + +const recorded = new WeakMap(); + +/** The fields a record was stored with, before any harness mapping. */ +export function recordedFields(row: LogRecord): LogRecord { + return recorded.get(row)?.row ?? row; +} +/** The harness whose native names were mapped onto this record, if any. */ +export function mappedHarness(row: LogRecord): AgentHarness | undefined { + return recorded.get(row)?.harness; +} + +function present(value: unknown) { + return value !== undefined && value !== null && value !== ''; +} +function assign(row: LogRecord, field: string, value: unknown) { + if (present(value) && !present(row[field])) row[field] = value; +} +function count(value: unknown) { + if (!present(value)) return; + const result = Number(value); + return Number.isSafeInteger(result) && result >= 0 ? result : undefined; +} +function id(row: LogRecord, field: string) { + const value = row[field]; + return typeof value === 'string' && value.trim() && !/^0+$/.test(value.trim()) + ? value.trim() + : undefined; +} +const traceOf = (row: LogRecord) => id(row, 'span_trace_id') ?? id(row, 'trace_id'); +const spanOf = (row: LogRecord) => id(row, 'span_span_id') ?? id(row, 'span_id'); +const parentOf = (row: LogRecord) => id(row, 'span_parent_span_id') ?? id(row, 'parent_span_id'); + +function mapClaudeCode(row: LogRecord) { + switch (row.span_name) { + case AGENT_HARNESSES['claude-code'].runSpan: + assign(row, 'gen_ai.agent.name', AGENT_HARNESSES['claude-code'].label); + assign(row, 'gen_ai.operation.name', 'invoke_agent'); + break; + case 'claude_code.llm_request': { + assign(row, 'gen_ai.operation.name', 'chat'); + assign(row, 'gen_ai.request.model', row.model); + // Anthropic reports cache reads and writes separately from input_tokens. + const input = ['input_tokens', 'cache_read_tokens', 'cache_creation_tokens'] + .map((field) => count(row[field])) + .filter((value): value is number => value !== undefined); + if (input.length) + assign( + row, + 'gen_ai.usage.input_tokens', + input.reduce((a, b) => a + b), + ); + assign(row, 'gen_ai.usage.output_tokens', count(row.output_tokens)); + break; + } + case 'claude_code.tool': + assign(row, 'gen_ai.operation.name', 'execute_tool'); + assign(row, 'gen_ai.tool.name', row.tool_name); + assign(row, 'gen_ai.tool.call.id', row.tool_use_id); + break; + } +} + +function mapCodex(rows: LogRecord[]) { + const spans = new Map(); + for (const row of rows) { + const trace = traceOf(row); + const span = spanOf(row); + if (trace && span) + spans.set(`${trace}/${span}`, [...(spans.get(`${trace}/${span}`) ?? []), row]); + } + const parent = (row: LogRecord) => { + const trace = traceOf(row); + const span = parentOf(row); + return trace && span ? spans.get(`${trace}/${span}`) : undefined; + }; + // Usage is reported on the response handler two levels below its model call. + for (const row of rows) { + if (row.span_name !== CODEX_RESPONSE) continue; + const stream = parent(row)?.find((entry) => entry.span_name === CODEX_STREAM); + const call = stream && parent(stream); + if (call?.[0]?.span_name !== CODEX_MODEL_CALL) continue; + for (const entry of call) + for (const field of ['gen_ai.usage.input_tokens', 'gen_ai.usage.output_tokens']) + assign(entry, field, row[field]); + } + const runs = new Map(); + for (const row of rows) { + const trace = traceOf(row); + const span = spanOf(row); + if (row.span_name !== AGENT_HARNESSES.codex.runSpan || !trace || !span) continue; + // A trace with several turns cannot attribute detached spans to one of them. + runs.set(trace, runs.has(trace) && runs.get(trace) !== span ? '' : span); + } + const kept = rows.filter( + (row) => row.span_name !== CODEX_STREAM && row.span_name !== CODEX_RESPONSE, + ); + const keptSet = new Set(kept); + for (const row of kept) { + if (row.span_name === AGENT_HARNESSES.codex.runSpan) { + assign(row, 'gen_ai.agent.name', AGENT_HARNESSES.codex.label); + assign(row, 'gen_ai.operation.name', 'invoke_agent'); + assign(row, 'gen_ai.request.model', row.model); + assign(row, 'gen_ai.usage.input_tokens', row['codex.turn.token_usage.input_tokens']); + assign(row, 'gen_ai.usage.output_tokens', row['codex.turn.token_usage.output_tokens']); + } else if (row.span_name === CODEX_MODEL_CALL) { + assign(row, 'gen_ai.operation.name', 'chat'); + assign(row, 'gen_ai.request.model', row.model); + } else if (present(row.tool_name)) { + assign(row, 'gen_ai.operation.name', 'execute_tool'); + assign(row, 'gen_ai.tool.name', row.tool_name); + assign(row, 'gen_ai.tool.call.id', row.call_id); + } + // Intermediate spans were not loaded. Attach the turn to the top of the tree + // and its model and tool calls directly to the turn. + const trace = traceOf(row); + if (!trace || !parentOf(row) || parent(row)?.some((entry) => keptSet.has(entry))) continue; + if (row.span_name === AGENT_HARNESSES.codex.runSpan) row.span_parent_span_id = ''; + else if (runs.get(trace)) row.span_parent_span_id = runs.get(trace); + } + return kept; +} + +/** Add GenAI fields derived from a harness's native telemetry. */ +export function mapHarnessRows(harness: AgentHarness | undefined, rows: LogRecord[]) { + if (!harness) return rows; + const mapped = rows.map((row) => { + const copy = { ...row }; + recorded.set(copy, { row, harness }); + return copy; + }); + if (harness === 'codex') return mapCodex(mapped); + mapped.forEach(mapClaudeCode); + return mapped; +} diff --git a/frontend/src/lib/agents.ts b/frontend/src/lib/agents.ts index 69ac7e3a8..9c2656a2b 100644 --- a/frontend/src/lib/agents.ts +++ b/frontend/src/lib/agents.ts @@ -1,3 +1,4 @@ +import { detectAgentHarness, harnessRowPredicate, harnessRunPredicate } from './agentHarnesses'; import { ApiError } from './apiError'; import { apmTransportBounds } from './apm'; import { mapConcurrent } from './concurrency'; @@ -17,7 +18,7 @@ export const AGENT_FIELDS = [ export const AGENT_RECORD_LIMIT = 1000; export const AGENT_RUN_LIMIT = 100; export function isAgentSchema(fields: string[]) { - return AGENT_FIELDS.some((field) => fields.includes(field)); + return AGENT_FIELDS.some((field) => fields.includes(field)) || !!detectAgentHarness(fields); } export function agentsPath(dataset?: string) { return dataset ? `/agents/${encodeURIComponent(dataset)}` : '/agents'; @@ -59,9 +60,14 @@ export function buildAgentRunQuery( ); if (!fields.includes(timeColumn)) throw new Error(`Dataset schema is missing the time field ${timeColumn}.`); - const agentPredicate = AGENT_FIELDS.filter((field) => fields.includes(field)) - .map((field) => `${quoteIdentifier(field)} IS NOT NULL`) - .join(' OR '); + const harness = detectAgentHarness(fields); + const agentPredicate = [ + ...AGENT_FIELDS.filter((field) => fields.includes(field)).map( + (field) => `${quoteIdentifier(field)} IS NOT NULL`, + ), + ...(harness ? [harnessRunPredicate(harness)] : []), + ].join(' OR '); + const rowPredicate = harness && harnessRowPredicate(harness, fields); const traceField = ['span_trace_id', 'trace_id'].find((field) => fields.includes(field)); const table = quoteIdentifier(dataset); const time = quoteIdentifier(timeColumn); @@ -74,7 +80,7 @@ export function buildAgentRunQuery( ? `${trace} IN (SELECT ${trace} FROM ${table} WHERE ${timePredicate} AND (${agentPredicate}) AND ${trace} IS NOT NULL AND ${trace} <> '' GROUP BY ${trace} ORDER BY MAX(${time}) DESC LIMIT ${AGENT_RUN_LIMIT}) OR ((${trace} IS NULL OR ${trace} = '') AND (${agentPredicate}))` : agentPredicate; return { - sql: `SELECT * FROM ${table} WHERE ${timePredicate} AND (${predicate}) ORDER BY ${time} DESC LIMIT ${AGENT_RECORD_LIMIT}`, + sql: `SELECT * FROM ${table} WHERE ${timePredicate} AND (${predicate})${rowPredicate ? ` AND ${rowPredicate}` : ''} ORDER BY ${time} DESC LIMIT ${AGENT_RECORD_LIMIT}`, ...apmTransportBounds(exactBounds), }; } From 9fb00cad9a7a9eee6da36e203d02fb4881929b46 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:22:23 -0500 Subject: [PATCH 2/4] fix(agents): handle mixed harness datasets and detached Codex calls safely Address review findings on native Codex and Claude Code telemetry: - Detect every harness in a dataset. Claude Code mapping always runs, and Codex mapping, dropping and reparenting apply only to rows from Codex services (or, without a service name, to a Codex-only dataset). - Keep Codex spans that already carry standard GenAI fields, such as tool spans whose tool_name a collector renamed to gen_ai.tool.name. - Attach a detached Codex call to a turn only when it started during exactly one loaded turn, so calls of an unexported turn stay detached and multi-turn traces attach each call to its own turn. - Rewrite whichever parent field a row uses (span_parent_span_id and/or parent_span_id). - Leave Claude Code input usage unknown unless input_tokens parses. - Show the "Mapped from native telemetry" note only on rows something was derived for, and drive the record-limit banner from the queried row count. Co-Authored-By: Claude Opus 5.5 --- frontend/e2e/agents.spec.ts | 49 ++++ frontend/src/features/agents/AgentsPage.tsx | 6 +- frontend/src/lib/agentHarnesses.test.ts | 300 ++++++++++++++++++-- frontend/src/lib/agentHarnesses.ts | 97 +++++-- frontend/src/lib/agentRuns.ts | 16 +- frontend/src/lib/agents.ts | 15 +- 6 files changed, 427 insertions(+), 56 deletions(-) diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index 8a65f793a..ab2e33fda 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -813,3 +813,52 @@ test('native Codex traces become runs with mapped model calls, tools and usage', await expect(span).toContainText('"tool_name": "exec"'); await expect(span).not.toContainText('gen_ai.tool.name'); }); + +test('discloses the record limit from the queried rows when Codex mapping drops some', async ({ + page, +}) => { + const trace = 'f6'.repeat(16); + const codex = (span: Omit) => + spanRows({ + ...span, + trace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); + const rows = [ + codex({ + id: 'f600000000000001', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { model: 'gpt-6-luna', 'codex.turn.token_usage.input_tokens': '10' }, + }), + codex({ + id: 'f600000000000002', + parent: 'f600000000000001', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 3000, + attributes: { model: 'gpt-6-luna' }, + }), + // Stream spans count toward the record limit but are not shown as run spans. + ...Array.from({ length: 998 }, (_, index) => + codex({ + id: (index + 1).toString(16).padStart(16, '0'), + parent: 'f600000000000002', + name: 'receiving_stream', + start: base + 1200, + duration: 10, + }), + ), + ].flat(); + await serveAgents(page, { + rows, + datasets: ['codex-traces'], + fields: [...new Set(rows.flatMap((row) => Object.keys(row)))], + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect(cell(runRow(page, 'Codex'), 'Spans')).toHaveText('2'); + await expect(page.getByRole('status').filter({ hasText: 'record limit' })).toContainText( + 'narrow the time range', + ); +}); diff --git a/frontend/src/features/agents/AgentsPage.tsx b/frontend/src/features/agents/AgentsPage.tsx index aeeccca93..fa7c4abe9 100644 --- a/frontend/src/features/agents/AgentsPage.tsx +++ b/frontend/src/features/agents/AgentsPage.tsx @@ -24,7 +24,7 @@ import { buildAgentRunQuery, discoverAgentDatasets, } from '../../lib/agents'; -import { detectAgentHarness, mapHarnessRows } from '../../lib/agentHarnesses'; +import { detectAgentHarnesses, mapHarnessRows } from '../../lib/agentHarnesses'; import { buildAgentRuns } from '../../lib/agentRuns'; import { agentValueMatchesSearch } from '../../lib/agentSearch'; import { @@ -117,7 +117,7 @@ export function AgentsPage() { signal, ); return { - rows: mapHarnessRows(detectAgentHarness(fields), rows), + rows: mapHarnessRows(detectAgentHarnesses(fields), rows), limitReached: rows.length >= AGENT_RECORD_LIMIT, timeColumn, traces: info.logSourceFormats.includes('otel-traces'), @@ -377,7 +377,7 @@ export function AgentsPage() { the time range to explore them.

    )} - {result.data.rows.length >= AGENT_RECORD_LIMIT && ( + {result.data.limitReached && (

    The record limit was reached. Runs may have additional spans or events; narrow the time range to improve coverage. diff --git a/frontend/src/lib/agentHarnesses.test.ts b/frontend/src/lib/agentHarnesses.test.ts index b69d67d6a..78a1b62eb 100644 --- a/frontend/src/lib/agentHarnesses.test.ts +++ b/frontend/src/lib/agentHarnesses.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { - detectAgentHarness, + detectAgentHarnesses, mapHarnessRows, mappedHarness, recordedFields, @@ -106,12 +106,12 @@ const claudeInteraction = [ describe('coding-agent harness detection', () => { it('recognizes Codex and Claude Code trace schemas without GenAI agent fields', () => { const codexFields = ['span_name', 'service.name', 'codex.turn.token_usage.input_tokens']; - expect(detectAgentHarness(codexFields)).toBe('codex'); + expect(detectAgentHarnesses(codexFields)).toEqual(['codex']); expect(isAgentSchema(codexFields)).toBe(true); - expect(detectAgentHarness(['span_name', 'interaction.sequence'])).toBe('claude-code'); + expect(detectAgentHarnesses(['span_name', 'interaction.sequence'])).toEqual(['claude-code']); // Logs carry no span names and cannot be grouped into runs. - expect(detectAgentHarness(['codex.turn.token_usage.input_tokens'])).toBeUndefined(); - expect(detectAgentHarness(['span_name', 'model', 'tool_name'])).toBeUndefined(); + expect(detectAgentHarnesses(['codex.turn.token_usage.input_tokens'])).toEqual([]); + expect(detectAgentHarnesses(['span_name', 'model', 'tool_name'])).toEqual([]); }); it('seeds Codex runs from turns and loads only run spans from Codex services', () => { const fields = [ @@ -142,6 +142,53 @@ describe('coding-agent harness detection', () => { expect(sparse).not.toContain('tool_name'); expect(sparse).not.toContain('gen_ai.usage'); }); + it('keeps Codex spans that carry standard GenAI fields', () => { + const { sql } = buildAgentRunQuery( + 'codex', + [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'gen_ai.operation.name', + 'gen_ai.tool.name', + 'codex.turn.phase', + ], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ); + expect(sql).toContain( + `NOT (COALESCE("service.name", '') LIKE 'codex%' AND NOT (COALESCE("span_name", '') IN ('session_task.turn', 'try_run_sampling_request', 'receiving_stream') OR "gen_ai.operation.name" IS NOT NULL OR "gen_ai.tool.name" IS NOT NULL))`, + ); + // A collector-renamed tool attribute and a renamed turn span still form a run. + const tool = { ...codexTurn[4], 'gen_ai.tool.name': 'exec', tool_name: undefined }; + const turn = { + ...codexTurn[0], + span_name: 'session_task.agent_turn', + 'gen_ai.operation.name': 'invoke_agent', + }; + const [run] = buildAgentRuns(mapHarnessRows(['codex'], [turn, tool])); + expect(run.roots[0].kind).toBe('agent'); + expect(run.spans.map((entry) => [entry.kind, entry.name])).toContainEqual(['tool', 'exec']); + }); + it('recognizes and queries a dataset holding both Codex and Claude Code traces', () => { + const fields = [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'codex.turn.phase', + 'interaction.sequence', + ]; + expect(detectAgentHarnesses(fields)).toEqual(['codex', 'claude-code']); + const { sql } = buildAgentRunQuery('mixed', fields, { + startTime: '2026-10-13T00:00:00Z', + endTime: '2026-10-14T00:00:00Z', + }); + expect(sql).toContain( + `AND ("span_name" = 'session_task.turn' OR "span_name" = 'claude_code.interaction') AND`, + ); + expect(sql).toContain(`LIKE 'codex%'`); + }); it('keeps every Claude Code span and seeds runs from interactions or model calls', () => { const { sql } = buildAgentRunQuery( 'claude', @@ -164,7 +211,7 @@ describe('coding-agent harness detection', () => { describe('coding-agent harness mapping', () => { it('builds a Codex turn with its model calls, tool call and per-request usage', () => { - const rows = mapHarnessRows('codex', codexTurn); + const rows = mapHarnessRows(['codex'], codexTurn); const [run] = buildAgentRuns(rows); expect(run.name).toBe('Codex'); expect(run.status).toBe('success'); @@ -187,7 +234,7 @@ describe('coding-agent harness mapping', () => { }); it('shows recorded Codex fields unchanged and does not modify the query result', () => { const original = structuredClone(codexTurn); - const rows = mapHarnessRows('codex', codexTurn); + const rows = mapHarnessRows(['codex'], codexTurn); expect(codexTurn).toEqual(original); const tool = rows.find((row) => row.span_name === 'exec')!; expect(tool['gen_ai.tool.name']).toBe('exec'); @@ -197,15 +244,16 @@ describe('coding-agent harness mapping', () => { expect(recordedFields(original[0])).toBe(original[0]); }); it('never replaces recorded GenAI values', () => { - const [turn] = mapHarnessRows('codex', [ - { ...codexTurn[0], 'gen_ai.agent.name': 'Reviewer', 'gen_ai.usage.input_tokens': 7 }, - ]); + const [turn] = mapHarnessRows( + ['codex'], + [{ ...codexTurn[0], 'gen_ai.agent.name': 'Reviewer', 'gen_ai.usage.input_tokens': 7 }], + ); expect(turn['gen_ai.agent.name']).toBe('Reviewer'); expect(turn['gen_ai.usage.input_tokens']).toBe(7); }); - it('does not attach detached spans to one of several turns in a trace', () => { + it('does not attach detached spans to one of overlapping turns', () => { const second = { ...codexTurn[0], span_span_id: 'b131b912f2251684' }; - const rows = mapHarnessRows('codex', [...codexTurn, second]); + const rows = mapHarnessRows(['codex'], [...codexTurn, second]); expect(rows.find((row) => row.span_name === 'exec')!.span_parent_span_id).toBe( '174bdda1bf2f0d51', ); @@ -213,8 +261,132 @@ describe('coding-agent harness mapping', () => { 'A parent span is outside the captured sample or missing.', ); }); + it('leaves calls after the loaded turn detached', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('call1', 'sampling1', 'try_run_sampling_request', 50, 10), + codex('call2', 'sampling2', 'try_run_sampling_request', 150, 10), + codex('tool2', 'tool-parent', 'exec', 170, 10, { tool_name: 'exec' }), + ], + ); + expect(rows.map((row) => [row.span_span_id, row.span_parent_span_id])).toEqual([ + ['turn1', ''], + ['call1', 'turn1'], + ['call2', 'sampling2'], + ['tool2', 'tool-parent'], + ]); + expect(buildAgentRuns(rows)[0].warnings).toContain( + 'A parent span is outside the captured sample or missing.', + ); + }); + it('attaches calls to the containing turn in a multi-turn trace', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('turn2', 'req', 'session_task.turn', 200, 100), + codex('call1', 'sampling1', 'try_run_sampling_request', 50, 10), + codex('call2', 'sampling2', 'try_run_sampling_request', 250, 10), + codex('tool2', 'tool-parent', 'exec', 270, 10, { tool_name: 'exec' }), + ], + ); + expect(rows.slice(2).map((row) => row.span_parent_span_id)).toEqual([ + 'turn1', + 'turn2', + 'turn2', + ]); + const [run] = buildAgentRuns(rows); + expect(run.warnings).not.toContain('A parent span is outside the captured sample or missing.'); + expect(run.roots.map((root) => root.children.map((child) => child.spanId))).toEqual([ + ['call1'], + ['call2', 'tool2'], + ]); + }); + it('leaves calls detached when several turn ranges contain them', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('turn2', 'req', 'session_task.turn', 20, 50), + codex('call', 'sampling', 'try_run_sampling_request', 30, 10), + ], + ); + expect(rows[2].span_parent_span_id).toBe('sampling'); + }); + it.each([ + ['missing call start', {}, { span_start_time_unix_nano: undefined }], + ['missing turn start', { span_start_time_unix_nano: undefined }, {}], + ['unknown turn end', { span_end_time_unix_nano: undefined, span_duration_ns: undefined }, {}], + ['invalid turn end', { span_end_time_unix_nano: 'invalid' }, {}], + ['reversed turn range', { span_end_time_unix_nano: '2026-10-13T02:50:33Z' }, {}], + ['absent call epoch', {}, { span_start_time_unix_nano_epoch: 0 }], + ['absent turn epoch', { span_end_time_unix_nano_epoch: 0 }, {}], + ])('leaves calls detached with %s', (_, turnFields, callFields) => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn', 'req', 'session_task.turn', 0, 100, turnFields), + codex('call', 'sampling', 'try_run_sampling_request', 50, 10, callFields), + ], + ); + expect(rows[1].span_parent_span_id).toBe('sampling'); + }); + it.each([ + ['numeric nanoseconds', 1791859834000000000, 1791859834100000000, '1791859834050000000'], + [ + 'UTC timestamp text', + '2026-10-13 02:50:34.000', + '2026-10-13 02:50:34.100', + '2026-10-13T02:50:34.050Z', + ], + ['duration without an end', '2026-10-13T02:50:34Z', undefined, '2026-10-13T02:50:34.050Z'], + ])('attaches calls using %s', (_, start, end, callStart) => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn', 'req', 'session_task.turn', 0, 100, { + span_start_time_unix_nano: start, + span_end_time_unix_nano: end, + }), + codex('call', 'sampling', 'try_run_sampling_request', 50, 10, { + span_start_time_unix_nano: callStart, + }), + ], + ); + expect(rows[1].span_parent_span_id).toBe('turn'); + }); + it.each(['fallback', 'both'])( + 'rewrites %s parent fields and preserves recorded parents', + (fields) => { + const turn = codex('turn', 'req', 'session_task.turn', 0, 100, { parent_span_id: 'req' }); + const tool = codex('tool', 'tool-parent', 'exec', 50, 10, { + parent_span_id: 'tool-parent', + tool_name: 'exec', + }); + if (fields === 'fallback') { + delete turn.span_parent_span_id; + delete tool.span_parent_span_id; + } + const rows = mapHarnessRows(['codex'], [turn, tool]); + expect(rows[0].parent_span_id).toBe(''); + expect(rows[1].parent_span_id).toBe('turn'); + if (fields === 'both') { + expect(rows[0].span_parent_span_id).toBe(''); + expect(rows[1].span_parent_span_id).toBe('turn'); + } + expect(recordedFields(rows[0])).toEqual(turn); + expect(recordedFields(rows[0]).parent_span_id).toBe('req'); + expect(recordedFields(rows[1]).parent_span_id).toBe('tool-parent'); + const [run] = buildAgentRuns(rows); + expect(run.status).toBe('success'); + expect(run.warnings).toEqual([]); + expect(run.roots[0].children[0].spanId).toBe('tool'); + }, + ); it('builds a Claude Code interaction with cache-inclusive input usage and tool names', () => { - const [run] = buildAgentRuns(mapHarnessRows('claude-code', claudeInteraction)); + const [run] = buildAgentRuns(mapHarnessRows(['claude-code'], claudeInteraction)); expect(run.name).toBe('Claude Code'); expect(run.status).toBe('success'); expect(run.models).toEqual(['claude-haiku-4-5']); @@ -229,14 +401,108 @@ describe('coding-agent harness mapping', () => { expect(run.roots[0].children[1].children[0].kind).toBe('span'); }); it('keeps absent Claude Code usage unknown', () => { - const [request] = mapHarnessRows('claude-code', [ - claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, claudeRequest), - ]); + const [request] = mapHarnessRows( + ['claude-code'], + [claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, claudeRequest)], + ); expect(request['gen_ai.usage.input_tokens']).toBeUndefined(); expect(request['gen_ai.usage.output_tokens']).toBeUndefined(); expect(buildAgentRuns([request])[0].inputTokens).toBeUndefined(); }); + it('maps each harness in a mixed dataset and leaves other services to themselves', () => { + const stray = claude('9a9a9a9a9a9a9a9a', '4b1b0ceaf03121e7', 'handle_responses', 1500, 10, { + tool_name: 'Read', + }); + const rows = mapHarnessRows( + ['codex', 'claude-code'], + [...codexTurn, ...claudeInteraction, stray], + ); + const runs = buildAgentRuns(rows); + const codexRun = runs.find((run) => run.traceId === codexTrace)!; + expect(codexRun.name).toBe('Codex'); + expect([codexRun.inputTokens, codexRun.outputTokens]).toEqual([32558, 38]); + expect(codexRun.roots[0].children.map((child) => child.kind)).toEqual([ + 'model', + 'tool', + 'model', + ]); + const claudeRun = runs.find((run) => run.traceId === claudeTrace)!; + expect(claudeRun.name).toBe('Claude Code'); + expect(claudeRun.roots[0].kind).toBe('agent'); + const [request, tool] = claudeRun.roots[0].children; + expect([request.inputTokens, request.outputTokens]).toEqual([21202, 102]); + expect([tool.kind, tool.toolName]).toEqual(['tool', 'Bash']); + // Only Codex services lose their stream spans or have tool_name mapped. + const kept = rows.find((row) => row.span_span_id === '9a9a9a9a9a9a9a9a')!; + expect(kept['gen_ai.tool.name']).toBeUndefined(); + expect(kept.span_parent_span_id).toBe('4b1b0ceaf03121e7'); + expect(mappedHarness(kept)).toBeUndefined(); + expect(mappedHarness(rows.find((row) => row.span_name === 'exec')!)).toBe('codex'); + expect(mappedHarness(rows.find((row) => row.span_name === 'claude_code.tool')!)).toBe( + 'claude-code', + ); + }); + it('marks only rows that something was mapped onto', () => { + const generic = codex('7777777777777777', 'a131b912f2251684', 'chat gpt-6-luna', 200, 50, { + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.model': 'gpt-6-luna', + }); + const rows = mapHarnessRows(['codex'], [...codexTurn, generic]); + const chat = rows.find((row) => row.span_span_id === '7777777777777777')!; + expect(mappedHarness(chat)).toBeUndefined(); + expect(recordedFields(chat)).toBe(generic); + expect(mappedHarness(rows[0])).toBe('codex'); + const claudeRows = mapHarnessRows(['claude-code'], claudeInteraction); + const execution = claudeRows.find((row) => row.span_name === 'claude_code.tool.execution')!; + expect(mappedHarness(execution)).toBeUndefined(); + expect(recordedFields(execution)).toBe(claudeInteraction[3]); + expect(mappedHarness(claudeRows[0])).toBe('claude-code'); + }); + it('reports Claude Code input usage only when input_tokens and present cache counts parse', () => { + const input = (usage: LogRecord) => + mapHarnessRows( + ['claude-code'], + [ + claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, { + ...claudeRequest, + ...usage, + }), + ], + )[0]['gen_ai.usage.input_tokens']; + expect(input({ input_tokens: '9' })).toBe(9); + expect(input({ input_tokens: '9', cache_read_tokens: '100' })).toBe(109); + expect(input({ input_tokens: '9', cache_read_tokens: '', cache_creation_tokens: 5 })).toBe(14); + // Cache counts alone are not the request's input usage. + expect(input({ cache_read_tokens: '5000', cache_creation_tokens: '20' })).toBeUndefined(); + for (const value of ['12.5', '-1', 'bad']) + expect(input({ input_tokens: value, cache_read_tokens: '5000' })).toBeUndefined(); + // A malformed cache count would understate the total, so input stays unknown. + expect(input({ input_tokens: '9', cache_creation_tokens: '-1' })).toBeUndefined(); + + const [run] = buildAgentRuns( + mapHarnessRows( + ['claude-code'], + [ + claudeInteraction[0], + claudeInteraction[1], + { ...claudeInteraction[4], input_tokens: '8.5' }, + ], + ), + ); + expect([run.inputTokens, run.outputTokens]).toEqual([9 + 12598 + 8595, 154]); + expect(run.warnings).toContain( + 'Token usage is partial because some model calls do not report usage.', + ); + }); + it('treats rows without a service name as Codex in a Codex-only dataset', () => { + const rows = mapHarnessRows( + ['codex'], + codexTurn.map(({ 'service.name': _, ...row }) => row), + ); + expect(rows.map((row) => row.span_name)).not.toContain('receiving_stream'); + expect(buildAgentRuns(rows)[0].name).toBe('Codex'); + }); it('leaves rows untouched when no harness is detected', () => { - expect(mapHarnessRows(undefined, codexTurn)).toBe(codexTurn); + expect(mapHarnessRows([], codexTurn)).toBe(codexTurn); }); }); diff --git a/frontend/src/lib/agentHarnesses.ts b/frontend/src/lib/agentHarnesses.ts index 4877f1b08..c0c88ce4f 100644 --- a/frontend/src/lib/agentHarnesses.ts +++ b/frontend/src/lib/agentHarnesses.ts @@ -1,3 +1,4 @@ +import { spanTiming } from './agentRuns'; import { quoteIdentifier, quoteLiteral } from './query'; import type { LogRecord } from './types'; @@ -34,9 +35,10 @@ const CODEX_STREAM = 'receiving_stream'; const CODEX_RESPONSE = 'handle_responses'; const CODEX_SPANS = [AGENT_HARNESSES.codex.runSpan, CODEX_MODEL_CALL, CODEX_STREAM]; -export function detectAgentHarness(fields: string[]): AgentHarness | undefined { - if (!fields.includes('span_name')) return; - return (Object.keys(AGENT_HARNESSES) as AgentHarness[]).find((harness) => +/** The harnesses whose telemetry a dataset holds; one dataset may hold several. */ +export function detectAgentHarnesses(fields: string[]): AgentHarness[] { + if (!fields.includes('span_name')) return []; + return (Object.keys(AGENT_HARNESSES) as AgentHarness[]).filter((harness) => AGENT_HARNESSES[harness].signature.some((field) => fields.includes(field)), ); } @@ -46,8 +48,15 @@ export function harnessRunPredicate(harness: AgentHarness) { return `${quoteIdentifier('span_name')} = ${quoteLiteral(AGENT_HARNESSES[harness].runSpan)}`; } -/** SQL keeping the run-relevant spans of a harness; other services' rows are untouched. */ -export function harnessRowPredicate(harness: AgentHarness, fields: string[]) { +/** + * SQL keeping the run-relevant spans of a harness; other services' rows are untouched. + * Spans carrying any of `agentFields` are always kept. + */ +export function harnessRowPredicate( + harness: AgentHarness, + fields: string[], + agentFields: readonly string[] = [], +) { if (harness !== 'codex' || !fields.includes('service.name')) return; const name = `COALESCE(${quoteIdentifier('span_name')}, '')`; const kept = [`${name} IN (${CODEX_SPANS.map(quoteLiteral).join(', ')})`]; @@ -55,11 +64,13 @@ export function harnessRowPredicate(harness: AgentHarness, fields: string[]) { kept.push( `(${name} = ${quoteLiteral(CODEX_RESPONSE)} AND ${quoteIdentifier('gen_ai.usage.input_tokens')} IS NOT NULL)`, ); - if (fields.includes('tool_name')) kept.push(`${quoteIdentifier('tool_name')} IS NOT NULL`); + for (const field of ['tool_name', ...agentFields]) + if (fields.includes(field)) kept.push(`${quoteIdentifier(field)} IS NOT NULL`); return `NOT (COALESCE(${quoteIdentifier('service.name')}, '') LIKE 'codex%' AND NOT (${kept.join(' OR ')}))`; } -const recorded = new WeakMap(); +const recorded = new WeakMap(); +const mapped = new WeakSet(); /** The fields a record was stored with, before any harness mapping. */ export function recordedFields(row: LogRecord): LogRecord { @@ -67,14 +78,20 @@ export function recordedFields(row: LogRecord): LogRecord { } /** The harness whose native names were mapped onto this record, if any. */ export function mappedHarness(row: LogRecord): AgentHarness | undefined { - return recorded.get(row)?.harness; + return mapped.has(row) ? recorded.get(row)?.harness : undefined; +} +/** Note that a field was derived for, or the parent rewritten on, a copied record. */ +function markMapped(row: LogRecord) { + if (recorded.has(row)) mapped.add(row); } function present(value: unknown) { return value !== undefined && value !== null && value !== ''; } function assign(row: LogRecord, field: string, value: unknown) { - if (present(value) && !present(row[field])) row[field] = value; + if (!present(value) || present(row[field])) return; + row[field] = value; + markMapped(row); } function count(value: unknown) { if (!present(value)) return; @@ -90,6 +107,12 @@ function id(row: LogRecord, field: string) { const traceOf = (row: LogRecord) => id(row, 'span_trace_id') ?? id(row, 'trace_id'); const spanOf = (row: LogRecord) => id(row, 'span_span_id') ?? id(row, 'span_id'); const parentOf = (row: LogRecord) => id(row, 'span_parent_span_id') ?? id(row, 'parent_span_id'); +// Rewrite whichever parent fields the record has, since readers fall back between them. +function setParent(row: LogRecord, value: string) { + const fields = ['span_parent_span_id', 'parent_span_id'].filter((field) => field in row); + for (const field of fields.length ? fields : ['span_parent_span_id']) row[field] = value; + markMapped(row); +} function mapClaudeCode(row: LogRecord) { switch (row.span_name) { @@ -100,11 +123,12 @@ function mapClaudeCode(row: LogRecord) { case 'claude_code.llm_request': { assign(row, 'gen_ai.operation.name', 'chat'); assign(row, 'gen_ai.request.model', row.model); - // Anthropic reports cache reads and writes separately from input_tokens. - const input = ['input_tokens', 'cache_read_tokens', 'cache_creation_tokens'] - .map((field) => count(row[field])) - .filter((value): value is number => value !== undefined); - if (input.length) + // Anthropic reports cache reads and writes separately from input_tokens, which + // must be reported. An absent cache count is zero; a malformed one leaves input unknown. + const input = ['input_tokens', 'cache_read_tokens', 'cache_creation_tokens'].map( + (field, index) => (index && !present(row[field]) ? 0 : count(row[field])), + ); + if (input.every((value): value is number => value !== undefined)) assign( row, 'gen_ai.usage.input_tokens', @@ -144,13 +168,14 @@ function mapCodex(rows: LogRecord[]) { for (const field of ['gen_ai.usage.input_tokens', 'gen_ai.usage.output_tokens']) assign(entry, field, row[field]); } - const runs = new Map(); + const runs = new Map(); for (const row of rows) { const trace = traceOf(row); const span = spanOf(row); if (row.span_name !== AGENT_HARNESSES.codex.runSpan || !trace || !span) continue; - // A trace with several turns cannot attribute detached spans to one of them. - runs.set(trace, runs.has(trace) && runs.get(trace) !== span ? '' : span); + const { start, durationMs } = spanTiming(row); + if (start === undefined || durationMs === undefined) continue; + runs.set(trace, [...(runs.get(trace) ?? []), { span, start, end: start + durationMs }]); } const kept = rows.filter( (row) => row.span_name !== CODEX_STREAM && row.span_name !== CODEX_RESPONSE, @@ -175,21 +200,41 @@ function mapCodex(rows: LogRecord[]) { // and its model and tool calls directly to the turn. const trace = traceOf(row); if (!trace || !parentOf(row) || parent(row)?.some((entry) => keptSet.has(entry))) continue; - if (row.span_name === AGENT_HARNESSES.codex.runSpan) row.span_parent_span_id = ''; - else if (runs.get(trace)) row.span_parent_span_id = runs.get(trace); + if (row.span_name === AGENT_HARNESSES.codex.runSpan) setParent(row, ''); + else { + // Only a call that started during exactly one loaded turn is known to belong to it; + // the calls of an unexported (still running) turn stay detached. + const { start } = spanTiming(row); + if (start === undefined) continue; + const matches = new Set( + (runs.get(trace) ?? []) + .filter((run) => start >= run.start && start <= run.end) + .map((run) => run.span), + ); + if (matches.size === 1) setParent(row, [...matches][0]); + } } return kept; } -/** Add GenAI fields derived from a harness's native telemetry. */ -export function mapHarnessRows(harness: AgentHarness | undefined, rows: LogRecord[]) { - if (!harness) return rows; - const mapped = rows.map((row) => { +/** Add GenAI fields derived from the native telemetry of the detected harnesses. */ +export function mapHarnessRows(harnesses: AgentHarness[], rows: LogRecord[]) { + if (!harnesses.length) return rows; + const codex = harnesses.includes('codex'); + const claude = harnesses.includes('claude-code'); + // Codex mapping and filtering apply to Codex services only, as in harnessRowPredicate. + // Without a service name, rows of a Codex-only dataset are taken to be Codex. + const isCodexRow = (row: LogRecord) => + codex && + (present(row['service.name']) ? String(row['service.name']).startsWith('codex') : !claude); + const copies = rows.map((row) => { const copy = { ...row }; + const harness = isCodexRow(row) ? 'codex' : claude ? 'claude-code' : undefined; recorded.set(copy, { row, harness }); return copy; }); - if (harness === 'codex') return mapCodex(mapped); - mapped.forEach(mapClaudeCode); - return mapped; + if (claude) copies.filter((row) => !isCodexRow(row)).forEach(mapClaudeCode); + if (!codex) return copies; + const kept = new Set(mapCodex(copies.filter(isCodexRow))); + return copies.filter((row) => !isCodexRow(row) || kept.has(row)); } diff --git a/frontend/src/lib/agentRuns.ts b/frontend/src/lib/agentRuns.ts index a397ea8c1..6e4f13569 100644 --- a/frontend/src/lib/agentRuns.ts +++ b/frontend/src/lib/agentRuns.ts @@ -87,7 +87,8 @@ function timestamp(row: LogRecord, field: string): number | undefined { const epoch = number(rawEpoch ?? raw); if (epoch !== undefined && epoch > 0) return epoch / 1e6; } -function timing(row: LogRecord, timeColumn: string) { +/** A span's start and end in epoch milliseconds, and its duration. */ +export function spanTiming(row: LogRecord, timeColumn = 'p_timestamp') { const start = timestamp(row, 'span_start_time_unix_nano'); const end = timestamp(row, 'span_end_time_unix_nano'); const hasEnd = @@ -312,7 +313,7 @@ export function buildAgentRuns( ), ), ) as LogRecord; - const { startTime, durationMs } = timing(row, timeColumn); + const { startTime, durationMs } = spanTiming(row, timeColumn); const errors = spanErrors(records); const operation = text(row, 'gen_ai.operation.name'); const model = text(row, 'gen_ai.response.model', 'gen_ai.request.model', 'llm.model_name'); @@ -343,7 +344,12 @@ export function buildAgentRuns( 'gen_ai.operation.name', ) ?? 'Unnamed span', kind, - status: status(records, durationMs, errors, timing(row, timeColumn).end !== undefined), + status: status( + records, + durationMs, + errors, + spanTiming(row, timeColumn).end !== undefined, + ), startTime, durationMs, model, @@ -464,11 +470,11 @@ export function buildAgentRuns( const inputTokens = tokenTotal('inputTokens'); const outputTokens = tokenTotal('outputTokens'); const starts = spans - .map((span) => timing(span.row, timeColumn).start) + .map((span) => spanTiming(span.row, timeColumn).start) .filter((value): value is number => value !== undefined); const ends = spans .map((span) => { - const t = timing(span.row, timeColumn); + const t = spanTiming(span.row, timeColumn); return t.start !== undefined && t.end !== undefined && t.durationMs !== undefined ? t.start + t.durationMs : undefined; diff --git a/frontend/src/lib/agents.ts b/frontend/src/lib/agents.ts index 9c2656a2b..2d9db9de1 100644 --- a/frontend/src/lib/agents.ts +++ b/frontend/src/lib/agents.ts @@ -1,4 +1,4 @@ -import { detectAgentHarness, harnessRowPredicate, harnessRunPredicate } from './agentHarnesses'; +import { detectAgentHarnesses, harnessRowPredicate, harnessRunPredicate } from './agentHarnesses'; import { ApiError } from './apiError'; import { apmTransportBounds } from './apm'; import { mapConcurrent } from './concurrency'; @@ -18,7 +18,9 @@ export const AGENT_FIELDS = [ export const AGENT_RECORD_LIMIT = 1000; export const AGENT_RUN_LIMIT = 100; export function isAgentSchema(fields: string[]) { - return AGENT_FIELDS.some((field) => fields.includes(field)) || !!detectAgentHarness(fields); + return ( + AGENT_FIELDS.some((field) => fields.includes(field)) || detectAgentHarnesses(fields).length > 0 + ); } export function agentsPath(dataset?: string) { return dataset ? `/agents/${encodeURIComponent(dataset)}` : '/agents'; @@ -60,14 +62,17 @@ export function buildAgentRunQuery( ); if (!fields.includes(timeColumn)) throw new Error(`Dataset schema is missing the time field ${timeColumn}.`); - const harness = detectAgentHarness(fields); + const harnesses = detectAgentHarnesses(fields); const agentPredicate = [ ...AGENT_FIELDS.filter((field) => fields.includes(field)).map( (field) => `${quoteIdentifier(field)} IS NOT NULL`, ), - ...(harness ? [harnessRunPredicate(harness)] : []), + ...harnesses.map(harnessRunPredicate), ].join(' OR '); - const rowPredicate = harness && harnessRowPredicate(harness, fields); + const rowPredicate = harnesses + .map((harness) => harnessRowPredicate(harness, fields, AGENT_FIELDS)) + .filter(Boolean) + .join(' AND '); const traceField = ['span_trace_id', 'trace_id'].find((field) => fields.includes(field)); const table = quoteIdentifier(dataset); const time = quoteIdentifier(timeColumn); From 4b9c1335e07ea424c1fabf4b98ca485bb97719f6 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:30:23 -0500 Subject: [PATCH 3/4] fix(agents): keep cross-service parents and descriptive Codex spans Map Codex rows in place among all loaded rows so a turn whose parent belongs to another service keeps that parent. Keep Codex stream and response spans that carry GenAI agent fields themselves, as the run query already does, and lift usage only from the ones that are dropped. Treat rows without a service name as Codex unless they are Claude Code spans, so mixed datasets without service names still map Codex turns. Co-Authored-By: Claude Opus 5.5 --- frontend/src/lib/agentHarnesses.test.ts | 40 +++++++++++++++++++ frontend/src/lib/agentHarnesses.ts | 52 ++++++++++++++++--------- frontend/src/lib/agents.ts | 20 ++++------ 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/frontend/src/lib/agentHarnesses.test.ts b/frontend/src/lib/agentHarnesses.test.ts index 78a1b62eb..10f2292a9 100644 --- a/frontend/src/lib/agentHarnesses.test.ts +++ b/frontend/src/lib/agentHarnesses.test.ts @@ -502,6 +502,46 @@ describe('coding-agent harness mapping', () => { expect(rows.map((row) => row.span_name)).not.toContain('receiving_stream'); expect(buildAgentRuns(rows)[0].name).toBe('Codex'); }); + it('keeps a Codex turn under a parent from another service', () => { + const outer = span(codexTrace, 'outer00000000001', '', 'gateway', -100, 5000, { + 'service.name': 'agent-gateway', + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.agent.name': 'Gateway', + }); + const turn = { ...codexTurn[0], span_parent_span_id: 'outer00000000001' }; + const rows = mapHarnessRows(['codex'], [outer, turn]); + expect(rows[1].span_parent_span_id).toBe('outer00000000001'); + expect(buildAgentRuns(rows)[0].roots).toHaveLength(1); + }); + it('keeps Codex stream and response spans that carry GenAI fields themselves', () => { + const fields = { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': 'exec' }; + const rows = mapHarnessRows( + ['codex'], + [ + codexTurn[0], + codex('5151515151515151', 'a131b912f2251684', 'receiving_stream', 200, 10, fields), + codex('5252525252525252', 'a131b912f2251684', 'handle_responses', 300, 10, { + ...fields, + 'gen_ai.usage.input_tokens': '5', + }), + ], + ); + const [run] = buildAgentRuns(rows); + expect(run.roots[0].children.map((child) => child.toolName)).toEqual(['exec', 'exec']); + // The kept response's usage is counted once, not also lifted onto a model call. + expect(run.inputTokens).toBe(5); + }); + it('recognizes Codex rows without a service name in a mixed dataset', () => { + const strip = ({ 'service.name': _, ...row }: LogRecord) => row; + const rows = mapHarnessRows( + ['codex', 'claude-code'], + [...codexTurn.map(strip), ...claudeInteraction.map(strip)], + ); + const runs = buildAgentRuns(rows); + expect(runs.find((run) => run.traceId === codexTrace)!.name).toBe('Codex'); + expect(runs.find((run) => run.traceId === codexTrace)!.inputTokens).toBe(32558); + expect(runs.find((run) => run.traceId === claudeTrace)!.name).toBe('Claude Code'); + }); it('leaves rows untouched when no harness is detected', () => { expect(mapHarnessRows([], codexTurn)).toBe(codexTurn); }); diff --git a/frontend/src/lib/agentHarnesses.ts b/frontend/src/lib/agentHarnesses.ts index c0c88ce4f..e6ddf031c 100644 --- a/frontend/src/lib/agentHarnesses.ts +++ b/frontend/src/lib/agentHarnesses.ts @@ -2,6 +2,17 @@ import { spanTiming } from './agentRuns'; import { quoteIdentifier, quoteLiteral } from './query'; import type { LogRecord } from './types'; +// Classic agents uses Enterprise /api/prism/v1/agent-observability APIs. +// Community can inspect the same flattened telemetry through its SQL endpoint. +export const AGENT_FIELDS = [ + 'gen_ai.agent.name', + 'gen_ai.agent.id', + 'gen_ai.operation.name', + 'gen_ai.request.model', + 'gen_ai.response.model', + 'gen_ai.tool.name', +] as const; + // Coding-agent CLIs export OTLP with their own attribute names. A harness profile // recognizes that telemetry and adds the GenAI span fields the Agents view reads. // Recorded values are never replaced, and absent values stay absent. @@ -49,14 +60,10 @@ export function harnessRunPredicate(harness: AgentHarness) { } /** - * SQL keeping the run-relevant spans of a harness; other services' rows are untouched. - * Spans carrying any of `agentFields` are always kept. + * SQL keeping the run-relevant spans of a harness; other services' rows, and spans + * already carrying GenAI fields, are untouched. */ -export function harnessRowPredicate( - harness: AgentHarness, - fields: string[], - agentFields: readonly string[] = [], -) { +export function harnessRowPredicate(harness: AgentHarness, fields: string[]) { if (harness !== 'codex' || !fields.includes('service.name')) return; const name = `COALESCE(${quoteIdentifier('span_name')}, '')`; const kept = [`${name} IN (${CODEX_SPANS.map(quoteLiteral).join(', ')})`]; @@ -64,7 +71,7 @@ export function harnessRowPredicate( kept.push( `(${name} = ${quoteLiteral(CODEX_RESPONSE)} AND ${quoteIdentifier('gen_ai.usage.input_tokens')} IS NOT NULL)`, ); - for (const field of ['tool_name', ...agentFields]) + for (const field of ['tool_name', ...AGENT_FIELDS]) if (fields.includes(field)) kept.push(`${quoteIdentifier(field)} IS NOT NULL`); return `NOT (COALESCE(${quoteIdentifier('service.name')}, '') LIKE 'codex%' AND NOT (${kept.join(' OR ')}))`; } @@ -145,7 +152,8 @@ function mapClaudeCode(row: LogRecord) { } } -function mapCodex(rows: LogRecord[]) { +// Rows of other services are kept as they are, but remain visible as parents. +function mapCodex(rows: LogRecord[], isCodex: (row: LogRecord) => boolean) { const spans = new Map(); for (const row of rows) { const trace = traceOf(row); @@ -158,9 +166,15 @@ function mapCodex(rows: LogRecord[]) { const span = parentOf(row); return trace && span ? spans.get(`${trace}/${span}`) : undefined; }; + // Stream and response spans only connect a model call to its usage, unless they + // already describe something themselves. + const plumbing = (row: LogRecord) => + isCodex(row) && + (row.span_name === CODEX_STREAM || row.span_name === CODEX_RESPONSE) && + !AGENT_FIELDS.some((field) => present(row[field])); // Usage is reported on the response handler two levels below its model call. for (const row of rows) { - if (row.span_name !== CODEX_RESPONSE) continue; + if (row.span_name !== CODEX_RESPONSE || !plumbing(row)) continue; const stream = parent(row)?.find((entry) => entry.span_name === CODEX_STREAM); const call = stream && parent(stream); if (call?.[0]?.span_name !== CODEX_MODEL_CALL) continue; @@ -172,16 +186,16 @@ function mapCodex(rows: LogRecord[]) { for (const row of rows) { const trace = traceOf(row); const span = spanOf(row); - if (row.span_name !== AGENT_HARNESSES.codex.runSpan || !trace || !span) continue; + if (row.span_name !== AGENT_HARNESSES.codex.runSpan || !isCodex(row) || !trace || !span) + continue; const { start, durationMs } = spanTiming(row); if (start === undefined || durationMs === undefined) continue; runs.set(trace, [...(runs.get(trace) ?? []), { span, start, end: start + durationMs }]); } - const kept = rows.filter( - (row) => row.span_name !== CODEX_STREAM && row.span_name !== CODEX_RESPONSE, - ); + const kept = rows.filter((row) => !plumbing(row)); const keptSet = new Set(kept); for (const row of kept) { + if (!isCodex(row)) continue; if (row.span_name === AGENT_HARNESSES.codex.runSpan) { assign(row, 'gen_ai.agent.name', AGENT_HARNESSES.codex.label); assign(row, 'gen_ai.operation.name', 'invoke_agent'); @@ -223,10 +237,12 @@ export function mapHarnessRows(harnesses: AgentHarness[], rows: LogRecord[]) { const codex = harnesses.includes('codex'); const claude = harnesses.includes('claude-code'); // Codex mapping and filtering apply to Codex services only, as in harnessRowPredicate. - // Without a service name, rows of a Codex-only dataset are taken to be Codex. + // Without a service name, any row that is not a Claude Code span is taken to be Codex. const isCodexRow = (row: LogRecord) => codex && - (present(row['service.name']) ? String(row['service.name']).startsWith('codex') : !claude); + (present(row['service.name']) + ? String(row['service.name']).startsWith('codex') + : !String(row.span_name ?? '').startsWith('claude_code.')); const copies = rows.map((row) => { const copy = { ...row }; const harness = isCodexRow(row) ? 'codex' : claude ? 'claude-code' : undefined; @@ -234,7 +250,5 @@ export function mapHarnessRows(harnesses: AgentHarness[], rows: LogRecord[]) { return copy; }); if (claude) copies.filter((row) => !isCodexRow(row)).forEach(mapClaudeCode); - if (!codex) return copies; - const kept = new Set(mapCodex(copies.filter(isCodexRow))); - return copies.filter((row) => !isCodexRow(row) || kept.has(row)); + return codex ? mapCodex(copies, isCodexRow) : copies; } diff --git a/frontend/src/lib/agents.ts b/frontend/src/lib/agents.ts index 2d9db9de1..2c88af507 100644 --- a/frontend/src/lib/agents.ts +++ b/frontend/src/lib/agents.ts @@ -1,20 +1,16 @@ -import { detectAgentHarnesses, harnessRowPredicate, harnessRunPredicate } from './agentHarnesses'; +import { + AGENT_FIELDS, + detectAgentHarnesses, + harnessRowPredicate, + harnessRunPredicate, +} from './agentHarnesses'; import { ApiError } from './apiError'; import { apmTransportBounds } from './apm'; import { mapConcurrent } from './concurrency'; import { paramsRange, quoteIdentifier, quoteLiteral, timeBounds } from './query'; import type { LogRecord, ParseableClient, QueryRequest, TimeRange } from './types'; -// Classic agents uses Enterprise /api/prism/v1/agent-observability APIs. -// Community can inspect the same flattened telemetry through its SQL endpoint. -export const AGENT_FIELDS = [ - 'gen_ai.agent.name', - 'gen_ai.agent.id', - 'gen_ai.operation.name', - 'gen_ai.request.model', - 'gen_ai.response.model', - 'gen_ai.tool.name', -] as const; +export { AGENT_FIELDS }; export const AGENT_RECORD_LIMIT = 1000; export const AGENT_RUN_LIMIT = 100; export function isAgentSchema(fields: string[]) { @@ -70,7 +66,7 @@ export function buildAgentRunQuery( ...harnesses.map(harnessRunPredicate), ].join(' OR '); const rowPredicate = harnesses - .map((harness) => harnessRowPredicate(harness, fields, AGENT_FIELDS)) + .map((harness) => harnessRowPredicate(harness, fields)) .filter(Boolean) .join(' AND '); const traceField = ['span_trace_id', 'trace_id'].find((field) => fields.includes(field)); From 3ace7e5d74500c67fc9f02637151775ed67a931b Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:34:02 -0500 Subject: [PATCH 4/4] fix(agents): attach kept Codex spans to their nearest kept ancestor A Codex response span kept for its own GenAI fields lost its dropped stream parent and was attached to the turn beside its model call, so its usage was counted twice. Walk dropped parents up to the nearest kept ancestor first. In mixed datasets, take a row without a service name to be Codex only if it has a Codex span name or a Codex tool call, so generic spans keep their recorded parent. Co-Authored-By: Claude Opus 5.5 --- frontend/src/lib/agentHarnesses.test.ts | 32 +++++++++++++++++++++++++ frontend/src/lib/agentHarnesses.ts | 22 ++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/agentHarnesses.test.ts b/frontend/src/lib/agentHarnesses.test.ts index 10f2292a9..81ded9bb7 100644 --- a/frontend/src/lib/agentHarnesses.test.ts +++ b/frontend/src/lib/agentHarnesses.test.ts @@ -542,6 +542,38 @@ describe('coding-agent harness mapping', () => { expect(runs.find((run) => run.traceId === codexTrace)!.inputTokens).toBe(32558); expect(runs.find((run) => run.traceId === claudeTrace)!.name).toBe('Claude Code'); }); + it('attaches a kept Codex response under its model call, not beside it', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codexTurn[0], + codex('6161616161616161', '6e055c16cf2ba610', 'try_run_sampling_request', 100, 500, { + model: 'gpt-6-luna', + 'gen_ai.usage.input_tokens': '5', + }), + codex('6262626262626262', '6161616161616161', 'receiving_stream', 102, 400), + codex('6363636363636363', '6262626262626262', 'handle_responses', 450, 10, { + 'gen_ai.operation.name': 'chat', + 'gen_ai.usage.input_tokens': '5', + }), + ], + ); + expect(rows.find((row) => row.span_span_id === '6363636363636363')!.span_parent_span_id).toBe( + '6161616161616161', + ); + expect(buildAgentRuns(rows)[0].inputTokens).toBe(5); + }); + it('leaves generic spans without a service name alone in a mixed dataset', () => { + const { 'service.name': _, ...turn } = codexTurn[0]; + const generic = span(codexTrace, '7171717171717171', 'gone', 'chat remote-model', 200, 10, { + 'gen_ai.operation.name': 'chat', + }); + const rows = mapHarnessRows(['codex', 'claude-code'], [turn, generic]); + const chat = rows.find((row) => row.span_span_id === '7171717171717171')!; + expect(chat.span_parent_span_id).toBe('gone'); + expect(mappedHarness(chat)).toBeUndefined(); + expect(mappedHarness(rows[0])).toBe('codex'); + }); it('leaves rows untouched when no harness is detected', () => { expect(mapHarnessRows([], codexTurn)).toBe(codexTurn); }); diff --git a/frontend/src/lib/agentHarnesses.ts b/frontend/src/lib/agentHarnesses.ts index e6ddf031c..72afdd34a 100644 --- a/frontend/src/lib/agentHarnesses.ts +++ b/frontend/src/lib/agentHarnesses.ts @@ -194,6 +194,15 @@ function mapCodex(rows: LogRecord[], isCodex: (row: LogRecord) => boolean) { } const kept = rows.filter((row) => !plumbing(row)); const keptSet = new Set(kept); + const keptAncestor = (row: LogRecord) => { + let ancestors = parent(row); + // Bounded, in case recorded parents form a cycle. + for (let hops = 0; ancestors?.length && hops < rows.length; hops++) { + const found = ancestors.find((entry) => keptSet.has(entry)); + if (found) return found; + ancestors = parent(ancestors[0]); + } + }; for (const row of kept) { if (!isCodex(row)) continue; if (row.span_name === AGENT_HARNESSES.codex.runSpan) { @@ -214,7 +223,10 @@ function mapCodex(rows: LogRecord[], isCodex: (row: LogRecord) => boolean) { // and its model and tool calls directly to the turn. const trace = traceOf(row); if (!trace || !parentOf(row) || parent(row)?.some((entry) => keptSet.has(entry))) continue; - if (row.span_name === AGENT_HARNESSES.codex.runSpan) setParent(row, ''); + // A loaded but dropped parent leads to the nearest kept ancestor. + const nearest = keptAncestor(row); + if (nearest) setParent(row, spanOf(nearest)!); + else if (row.span_name === AGENT_HARNESSES.codex.runSpan) setParent(row, ''); else { // Only a call that started during exactly one loaded turn is known to belong to it; // the calls of an unexported (still running) turn stay detached. @@ -237,12 +249,16 @@ export function mapHarnessRows(harnesses: AgentHarness[], rows: LogRecord[]) { const codex = harnesses.includes('codex'); const claude = harnesses.includes('claude-code'); // Codex mapping and filtering apply to Codex services only, as in harnessRowPredicate. - // Without a service name, any row that is not a Claude Code span is taken to be Codex. + // Without a service name, a Codex-only dataset is all Codex; a mixed one needs a + // Codex span name or a Codex tool call (Claude Code tools carry tool_use_id instead). const isCodexRow = (row: LogRecord) => codex && (present(row['service.name']) ? String(row['service.name']).startsWith('codex') - : !String(row.span_name ?? '').startsWith('claude_code.')); + : !claude || + CODEX_SPANS.includes(String(row.span_name)) || + row.span_name === CODEX_RESPONSE || + (present(row.tool_name) && present(row.call_id))); const copies = rows.map((row) => { const copy = { ...row }; const harness = isCodexRow(row) ? 'codex' : claude ? 'claude-code' : undefined;