diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index d8878ac4a..ab2e33fda 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -294,7 +294,7 @@ const agentFields = [...new Set(agentRows.flatMap((row) => Object.keys(row)))]; type Captured = { query: string; startTime: string; endTime: string }; async function serveAgents( page: Page, - options: { status?: number; rows?: LogRecord[]; datasets?: string[] } = {}, + options: { status?: number; rows?: LogRecord[]; datasets?: string[]; fields?: string[] } = {}, ) { const captured: Captured[] = []; const state = { status: options.status ?? 200, rows: options.rows ?? agentRows }; @@ -305,7 +305,9 @@ async function serveAgents( json: (options.datasets ?? ['agent_telemetry']).map((name) => ({ name })), }); if (path.endsWith('/schema')) - return route.fulfill({ json: { fields: agentFields.map((name) => ({ name })) } }); + return route.fulfill({ + json: { fields: (options.fields ?? agentFields).map((name) => ({ name })) }, + }); if (path.endsWith('/info')) return route.fulfill({ json: { telemetryType: 'traces', logSource: [{ log_source_format: 'otel-traces' }] }, @@ -723,3 +725,140 @@ for (const trace of ['trace-with-dashes', '']) { await expect(details.getByRole('tab', { name: 'Span tree (1)' })).toBeVisible(); }); } + +test('native Codex traces become runs with mapped model calls, tools and usage', async ({ + page, +}) => { + // Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the + // response handler below each model call, and intermediate spans filtered out. + const trace = 'e5'.repeat(16); + const codex = (span: Omit) => + spanRows({ + ...span, + trace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); + const rows = [ + codex({ + id: 'e500000000000001', + parent: 'e5000000000000aa', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.output_tokens': '38', + }, + }), + codex({ + id: 'e500000000000002', + parent: 'e5000000000000bb', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 1600, + attributes: { model: 'gpt-6-luna' }, + }), + codex({ + id: 'e500000000000003', + parent: 'e500000000000002', + name: 'receiving_stream', + start: base + 1110, + duration: 1580, + }), + codex({ + id: 'e500000000000004', + parent: 'e500000000000003', + name: 'handle_responses', + start: base + 2600, + duration: 40, + attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, + }), + codex({ + id: 'e500000000000005', + parent: 'e5000000000000cc', + name: 'exec', + start: base + 2700, + duration: 137, + attributes: { tool_name: 'exec', call_id: 'call_codex' }, + }), + ].flat(); + const { captured } = await serveAgents(page, { + rows, + datasets: ['codex-traces'], + fields: [...new Set([...rows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens'])], + }); + await page.goto(`/agents?${range}`); + await page.getByLabel('Agents dataset').selectOption('codex-traces'); + const row = runRow(page, 'Codex'); + await expect(cell(row, 'Status')).toHaveText('Successful'); + await expect(cell(row, 'Model')).toHaveText('gpt-6-luna'); + await expect(cell(row, 'Tokens')).toContainText('32,558 in'); + await expect(cell(row, 'Tokens')).toContainText('38 out'); + await expect(cell(row, 'Spans')).toHaveText('3'); + expect(captured[0].query).toContain(`"span_name" = 'session_task.turn'`); + expect(captured[0].query).toContain(`LIKE 'codex%'`); + + const details = await openRun(page, 'Codex'); + const children = details + .getByRole('list', { name: 'Run span tree' }) + .getByRole('list', { name: 'Child spans' }); + await expect(children.getByRole('button')).toHaveCount(2); + await children.getByRole('button', { name: /exec/ }).click(); + const span = details.getByRole('region', { name: 'Selected span' }); + await expect(span).toContainText('Mapped from native Codex telemetry'); + // Facts and raw records show what Codex recorded, not the attached tree position. + await expect(fact(span, 'Parent span ID')).toHaveText('e5000000000000cc'); + await span.getByText('Recorded fields (1 record)').click(); + await expect(span).toContainText('"tool_name": "exec"'); + await expect(span).not.toContainText('gen_ai.tool.name'); +}); + +test('discloses the record limit from the queried rows when Codex mapping drops some', async ({ + page, +}) => { + const trace = 'f6'.repeat(16); + const codex = (span: Omit) => + spanRows({ + ...span, + trace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); + const rows = [ + codex({ + id: 'f600000000000001', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { model: 'gpt-6-luna', 'codex.turn.token_usage.input_tokens': '10' }, + }), + codex({ + id: 'f600000000000002', + parent: 'f600000000000001', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 3000, + attributes: { model: 'gpt-6-luna' }, + }), + // Stream spans count toward the record limit but are not shown as run spans. + ...Array.from({ length: 998 }, (_, index) => + codex({ + id: (index + 1).toString(16).padStart(16, '0'), + parent: 'f600000000000002', + name: 'receiving_stream', + start: base + 1200, + duration: 10, + }), + ), + ].flat(); + await serveAgents(page, { + rows, + datasets: ['codex-traces'], + fields: [...new Set(rows.flatMap((row) => Object.keys(row)))], + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect(cell(runRow(page, 'Codex'), 'Spans')).toHaveText('2'); + await expect(page.getByRole('status').filter({ hasText: 'record limit' })).toContainText( + 'narrow the time range', + ); +}); diff --git a/frontend/src/features/agents/AgentRunDetails.tsx b/frontend/src/features/agents/AgentRunDetails.tsx index 2d78b0624..9d4e99640 100644 --- a/frontend/src/features/agents/AgentRunDetails.tsx +++ b/frontend/src/features/agents/AgentRunDetails.tsx @@ -1,6 +1,7 @@ import { useState } from 'react'; import { Bot, GitBranch, MessageSquare, Wrench } from 'lucide-react'; import { Badge, Button, Tabs } from '../../components/ui'; +import { AGENT_HARNESSES, mappedHarness, recordedFields } from '../../lib/agentHarnesses'; import type { AgentRun, AgentRunStatus, AgentSpan } from '../../lib/agentRuns'; export function formatAgentDuration(value?: number) { @@ -57,6 +58,11 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { const [tab, setTab] = useState('tree'); const [spanId, setSpanId] = useState(run.roots[0]?.id ?? run.spans[0]?.id); const selected = run.spans.find((span) => span.id === spanId) ?? run.spans[0]; + const selectedHarness = selected?.rows.map(mappedHarness).find(Boolean); + // Harness mapping can attach a span to its run when intermediate spans were not loaded. + const recordedParent = selected?.rows + .map((row) => recordedFields(row).span_parent_span_id ?? recordedFields(row).parent_span_id) + .find((value) => typeof value === 'string' && value.trim() && !/^0+$/.test(value.trim())); const models = run.spans.filter((span) => span.kind === 'model'); const tools = run.spans.filter((span) => span.kind === 'tool'); const errors = run.spans.filter((span) => span.errors.length || span.status === 'failed'); @@ -233,7 +239,11 @@ export function AgentRunDetails({ run }: { run: AgentRun }) {
Parent span ID
- {selected.parentSpanId ?? 'Not recorded'} + + {String( + (selectedHarness ? recordedParent : selected.parentSpanId) ?? 'Not recorded', + )} +
@@ -255,6 +265,12 @@ export function AgentRunDetails({ run }: { run: AgentRun }) {
+ {selectedHarness && ( +

+ Mapped from native {AGENT_HARNESSES[selectedHarness].label} telemetry. Recorded fields + are shown unchanged. +

+ )}
Recorded fields ({selected.rows.length}{' '} @@ -262,7 +278,7 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { {selected.rows.map((row, index) => ( 
-                {JSON.stringify(row, null, 2)}
+                {JSON.stringify(recordedFields(row), null, 2)}
))}
diff --git a/frontend/src/features/agents/AgentTelemetryGuide.tsx b/frontend/src/features/agents/AgentTelemetryGuide.tsx index a292535c4..be1db2445 100644 --- a/frontend/src/features/agents/AgentTelemetryGuide.tsx +++ b/frontend/src/features/agents/AgentTelemetryGuide.tsx @@ -75,24 +75,23 @@ export function AgentTelemetryGuide() {

3. Validate real telemetry

Confirm collector receipt and stored rows first, then run discovery, parent links, models, - tools and usage in Agents. Test successful, failed and interrupted work. Native CLI - compatibility has not been verified end to end; capture Codex’s actual trace schema from - your installed runtime. + tools and usage in Agents. Test successful, failed and interrupted work. Agents reads + native Claude Code and Codex trace fields directly, so their trace datasets need no field + mapping. Runs come from traces; a logs dataset is not listed on its own.

Mapping and validation checklist +

For other agents and frameworks:

  • Map agent and operation to gen_ai.agent.name and{' '} gen_ai.operation.name; models to gen_ai.request.model /{' '} - gen_ai.response.model; tools to gen_ai.tool.name (Claude:{' '} - tool_name). + gen_ai.response.model; tools to gen_ai.tool.name.
  • Map usage to gen_ai.usage.input_tokens and{' '} - gen_ai.usage.output_tokens. Claude input totals include{' '} - input_tokens + cache_read_tokens + cache_creation_tokens. Preserve absent - values as unknown. + gen_ai.usage.output_tokens, including cached input tokens in the input + total. Preserve absent values as unknown.
  • Validate 32-digit hexadecimal trace IDs, 16-digit hexadecimal span IDs, parent IDs, diff --git a/frontend/src/features/agents/AgentsPage.tsx b/frontend/src/features/agents/AgentsPage.tsx index 56bcbfd97..fa7c4abe9 100644 --- a/frontend/src/features/agents/AgentsPage.tsx +++ b/frontend/src/features/agents/AgentsPage.tsx @@ -24,6 +24,7 @@ import { buildAgentRunQuery, discoverAgentDatasets, } from '../../lib/agents'; +import { detectAgentHarnesses, mapHarnessRows } from '../../lib/agentHarnesses'; import { buildAgentRuns } from '../../lib/agentRuns'; import { agentValueMatchesSearch } from '../../lib/agentSearch'; import { @@ -101,6 +102,7 @@ export function AgentsPage() { rows: agentDemoRows(Date.parse(bounds.endTime)).filter( (row) => Date.parse(String(row.p_timestamp)) >= Date.parse(bounds.startTime), ), + limitReached: false, timeColumn: 'p_timestamp', traces: false, }; @@ -114,7 +116,12 @@ export function AgentsPage() { buildAgentRunQuery(dataset, fields, bounds, timeColumn), signal, ); - return { rows, timeColumn, traces: info.logSourceFormats.includes('otel-traces') }; + return { + rows: mapHarnessRows(detectAgentHarnesses(fields), rows), + limitReached: rows.length >= AGENT_RECORD_LIMIT, + timeColumn, + traces: info.logSourceFormats.includes('otel-traces'), + }; }, [client, mode, dataset, bounds], ), @@ -132,7 +139,7 @@ export function AgentsPage() { const runs = useMemo( () => buildAgentRuns(result.data?.rows ?? [], result.data?.timeColumn, { - limitReached: (result.data?.rows.length ?? 0) >= AGENT_RECORD_LIMIT, + limitReached: result.data?.limitReached, bounds, }), [result.data, bounds], @@ -370,7 +377,7 @@ export function AgentsPage() { the time range to explore them.

    )} - {result.data.rows.length >= AGENT_RECORD_LIMIT && ( + {result.data.limitReached && (

    The record limit was reached. Runs may have additional spans or events; narrow the time range to improve coverage. diff --git a/frontend/src/features/agents/agentTelemetrySetup.ts b/frontend/src/features/agents/agentTelemetrySetup.ts index 55619e7ae..94dee5d78 100644 --- a/frontend/src/features/agents/agentTelemetrySetup.ts +++ b/frontend/src/features/agents/agentTelemetrySetup.ts @@ -62,7 +62,7 @@ Inspect the installed runtime/version and its actual config location first. Merg ${config} -Use an already configured collector; configure its authenticated Parseable exporter separately with the destination trace dataset and X-P-Stream. Use separate collector logs and traces pipelines/exporters with distinct X-P-Stream values for their destination datasets. Keep credentials in the existing secret mechanism. Inspect real payloads from this installed runtime, map provider fields to the dashboard GenAI span contract, and preserve missing values as unknown. Claude input token totals include input_tokens + cache_read_tokens + cache_creation_tokens, only when those values are present. Map tool_name to gen_ai.tool.name. Codex trace schema must be captured and checked from this installed runtime. Prompt capture does not guarantee a full transcript. +Use an already configured collector; configure its authenticated Parseable exporter separately with the destination trace dataset and X-P-Stream. Use separate collector logs and traces pipelines/exporters with distinct X-P-Stream values for their destination datasets. Keep credentials in the existing secret mechanism. Parseable Agents reads native ${provider === 'claude' ? 'Claude Code' : 'Codex'} trace fields directly, so forward them unchanged rather than renaming them; runs come from the traces dataset. Prompt capture does not guarantee a full transcript. Validate transport first (collector receipt and stored rows), then dashboard compatibility: run discovery, trace/parent relationships, models, tools, usage, and any captured messages. Test successful, failed, and interrupted/incomplete work; missing telemetry remains unknown/incomplete. Do not invent content, tokens, costs, evaluations, or live state. A synthetic fixture alone does not prove native CLI compatibility. Report changed files, restart steps, observed payload mappings, evidence, limitations, and how to restore previous exporter settings.`; } diff --git a/frontend/src/lib/agentHarnesses.test.ts b/frontend/src/lib/agentHarnesses.test.ts new file mode 100644 index 000000000..81ded9bb7 --- /dev/null +++ b/frontend/src/lib/agentHarnesses.test.ts @@ -0,0 +1,580 @@ +import { describe, expect, it } from 'vitest'; +import { + detectAgentHarnesses, + mapHarnessRows, + mappedHarness, + recordedFields, +} from './agentHarnesses'; +import { buildAgentRunQuery, isAgentSchema } from './agents'; +import { buildAgentRuns } from './agentRuns'; +import type { LogRecord } from './types'; + +// Trimmed from spans exported by Codex CLI 0.162.1 (app server) and Claude Code +// 2.1.296 with CLAUDE_CODE_ENHANCED_TELEMETRY_BETA, as stored by Parseable. +function span( + trace: string, + id: string, + parent: string, + name: string, + start: number, + durationMs: number, + fields: LogRecord = {}, +): LogRecord { + const begin = Date.parse('2026-10-13T02:50:34Z') + start; + return { + p_timestamp: new Date(begin + durationMs).toISOString(), + span_trace_id: trace, + span_span_id: id, + span_parent_span_id: parent, + span_name: name, + span_start_time_unix_nano: new Date(begin).toISOString(), + span_end_time_unix_nano: new Date(begin + durationMs).toISOString(), + span_duration_ns: durationMs * 1e6, + span_status_code: 0, + span_status_description: 'UNSET', + ...fields, + }; +} +const codexTrace = '3eead3c4c7897ddb3613163ebdeaa6c0'; +const codex = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(codexTrace, id, parent, name, start, ms, { 'service.name': 'codex-app-server', ...f }); +const codexTurn = [ + codex('a131b912f2251684', '0d2585ff2dc6a775', 'session_task.turn', 0, 4027, { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.cached_input_tokens': '27136', + 'codex.turn.token_usage.output_tokens': '38', + }), + codex('f29e280320d84c8e', '6e055c16cf2ba610', 'try_run_sampling_request', 100, 1648, { + model: 'gpt-6-luna', + }), + codex('845411c03a89e0ff', 'f29e280320d84c8e', 'receiving_stream', 102, 1645), + codex('5e5901e207ece80e', '845411c03a89e0ff', 'handle_responses', 1700, 38, { + 'gen_ai.usage.input_tokens': '16251', + 'gen_ai.usage.output_tokens': '33', + 'gen_ai.usage.cache_read.input_tokens': '11008', + }), + codex('a9a8355c5b8e769e', '174bdda1bf2f0d51', 'exec', 1760, 137, { + tool_name: 'exec', + call_id: 'call_vRR2GrjSnDyX8FWz50ONlZqM', + }), + codex('ff4f6fbba143b1c0', '638f0e6a3947c618', 'try_run_sampling_request', 2900, 967, { + model: 'gpt-6-luna', + }), + codex('761175078dbf3f02', 'ff4f6fbba143b1c0', 'receiving_stream', 2902, 965), + codex('297376cdba4bc3cc', '761175078dbf3f02', 'handle_responses', 3800, 37, { + 'gen_ai.usage.input_tokens': '16307', + 'gen_ai.usage.output_tokens': '5', + }), +]; +const claudeTrace = '0e638bc7089fb020dfd801f6c58a5dc6'; +const claude = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(claudeTrace, id, parent, name, start, ms, { 'service.name': 'claude-code', ...f }); +const claudeRequest = { + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-haiku-4-5', + model: 'claude-haiku-4-5', + 'llm_request.context': 'interaction', +}; +const claudeInteraction = [ + claude('4b1b0ceaf03121e7', '', 'claude_code.interaction', 0, 3162, { + 'interaction.sequence': '1', + user_prompt: '', + }), + claude('06675bb5f58964d7', '4b1b0ceaf03121e7', 'claude_code.llm_request', 48, 1362, { + ...claudeRequest, + input_tokens: '9', + cache_read_tokens: '12598', + cache_creation_tokens: '8595', + output_tokens: '102', + }), + claude('cf906dbe161dcbc5', '4b1b0ceaf03121e7', 'claude_code.tool', 1391, 43, { + tool_name: 'Bash', + tool_use_id: 'toolu_017FNPpeGEpbuRnKoQNMyXgb', + 'gen_ai.tool.call.id': 'toolu_017FNPpeGEpbuRnKoQNMyXgb', + }), + claude('c3b6fd085798feb9', 'cf906dbe161dcbc5', 'claude_code.tool.execution', 1396, 38), + claude('8062decff7ff63cf', '4b1b0ceaf03121e7', 'claude_code.llm_request', 1448, 1707, { + ...claudeRequest, + input_tokens: '8', + cache_read_tokens: '21193', + cache_creation_tokens: '151', + output_tokens: '52', + }), +]; + +describe('coding-agent harness detection', () => { + it('recognizes Codex and Claude Code trace schemas without GenAI agent fields', () => { + const codexFields = ['span_name', 'service.name', 'codex.turn.token_usage.input_tokens']; + expect(detectAgentHarnesses(codexFields)).toEqual(['codex']); + expect(isAgentSchema(codexFields)).toBe(true); + expect(detectAgentHarnesses(['span_name', 'interaction.sequence'])).toEqual(['claude-code']); + // Logs carry no span names and cannot be grouped into runs. + expect(detectAgentHarnesses(['codex.turn.token_usage.input_tokens'])).toEqual([]); + expect(detectAgentHarnesses(['span_name', 'model', 'tool_name'])).toEqual([]); + }); + it('seeds Codex runs from turns and loads only run spans from Codex services', () => { + const fields = [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'tool_name', + 'gen_ai.usage.input_tokens', + 'codex.turn.token_usage.input_tokens', + ]; + const { sql } = buildAgentRunQuery('codex', fields, { + startTime: '2026-10-13T00:00:00Z', + endTime: '2026-10-14T00:00:00Z', + }); + expect(sql).toContain( + `AND ("span_name" = 'session_task.turn') AND "span_trace_id" IS NOT NULL`, + ); + expect(sql).toContain( + `AND NOT (COALESCE("service.name", '') LIKE 'codex%' AND NOT (COALESCE("span_name", '') IN ('session_task.turn', 'try_run_sampling_request', 'receiving_stream') OR (COALESCE("span_name", '') = 'handle_responses' AND "gen_ai.usage.input_tokens" IS NOT NULL) OR "tool_name" IS NOT NULL)) ORDER BY`, + ); + // Absent columns are never referenced. + const sparse = buildAgentRunQuery( + 'codex', + ['p_timestamp', 'span_trace_id', 'span_name', 'service.name', 'codex.turn.phase'], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ).sql; + expect(sparse).not.toContain('tool_name'); + expect(sparse).not.toContain('gen_ai.usage'); + }); + it('keeps Codex spans that carry standard GenAI fields', () => { + const { sql } = buildAgentRunQuery( + 'codex', + [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'gen_ai.operation.name', + 'gen_ai.tool.name', + 'codex.turn.phase', + ], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ); + expect(sql).toContain( + `NOT (COALESCE("service.name", '') LIKE 'codex%' AND NOT (COALESCE("span_name", '') IN ('session_task.turn', 'try_run_sampling_request', 'receiving_stream') OR "gen_ai.operation.name" IS NOT NULL OR "gen_ai.tool.name" IS NOT NULL))`, + ); + // A collector-renamed tool attribute and a renamed turn span still form a run. + const tool = { ...codexTurn[4], 'gen_ai.tool.name': 'exec', tool_name: undefined }; + const turn = { + ...codexTurn[0], + span_name: 'session_task.agent_turn', + 'gen_ai.operation.name': 'invoke_agent', + }; + const [run] = buildAgentRuns(mapHarnessRows(['codex'], [turn, tool])); + expect(run.roots[0].kind).toBe('agent'); + expect(run.spans.map((entry) => [entry.kind, entry.name])).toContainEqual(['tool', 'exec']); + }); + it('recognizes and queries a dataset holding both Codex and Claude Code traces', () => { + const fields = [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'codex.turn.phase', + 'interaction.sequence', + ]; + expect(detectAgentHarnesses(fields)).toEqual(['codex', 'claude-code']); + const { sql } = buildAgentRunQuery('mixed', fields, { + startTime: '2026-10-13T00:00:00Z', + endTime: '2026-10-14T00:00:00Z', + }); + expect(sql).toContain( + `AND ("span_name" = 'session_task.turn' OR "span_name" = 'claude_code.interaction') AND`, + ); + expect(sql).toContain(`LIKE 'codex%'`); + }); + it('keeps every Claude Code span and seeds runs from interactions or model calls', () => { + const { sql } = buildAgentRunQuery( + 'claude', + [ + 'p_timestamp', + 'span_trace_id', + 'span_name', + 'service.name', + 'gen_ai.request.model', + 'interaction.sequence', + ], + { startTime: '2026-10-13T00:00:00Z', endTime: '2026-10-14T00:00:00Z' }, + ); + expect(sql).toContain( + `("gen_ai.request.model" IS NOT NULL OR "span_name" = 'claude_code.interaction')`, + ); + expect(sql).not.toContain('COALESCE'); + }); +}); + +describe('coding-agent harness mapping', () => { + it('builds a Codex turn with its model calls, tool call and per-request usage', () => { + const rows = mapHarnessRows(['codex'], codexTurn); + const [run] = buildAgentRuns(rows); + expect(run.name).toBe('Codex'); + expect(run.status).toBe('success'); + expect(run.warnings).toEqual([]); + expect(run.models).toEqual(['gpt-6-luna']); + // Per-request usage sums to the turn's own total; the rollup is not added twice. + expect([run.inputTokens, run.outputTokens]).toEqual([32558, 38]); + expect(run.roots).toHaveLength(1); + expect(run.roots[0].kind).toBe('agent'); + expect( + run.roots[0].children.map((child) => [child.kind, child.name, child.inputTokens]), + ).toEqual([ + ['model', 'try_run_sampling_request', 16251], + ['tool', 'exec', undefined], + ['model', 'try_run_sampling_request', 16307], + ]); + expect(run.roots[0].children[1].toolName).toBe('exec'); + expect(run.roots[0].children[1].durationMs).toBe(137); + expect(run.spans.some((entry) => entry.name === 'handle_responses')).toBe(false); + }); + it('shows recorded Codex fields unchanged and does not modify the query result', () => { + const original = structuredClone(codexTurn); + const rows = mapHarnessRows(['codex'], codexTurn); + expect(codexTurn).toEqual(original); + const tool = rows.find((row) => row.span_name === 'exec')!; + expect(tool['gen_ai.tool.name']).toBe('exec'); + expect(tool.span_parent_span_id).toBe('a131b912f2251684'); + expect(mappedHarness(tool)).toBe('codex'); + expect(recordedFields(tool)).toEqual(original[4]); + expect(recordedFields(original[0])).toBe(original[0]); + }); + it('never replaces recorded GenAI values', () => { + const [turn] = mapHarnessRows( + ['codex'], + [{ ...codexTurn[0], 'gen_ai.agent.name': 'Reviewer', 'gen_ai.usage.input_tokens': 7 }], + ); + expect(turn['gen_ai.agent.name']).toBe('Reviewer'); + expect(turn['gen_ai.usage.input_tokens']).toBe(7); + }); + it('does not attach detached spans to one of overlapping turns', () => { + const second = { ...codexTurn[0], span_span_id: 'b131b912f2251684' }; + const rows = mapHarnessRows(['codex'], [...codexTurn, second]); + expect(rows.find((row) => row.span_name === 'exec')!.span_parent_span_id).toBe( + '174bdda1bf2f0d51', + ); + expect(buildAgentRuns(rows)[0].warnings).toContain( + 'A parent span is outside the captured sample or missing.', + ); + }); + it('leaves calls after the loaded turn detached', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('call1', 'sampling1', 'try_run_sampling_request', 50, 10), + codex('call2', 'sampling2', 'try_run_sampling_request', 150, 10), + codex('tool2', 'tool-parent', 'exec', 170, 10, { tool_name: 'exec' }), + ], + ); + expect(rows.map((row) => [row.span_span_id, row.span_parent_span_id])).toEqual([ + ['turn1', ''], + ['call1', 'turn1'], + ['call2', 'sampling2'], + ['tool2', 'tool-parent'], + ]); + expect(buildAgentRuns(rows)[0].warnings).toContain( + 'A parent span is outside the captured sample or missing.', + ); + }); + it('attaches calls to the containing turn in a multi-turn trace', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('turn2', 'req', 'session_task.turn', 200, 100), + codex('call1', 'sampling1', 'try_run_sampling_request', 50, 10), + codex('call2', 'sampling2', 'try_run_sampling_request', 250, 10), + codex('tool2', 'tool-parent', 'exec', 270, 10, { tool_name: 'exec' }), + ], + ); + expect(rows.slice(2).map((row) => row.span_parent_span_id)).toEqual([ + 'turn1', + 'turn2', + 'turn2', + ]); + const [run] = buildAgentRuns(rows); + expect(run.warnings).not.toContain('A parent span is outside the captured sample or missing.'); + expect(run.roots.map((root) => root.children.map((child) => child.spanId))).toEqual([ + ['call1'], + ['call2', 'tool2'], + ]); + }); + it('leaves calls detached when several turn ranges contain them', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn1', 'req', 'session_task.turn', 0, 100), + codex('turn2', 'req', 'session_task.turn', 20, 50), + codex('call', 'sampling', 'try_run_sampling_request', 30, 10), + ], + ); + expect(rows[2].span_parent_span_id).toBe('sampling'); + }); + it.each([ + ['missing call start', {}, { span_start_time_unix_nano: undefined }], + ['missing turn start', { span_start_time_unix_nano: undefined }, {}], + ['unknown turn end', { span_end_time_unix_nano: undefined, span_duration_ns: undefined }, {}], + ['invalid turn end', { span_end_time_unix_nano: 'invalid' }, {}], + ['reversed turn range', { span_end_time_unix_nano: '2026-10-13T02:50:33Z' }, {}], + ['absent call epoch', {}, { span_start_time_unix_nano_epoch: 0 }], + ['absent turn epoch', { span_end_time_unix_nano_epoch: 0 }, {}], + ])('leaves calls detached with %s', (_, turnFields, callFields) => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn', 'req', 'session_task.turn', 0, 100, turnFields), + codex('call', 'sampling', 'try_run_sampling_request', 50, 10, callFields), + ], + ); + expect(rows[1].span_parent_span_id).toBe('sampling'); + }); + it.each([ + ['numeric nanoseconds', 1791859834000000000, 1791859834100000000, '1791859834050000000'], + [ + 'UTC timestamp text', + '2026-10-13 02:50:34.000', + '2026-10-13 02:50:34.100', + '2026-10-13T02:50:34.050Z', + ], + ['duration without an end', '2026-10-13T02:50:34Z', undefined, '2026-10-13T02:50:34.050Z'], + ])('attaches calls using %s', (_, start, end, callStart) => { + const rows = mapHarnessRows( + ['codex'], + [ + codex('turn', 'req', 'session_task.turn', 0, 100, { + span_start_time_unix_nano: start, + span_end_time_unix_nano: end, + }), + codex('call', 'sampling', 'try_run_sampling_request', 50, 10, { + span_start_time_unix_nano: callStart, + }), + ], + ); + expect(rows[1].span_parent_span_id).toBe('turn'); + }); + it.each(['fallback', 'both'])( + 'rewrites %s parent fields and preserves recorded parents', + (fields) => { + const turn = codex('turn', 'req', 'session_task.turn', 0, 100, { parent_span_id: 'req' }); + const tool = codex('tool', 'tool-parent', 'exec', 50, 10, { + parent_span_id: 'tool-parent', + tool_name: 'exec', + }); + if (fields === 'fallback') { + delete turn.span_parent_span_id; + delete tool.span_parent_span_id; + } + const rows = mapHarnessRows(['codex'], [turn, tool]); + expect(rows[0].parent_span_id).toBe(''); + expect(rows[1].parent_span_id).toBe('turn'); + if (fields === 'both') { + expect(rows[0].span_parent_span_id).toBe(''); + expect(rows[1].span_parent_span_id).toBe('turn'); + } + expect(recordedFields(rows[0])).toEqual(turn); + expect(recordedFields(rows[0]).parent_span_id).toBe('req'); + expect(recordedFields(rows[1]).parent_span_id).toBe('tool-parent'); + const [run] = buildAgentRuns(rows); + expect(run.status).toBe('success'); + expect(run.warnings).toEqual([]); + expect(run.roots[0].children[0].spanId).toBe('tool'); + }, + ); + it('builds a Claude Code interaction with cache-inclusive input usage and tool names', () => { + const [run] = buildAgentRuns(mapHarnessRows(['claude-code'], claudeInteraction)); + expect(run.name).toBe('Claude Code'); + expect(run.status).toBe('success'); + expect(run.models).toEqual(['claude-haiku-4-5']); + expect([run.inputTokens, run.outputTokens]).toEqual([9 + 12598 + 8595 + 8 + 21193 + 151, 154]); + expect( + run.roots[0].children.map((child) => [child.kind, child.toolName ?? child.model]), + ).toEqual([ + ['model', 'claude-haiku-4-5'], + ['tool', 'Bash'], + ['model', 'claude-haiku-4-5'], + ]); + expect(run.roots[0].children[1].children[0].kind).toBe('span'); + }); + it('keeps absent Claude Code usage unknown', () => { + const [request] = mapHarnessRows( + ['claude-code'], + [claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, claudeRequest)], + ); + expect(request['gen_ai.usage.input_tokens']).toBeUndefined(); + expect(request['gen_ai.usage.output_tokens']).toBeUndefined(); + expect(buildAgentRuns([request])[0].inputTokens).toBeUndefined(); + }); + it('maps each harness in a mixed dataset and leaves other services to themselves', () => { + const stray = claude('9a9a9a9a9a9a9a9a', '4b1b0ceaf03121e7', 'handle_responses', 1500, 10, { + tool_name: 'Read', + }); + const rows = mapHarnessRows( + ['codex', 'claude-code'], + [...codexTurn, ...claudeInteraction, stray], + ); + const runs = buildAgentRuns(rows); + const codexRun = runs.find((run) => run.traceId === codexTrace)!; + expect(codexRun.name).toBe('Codex'); + expect([codexRun.inputTokens, codexRun.outputTokens]).toEqual([32558, 38]); + expect(codexRun.roots[0].children.map((child) => child.kind)).toEqual([ + 'model', + 'tool', + 'model', + ]); + const claudeRun = runs.find((run) => run.traceId === claudeTrace)!; + expect(claudeRun.name).toBe('Claude Code'); + expect(claudeRun.roots[0].kind).toBe('agent'); + const [request, tool] = claudeRun.roots[0].children; + expect([request.inputTokens, request.outputTokens]).toEqual([21202, 102]); + expect([tool.kind, tool.toolName]).toEqual(['tool', 'Bash']); + // Only Codex services lose their stream spans or have tool_name mapped. + const kept = rows.find((row) => row.span_span_id === '9a9a9a9a9a9a9a9a')!; + expect(kept['gen_ai.tool.name']).toBeUndefined(); + expect(kept.span_parent_span_id).toBe('4b1b0ceaf03121e7'); + expect(mappedHarness(kept)).toBeUndefined(); + expect(mappedHarness(rows.find((row) => row.span_name === 'exec')!)).toBe('codex'); + expect(mappedHarness(rows.find((row) => row.span_name === 'claude_code.tool')!)).toBe( + 'claude-code', + ); + }); + it('marks only rows that something was mapped onto', () => { + const generic = codex('7777777777777777', 'a131b912f2251684', 'chat gpt-6-luna', 200, 50, { + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.model': 'gpt-6-luna', + }); + const rows = mapHarnessRows(['codex'], [...codexTurn, generic]); + const chat = rows.find((row) => row.span_span_id === '7777777777777777')!; + expect(mappedHarness(chat)).toBeUndefined(); + expect(recordedFields(chat)).toBe(generic); + expect(mappedHarness(rows[0])).toBe('codex'); + const claudeRows = mapHarnessRows(['claude-code'], claudeInteraction); + const execution = claudeRows.find((row) => row.span_name === 'claude_code.tool.execution')!; + expect(mappedHarness(execution)).toBeUndefined(); + expect(recordedFields(execution)).toBe(claudeInteraction[3]); + expect(mappedHarness(claudeRows[0])).toBe('claude-code'); + }); + it('reports Claude Code input usage only when input_tokens and present cache counts parse', () => { + const input = (usage: LogRecord) => + mapHarnessRows( + ['claude-code'], + [ + claude('0997d8a433ad3456', '', 'claude_code.llm_request', 0, 834, { + ...claudeRequest, + ...usage, + }), + ], + )[0]['gen_ai.usage.input_tokens']; + expect(input({ input_tokens: '9' })).toBe(9); + expect(input({ input_tokens: '9', cache_read_tokens: '100' })).toBe(109); + expect(input({ input_tokens: '9', cache_read_tokens: '', cache_creation_tokens: 5 })).toBe(14); + // Cache counts alone are not the request's input usage. + expect(input({ cache_read_tokens: '5000', cache_creation_tokens: '20' })).toBeUndefined(); + for (const value of ['12.5', '-1', 'bad']) + expect(input({ input_tokens: value, cache_read_tokens: '5000' })).toBeUndefined(); + // A malformed cache count would understate the total, so input stays unknown. + expect(input({ input_tokens: '9', cache_creation_tokens: '-1' })).toBeUndefined(); + + const [run] = buildAgentRuns( + mapHarnessRows( + ['claude-code'], + [ + claudeInteraction[0], + claudeInteraction[1], + { ...claudeInteraction[4], input_tokens: '8.5' }, + ], + ), + ); + expect([run.inputTokens, run.outputTokens]).toEqual([9 + 12598 + 8595, 154]); + expect(run.warnings).toContain( + 'Token usage is partial because some model calls do not report usage.', + ); + }); + it('treats rows without a service name as Codex in a Codex-only dataset', () => { + const rows = mapHarnessRows( + ['codex'], + codexTurn.map(({ 'service.name': _, ...row }) => row), + ); + expect(rows.map((row) => row.span_name)).not.toContain('receiving_stream'); + expect(buildAgentRuns(rows)[0].name).toBe('Codex'); + }); + it('keeps a Codex turn under a parent from another service', () => { + const outer = span(codexTrace, 'outer00000000001', '', 'gateway', -100, 5000, { + 'service.name': 'agent-gateway', + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.agent.name': 'Gateway', + }); + const turn = { ...codexTurn[0], span_parent_span_id: 'outer00000000001' }; + const rows = mapHarnessRows(['codex'], [outer, turn]); + expect(rows[1].span_parent_span_id).toBe('outer00000000001'); + expect(buildAgentRuns(rows)[0].roots).toHaveLength(1); + }); + it('keeps Codex stream and response spans that carry GenAI fields themselves', () => { + const fields = { 'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': 'exec' }; + const rows = mapHarnessRows( + ['codex'], + [ + codexTurn[0], + codex('5151515151515151', 'a131b912f2251684', 'receiving_stream', 200, 10, fields), + codex('5252525252525252', 'a131b912f2251684', 'handle_responses', 300, 10, { + ...fields, + 'gen_ai.usage.input_tokens': '5', + }), + ], + ); + const [run] = buildAgentRuns(rows); + expect(run.roots[0].children.map((child) => child.toolName)).toEqual(['exec', 'exec']); + // The kept response's usage is counted once, not also lifted onto a model call. + expect(run.inputTokens).toBe(5); + }); + it('recognizes Codex rows without a service name in a mixed dataset', () => { + const strip = ({ 'service.name': _, ...row }: LogRecord) => row; + const rows = mapHarnessRows( + ['codex', 'claude-code'], + [...codexTurn.map(strip), ...claudeInteraction.map(strip)], + ); + const runs = buildAgentRuns(rows); + expect(runs.find((run) => run.traceId === codexTrace)!.name).toBe('Codex'); + expect(runs.find((run) => run.traceId === codexTrace)!.inputTokens).toBe(32558); + expect(runs.find((run) => run.traceId === claudeTrace)!.name).toBe('Claude Code'); + }); + it('attaches a kept Codex response under its model call, not beside it', () => { + const rows = mapHarnessRows( + ['codex'], + [ + codexTurn[0], + codex('6161616161616161', '6e055c16cf2ba610', 'try_run_sampling_request', 100, 500, { + model: 'gpt-6-luna', + 'gen_ai.usage.input_tokens': '5', + }), + codex('6262626262626262', '6161616161616161', 'receiving_stream', 102, 400), + codex('6363636363636363', '6262626262626262', 'handle_responses', 450, 10, { + 'gen_ai.operation.name': 'chat', + 'gen_ai.usage.input_tokens': '5', + }), + ], + ); + expect(rows.find((row) => row.span_span_id === '6363636363636363')!.span_parent_span_id).toBe( + '6161616161616161', + ); + expect(buildAgentRuns(rows)[0].inputTokens).toBe(5); + }); + it('leaves generic spans without a service name alone in a mixed dataset', () => { + const { 'service.name': _, ...turn } = codexTurn[0]; + const generic = span(codexTrace, '7171717171717171', 'gone', 'chat remote-model', 200, 10, { + 'gen_ai.operation.name': 'chat', + }); + const rows = mapHarnessRows(['codex', 'claude-code'], [turn, generic]); + const chat = rows.find((row) => row.span_span_id === '7171717171717171')!; + expect(chat.span_parent_span_id).toBe('gone'); + expect(mappedHarness(chat)).toBeUndefined(); + expect(mappedHarness(rows[0])).toBe('codex'); + }); + it('leaves rows untouched when no harness is detected', () => { + expect(mapHarnessRows([], codexTurn)).toBe(codexTurn); + }); +}); diff --git a/frontend/src/lib/agentHarnesses.ts b/frontend/src/lib/agentHarnesses.ts new file mode 100644 index 000000000..72afdd34a --- /dev/null +++ b/frontend/src/lib/agentHarnesses.ts @@ -0,0 +1,270 @@ +import { spanTiming } from './agentRuns'; +import { quoteIdentifier, quoteLiteral } from './query'; +import type { LogRecord } from './types'; + +// Classic agents uses Enterprise /api/prism/v1/agent-observability APIs. +// Community can inspect the same flattened telemetry through its SQL endpoint. +export const AGENT_FIELDS = [ + 'gen_ai.agent.name', + 'gen_ai.agent.id', + 'gen_ai.operation.name', + 'gen_ai.request.model', + 'gen_ai.response.model', + 'gen_ai.tool.name', +] as const; + +// Coding-agent CLIs export OTLP with their own attribute names. A harness profile +// recognizes that telemetry and adds the GenAI span fields the Agents view reads. +// Recorded values are never replaced, and absent values stay absent. +export type AgentHarness = 'codex' | 'claude-code'; + +type HarnessProfile = { + label: string; + // Any of these fields identifies the harness in a dataset schema. + signature: string[]; + // The span that represents one agent run (a user turn). + runSpan: string; +}; + +export const AGENT_HARNESSES: Record = { + codex: { + label: 'Codex', + signature: ['codex.turn.token_usage.input_tokens', 'codex.turn.phase'], + runSpan: 'session_task.turn', + }, + 'claude-code': { + label: 'Claude Code', + signature: ['interaction.sequence', 'llm_request.context'], + runSpan: 'claude_code.interaction', + }, +}; + +// Codex traces a turn with hundreds of internal spans. Only these, plus spans +// carrying a tool name, describe the run; the others would exhaust the record limit. +const CODEX_MODEL_CALL = 'try_run_sampling_request'; +const CODEX_STREAM = 'receiving_stream'; +const CODEX_RESPONSE = 'handle_responses'; +const CODEX_SPANS = [AGENT_HARNESSES.codex.runSpan, CODEX_MODEL_CALL, CODEX_STREAM]; + +/** The harnesses whose telemetry a dataset holds; one dataset may hold several. */ +export function detectAgentHarnesses(fields: string[]): AgentHarness[] { + if (!fields.includes('span_name')) return []; + return (Object.keys(AGENT_HARNESSES) as AgentHarness[]).filter((harness) => + AGENT_HARNESSES[harness].signature.some((field) => fields.includes(field)), + ); +} + +/** SQL selecting the spans that start a run of this harness. */ +export function harnessRunPredicate(harness: AgentHarness) { + return `${quoteIdentifier('span_name')} = ${quoteLiteral(AGENT_HARNESSES[harness].runSpan)}`; +} + +/** + * SQL keeping the run-relevant spans of a harness; other services' rows, and spans + * already carrying GenAI fields, are untouched. + */ +export function harnessRowPredicate(harness: AgentHarness, fields: string[]) { + if (harness !== 'codex' || !fields.includes('service.name')) return; + const name = `COALESCE(${quoteIdentifier('span_name')}, '')`; + const kept = [`${name} IN (${CODEX_SPANS.map(quoteLiteral).join(', ')})`]; + if (fields.includes('gen_ai.usage.input_tokens')) + kept.push( + `(${name} = ${quoteLiteral(CODEX_RESPONSE)} AND ${quoteIdentifier('gen_ai.usage.input_tokens')} IS NOT NULL)`, + ); + for (const field of ['tool_name', ...AGENT_FIELDS]) + if (fields.includes(field)) kept.push(`${quoteIdentifier(field)} IS NOT NULL`); + return `NOT (COALESCE(${quoteIdentifier('service.name')}, '') LIKE 'codex%' AND NOT (${kept.join(' OR ')}))`; +} + +const recorded = new WeakMap(); +const mapped = new WeakSet(); + +/** The fields a record was stored with, before any harness mapping. */ +export function recordedFields(row: LogRecord): LogRecord { + return recorded.get(row)?.row ?? row; +} +/** The harness whose native names were mapped onto this record, if any. */ +export function mappedHarness(row: LogRecord): AgentHarness | undefined { + return mapped.has(row) ? recorded.get(row)?.harness : undefined; +} +/** Note that a field was derived for, or the parent rewritten on, a copied record. */ +function markMapped(row: LogRecord) { + if (recorded.has(row)) mapped.add(row); +} + +function present(value: unknown) { + return value !== undefined && value !== null && value !== ''; +} +function assign(row: LogRecord, field: string, value: unknown) { + if (!present(value) || present(row[field])) return; + row[field] = value; + markMapped(row); +} +function count(value: unknown) { + if (!present(value)) return; + const result = Number(value); + return Number.isSafeInteger(result) && result >= 0 ? result : undefined; +} +function id(row: LogRecord, field: string) { + const value = row[field]; + return typeof value === 'string' && value.trim() && !/^0+$/.test(value.trim()) + ? value.trim() + : undefined; +} +const traceOf = (row: LogRecord) => id(row, 'span_trace_id') ?? id(row, 'trace_id'); +const spanOf = (row: LogRecord) => id(row, 'span_span_id') ?? id(row, 'span_id'); +const parentOf = (row: LogRecord) => id(row, 'span_parent_span_id') ?? id(row, 'parent_span_id'); +// Rewrite whichever parent fields the record has, since readers fall back between them. +function setParent(row: LogRecord, value: string) { + const fields = ['span_parent_span_id', 'parent_span_id'].filter((field) => field in row); + for (const field of fields.length ? fields : ['span_parent_span_id']) row[field] = value; + markMapped(row); +} + +function mapClaudeCode(row: LogRecord) { + switch (row.span_name) { + case AGENT_HARNESSES['claude-code'].runSpan: + assign(row, 'gen_ai.agent.name', AGENT_HARNESSES['claude-code'].label); + assign(row, 'gen_ai.operation.name', 'invoke_agent'); + break; + case 'claude_code.llm_request': { + assign(row, 'gen_ai.operation.name', 'chat'); + assign(row, 'gen_ai.request.model', row.model); + // Anthropic reports cache reads and writes separately from input_tokens, which + // must be reported. An absent cache count is zero; a malformed one leaves input unknown. + const input = ['input_tokens', 'cache_read_tokens', 'cache_creation_tokens'].map( + (field, index) => (index && !present(row[field]) ? 0 : count(row[field])), + ); + if (input.every((value): value is number => value !== undefined)) + assign( + row, + 'gen_ai.usage.input_tokens', + input.reduce((a, b) => a + b), + ); + assign(row, 'gen_ai.usage.output_tokens', count(row.output_tokens)); + break; + } + case 'claude_code.tool': + assign(row, 'gen_ai.operation.name', 'execute_tool'); + assign(row, 'gen_ai.tool.name', row.tool_name); + assign(row, 'gen_ai.tool.call.id', row.tool_use_id); + break; + } +} + +// Rows of other services are kept as they are, but remain visible as parents. +function mapCodex(rows: LogRecord[], isCodex: (row: LogRecord) => boolean) { + const spans = new Map(); + for (const row of rows) { + const trace = traceOf(row); + const span = spanOf(row); + if (trace && span) + spans.set(`${trace}/${span}`, [...(spans.get(`${trace}/${span}`) ?? []), row]); + } + const parent = (row: LogRecord) => { + const trace = traceOf(row); + const span = parentOf(row); + return trace && span ? spans.get(`${trace}/${span}`) : undefined; + }; + // Stream and response spans only connect a model call to its usage, unless they + // already describe something themselves. + const plumbing = (row: LogRecord) => + isCodex(row) && + (row.span_name === CODEX_STREAM || row.span_name === CODEX_RESPONSE) && + !AGENT_FIELDS.some((field) => present(row[field])); + // Usage is reported on the response handler two levels below its model call. + for (const row of rows) { + if (row.span_name !== CODEX_RESPONSE || !plumbing(row)) continue; + const stream = parent(row)?.find((entry) => entry.span_name === CODEX_STREAM); + const call = stream && parent(stream); + if (call?.[0]?.span_name !== CODEX_MODEL_CALL) continue; + for (const entry of call) + for (const field of ['gen_ai.usage.input_tokens', 'gen_ai.usage.output_tokens']) + assign(entry, field, row[field]); + } + const runs = new Map(); + for (const row of rows) { + const trace = traceOf(row); + const span = spanOf(row); + if (row.span_name !== AGENT_HARNESSES.codex.runSpan || !isCodex(row) || !trace || !span) + continue; + const { start, durationMs } = spanTiming(row); + if (start === undefined || durationMs === undefined) continue; + runs.set(trace, [...(runs.get(trace) ?? []), { span, start, end: start + durationMs }]); + } + const kept = rows.filter((row) => !plumbing(row)); + const keptSet = new Set(kept); + const keptAncestor = (row: LogRecord) => { + let ancestors = parent(row); + // Bounded, in case recorded parents form a cycle. + for (let hops = 0; ancestors?.length && hops < rows.length; hops++) { + const found = ancestors.find((entry) => keptSet.has(entry)); + if (found) return found; + ancestors = parent(ancestors[0]); + } + }; + for (const row of kept) { + if (!isCodex(row)) continue; + if (row.span_name === AGENT_HARNESSES.codex.runSpan) { + assign(row, 'gen_ai.agent.name', AGENT_HARNESSES.codex.label); + assign(row, 'gen_ai.operation.name', 'invoke_agent'); + assign(row, 'gen_ai.request.model', row.model); + assign(row, 'gen_ai.usage.input_tokens', row['codex.turn.token_usage.input_tokens']); + assign(row, 'gen_ai.usage.output_tokens', row['codex.turn.token_usage.output_tokens']); + } else if (row.span_name === CODEX_MODEL_CALL) { + assign(row, 'gen_ai.operation.name', 'chat'); + assign(row, 'gen_ai.request.model', row.model); + } else if (present(row.tool_name)) { + assign(row, 'gen_ai.operation.name', 'execute_tool'); + assign(row, 'gen_ai.tool.name', row.tool_name); + assign(row, 'gen_ai.tool.call.id', row.call_id); + } + // Intermediate spans were not loaded. Attach the turn to the top of the tree + // and its model and tool calls directly to the turn. + const trace = traceOf(row); + if (!trace || !parentOf(row) || parent(row)?.some((entry) => keptSet.has(entry))) continue; + // A loaded but dropped parent leads to the nearest kept ancestor. + const nearest = keptAncestor(row); + if (nearest) setParent(row, spanOf(nearest)!); + else if (row.span_name === AGENT_HARNESSES.codex.runSpan) setParent(row, ''); + else { + // Only a call that started during exactly one loaded turn is known to belong to it; + // the calls of an unexported (still running) turn stay detached. + const { start } = spanTiming(row); + if (start === undefined) continue; + const matches = new Set( + (runs.get(trace) ?? []) + .filter((run) => start >= run.start && start <= run.end) + .map((run) => run.span), + ); + if (matches.size === 1) setParent(row, [...matches][0]); + } + } + return kept; +} + +/** Add GenAI fields derived from the native telemetry of the detected harnesses. */ +export function mapHarnessRows(harnesses: AgentHarness[], rows: LogRecord[]) { + if (!harnesses.length) return rows; + const codex = harnesses.includes('codex'); + const claude = harnesses.includes('claude-code'); + // Codex mapping and filtering apply to Codex services only, as in harnessRowPredicate. + // Without a service name, a Codex-only dataset is all Codex; a mixed one needs a + // Codex span name or a Codex tool call (Claude Code tools carry tool_use_id instead). + const isCodexRow = (row: LogRecord) => + codex && + (present(row['service.name']) + ? String(row['service.name']).startsWith('codex') + : !claude || + CODEX_SPANS.includes(String(row.span_name)) || + row.span_name === CODEX_RESPONSE || + (present(row.tool_name) && present(row.call_id))); + const copies = rows.map((row) => { + const copy = { ...row }; + const harness = isCodexRow(row) ? 'codex' : claude ? 'claude-code' : undefined; + recorded.set(copy, { row, harness }); + return copy; + }); + if (claude) copies.filter((row) => !isCodexRow(row)).forEach(mapClaudeCode); + return codex ? mapCodex(copies, isCodexRow) : copies; +} diff --git a/frontend/src/lib/agentRuns.ts b/frontend/src/lib/agentRuns.ts index a397ea8c1..6e4f13569 100644 --- a/frontend/src/lib/agentRuns.ts +++ b/frontend/src/lib/agentRuns.ts @@ -87,7 +87,8 @@ function timestamp(row: LogRecord, field: string): number | undefined { const epoch = number(rawEpoch ?? raw); if (epoch !== undefined && epoch > 0) return epoch / 1e6; } -function timing(row: LogRecord, timeColumn: string) { +/** A span's start and end in epoch milliseconds, and its duration. */ +export function spanTiming(row: LogRecord, timeColumn = 'p_timestamp') { const start = timestamp(row, 'span_start_time_unix_nano'); const end = timestamp(row, 'span_end_time_unix_nano'); const hasEnd = @@ -312,7 +313,7 @@ export function buildAgentRuns( ), ), ) as LogRecord; - const { startTime, durationMs } = timing(row, timeColumn); + const { startTime, durationMs } = spanTiming(row, timeColumn); const errors = spanErrors(records); const operation = text(row, 'gen_ai.operation.name'); const model = text(row, 'gen_ai.response.model', 'gen_ai.request.model', 'llm.model_name'); @@ -343,7 +344,12 @@ export function buildAgentRuns( 'gen_ai.operation.name', ) ?? 'Unnamed span', kind, - status: status(records, durationMs, errors, timing(row, timeColumn).end !== undefined), + status: status( + records, + durationMs, + errors, + spanTiming(row, timeColumn).end !== undefined, + ), startTime, durationMs, model, @@ -464,11 +470,11 @@ export function buildAgentRuns( const inputTokens = tokenTotal('inputTokens'); const outputTokens = tokenTotal('outputTokens'); const starts = spans - .map((span) => timing(span.row, timeColumn).start) + .map((span) => spanTiming(span.row, timeColumn).start) .filter((value): value is number => value !== undefined); const ends = spans .map((span) => { - const t = timing(span.row, timeColumn); + const t = spanTiming(span.row, timeColumn); return t.start !== undefined && t.end !== undefined && t.durationMs !== undefined ? t.start + t.durationMs : undefined; diff --git a/frontend/src/lib/agents.ts b/frontend/src/lib/agents.ts index 69ac7e3a8..2c88af507 100644 --- a/frontend/src/lib/agents.ts +++ b/frontend/src/lib/agents.ts @@ -1,23 +1,22 @@ +import { + AGENT_FIELDS, + detectAgentHarnesses, + harnessRowPredicate, + harnessRunPredicate, +} from './agentHarnesses'; import { ApiError } from './apiError'; import { apmTransportBounds } from './apm'; import { mapConcurrent } from './concurrency'; import { paramsRange, quoteIdentifier, quoteLiteral, timeBounds } from './query'; import type { LogRecord, ParseableClient, QueryRequest, TimeRange } from './types'; -// Classic agents uses Enterprise /api/prism/v1/agent-observability APIs. -// Community can inspect the same flattened telemetry through its SQL endpoint. -export const AGENT_FIELDS = [ - 'gen_ai.agent.name', - 'gen_ai.agent.id', - 'gen_ai.operation.name', - 'gen_ai.request.model', - 'gen_ai.response.model', - 'gen_ai.tool.name', -] as const; +export { AGENT_FIELDS }; export const AGENT_RECORD_LIMIT = 1000; export const AGENT_RUN_LIMIT = 100; export function isAgentSchema(fields: string[]) { - return AGENT_FIELDS.some((field) => fields.includes(field)); + return ( + AGENT_FIELDS.some((field) => fields.includes(field)) || detectAgentHarnesses(fields).length > 0 + ); } export function agentsPath(dataset?: string) { return dataset ? `/agents/${encodeURIComponent(dataset)}` : '/agents'; @@ -59,9 +58,17 @@ export function buildAgentRunQuery( ); if (!fields.includes(timeColumn)) throw new Error(`Dataset schema is missing the time field ${timeColumn}.`); - const agentPredicate = AGENT_FIELDS.filter((field) => fields.includes(field)) - .map((field) => `${quoteIdentifier(field)} IS NOT NULL`) - .join(' OR '); + const harnesses = detectAgentHarnesses(fields); + const agentPredicate = [ + ...AGENT_FIELDS.filter((field) => fields.includes(field)).map( + (field) => `${quoteIdentifier(field)} IS NOT NULL`, + ), + ...harnesses.map(harnessRunPredicate), + ].join(' OR '); + const rowPredicate = harnesses + .map((harness) => harnessRowPredicate(harness, fields)) + .filter(Boolean) + .join(' AND '); const traceField = ['span_trace_id', 'trace_id'].find((field) => fields.includes(field)); const table = quoteIdentifier(dataset); const time = quoteIdentifier(timeColumn); @@ -74,7 +81,7 @@ export function buildAgentRunQuery( ? `${trace} IN (SELECT ${trace} FROM ${table} WHERE ${timePredicate} AND (${agentPredicate}) AND ${trace} IS NOT NULL AND ${trace} <> '' GROUP BY ${trace} ORDER BY MAX(${time}) DESC LIMIT ${AGENT_RUN_LIMIT}) OR ((${trace} IS NULL OR ${trace} = '') AND (${agentPredicate}))` : agentPredicate; return { - sql: `SELECT * FROM ${table} WHERE ${timePredicate} AND (${predicate}) ORDER BY ${time} DESC LIMIT ${AGENT_RECORD_LIMIT}`, + sql: `SELECT * FROM ${table} WHERE ${timePredicate} AND (${predicate})${rowPredicate ? ` AND ${rowPredicate}` : ''} ORDER BY ${time} DESC LIMIT ${AGENT_RECORD_LIMIT}`, ...apmTransportBounds(exactBounds), }; }