From 64925c231ffdc0df6334cf653b57a960a11cb027 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:54:34 -0500 Subject: [PATCH 1/5] feat(agents): join Codex and Claude Code logs into run conversations Both harnesses export prompts, responses and tool results as log records stamped with the trace and span they happened in. For a harness traces dataset, Agents now finds the matching logs dataset (preferring the one named like the traces dataset), queries only conversation events for the loaded runs' trace IDs, and attaches them to their spans in order: - Codex: the user prompt on its turn; tool arguments and output on the tool span, matched by call ID. - Claude Code: the user prompt, assistant responses on their model call (by request ID), tool input and approval decisions on the tool span (by tool use ID), and tool output from the tool span's trace event. Redacted content is labelled rather than hidden, and records for runs that are not loaded are ignored. A "Conversation logs" selector shows the paired dataset and can turn the join off; search also matches conversation text. Co-Authored-By: Claude Opus 5.5 --- frontend/e2e/agents.spec.ts | 196 +++++++++---- .../src/features/agents/AgentRunDetails.tsx | 4 +- .../features/agents/AgentTelemetryGuide.tsx | 4 +- frontend/src/features/agents/AgentsPage.tsx | 110 ++++++- frontend/src/lib/agentHarnessLogs.test.ts | 273 ++++++++++++++++++ frontend/src/lib/agentHarnessLogs.ts | 212 ++++++++++++++ 6 files changed, 729 insertions(+), 70 deletions(-) create mode 100644 frontend/src/lib/agentHarnessLogs.test.ts create mode 100644 frontend/src/lib/agentHarnessLogs.ts diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index 8a65f793a..194aec513 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -294,7 +294,13 @@ const agentFields = [...new Set(agentRows.flatMap((row) => Object.keys(row)))]; type Captured = { query: string; startTime: string; endTime: string }; async function serveAgents( page: Page, - options: { status?: number; rows?: LogRecord[]; datasets?: string[]; fields?: string[] } = {}, + options: { + status?: number; + rows?: LogRecord[]; + datasets?: string[]; + fields?: string[]; + logs?: { dataset: string; fields: string[]; rows: LogRecord[] }; + } = {}, ) { const captured: Captured[] = []; const state = { status: options.status ?? 200, rows: options.rows ?? agentRows }; @@ -306,7 +312,12 @@ async function serveAgents( }); if (path.endsWith('/schema')) return route.fulfill({ - json: { fields: (options.fields ?? agentFields).map((name) => ({ name })) }, + json: { + fields: (path.includes(`/${options.logs?.dataset}/`) + ? options.logs!.fields + : (options.fields ?? agentFields) + ).map((name) => ({ name })), + }, }); if (path.endsWith('/info')) return route.fulfill({ @@ -314,7 +325,10 @@ async function serveAgents( }); if (path === '/api/v1/about') return route.fulfill({ json: { capabilities: {} } }); if (path === '/api/v1/query') { - captured.push(route.request().postDataJSON()); + const request: Captured = route.request().postDataJSON(); + captured.push(request); + if (options.logs && request.query.includes(`FROM "${options.logs.dataset}"`)) + return route.fulfill({ json: options.logs.rows }); return state.status === 200 ? route.fulfill({ json: state.rows }) : route.fulfill({ status: state.status, body: 'Access denied' }); @@ -726,67 +740,71 @@ for (const trace of ['trace-with-dashes', '']) { }); } +// Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the +// response handler below each model call, and intermediate spans filtered out. +const codexTrace = 'e5'.repeat(16); +const codex = (span: Omit) => + spanRows({ + ...span, + trace: codexTrace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); +const codexRows = [ + codex({ + id: 'e500000000000001', + parent: 'e5000000000000aa', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.output_tokens': '38', + }, + }), + codex({ + id: 'e500000000000002', + parent: 'e5000000000000bb', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 1600, + attributes: { model: 'gpt-6-luna' }, + }), + codex({ + id: 'e500000000000003', + parent: 'e500000000000002', + name: 'receiving_stream', + start: base + 1110, + duration: 1580, + }), + codex({ + id: 'e500000000000004', + parent: 'e500000000000003', + name: 'handle_responses', + start: base + 2600, + duration: 40, + attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, + }), + codex({ + id: 'e500000000000005', + parent: 'e5000000000000cc', + name: 'exec', + start: base + 2700, + duration: 137, + attributes: { tool_name: 'exec', call_id: 'call_codex' }, + }), +].flat(); +const codexFields = [ + ...new Set([...codexRows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens']), +]; + test('native Codex traces become runs with mapped model calls, tools and usage', async ({ page, }) => { - // Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the - // response handler below each model call, and intermediate spans filtered out. - const trace = 'e5'.repeat(16); - const codex = (span: Omit) => - spanRows({ - ...span, - trace, - attributes: { 'service.name': 'codex-app-server', ...span.attributes }, - }); - const rows = [ - codex({ - id: 'e500000000000001', - parent: 'e5000000000000aa', - name: 'session_task.turn', - start: base + 1000, - duration: 4000, - attributes: { - model: 'gpt-6-luna', - 'codex.turn.token_usage.input_tokens': '32558', - 'codex.turn.token_usage.output_tokens': '38', - }, - }), - codex({ - id: 'e500000000000002', - parent: 'e5000000000000bb', - name: 'try_run_sampling_request', - start: base + 1100, - duration: 1600, - attributes: { model: 'gpt-6-luna' }, - }), - codex({ - id: 'e500000000000003', - parent: 'e500000000000002', - name: 'receiving_stream', - start: base + 1110, - duration: 1580, - }), - codex({ - id: 'e500000000000004', - parent: 'e500000000000003', - name: 'handle_responses', - start: base + 2600, - duration: 40, - attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, - }), - codex({ - id: 'e500000000000005', - parent: 'e5000000000000cc', - name: 'exec', - start: base + 2700, - duration: 137, - attributes: { tool_name: 'exec', call_id: 'call_codex' }, - }), - ].flat(); const { captured } = await serveAgents(page, { - rows, + rows: codexRows, datasets: ['codex-traces'], - fields: [...new Set([...rows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens'])], + fields: codexFields, }); await page.goto(`/agents?${range}`); await page.getByLabel('Agents dataset').selectOption('codex-traces'); @@ -813,3 +831,63 @@ test('native Codex traces become runs with mapped model calls, tools and usage', await expect(span).toContainText('"tool_name": "exec"'); await expect(span).not.toContainText('gen_ai.tool.name'); }); + +test('Codex conversation logs fill run details from the paired logs dataset', async ({ page }) => { + // Log records as Codex exports them with log_user_prompt on, stamped with the + // trace and span they happened in. The tool result names the call it answers. + const logs = [ + { + 'event.name': 'codex.user_prompt', + 'event.timestamp': ingested(base + 1000), + trace_id: codexTrace, + span_id: 'e5000000000000aa', + 'conversation.id': 'conversation', + prompt: 'List the files here', + }, + { + 'event.name': 'codex.tool_result', + 'event.timestamp': ingested(base + 2840), + trace_id: codexTrace, + span_id: 'e5000000000000dd', + call_id: 'call_codex', + tool_name: 'exec', + arguments: 'ls', + output: 'README.md', + }, + ]; + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs', 'archive-logs'], + fields: codexFields, + logs: { + dataset: 'codex-logs', + fields: [ + ...new Set([...logs.flatMap((row) => Object.keys(row)), 'originator', 'p_timestamp']), + ], + rows: logs, + }, + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + const details = await openRun(page, 'Codex'); + await details.getByRole('tab', { name: 'Conversation' }).click(); + const turn = details.getByRole('region', { name: 'Conversation from session_task.turn' }); + await expect(turn).toContainText('User prompt'); + await expect(turn).toContainText('List the files here'); + const tool = details.getByRole('region', { name: 'Conversation from exec' }); + await expect(tool).toContainText('Tool arguments · exec'); + await expect(tool).toContainText('README.md'); + const logQuery = captured.find((request) => request.query.includes('FROM "codex-logs"')); + expect(logQuery?.query).toContain(`"trace_id" IN ('${codexTrace}')`); + expect(sqlBounds(logQuery!.query)).toEqual([bounds.startTime, bounds.endTime]); + + // Conversation text is searchable, and logs can be turned off. + await page.keyboard.press('Escape'); + await expect(details).toBeHidden(); + await page.getByLabel('Search agent runs').fill('README.md'); + await expect(page).toHaveURL(/search=README\.md/); + await expect(runRow(page, 'Codex')).toBeVisible(); + await page.getByLabel('Conversation logs').selectOption('none'); + await expect(page).toHaveURL(/logs=none/); + await expect(page.getByRole('heading', { name: 'No matching runs' })).toBeVisible(); +}); diff --git a/frontend/src/features/agents/AgentRunDetails.tsx b/frontend/src/features/agents/AgentRunDetails.tsx index 9d4e99640..f7bb3d18e 100644 --- a/frontend/src/features/agents/AgentRunDetails.tsx +++ b/frontend/src/features/agents/AgentRunDetails.tsx @@ -66,7 +66,9 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { const models = run.spans.filter((span) => span.kind === 'model'); const tools = run.spans.filter((span) => span.kind === 'tool'); const errors = run.spans.filter((span) => span.errors.length || span.status === 'failed'); - const conversation = run.spans.filter((span) => span.conversation.length); + const conversation = run.spans + .filter((span) => span.conversation.length) + .sort((a, b) => (Date.parse(a.startTime ?? '') || 0) - (Date.parse(b.startTime ?? '') || 0)); const spanButton = (span: AgentSpan) => ( +

+ )} + {harness && logsCandidates.data && !logsCandidates.data.length && !logsParam && ( +

+ No {AGENT_HARNESSES[harness].label} logs dataset found. Export its logs to Parseable to + see prompts and tool results in run details. +

+ )} {dataset && discoveryEnabled && discovery.loading && (

Finding compatible datasets… diff --git a/frontend/src/lib/agentHarnessLogs.test.ts b/frontend/src/lib/agentHarnessLogs.test.ts new file mode 100644 index 000000000..a560632e5 --- /dev/null +++ b/frontend/src/lib/agentHarnessLogs.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from 'vitest'; +import { mapHarnessRows } from './agentHarnesses'; +import { + attachHarnessConversation, + buildHarnessLogQuery, + detectHarnessLogs, + pairedLogsDataset, +} from './agentHarnessLogs'; +import { buildAgentRuns, type AgentRun } from './agentRuns'; +import type { LogRecord } from './types'; + +// Trimmed from logs exported by Codex CLI 0.162.1 (`codex exec`, log_user_prompt +// on) and Claude Code 2.1.296 (OTEL_LOG_USER_PROMPTS, _ASSISTANT_RESPONSES, +// _TOOL_DETAILS and _TOOL_CONTENT on), as stored by Parseable. +const at = (seconds: number) => new Date(Date.parse('2026-10-13T03:38:00Z') + seconds * 1000); +const stamp = (seconds: number) => at(seconds).toISOString().replace('Z', ''); +function span( + trace: string, + id: string, + parent: string, + name: string, + start: number, + duration: number, + fields: LogRecord = {}, +): LogRecord { + return { + p_timestamp: stamp(start + duration), + span_trace_id: trace, + span_span_id: id, + span_parent_span_id: parent, + span_name: name, + span_start_time_unix_nano: stamp(start), + span_end_time_unix_nano: stamp(start + duration), + span_duration_ns: duration * 1e9, + span_status_code: 0, + ...fields, + }; +} +const conversations = (run: AgentRun) => + run.spans + .filter((entry) => entry.conversation.length) + .map((entry) => [entry.name, entry.conversation.map((item) => [item.label, item.value])]); + +const codexTrace = '1b1ef19838e4d610208a394b9af94384'; +const codexRuns = () => + buildAgentRuns( + mapHarnessRows('codex', [ + span(codexTrace, 'fe5417199aa59ebf', '57d7ac8056fe8b6d', 'session_task.turn', 15.8, 4, { + 'service.name': 'codex_exec', + model: 'gpt-6-luna', + }), + span(codexTrace, '67c205c2bacb4518', 'e0d1c4f0b19c5a7e', 'exec', 18, 0.2, { + 'service.name': 'codex_exec', + tool_name: 'exec', + call_id: 'call_7b616c08bc894640a843cff3e0f7b38a', + }), + ]), + ); +const codexLog = (fields: LogRecord): LogRecord => ({ + 'service.name': 'codex_exec', + 'conversation.id': '01a13356-e711-7291-829d-0051eaa8578d', + model: 'gpt-6-luna', + trace_id: codexTrace, + ...fields, +}); +const codexLogs = [ + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(15.79), + span_id: '57d7ac8056fe8b6d', + prompt: 'Run: cat sample.txt then reply with just the file contents.', + }), + codexLog({ + 'event.name': 'codex.tool_result', + 'event.timestamp': stamp(18.23), + span_id: '07d5bd18e22b0db8', + call_id: 'call_7b616c08bc894640a843cff3e0f7b38a', + tool_name: 'exec', + arguments: 'const r = await tools.exec_command({cmd:"cat sample.txt"});\ntext(r.output);\n', + output: 'Script completed\nWall time 0.2 seconds\nOutput:\n\nhello\n', + }), + // Code-mode tool calls run in a separate trace that has no turn. + codexLog({ + 'event.name': 'codex.tool_result', + 'event.timestamp': stamp(18.22), + trace_id: 'b5039406affbf2550d35a030cd3f6986', + span_id: 'f77605377c432cde', + call_id: 'exec-65b04fe7-d62b-4d47-9411-ef6a1c283dde', + tool_name: 'exec_command', + output: 'hello\n', + }), +]; + +const claudeTrace = '677fd41f03b0e3928c18d90bd38a609c'; +const claudeSpan = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(claudeTrace, id, parent, name, start, ms, { + 'service.name': 'claude-code', + 'interaction.sequence': '1', + ...f, + }); +const claudeRequest = (id: string, start: number) => + claudeSpan(id, '671ed833b6b2a759', 'claude_code.llm_request', start, 2.8, { + 'gen_ai.request.model': 'claude-haiku-4-5', + 'gen_ai.response.id': `req_${id}`, + request_id: `req_${id}`, + }); +const claudeToolSpan = claudeSpan( + '45801e168f0029ae', + '671ed833b6b2a759', + 'claude_code.tool', + 12.42, + 0.05, + { tool_name: 'Bash', tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx' }, +); +const claudeToolOutput = { + ...claudeToolSpan, + event_name: 'tool.output', + event_time_unix_nano: stamp(12.47), + event_output: 'hello', + event_bash_command: 'cat sample.txt', +}; +const claudeRuns = () => + buildAgentRuns( + mapHarnessRows('claude-code', [ + claudeSpan('671ed833b6b2a759', '', 'claude_code.interaction', 9.5, 6), + claudeRequest('first', 9.6), + claudeToolSpan, + // Flattened span events repeat; the output is shown once. + claudeToolOutput, + { ...claudeToolOutput }, + claudeRequest('second', 12.5), + ]), + ); +const claudeLog = (sequence: number, seconds: number, fields: LogRecord): LogRecord => ({ + 'service.name': 'claude-code', + 'session.id': 'bf8fbff9-ba17-486f-9385-b22d997bf213', + 'prompt.id': 'c0c7a199-8d62-4a46-9575-2654870f8ad0', + 'event.sequence': String(sequence), + 'event.timestamp': stamp(seconds), + trace_id: claudeTrace, + span_id: '671ed833b6b2a759', + ...fields, +}); +const claudeLogs = [ + claudeLog(4, 9.52, { + 'event.name': 'user_prompt', + prompt: 'Run cat sample.txt with the Bash tool, then reply with just the file contents.\n', + }), + claudeLog(8, 12.428, { + 'event.name': 'tool_decision', + span_id: '55648ab0912c53ee', + decision: 'accept', + source: 'config', + tool_name: 'Bash', + tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx', + }), + // Logged when the tool finished; its input belongs before the output. + claudeLog(10, 12.471, { + 'event.name': 'tool_result', + duration_ms: '42', + tool_name: 'Bash', + tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx', + tool_input: '{"command":"cat sample.txt"}', + }), + claudeLog(12, 15.358, { + 'event.name': 'assistant_response', + request_id: 'req_second', + response: 'hello', + }), + claudeLog(13, 15.4, { + 'event.name': 'assistant_response', + request_id: 'req_not_loaded', + response: '', + }), +]; + +describe('coding-agent logs datasets', () => { + it('recognizes Codex and Claude Code logs, but not their traces', () => { + const logs = ['event.name', 'trace_id', 'span_id']; + expect(detectHarnessLogs([...logs, 'conversation.id', 'originator'])).toBe('codex'); + expect(detectHarnessLogs([...logs, 'prompt.id', 'event.sequence'])).toBe('claude-code'); + expect( + detectHarnessLogs([...logs, 'prompt.id', 'event.sequence', 'span_name']), + ).toBeUndefined(); + expect(detectHarnessLogs(['event.name', 'conversation.id', 'originator'])).toBeUndefined(); + }); + it('pairs a traces dataset with the logs dataset named like it', () => { + expect(pairedLogsDataset('codex-traces', ['archive', 'codex-logs'])).toBe('codex-logs'); + expect(pairedLogsDataset('claude_trace', ['claude_log', 'z'])).toBe('claude_log'); + expect(pairedLogsDataset('agents', ['z-logs', 'a-logs'])).toBe('a-logs'); + }); + it('queries only conversation events for the loaded traces with quoted values', () => { + const query = buildHarnessLogQuery( + 'codex"logs', + 'codex', + [codexTrace, "x' OR '1'='1", codexTrace, ' '], + { startTime: '2026-10-13T03:00:00Z', endTime: '2026-10-13T04:00:00Z' }, + )!; + expect(query.sql).toBe( + `SELECT * FROM "codex""logs" WHERE "p_timestamp" >= TIMESTAMP '2026-10-13T03:00:00.000Z' AND "p_timestamp" < TIMESTAMP '2026-10-13T04:00:00.000Z' AND "trace_id" IN ('${codexTrace}', 'x'' OR ''1''=''1') AND "event.name" IN ('codex.user_prompt', 'codex.tool_result', 'codex.tool_decision') ORDER BY "p_timestamp" ASC LIMIT 1000`, + ); + expect( + buildHarnessLogQuery('logs', 'codex', [], { + startTime: '2026-10-13T03:00:00Z', + endTime: '2026-10-13T04:00:00Z', + }), + ).toBeUndefined(); + }); +}); + +describe('coding-agent conversation', () => { + it('puts the Codex prompt on its turn and tool calls on their tool span', () => { + const [run] = attachHarnessConversation(codexRuns(), 'codex', codexLogs); + expect(conversations(run)).toEqual([ + [ + 'session_task.turn', + [['User prompt', 'Run: cat sample.txt then reply with just the file contents.']], + ], + [ + 'exec', + [ + ['Tool arguments · exec', codexLogs[1].arguments], + ['Tool output · exec', codexLogs[1].output], + ], + ], + ]); + }); + it('labels redacted content instead of hiding it', () => { + const [run] = attachHarnessConversation(codexRuns(), 'codex', [ + { ...codexLogs[0], prompt: '[REDACTED]' }, + ]); + expect(run.roots[0].conversation).toEqual([ + { label: 'User prompt (redacted)', value: '[REDACTED]' }, + ]); + }); + it('orders Claude Code prompts, tool activity and responses as they happened', () => { + const [run] = attachHarnessConversation(claudeRuns(), 'claude-code', claudeLogs); + expect(conversations(run)).toEqual([ + [ + 'claude_code.interaction', + [ + ['User prompt', claudeLogs[0].prompt], + // A response whose model call is not loaded stays on the span that logged it. + ['Assistant response (redacted)', ''], + ], + ], + [ + 'claude_code.tool', + [ + ['Tool decision · Bash', 'accept (config)'], + ['Tool input · Bash', '{"command":"cat sample.txt"}'], + ['Tool output · Bash', 'hello'], + ], + ], + ['claude_code.llm_request', [['Assistant response', 'hello']]], + ]); + }); + it('shows Claude Code tool output from traces without a logs dataset', () => { + const [run] = attachHarnessConversation(claudeRuns(), 'claude-code'); + expect(conversations(run)).toEqual([['claude_code.tool', [['Tool output · Bash', 'hello']]]]); + }); + it('ignores records for runs that are not loaded, and other datasets', () => { + const runs = codexRuns(); + expect( + attachHarnessConversation(runs, 'codex', [codexLogs[2]]).flatMap((run) => + run.spans.flatMap((entry) => entry.conversation), + ), + ).toEqual([]); + const plain = codexRuns(); + expect(attachHarnessConversation(plain, undefined, codexLogs)).toBe(plain); + expect(plain[0].roots[0].conversation).toEqual([]); + }); +}); diff --git a/frontend/src/lib/agentHarnessLogs.ts b/frontend/src/lib/agentHarnessLogs.ts new file mode 100644 index 000000000..8d8d1b701 --- /dev/null +++ b/frontend/src/lib/agentHarnessLogs.ts @@ -0,0 +1,212 @@ +import { parseEventTimestamp } from '../components/explorer/timestamp'; +import type { AgentHarness } from './agentHarnesses'; +import type { AgentConversationEntry, AgentRun, AgentSpan } from './agentRuns'; +import { AGENT_RECORD_LIMIT } from './agents'; +import { ApiError } from './apiError'; +import { apmTransportBounds } from './apm'; +import { mapConcurrent } from './concurrency'; +import { quoteIdentifier, quoteLiteral, timeBounds } from './query'; +import type { LogRecord, ParseableClient, QueryRequest } from './types'; + +// Coding-agent CLIs export prompts, responses and tool results as log records +// stamped with the trace and span they happened in. Joining them by trace ID +// fills the Conversation tab for runs built from the harness's traces. +const LOG_PROFILES: Record = { + codex: { + signature: ['conversation.id', 'originator'], + events: ['codex.user_prompt', 'codex.tool_result', 'codex.tool_decision'], + }, + 'claude-code': { + signature: ['prompt.id', 'event.sequence'], + events: ['user_prompt', 'assistant_response', 'tool_result', 'tool_decision'], + }, +}; +// Claude Code records tool output as an event on the tool span, not as a log. +const CLAUDE_TOOL_OUTPUT = 'tool.output'; +const REDACTED = new Set(['[REDACTED]', '']); + +/** The harness whose logs this dataset holds; traces datasets are not logs. */ +export function detectHarnessLogs(fields: string[]): AgentHarness | undefined { + if (fields.includes('span_name') || !fields.includes('trace_id')) return; + if (!fields.includes('event.name')) return; + return (Object.keys(LOG_PROFILES) as AgentHarness[]).find((harness) => + LOG_PROFILES[harness].signature.every((field) => fields.includes(field)), + ); +} + +/** Logs datasets holding this harness's events, skipping datasets that cannot be read. */ +export async function discoverHarnessLogs( + client: ParseableClient, + harness: AgentHarness, + signal?: AbortSignal, +) { + const datasets = await client.listDatasets(signal); + const matches = await mapConcurrent(datasets, 4, async ({ name }) => { + signal?.throwIfAborted(); + try { + return detectHarnessLogs(await client.schema(name, signal)) === harness ? name : undefined; + } catch (error) { + if (signal?.aborted || (error instanceof ApiError && error.status === 401)) throw error; + return undefined; + } + }); + return matches.filter((name): name is string => !!name); +} + +/** Prefer the logs dataset named like the traces dataset, e.g. codex-traces → codex-logs. */ +export function pairedLogsDataset(traces: string, candidates: string[]) { + const sorted = [...candidates].sort(); + return ( + sorted.find((name) => name === traces.replace(/traces?/i, 'logs')) ?? + sorted.find((name) => name === traces.replace(/traces?/i, 'log')) ?? + sorted[0] + ); +} + +/** Conversation log records for the loaded runs' traces. */ +export function buildHarnessLogQuery( + dataset: string, + harness: AgentHarness, + traceIds: string[], + bounds: { startTime: string; endTime: string }, + timeColumn = 'p_timestamp', +): QueryRequest | undefined { + const ids = [...new Set(traceIds.filter((id) => id.trim()))]; + if (!ids.length) return; + const exact = timeBounds(bounds); + const time = quoteIdentifier(timeColumn); + const events = LOG_PROFILES[harness].events.map(quoteLiteral).join(', '); + return { + sql: `SELECT * FROM ${quoteIdentifier(dataset)} WHERE ${time} >= TIMESTAMP ${quoteLiteral(exact.startTime)} AND ${time} < TIMESTAMP ${quoteLiteral(exact.endTime)} AND ${quoteIdentifier('trace_id')} IN (${ids.map(quoteLiteral).join(', ')}) AND ${quoteIdentifier('event.name')} IN (${events}) ORDER BY ${time} ASC LIMIT ${AGENT_RECORD_LIMIT}`, + ...apmTransportBounds(exact), + }; +} + +function text(row: LogRecord, ...fields: string[]) { + for (const field of fields) { + const value = row[field]; + if (typeof value === 'string' && value.trim()) return value.trim(); + } +} +function entry(label: string, value: unknown): AgentConversationEntry[] { + if (value === undefined || value === null || value === '') return []; + return [ + { + label: + typeof value === 'string' && REDACTED.has(value.trim()) ? `${label} (redacted)` : label, + value, + }, + ]; +} +function withTool(label: string, row: LogRecord) { + const tool = text(row, 'tool_name'); + return tool ? `${label} · ${tool}` : label; +} + +type Occurrence = { + time: number; + sequence: number; + target?: AgentSpan; + entries: AgentConversationEntry[]; +}; + +/** + * Append harness conversation events to the spans they belong to, in the order + * they happened. Records whose run is not loaded are ignored, never guessed. + */ +export function attachHarnessConversation( + runs: AgentRun[], + harness: AgentHarness | undefined, + logs: LogRecord[] = [], +): AgentRun[] { + if (!harness) return runs; + const spans = new Map(); + const calls = new Map(); + const requests = new Map(); + const roots = new Map(); + for (const run of runs) { + if (run.traceId) + roots.set(run.traceId, run.roots.find((span) => span.kind === 'agent') ?? run.roots[0]); + for (const span of run.spans) { + if (run.traceId && span.spanId) spans.set(`${run.traceId}/${span.spanId}`, span); + const call = text(span.row, 'gen_ai.tool.call.id', 'tool_use_id', 'call_id'); + if (span.kind === 'tool' && call) calls.set(call, span); + const request = text(span.row, 'gen_ai.response.id', 'request_id'); + if (span.kind === 'model' && request) requests.set(request, span); + } + } + const occurrences: Occurrence[] = []; + const at = (row: LogRecord, ...fields: string[]) => { + for (const field of fields) { + const parsed = parseEventTimestamp(row[field]); + if (Number.isFinite(parsed)) return parsed; + } + return Number.POSITIVE_INFINITY; + }; + for (const row of logs) { + const trace = text(row, 'trace_id'); + if (!trace || !roots.has(trace)) continue; + const own = spans.get(`${trace}/${text(row, 'span_id')}`); + const event = String(row['event.name'] ?? '').replace(/^claude_code\./, ''); + const tool = calls.get(text(row, 'tool_use_id', 'call_id') ?? ''); + let target = own ?? roots.get(trace); + let entries: AgentConversationEntry[] = []; + if (event === 'codex.user_prompt' || event === 'user_prompt') { + entries = entry('User prompt', row.prompt ?? row.prompt_text); + } else if (event === 'assistant_response') { + // A response belongs to its model call, which may be a separate trace. + target = requests.get(text(row, 'request_id') ?? '') ?? target; + entries = entry('Assistant response', row.response); + } else if (event === 'codex.tool_result') { + target = tool ?? target; + entries = [ + ...entry(withTool('Tool arguments', row), row.arguments), + ...entry(withTool('Tool output', row), row.output), + ]; + } else if (event === 'tool_result') { + target = tool ?? target; + entries = entry(withTool('Tool input', row), row.tool_input ?? row.tool_parameters); + } else if (event === 'codex.tool_decision' || event === 'tool_decision') { + target = tool ?? target; + const source = text(row, 'source'); + entries = entry( + withTool('Tool decision', row), + row.decision === undefined || !source ? row.decision : `${row.decision} (${source})`, + ); + } + if (entries.length) + occurrences.push({ + // Claude Code logs a tool's input when it finishes; order it by the call's start. + time: + at(row, 'event.timestamp', 'time_unix_nano', 'p_timestamp') - + (event === 'tool_result' ? Number(row.duration_ms) || 0 : 0), + sequence: Number(row['event.sequence'] ?? Number.POSITIVE_INFINITY), + target, + entries, + }); + } + if (harness === 'claude-code') + for (const run of runs) + for (const span of run.spans) + for (const row of span.rows) + if (row.event_name === CLAUDE_TOOL_OUTPUT) + occurrences.push({ + time: at(row, 'event_time_unix_nano', 'p_timestamp'), + sequence: Number.POSITIVE_INFINITY, + target: span, + entries: entry(withTool('Tool output', span.row), row.event_output), + }); + occurrences.sort((a, b) => a.time - b.time || a.sequence - b.sequence); + const seen = new Set(); + for (const { target, entries } of occurrences) { + if (!target) continue; + for (const item of entries) { + // Flattened span rows repeat events; keep each recorded value once per span. + const key = JSON.stringify([target.id, item.label, item.value]); + if (seen.has(key)) continue; + seen.add(key); + target.conversation.push(item); + } + } + return runs; +} From f4e4c43bc72fbae43e44a28c160c949f6e40c136 Mon Sep 17 00:00:00 2001 From: VibeSWE <338779468+VibeSWE@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:50:23 -0500 Subject: [PATCH 2/5] fix(agents): keep newest conversation logs and stable logs discovery Address review findings on joining harness logs into run conversations: - Query logs newest first so the most recent runs keep their events when the record limit is reached, and say when the limit was reached. - Discover logs datasets once for all harnesses, keyed on a stable "needed" flag rather than the reloading run data, and reload it from Rescan datasets. Skip 403/404 schemas, report other failures with Retry, and only say no logs dataset exists when the scan succeeded. - Drop the logs choice when switching traces datasets, and ignore a logs param that is not a candidate for the current dataset's harnesses. Mixed traces datasets offer logs of every detected harness and join the selected one with its own harness. - Attach prompts to the turn whose time range contains them, scope tool call IDs to their trace and prefer the record's own tool span, and keep repeated log events while still collapsing flattened span events. Co-Authored-By: Claude Opus 5.5 --- frontend/e2e/agents.spec.ts | 210 ++++++++++--- frontend/src/features/agents/AgentsPage.tsx | 132 ++++++--- frontend/src/lib/agentHarnessLogs.test.ts | 307 +++++++++++++++++++- frontend/src/lib/agentHarnessLogs.ts | 101 ++++--- 4 files changed, 640 insertions(+), 110 deletions(-) diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index e3d1a22bf..82c5f4a34 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -300,16 +300,28 @@ async function serveAgents( datasets?: string[]; fields?: string[]; logs?: { dataset: string; fields: string[]; rows: LogRecord[] }; + /** Per-dataset schema fields, or an error status. */ + schemas?: Record; } = {}, ) { const captured: Captured[] = []; - const state = { status: options.status ?? 200, rows: options.rows ?? agentRows }; + const requests: string[] = []; + const state = { + status: options.status ?? 200, + rows: options.rows ?? agentRows, + datasets: options.datasets ?? ['agent_telemetry'], + schemas: options.schemas ?? {}, + }; await page.route('**/api/**', async (route) => { const path = new URL(route.request().url()).pathname; + requests.push(path); if (path === '/api/v1/logstream') - return route.fulfill({ - json: (options.datasets ?? ['agent_telemetry']).map((name) => ({ name })), - }); + return route.fulfill({ json: state.datasets.map((name) => ({ name })) }); + const schema = state.schemas[decodeURIComponent(path.split('/').at(-2) ?? '')]; + if (path.endsWith('/schema') && typeof schema === 'number') + return route.fulfill({ status: schema, body: 'Unavailable' }); + if (path.endsWith('/schema') && Array.isArray(schema)) + return route.fulfill({ json: { fields: schema.map((name) => ({ name })) } }); if (path.endsWith('/schema')) return route.fulfill({ json: { @@ -335,7 +347,7 @@ async function serveAgents( } return route.fulfill({ status: 404, body: 'No fixture' }); }); - return { captured, state }; + return { captured, requests, state }; } const columns = ['Run', 'Status', 'Started (UTC)', 'Duration', 'Model', 'Tokens', 'Spans']; @@ -797,6 +809,35 @@ const codexRows = [ const codexFields = [ ...new Set([...codexRows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens']), ]; +// Log records as Codex exports them with log_user_prompt on, stamped with the +// trace and span they happened in. The tool result names the call it answers. +const codexLogs: LogRecord[] = [ + { + 'event.name': 'codex.user_prompt', + 'event.timestamp': ingested(base + 1000), + trace_id: codexTrace, + span_id: 'e5000000000000aa', + 'conversation.id': 'conversation', + prompt: 'List the files here', + }, + { + 'event.name': 'codex.tool_result', + 'event.timestamp': ingested(base + 2840), + trace_id: codexTrace, + span_id: 'e5000000000000dd', + call_id: 'call_codex', + tool_name: 'exec', + arguments: 'ls', + output: 'README.md', + }, +]; +const codexLogFields = [ + ...new Set([...codexLogs.flatMap((row) => Object.keys(row)), 'originator', 'p_timestamp']), +]; +const claudeFields = [ + ...codexFields.filter((field) => !field.startsWith('codex.')), + 'interaction.sequence', +]; test('native Codex traces become runs with mapped model calls, tools and usage', async ({ page, @@ -833,39 +874,11 @@ test('native Codex traces become runs with mapped model calls, tools and usage', }); test('Codex conversation logs fill run details from the paired logs dataset', async ({ page }) => { - // Log records as Codex exports them with log_user_prompt on, stamped with the - // trace and span they happened in. The tool result names the call it answers. - const logs = [ - { - 'event.name': 'codex.user_prompt', - 'event.timestamp': ingested(base + 1000), - trace_id: codexTrace, - span_id: 'e5000000000000aa', - 'conversation.id': 'conversation', - prompt: 'List the files here', - }, - { - 'event.name': 'codex.tool_result', - 'event.timestamp': ingested(base + 2840), - trace_id: codexTrace, - span_id: 'e5000000000000dd', - call_id: 'call_codex', - tool_name: 'exec', - arguments: 'ls', - output: 'README.md', - }, - ]; const { captured } = await serveAgents(page, { rows: codexRows, datasets: ['codex-traces', 'codex-logs', 'archive-logs'], fields: codexFields, - logs: { - dataset: 'codex-logs', - fields: [ - ...new Set([...logs.flatMap((row) => Object.keys(row)), 'originator', 'p_timestamp']), - ], - rows: logs, - }, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, }); await page.goto(`/agents/codex-traces?${range}`); await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); @@ -940,3 +953,132 @@ test('discloses the record limit from the queried rows when Codex mapping drops 'narrow the time range', ); }); + +test('a logs choice does not carry over to a traces dataset of another harness', async ({ + page, +}) => { + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'claude-code-traces', 'codex-logs'], + fields: codexFields, + schemas: { 'claude-code-traces': claudeFields }, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + const logQueries = () => + captured.filter((request) => request.query.includes('FROM "codex-logs"')).length; + await page.goto(`/agents/codex-traces?${range}&logs=codex-logs`); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect.poll(logQueries).toBe(1); + + await page.getByLabel('Agents dataset').focus(); + await page.getByLabel('Agents dataset').selectOption('claude-code-traces'); + await expect(page).toHaveURL(/\/agents\/claude-code-traces\?/); + expect(new URL(page.url()).searchParams.get('logs')).toBeNull(); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toHaveText( + /No Claude Code logs dataset found/, + ); + await expect(page.getByLabel('Conversation logs')).toHaveValue('none'); + expect(logQueries()).toBe(1); + + // A stale link naming another harness's logs is ignored too. + await page.goto(`/agents/claude-code-traces?${range}&logs=codex-logs`); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toBeVisible(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('none'); + await expect(page.getByLabel('Conversation logs').locator('option')).toHaveText([ + 'No conversation logs', + ]); + expect(logQueries()).toBe(1); +}); + +test('Refresh keeps logs discovery, and Rescan finds a new logs dataset', async ({ page }) => { + const { captured, requests, state } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces'], + fields: codexFields, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + const count = (match: (path: string) => boolean) => requests.filter(match).length; + const lists = () => count((path) => path === '/api/v1/logstream'); + const schemas = () => count((path) => path.endsWith('/schema')); + await page.goto(`/agents/codex-traces?${range}`); + const missing = page.getByRole('status').filter({ hasText: 'No Codex logs dataset found' }); + await expect(missing).toBeVisible(); + const [listsBefore, schemasBefore] = [lists(), schemas()]; + + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect.poll(() => captured.length).toBe(2); + await expect(runRow(page, 'Codex')).toBeVisible(); + await expect(page.getByLabel('Conversation logs')).toBeVisible(); + // Only the traces dataset's own schema is read again. + expect(lists()).toBe(listsBefore); + expect(schemas()).toBe(schemasBefore + 1); + + state.datasets.push('codex-logs'); + await page.getByRole('button', { name: 'Rescan datasets' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect(missing).toBeHidden(); + await expect + .poll(() => captured.filter((request) => request.query.includes('FROM "codex-logs"')).length) + .toBe(1); +}); + +test('discloses when conversation logs reach the record limit', async ({ page }) => { + const rows = Array.from({ length: 1000 }, (_, index) => ({ + ...codexLogs[1], + 'event.timestamp': ingested(base + 2000 + index), + })); + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs'], + fields: codexFields, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows }, + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect( + page.getByRole('status').filter({ hasText: 'Conversation logs reached the record limit' }), + ).toContainText('the oldest events may be missing'); + // The newest events are kept when the limit cuts the result. + const logQuery = captured.find((request) => request.query.includes('FROM "codex-logs"')); + expect(logQuery?.query).toMatch(/ORDER BY "p_timestamp" DESC LIMIT 1000$/); +}); + +test('a failed logs scan offers Retry instead of reporting no logs dataset', async ({ page }) => { + const { state } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs'], + fields: codexFields, + schemas: { 'codex-logs': 503 }, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + await page.goto(`/agents/codex-traces?${range}`); + const unchecked = page.getByRole('status').filter({ hasText: 'Could not inspect codex-logs' }); + await expect(unchecked).toContainText('for Codex logs'); + await expect(page.getByText('No Codex logs dataset found')).toHaveCount(0); + + delete state.schemas['codex-logs']; + await unchecked.getByRole('button', { name: 'Retry' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect(unchecked).toBeHidden(); +}); + +test('a mixed traces dataset joins logs of whichever harness is selected', async ({ page }) => { + const { state } = await serveAgents(page, { + rows: codexRows, + datasets: ['agent-traces'], + fields: [...codexFields, 'interaction.sequence'], + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + await page.goto(`/agents/agent-traces?${range}`); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toHaveText( + /No Codex or Claude Code logs dataset found/, + ); + + state.datasets.push('codex-logs'); + await page.getByRole('button', { name: 'Rescan datasets' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + const details = await openRun(page, 'Codex'); + await details.getByRole('tab', { name: 'Conversation' }).click(); + await expect( + details.getByRole('region', { name: 'Conversation from session_task.turn' }), + ).toContainText('List the files here'); +}); diff --git a/frontend/src/features/agents/AgentsPage.tsx b/frontend/src/features/agents/AgentsPage.tsx index d73102575..d59aec7d1 100644 --- a/frontend/src/features/agents/AgentsPage.tsx +++ b/frontend/src/features/agents/AgentsPage.tsx @@ -24,7 +24,12 @@ import { buildAgentRunQuery, discoverAgentDatasets, } from '../../lib/agents'; -import { AGENT_HARNESSES, detectAgentHarnesses, mapHarnessRows } from '../../lib/agentHarnesses'; +import { + AGENT_HARNESSES, + detectAgentHarnesses, + mapHarnessRows, + type AgentHarness, +} from '../../lib/agentHarnesses'; import { attachHarnessConversation, buildHarnessLogQuery, @@ -111,7 +116,7 @@ export function AgentsPage() { limitReached: false, timeColumn: 'p_timestamp', traces: false, - harness: undefined, + harnesses: [] as AgentHarness[], }; } const [fields, info] = await Promise.all([ @@ -124,35 +129,60 @@ export function AgentsPage() { signal, ); const harnesses = detectAgentHarnesses(fields); - // Conversation logs are joined for one harness; a mixed dataset has none yet. - const harness = harnesses.length === 1 ? harnesses[0] : undefined; return { rows: mapHarnessRows(harnesses, rows), limitReached: rows.length >= AGENT_RECORD_LIMIT, timeColumn, traces: info.logSourceFormats.includes('otel-traces'), - harness, + harnesses, }; }, [client, mode, dataset, bounds], ), ); - const harness = result.data?.harness; + // Held across reloads of the same dataset, so logs discovery and the logs + // selector do not restart on every Refresh. + const [detected, setDetected] = useState<{ dataset?: string; harnesses: AgentHarness[] }>({ + harnesses: [], + }); + const loadedHarnesses = result.data?.harnesses; + if ( + loadedHarnesses && + (detected.dataset !== dataset || detected.harnesses.join() !== loadedHarnesses.join()) + ) + setDetected({ dataset, harnesses: loadedHarnesses }); + const harnesses = detected.dataset === dataset ? detected.harnesses : []; + const harnessLabels = harnesses.map((harness) => AGENT_HARNESSES[harness].label).join(' or '); + const [logsDiscoveryEnabled, setLogsDiscoveryEnabled] = useState(false); + if (harnesses.length && !logsDiscoveryEnabled) setLogsDiscoveryEnabled(true); // Coding agents export prompts and tool results as logs in a separate dataset. - const logsCandidates = useAsync( + const logsDiscovery = useAsync( useCallback( - async (signal) => (harness ? discoverHarnessLogs(client, harness, signal) : undefined), - [client, harness], + async (signal) => (logsDiscoveryEnabled ? discoverHarnessLogs(client, signal) : undefined), + [client, logsDiscoveryEnabled], ), + { keepPreviousData: true }, + ); + const logsCandidates = logsDiscovery.data?.datasets.filter((entry) => + harnesses.includes(entry.harness), ); const logsParam = params.get('logs'); + // A logs choice only applies while it is a candidate for the current dataset. const logsDataset = - logsParam === 'none' + logsParam === 'none' || !dataset || !logsCandidates?.length ? undefined - : logsParam || - (dataset && logsCandidates.data?.length - ? pairedLogsDataset(dataset, logsCandidates.data) - : undefined); + : logsCandidates.some((entry) => entry.name === logsParam) + ? logsParam! + : pairedLogsDataset( + dataset, + logsCandidates.map((entry) => entry.name), + ); + const logsHarness = logsCandidates?.find((entry) => entry.name === logsDataset)?.harness; + const rescan = () => { + if (discoveryEnabled) discovery.reload(); + else setDiscoveryRequested(true); + logsDiscovery.reload(); + }; const traceIds = useMemo( () => JSON.stringify([ @@ -167,18 +197,18 @@ export function AgentsPage() { const logs = useAsync( useCallback( async (signal) => { - if (!harness || !logsDataset) return undefined; + if (!logsHarness || !logsDataset) return undefined; const info = await client.datasetInfo(logsDataset, signal); const request = buildHarnessLogQuery( logsDataset, - harness, + logsHarness, JSON.parse(traceIds) as string[], bounds, info.timePartition || 'p_timestamp', ); return request ? client.query(request, signal) : []; }, - [client, harness, logsDataset, traceIds, bounds], + [client, logsHarness, logsDataset, traceIds, bounds], ), ); const authorizedDataset = useRef(undefined); @@ -193,15 +223,17 @@ export function AgentsPage() { }); const runs = useMemo( () => - attachHarnessConversation( + // The selected logs join their own harness; each harness still attaches + // what its traces record, e.g. Claude Code tool output. + (result.data?.harnesses ?? []).reduce( + (built, harness) => + attachHarnessConversation(built, harness, harness === logsHarness ? logs.data : []), buildAgentRuns(result.data?.rows ?? [], result.data?.timeColumn, { limitReached: result.data?.limitReached, bounds, }), - result.data?.harness, - logs.data, ), - [result.data, bounds, logs.data], + [result.data, bounds, logsHarness, logs.data], ); const visibleRuns = useMemo( () => @@ -273,6 +305,7 @@ export function AgentsPage() { currentObservabilitySearch(location.pathname, params.toString()), ); next.delete('run'); + next.delete('logs'); next.set('start', bounds.startTime); next.set('end', bounds.endTime); navigate(`${agentsPath(event.target.value)}?${next}`); @@ -290,18 +323,15 @@ export function AgentsPage() { ))} - {harness && mode !== 'demo' && ( + {!!harnesses.length && mode !== 'demo' && (