diff --git a/frontend/e2e/agents.spec.ts b/frontend/e2e/agents.spec.ts index ab2e33fda..82c5f4a34 100644 --- a/frontend/e2e/agents.spec.ts +++ b/frontend/e2e/agents.spec.ts @@ -294,19 +294,42 @@ const agentFields = [...new Set(agentRows.flatMap((row) => Object.keys(row)))]; type Captured = { query: string; startTime: string; endTime: string }; async function serveAgents( page: Page, - options: { status?: number; rows?: LogRecord[]; datasets?: string[]; fields?: string[] } = {}, + options: { + status?: number; + rows?: LogRecord[]; + datasets?: string[]; + fields?: string[]; + logs?: { dataset: string; fields: string[]; rows: LogRecord[] }; + /** Per-dataset schema fields, or an error status. */ + schemas?: Record; + } = {}, ) { const captured: Captured[] = []; - const state = { status: options.status ?? 200, rows: options.rows ?? agentRows }; + const requests: string[] = []; + const state = { + status: options.status ?? 200, + rows: options.rows ?? agentRows, + datasets: options.datasets ?? ['agent_telemetry'], + schemas: options.schemas ?? {}, + }; await page.route('**/api/**', async (route) => { const path = new URL(route.request().url()).pathname; + requests.push(path); if (path === '/api/v1/logstream') - return route.fulfill({ - json: (options.datasets ?? ['agent_telemetry']).map((name) => ({ name })), - }); + return route.fulfill({ json: state.datasets.map((name) => ({ name })) }); + const schema = state.schemas[decodeURIComponent(path.split('/').at(-2) ?? '')]; + if (path.endsWith('/schema') && typeof schema === 'number') + return route.fulfill({ status: schema, body: 'Unavailable' }); + if (path.endsWith('/schema') && Array.isArray(schema)) + return route.fulfill({ json: { fields: schema.map((name) => ({ name })) } }); if (path.endsWith('/schema')) return route.fulfill({ - json: { fields: (options.fields ?? agentFields).map((name) => ({ name })) }, + json: { + fields: (path.includes(`/${options.logs?.dataset}/`) + ? options.logs!.fields + : (options.fields ?? agentFields) + ).map((name) => ({ name })), + }, }); if (path.endsWith('/info')) return route.fulfill({ @@ -314,14 +337,17 @@ async function serveAgents( }); if (path === '/api/v1/about') return route.fulfill({ json: { capabilities: {} } }); if (path === '/api/v1/query') { - captured.push(route.request().postDataJSON()); + const request: Captured = route.request().postDataJSON(); + captured.push(request); + if (options.logs && request.query.includes(`FROM "${options.logs.dataset}"`)) + return route.fulfill({ json: options.logs.rows }); return state.status === 200 ? route.fulfill({ json: state.rows }) : route.fulfill({ status: state.status, body: 'Access denied' }); } return route.fulfill({ status: 404, body: 'No fixture' }); }); - return { captured, state }; + return { captured, requests, state }; } const columns = ['Run', 'Status', 'Started (UTC)', 'Duration', 'Model', 'Tokens', 'Spans']; @@ -726,67 +752,100 @@ for (const trace of ['trace-with-dashes', '']) { }); } +// Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the +// response handler below each model call, and intermediate spans filtered out. +const codexTrace = 'e5'.repeat(16); +const codex = (span: Omit) => + spanRows({ + ...span, + trace: codexTrace, + attributes: { 'service.name': 'codex-app-server', ...span.attributes }, + }); +const codexRows = [ + codex({ + id: 'e500000000000001', + parent: 'e5000000000000aa', + name: 'session_task.turn', + start: base + 1000, + duration: 4000, + attributes: { + model: 'gpt-6-luna', + 'codex.turn.token_usage.input_tokens': '32558', + 'codex.turn.token_usage.output_tokens': '38', + }, + }), + codex({ + id: 'e500000000000002', + parent: 'e5000000000000bb', + name: 'try_run_sampling_request', + start: base + 1100, + duration: 1600, + attributes: { model: 'gpt-6-luna' }, + }), + codex({ + id: 'e500000000000003', + parent: 'e500000000000002', + name: 'receiving_stream', + start: base + 1110, + duration: 1580, + }), + codex({ + id: 'e500000000000004', + parent: 'e500000000000003', + name: 'handle_responses', + start: base + 2600, + duration: 40, + attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, + }), + codex({ + id: 'e500000000000005', + parent: 'e5000000000000cc', + name: 'exec', + start: base + 2700, + duration: 137, + attributes: { tool_name: 'exec', call_id: 'call_codex' }, + }), +].flat(); +const codexFields = [ + ...new Set([...codexRows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens']), +]; +// Log records as Codex exports them with log_user_prompt on, stamped with the +// trace and span they happened in. The tool result names the call it answers. +const codexLogs: LogRecord[] = [ + { + 'event.name': 'codex.user_prompt', + 'event.timestamp': ingested(base + 1000), + trace_id: codexTrace, + span_id: 'e5000000000000aa', + 'conversation.id': 'conversation', + prompt: 'List the files here', + }, + { + 'event.name': 'codex.tool_result', + 'event.timestamp': ingested(base + 2840), + trace_id: codexTrace, + span_id: 'e5000000000000dd', + call_id: 'call_codex', + tool_name: 'exec', + arguments: 'ls', + output: 'README.md', + }, +]; +const codexLogFields = [ + ...new Set([...codexLogs.flatMap((row) => Object.keys(row)), 'originator', 'p_timestamp']), +]; +const claudeFields = [ + ...codexFields.filter((field) => !field.startsWith('codex.')), + 'interaction.sequence', +]; + test('native Codex traces become runs with mapped model calls, tools and usage', async ({ page, }) => { - // Spans as Codex CLI 0.162.1 exports them: no GenAI agent fields, usage on the - // response handler below each model call, and intermediate spans filtered out. - const trace = 'e5'.repeat(16); - const codex = (span: Omit) => - spanRows({ - ...span, - trace, - attributes: { 'service.name': 'codex-app-server', ...span.attributes }, - }); - const rows = [ - codex({ - id: 'e500000000000001', - parent: 'e5000000000000aa', - name: 'session_task.turn', - start: base + 1000, - duration: 4000, - attributes: { - model: 'gpt-6-luna', - 'codex.turn.token_usage.input_tokens': '32558', - 'codex.turn.token_usage.output_tokens': '38', - }, - }), - codex({ - id: 'e500000000000002', - parent: 'e5000000000000bb', - name: 'try_run_sampling_request', - start: base + 1100, - duration: 1600, - attributes: { model: 'gpt-6-luna' }, - }), - codex({ - id: 'e500000000000003', - parent: 'e500000000000002', - name: 'receiving_stream', - start: base + 1110, - duration: 1580, - }), - codex({ - id: 'e500000000000004', - parent: 'e500000000000003', - name: 'handle_responses', - start: base + 2600, - duration: 40, - attributes: { 'gen_ai.usage.input_tokens': '32558', 'gen_ai.usage.output_tokens': '38' }, - }), - codex({ - id: 'e500000000000005', - parent: 'e5000000000000cc', - name: 'exec', - start: base + 2700, - duration: 137, - attributes: { tool_name: 'exec', call_id: 'call_codex' }, - }), - ].flat(); const { captured } = await serveAgents(page, { - rows, + rows: codexRows, datasets: ['codex-traces'], - fields: [...new Set([...rows.flatMap((row) => Object.keys(row)), 'gen_ai.usage.input_tokens'])], + fields: codexFields, }); await page.goto(`/agents?${range}`); await page.getByLabel('Agents dataset').selectOption('codex-traces'); @@ -814,6 +873,38 @@ test('native Codex traces become runs with mapped model calls, tools and usage', await expect(span).not.toContainText('gen_ai.tool.name'); }); +test('Codex conversation logs fill run details from the paired logs dataset', async ({ page }) => { + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs', 'archive-logs'], + fields: codexFields, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + const details = await openRun(page, 'Codex'); + await details.getByRole('tab', { name: 'Conversation' }).click(); + const turn = details.getByRole('region', { name: 'Conversation from session_task.turn' }); + await expect(turn).toContainText('User prompt'); + await expect(turn).toContainText('List the files here'); + const tool = details.getByRole('region', { name: 'Conversation from exec' }); + await expect(tool).toContainText('Tool arguments · exec'); + await expect(tool).toContainText('README.md'); + const logQuery = captured.find((request) => request.query.includes('FROM "codex-logs"')); + expect(logQuery?.query).toContain(`"trace_id" IN ('${codexTrace}')`); + expect(sqlBounds(logQuery!.query)).toEqual([bounds.startTime, bounds.endTime]); + + // Conversation text is searchable, and logs can be turned off. + await page.keyboard.press('Escape'); + await expect(details).toBeHidden(); + await page.getByLabel('Search agent runs').fill('README.md'); + await expect(page).toHaveURL(/search=README\.md/); + await expect(runRow(page, 'Codex')).toBeVisible(); + await page.getByLabel('Conversation logs').selectOption('none'); + await expect(page).toHaveURL(/logs=none/); + await expect(page.getByRole('heading', { name: 'No matching runs' })).toBeVisible(); +}); + test('discloses the record limit from the queried rows when Codex mapping drops some', async ({ page, }) => { @@ -862,3 +953,132 @@ test('discloses the record limit from the queried rows when Codex mapping drops 'narrow the time range', ); }); + +test('a logs choice does not carry over to a traces dataset of another harness', async ({ + page, +}) => { + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'claude-code-traces', 'codex-logs'], + fields: codexFields, + schemas: { 'claude-code-traces': claudeFields }, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + const logQueries = () => + captured.filter((request) => request.query.includes('FROM "codex-logs"')).length; + await page.goto(`/agents/codex-traces?${range}&logs=codex-logs`); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect.poll(logQueries).toBe(1); + + await page.getByLabel('Agents dataset').focus(); + await page.getByLabel('Agents dataset').selectOption('claude-code-traces'); + await expect(page).toHaveURL(/\/agents\/claude-code-traces\?/); + expect(new URL(page.url()).searchParams.get('logs')).toBeNull(); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toHaveText( + /No Claude Code logs dataset found/, + ); + await expect(page.getByLabel('Conversation logs')).toHaveValue('none'); + expect(logQueries()).toBe(1); + + // A stale link naming another harness's logs is ignored too. + await page.goto(`/agents/claude-code-traces?${range}&logs=codex-logs`); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toBeVisible(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('none'); + await expect(page.getByLabel('Conversation logs').locator('option')).toHaveText([ + 'No conversation logs', + ]); + expect(logQueries()).toBe(1); +}); + +test('Refresh keeps logs discovery, and Rescan finds a new logs dataset', async ({ page }) => { + const { captured, requests, state } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces'], + fields: codexFields, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + const count = (match: (path: string) => boolean) => requests.filter(match).length; + const lists = () => count((path) => path === '/api/v1/logstream'); + const schemas = () => count((path) => path.endsWith('/schema')); + await page.goto(`/agents/codex-traces?${range}`); + const missing = page.getByRole('status').filter({ hasText: 'No Codex logs dataset found' }); + await expect(missing).toBeVisible(); + const [listsBefore, schemasBefore] = [lists(), schemas()]; + + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect.poll(() => captured.length).toBe(2); + await expect(runRow(page, 'Codex')).toBeVisible(); + await expect(page.getByLabel('Conversation logs')).toBeVisible(); + // Only the traces dataset's own schema is read again. + expect(lists()).toBe(listsBefore); + expect(schemas()).toBe(schemasBefore + 1); + + state.datasets.push('codex-logs'); + await page.getByRole('button', { name: 'Rescan datasets' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect(missing).toBeHidden(); + await expect + .poll(() => captured.filter((request) => request.query.includes('FROM "codex-logs"')).length) + .toBe(1); +}); + +test('discloses when conversation logs reach the record limit', async ({ page }) => { + const rows = Array.from({ length: 1000 }, (_, index) => ({ + ...codexLogs[1], + 'event.timestamp': ingested(base + 2000 + index), + })); + const { captured } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs'], + fields: codexFields, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows }, + }); + await page.goto(`/agents/codex-traces?${range}`); + await expect( + page.getByRole('status').filter({ hasText: 'Conversation logs reached the record limit' }), + ).toContainText('the oldest events may be missing'); + // The newest events are kept when the limit cuts the result. + const logQuery = captured.find((request) => request.query.includes('FROM "codex-logs"')); + expect(logQuery?.query).toMatch(/ORDER BY "p_timestamp" DESC LIMIT 1000$/); +}); + +test('a failed logs scan offers Retry instead of reporting no logs dataset', async ({ page }) => { + const { state } = await serveAgents(page, { + rows: codexRows, + datasets: ['codex-traces', 'codex-logs'], + fields: codexFields, + schemas: { 'codex-logs': 503 }, + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + await page.goto(`/agents/codex-traces?${range}`); + const unchecked = page.getByRole('status').filter({ hasText: 'Could not inspect codex-logs' }); + await expect(unchecked).toContainText('for Codex logs'); + await expect(page.getByText('No Codex logs dataset found')).toHaveCount(0); + + delete state.schemas['codex-logs']; + await unchecked.getByRole('button', { name: 'Retry' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + await expect(unchecked).toBeHidden(); +}); + +test('a mixed traces dataset joins logs of whichever harness is selected', async ({ page }) => { + const { state } = await serveAgents(page, { + rows: codexRows, + datasets: ['agent-traces'], + fields: [...codexFields, 'interaction.sequence'], + logs: { dataset: 'codex-logs', fields: codexLogFields, rows: codexLogs }, + }); + await page.goto(`/agents/agent-traces?${range}`); + await expect(page.getByRole('status').filter({ hasText: 'logs dataset found' })).toHaveText( + /No Codex or Claude Code logs dataset found/, + ); + + state.datasets.push('codex-logs'); + await page.getByRole('button', { name: 'Rescan datasets' }).click(); + await expect(page.getByLabel('Conversation logs')).toHaveValue('codex-logs'); + const details = await openRun(page, 'Codex'); + await details.getByRole('tab', { name: 'Conversation' }).click(); + await expect( + details.getByRole('region', { name: 'Conversation from session_task.turn' }), + ).toContainText('List the files here'); +}); diff --git a/frontend/src/features/agents/AgentRunDetails.tsx b/frontend/src/features/agents/AgentRunDetails.tsx index 9d4e99640..f7bb3d18e 100644 --- a/frontend/src/features/agents/AgentRunDetails.tsx +++ b/frontend/src/features/agents/AgentRunDetails.tsx @@ -66,7 +66,9 @@ export function AgentRunDetails({ run }: { run: AgentRun }) { const models = run.spans.filter((span) => span.kind === 'model'); const tools = run.spans.filter((span) => span.kind === 'tool'); const errors = run.spans.filter((span) => span.errors.length || span.status === 'failed'); - const conversation = run.spans.filter((span) => span.conversation.length); + const conversation = run.spans + .filter((span) => span.conversation.length) + .sort((a, b) => (Date.parse(a.startTime ?? '') || 0) - (Date.parse(b.startTime ?? '') || 0)); const spanButton = (span: AgentSpan) => ( {dataset && ( - )} @@ -266,6 +369,49 @@ export function AgentsPage() { )} )} + {logsHarness && logs.error && ( +

+ Could not load {AGENT_HARNESSES[logsHarness].label} conversation logs from {logsDataset}:{' '} + {logs.error.message}{' '} + +

+ )} + {logsHarness && logs.data && logs.data.length >= AGENT_RECORD_LIMIT && ( +

+ Conversation logs reached the record limit; the oldest events may be missing. Narrow the + time range to load them. +

+ )} + {!!harnesses.length && logsDiscovery.error && ( +

+ Could not find {harnessLabels} logs datasets: {logsDiscovery.error.message}{' '} + +

+ )} + {!!harnesses.length && !!logsDiscovery.data?.unchecked.length && ( +

+ Could not inspect {logsDiscovery.data.unchecked.map((entry) => entry.name).join(', ')} for{' '} + {harnessLabels} logs.{' '} + +

+ )} + {!!harnesses.length && + logsCandidates && + !logsCandidates.length && + !logsDiscovery.error && + !logsDiscovery.data?.unchecked.length && + logsParam !== 'none' && ( +

+ No {harnessLabels} logs dataset found. Export its logs to Parseable to see prompts and + tool results in run details. +

+ )} {dataset && discoveryEnabled && discovery.loading && (

Finding compatible datasets… @@ -279,7 +425,7 @@ export function AgentsPage() { {!!discovery.data?.unchecked.length && (

Could not inspect {discovery.data.unchecked.map((entry) => entry.name).join(', ')}.{' '} -

diff --git a/frontend/src/lib/agentHarnessLogs.test.ts b/frontend/src/lib/agentHarnessLogs.test.ts new file mode 100644 index 000000000..84da6a2a0 --- /dev/null +++ b/frontend/src/lib/agentHarnessLogs.test.ts @@ -0,0 +1,748 @@ +import { describe, expect, it, vi } from 'vitest'; +import { mapHarnessRows } from './agentHarnesses'; +import { + attachHarnessConversation, + buildHarnessLogQuery, + detectHarnessLogs, + discoverHarnessLogs, + pairedLogsDataset, +} from './agentHarnessLogs'; +import { buildAgentRuns, type AgentRun } from './agentRuns'; +import { AGENT_RECORD_LIMIT } from './agents'; +import { ApiError } from './apiError'; +import { createDemoClient } from './demo'; +import type { LogRecord } from './types'; + +// Trimmed from logs exported by Codex CLI 0.162.1 (`codex exec`, log_user_prompt +// on) and Claude Code 2.1.296 (OTEL_LOG_USER_PROMPTS, _ASSISTANT_RESPONSES, +// _TOOL_DETAILS and _TOOL_CONTENT on), as stored by Parseable. +const at = (seconds: number) => new Date(Date.parse('2026-10-13T03:38:00Z') + seconds * 1000); +const stamp = (seconds: number) => at(seconds).toISOString().replace('Z', ''); +function span( + trace: string, + id: string, + parent: string, + name: string, + start: number, + duration: number, + fields: LogRecord = {}, +): LogRecord { + return { + p_timestamp: stamp(start + duration), + span_trace_id: trace, + span_span_id: id, + span_parent_span_id: parent, + span_name: name, + span_start_time_unix_nano: stamp(start), + span_end_time_unix_nano: stamp(start + duration), + span_duration_ns: duration * 1e9, + span_status_code: 0, + ...fields, + }; +} +const conversations = (run: AgentRun) => + run.spans + .filter((entry) => entry.conversation.length) + .map((entry) => [entry.name, entry.conversation.map((item) => [item.label, item.value])]); + +const codexTrace = '1b1ef19838e4d610208a394b9af94384'; +const codexRuns = () => + buildAgentRuns( + mapHarnessRows( + ['codex'], + [ + span(codexTrace, 'fe5417199aa59ebf', '57d7ac8056fe8b6d', 'session_task.turn', 15.8, 4, { + 'service.name': 'codex_exec', + model: 'gpt-6-luna', + }), + span(codexTrace, '67c205c2bacb4518', 'e0d1c4f0b19c5a7e', 'exec', 18, 0.2, { + 'service.name': 'codex_exec', + tool_name: 'exec', + call_id: 'call_7b616c08bc894640a843cff3e0f7b38a', + }), + ], + ), + ); +const codexLog = (fields: LogRecord): LogRecord => ({ + 'service.name': 'codex_exec', + 'conversation.id': '01a13356-e711-7291-829d-0051eaa8578d', + model: 'gpt-6-luna', + trace_id: codexTrace, + ...fields, +}); +const codexTurns = ( + ranges = [ + [15.8, 4], + [25.8, 4], + ], +) => + buildAgentRuns( + mapHarnessRows( + ['codex'], + ranges.map(([start, duration], index) => + span(codexTrace, `turn-${index}`, '', 'session_task.turn', start, duration, { + 'service.name': 'codex_exec', + }), + ), + ), + ); +const codexLogs = [ + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(15.79), + span_id: '57d7ac8056fe8b6d', + prompt: 'Run: cat sample.txt then reply with just the file contents.', + }), + codexLog({ + 'event.name': 'codex.tool_result', + 'event.timestamp': stamp(18.23), + span_id: '07d5bd18e22b0db8', + call_id: 'call_7b616c08bc894640a843cff3e0f7b38a', + tool_name: 'exec', + arguments: 'const r = await tools.exec_command({cmd:"cat sample.txt"});\ntext(r.output);\n', + output: 'Script completed\nWall time 0.2 seconds\nOutput:\n\nhello\n', + }), + // Code-mode tool calls run in a separate trace that has no turn. + codexLog({ + 'event.name': 'codex.tool_result', + 'event.timestamp': stamp(18.22), + trace_id: 'b5039406affbf2550d35a030cd3f6986', + span_id: 'f77605377c432cde', + call_id: 'exec-65b04fe7-d62b-4d47-9411-ef6a1c283dde', + tool_name: 'exec_command', + output: 'hello\n', + }), +]; + +const claudeTrace = '677fd41f03b0e3928c18d90bd38a609c'; +const claudeSpan = (id: string, parent: string, name: string, start: number, ms: number, f = {}) => + span(claudeTrace, id, parent, name, start, ms, { + 'service.name': 'claude-code', + 'interaction.sequence': '1', + ...f, + }); +const claudeRequest = (id: string, start: number) => + claudeSpan(id, '671ed833b6b2a759', 'claude_code.llm_request', start, 2.8, { + 'gen_ai.request.model': 'claude-haiku-4-5', + 'gen_ai.response.id': `req_${id}`, + request_id: `req_${id}`, + }); +const claudeToolSpan = claudeSpan( + '45801e168f0029ae', + '671ed833b6b2a759', + 'claude_code.tool', + 12.42, + 0.05, + { tool_name: 'Bash', tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx' }, +); +const claudeToolOutput = { + ...claudeToolSpan, + event_name: 'tool.output', + event_time_unix_nano: stamp(12.47), + event_output: 'hello', + event_bash_command: 'cat sample.txt', +}; +const claudeRuns = () => + buildAgentRuns( + mapHarnessRows( + ['claude-code'], + [ + claudeSpan('671ed833b6b2a759', '', 'claude_code.interaction', 9.5, 6), + claudeRequest('first', 9.6), + claudeToolSpan, + // Flattened span events repeat; the output is shown once. + claudeToolOutput, + { ...claudeToolOutput }, + claudeRequest('second', 12.5), + ], + ), + ); +const claudeLog = (sequence: number, seconds: number, fields: LogRecord): LogRecord => ({ + 'service.name': 'claude-code', + 'session.id': 'bf8fbff9-ba17-486f-9385-b22d997bf213', + 'prompt.id': 'c0c7a199-8d62-4a46-9575-2654870f8ad0', + 'event.sequence': String(sequence), + 'event.timestamp': stamp(seconds), + trace_id: claudeTrace, + span_id: '671ed833b6b2a759', + ...fields, +}); +const claudeLogs = [ + claudeLog(4, 9.52, { + 'event.name': 'user_prompt', + prompt: 'Run cat sample.txt with the Bash tool, then reply with just the file contents.\n', + }), + claudeLog(8, 12.428, { + 'event.name': 'tool_decision', + span_id: '55648ab0912c53ee', + decision: 'accept', + source: 'config', + tool_name: 'Bash', + tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx', + }), + // Logged when the tool finished; its input belongs before the output. + claudeLog(10, 12.471, { + 'event.name': 'tool_result', + duration_ms: '42', + tool_name: 'Bash', + tool_use_id: 'toolu_01HvwCWjo8bJsGSinB2vRVrx', + tool_input: '{"command":"cat sample.txt"}', + }), + claudeLog(12, 15.358, { + 'event.name': 'assistant_response', + request_id: 'req_second', + response: 'hello', + }), + claudeLog(13, 15.4, { + 'event.name': 'assistant_response', + request_id: 'req_not_loaded', + response: '', + }), +]; + +describe('coding-agent logs datasets', () => { + it('recognizes Codex and Claude Code logs, but not their traces', () => { + const logs = ['event.name', 'trace_id', 'span_id']; + expect(detectHarnessLogs([...logs, 'conversation.id', 'originator'])).toBe('codex'); + expect(detectHarnessLogs([...logs, 'prompt.id', 'event.sequence'])).toBe('claude-code'); + expect( + detectHarnessLogs([...logs, 'prompt.id', 'event.sequence', 'span_name']), + ).toBeUndefined(); + expect(detectHarnessLogs(['event.name', 'conversation.id', 'originator'])).toBeUndefined(); + }); + it('pairs a traces dataset with the logs dataset named like it', () => { + expect(pairedLogsDataset('codex-traces', ['archive', 'codex-logs'])).toBe('codex-logs'); + expect(pairedLogsDataset('claude_trace', ['claude_log', 'z'])).toBe('claude_log'); + expect(pairedLogsDataset('agents', ['z-logs', 'a-logs'])).toBe('a-logs'); + }); + it('queries only conversation events for the loaded traces with quoted values', () => { + const query = buildHarnessLogQuery( + 'codex"logs', + 'codex', + [codexTrace, "x' OR '1'='1", codexTrace, ' '], + { startTime: '2026-10-13T03:00:00Z', endTime: '2026-10-13T04:00:00Z' }, + )!; + expect(query.sql).toBe( + `SELECT * FROM "codex""logs" WHERE "p_timestamp" >= TIMESTAMP '2026-10-13T03:00:00.000Z' AND "p_timestamp" < TIMESTAMP '2026-10-13T04:00:00.000Z' AND "trace_id" IN ('${codexTrace}', 'x'' OR ''1''=''1') AND "event.name" IN ('codex.user_prompt', 'codex.tool_result', 'codex.tool_decision') ORDER BY "p_timestamp" DESC LIMIT ${AGENT_RECORD_LIMIT}`, + ); + expect( + buildHarnessLogQuery('logs', 'codex', [], { + startTime: '2026-10-13T03:00:00Z', + endTime: '2026-10-13T04:00:00Z', + }), + ).toBeUndefined(); + }); +}); + +describe('harness logs discovery', () => { + const codexFields = ['event.name', 'trace_id', 'conversation.id', 'originator']; + const clientFor = (names: string[]) => ({ + ...createDemoClient(), + listDatasets: vi.fn(async () => names.map((name) => ({ name, type: 'logs' as const }))), + schema: vi.fn(async (_name: string, _signal?: AbortSignal) => codexFields), + }); + + it('discovers both harnesses in one scan and excludes non-logs datasets', async () => { + const client = clientFor(['codex-logs', 'claude-logs', 'traces', 'other']); + client.schema.mockImplementation(async (name) => { + if (name === 'claude-logs') return ['event.name', 'trace_id', 'prompt.id', 'event.sequence']; + if (name === 'traces') return [...codexFields, 'span_name']; + if (name === 'other') return ['p_timestamp', 'message']; + return codexFields; + }); + const signal = new AbortController().signal; + await expect(discoverHarnessLogs(client, signal)).resolves.toEqual({ + datasets: [ + { name: 'codex-logs', harness: 'codex' }, + { name: 'claude-logs', harness: 'claude-code' }, + ], + unchecked: [], + }); + expect(client.listDatasets).toHaveBeenCalledExactlyOnceWith(signal); + expect(client.schema).toHaveBeenCalledTimes(4); + for (const name of ['codex-logs', 'claude-logs', 'traces', 'other']) + expect(client.schema).toHaveBeenCalledWith(name, signal); + }); + + it.each([403, 404])('silently skips schema status %s', async (status) => { + const client = clientFor(['allowed', 'skipped']); + client.schema.mockImplementation(async (name) => { + if (name === 'skipped') throw new ApiError('Cannot read', status); + return codexFields; + }); + await expect(discoverHarnessLogs(client)).resolves.toEqual({ + datasets: [{ name: 'allowed', harness: 'codex' }], + unchecked: [], + }); + }); + + it('reports unavailable schemas while keeping discovered datasets', async () => { + const client = clientFor(['allowed', 'unavailable']); + client.schema.mockImplementation(async (name) => { + if (name === 'unavailable') throw new ApiError('Unavailable', 503); + return codexFields; + }); + await expect(discoverHarnessLogs(client)).resolves.toEqual({ + datasets: [{ name: 'allowed', harness: 'codex' }], + unchecked: [{ name: 'unavailable', error: 'Unavailable' }], + }); + }); + + it('reports non-Error schema failures', async () => { + const client = clientFor(['unavailable']); + client.schema.mockRejectedValue('Connection lost'); + await expect(discoverHarnessLogs(client)).resolves.toEqual({ + datasets: [], + unchecked: [{ name: 'unavailable', error: 'Connection lost' }], + }); + }); + + it('rethrows expired sessions', async () => { + const client = clientFor(['logs']); + const error = new ApiError('Expired', 401); + client.schema.mockRejectedValue(error); + await expect(discoverHarnessLogs(client)).rejects.toBe(error); + }); + + it('rethrows dataset listing failures without scanning schemas', async () => { + const client = clientFor(['logs']); + const error = new ApiError('Unavailable', 503); + client.listDatasets.mockRejectedValue(error); + await expect(discoverHarnessLogs(client)).rejects.toBe(error); + expect(client.schema).not.toHaveBeenCalled(); + }); + + it('rethrows an aborted schema request', async () => { + const client = clientFor(['logs']); + const controller = new AbortController(); + const error = new DOMException('Aborted', 'AbortError'); + client.schema.mockImplementation(async () => { + controller.abort(error); + throw error; + }); + await expect(discoverHarnessLogs(client, controller.signal)).rejects.toBe(error); + }); +}); + +describe('coding-agent conversation', () => { + it('restores time and sequence order for a full descending page of logs', () => { + const prompts = Array.from({ length: AGENT_RECORD_LIMIT + 1 }, (_, index) => + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(16 + Math.floor(index / 2) / 1000), + 'event.sequence': index, + prompt: `Prompt ${index}`, + }), + ); + const newest = [...prompts].reverse().slice(0, AGENT_RECORD_LIMIT); + const [run] = attachHarnessConversation(codexRuns(), 'codex', newest); + expect(run.roots[0].conversation.map((entry) => entry.value)).toEqual( + prompts.slice(1).map((row) => row.prompt), + ); + }); + + it('puts the Codex prompt on its turn and tool calls on their tool span', () => { + const [run] = attachHarnessConversation(codexRuns(), 'codex', codexLogs); + expect(conversations(run)).toEqual([ + [ + 'session_task.turn', + [['User prompt', 'Run: cat sample.txt then reply with just the file contents.']], + ], + [ + 'exec', + [ + ['Tool arguments · exec', codexLogs[1].arguments], + ['Tool output · exec', codexLogs[1].output], + ], + ], + ]); + }); + it('assigns prompts to their turns, including a lead before the turn and the turn end', () => { + const [run] = attachHarnessConversation(codexTurns(), 'codex', [ + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(15.79), + span_id: 'turn-input-0', + prompt: 'first', + }), + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(29.8), + span_id: 'turn-input-1', + prompt: 'second', + }), + ]); + expect(run.spans.map((span) => span.conversation)).toEqual([ + [{ label: 'User prompt', value: 'first' }], + [{ label: 'User prompt', value: 'second' }], + ]); + }); + it('leaves unmatched prompts unattached when several turns are loaded', () => { + const [run] = attachHarnessConversation( + codexTurns(), + 'codex', + [14.7, 22, undefined].map((seconds) => + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': seconds === undefined ? undefined : stamp(seconds), + prompt: 'unmatched', + }), + ), + ); + expect(conversations(run)).toEqual([]); + }); + it('does not give a later turn prompt to the only loaded turn', () => { + // The second turn is still running, so its span has not been exported. + const [run] = attachHarnessConversation(codexTurns([[15.8, 4]]), 'codex', [ + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(25.79), + prompt: 'later', + }), + ]); + expect(conversations(run)).toEqual([]); + }); + it('attributes prompts to Codex turns under an agent span of another service', () => { + const rows = [ + span(codexTrace, 'orchestrator0001', '', 'orchestrate', 10, 30, { + 'service.name': 'orchestrator', + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.agent.name': 'Orchestrator', + }), + ...[15.8, 25.8].map((start, index) => + span(codexTrace, `turn-${index}`, 'orchestrator0001', 'session_task.turn', start, 4, { + 'service.name': 'codex_exec', + }), + ), + ]; + const [run] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], rows)), + 'codex', + [15.79, 25.79].map((seconds, index) => + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(seconds), + prompt: `prompt ${index}`, + }), + ), + ); + expect(conversations(run).map(([, items]) => items)).toEqual([ + [['User prompt', 'prompt 0']], + [['User prompt', 'prompt 1']], + ]); + }); + it('falls back to the only turn, not an enclosing span of another service', () => { + const rows = [ + span(codexTrace, 'cijob00000000001', '', 'ci.job', 0, 60, { 'service.name': 'ci' }), + span(codexTrace, 'turn-0', 'cijob00000000001', 'session_task.turn', 15.8, 4, { + 'service.name': 'codex_exec', + }), + ]; + const [run] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], rows)), + 'codex', + [codexLog({ 'event.name': 'codex.user_prompt', prompt: 'untimed' })], + ); + expect(run.spans.find((entry) => entry.spanId === 'turn-0')!.conversation).toEqual([ + { label: 'User prompt', value: 'untimed' }, + ]); + }); + it('gives each prompt to the turn starting right after it, even for close turns', () => { + const prompts = (...seconds: number[]) => + seconds.map((value, index) => + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(value), + span_id: `turn-input-${index}`, + prompt: `prompt ${index}`, + }), + ); + const [close] = attachHarnessConversation( + codexTurns([ + [10, 0.2], + [10.7, 0.2], + ]), + 'codex', + prompts(9.99, 10.69), + ); + expect(close.spans.map((span) => span.conversation)).toEqual([ + [{ label: 'User prompt', value: 'prompt 0' }], + [{ label: 'User prompt', value: 'prompt 1' }], + ]); + // The prompt of an earlier, unloaded turn stays unattached. + const [later] = attachHarnessConversation( + codexTurns([ + [10.7, 0.2], + [15, 0.2], + ]), + 'codex', + prompts(9.99), + ); + expect(conversations(later)).toEqual([]); + }); + it('matches a prompt to the turn whose recorded parent is its input span', () => { + const turn = (id: string, parent: string, start: number) => + span(codexTrace, id, parent, 'session_task.turn', start, 4, { 'service.name': 'codex_exec' }); + const prompt = (spanId: string, seconds: number, text: string) => + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(seconds), + span_id: spanId, + prompt: text, + }); + // Logged well before the turn starts, so only the recorded parent places it. + const [slow] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], [turn('turn-0', 'input-0', 15.8)])), + 'codex', + [prompt('input-0', 13.8, 'slow start')], + ); + expect(slow.roots[0].conversation).toEqual([{ label: 'User prompt', value: 'slow start' }]); + // A parent shared by several turns falls back to their time ranges. + const [shared] = attachHarnessConversation( + buildAgentRuns( + mapHarnessRows(['codex'], [turn('turn-0', 'shared', 15.8), turn('turn-1', 'shared', 25.8)]), + ), + 'codex', + [prompt('shared', 16, 'first'), prompt('shared', 26, 'second')], + ); + expect(shared.spans.map((entry) => entry.conversation)).toEqual([ + [{ label: 'User prompt', value: 'first' }], + [{ label: 'User prompt', value: 'second' }], + ]); + }); + it('leaves a prompt unattached when turns start together or only foreign agents are loaded', () => { + const prompt = codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(9.99), + prompt: 'tied', + }); + const [tied] = attachHarnessConversation( + codexTurns([ + [10, 1], + [10, 2], + ]), + 'codex', + [prompt], + ); + expect(conversations(tied)).toEqual([]); + const orchestrator = span(codexTrace, 'orchestrator0001', '', 'orchestrate', 0, 60, { + 'service.name': 'orchestrator', + 'gen_ai.operation.name': 'invoke_agent', + }); + const [foreign] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], [orchestrator])), + 'codex', + [codexLog({ 'event.name': 'codex.user_prompt', 'event.timestamp': stamp(20), prompt: 'x' })], + ); + expect(conversations(foreign)).toEqual([]); + }); + it('uses a turn input span only for Codex prompts', () => { + const rows = [ + span(codexTrace, 'turn-0', 'external-parent', 'session_task.turn', 10, 2, { + 'service.name': 'codex_exec', + }), + span(codexTrace, 'turn-1', '', 'session_task.turn', 25, 2, { 'service.name': 'codex_exec' }), + ]; + const [run] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], rows)), + 'codex', + [ + codexLog({ + 'event.name': 'codex.tool_decision', + 'event.timestamp': stamp(26), + span_id: 'external-parent', + decision: 'approved', + }), + ], + ); + expect(run.spans.map((entry) => entry.conversation.length)).toEqual([0, 1]); + }); + it('leaves prompts unattached when turn ranges overlap', () => { + const [run] = attachHarnessConversation( + codexTurns([ + [15.8, 4], + [17, 4], + ]), + 'codex', + [ + codexLog({ + 'event.name': 'codex.user_prompt', + 'event.timestamp': stamp(18), + prompt: 'ambiguous', + }), + ], + ); + expect(conversations(run)).toEqual([]); + }); + it('does not fall back to a tool root when no agent turn is loaded', () => { + const runs = buildAgentRuns(mapHarnessRows(['codex'], [codexRuns()[0].spans[1].row])); + const [run] = attachHarnessConversation(runs, 'codex', [codexLogs[0]]); + expect(conversations(run)).toEqual([]); + }); + it('falls back to the root for an unmatched prompt only with one agent turn', () => { + const [run] = attachHarnessConversation(codexTurns([[15.8, 4]]), 'codex', [ + codexLog({ 'event.name': 'codex.user_prompt', prompt: 'single turn' }), + ]); + expect(run.roots[0].conversation).toEqual([{ label: 'User prompt', value: 'single turn' }]); + }); + it('uses the same turn fallback for tool records without a loaded tool', () => { + const [run] = attachHarnessConversation(codexTurns(), 'codex', [ + codexLog({ + 'event.name': 'codex.tool_result', + 'event.timestamp': stamp(26), + output: 'second turn', + }), + codexLog({ + 'event.name': 'codex.tool_decision', + 'event.timestamp': stamp(22), + decision: 'unmatched', + }), + ]); + expect(run.spans.map((span) => span.conversation)).toEqual([ + [], + [{ label: 'Tool output', value: 'second turn' }], + ]); + }); + it.each([false, true])('scopes shared call IDs by trace (own tool span: %s)', (own) => { + const rows = [codexTrace, 'other-trace'].flatMap((trace) => + codexRuns()[0].spans.map((span) => ({ ...span.row, span_trace_id: trace })), + ); + const logs = [codexTrace, 'other-trace'].map((trace) => ({ + ...codexLogs[1], + trace_id: trace, + span_id: own ? '67c205c2bacb4518' : 'not-loaded', + arguments: undefined, + output: trace, + })); + const runs = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], rows)), + 'codex', + logs, + ); + for (const run of runs) + expect(run.spans.find((span) => span.kind === 'tool')?.conversation).toEqual([ + { label: 'Tool output · exec', value: run.traceId }, + ]); + }); + it.each(['codex.tool_result', 'tool_result', 'codex.tool_decision'])( + 'prefers the own tool span over a conflicting call ID for %s', + (event) => { + const rows = codexRuns()[0].spans.map((span) => span.row); + rows.push({ ...rows[1], span_span_id: 'colliding-tool' }); + const [run] = attachHarnessConversation( + buildAgentRuns(mapHarnessRows(['codex'], rows)), + 'codex', + [ + codexLog({ + 'event.name': event, + span_id: '67c205c2bacb4518', + call_id: codexLogs[1].call_id, + output: 'own tool', + tool_input: 'own tool', + decision: 'own tool', + }), + ], + ); + expect(run.spans.find((span) => span.spanId === '67c205c2bacb4518')?.conversation).toEqual([ + { + label: + event === 'codex.tool_result' + ? 'Tool output' + : event === 'tool_result' + ? 'Tool input' + : 'Tool decision', + value: 'own tool', + }, + ]); + expect(run.spans.find((span) => span.spanId === 'colliding-tool')?.conversation).toEqual([]); + }, + ); + it.each(['continue', '[REDACTED]'])( + 'keeps distinct prompts with identical text: %s', + (prompt) => { + const [run] = attachHarnessConversation(codexRuns(), 'codex', [ + { ...codexLogs[0], prompt }, + { ...codexLogs[0], 'event.timestamp': stamp(16), prompt }, + ]); + const item = { + label: prompt === '[REDACTED]' ? 'User prompt (redacted)' : 'User prompt', + value: prompt, + }; + expect(run.roots[0].conversation).toEqual([item, item]); + }, + ); + it('keeps identical prompts at the same time with distinct event sequences', () => { + const [run] = attachHarnessConversation(claudeRuns(), 'claude-code', [ + claudeLog(1, 10, { 'event.name': 'user_prompt', prompt: 'continue' }), + claudeLog(2, 10, { 'event.name': 'user_prompt', prompt: 'continue' }), + ]); + expect(run.roots[0].conversation).toEqual([ + { label: 'User prompt', value: 'continue' }, + { label: 'User prompt', value: 'continue' }, + ]); + }); + it('deduplicates flattened outputs by occurrence while keeping later identical output', () => { + const runs = claudeRuns(); + runs[0].spans + .find((span) => span.kind === 'tool')! + .rows.push({ + ...claudeToolOutput, + event_time_unix_nano: stamp(12.48), + }); + const [run] = attachHarnessConversation(runs, 'claude-code'); + expect(conversations(run)).toEqual([ + [ + 'claude_code.tool', + [ + ['Tool output · Bash', 'hello'], + ['Tool output · Bash', 'hello'], + ], + ], + ]); + }); + it('labels redacted content instead of hiding it', () => { + const [run] = attachHarnessConversation(codexRuns(), 'codex', [ + { ...codexLogs[0], prompt: '[REDACTED]' }, + ]); + expect(run.roots[0].conversation).toEqual([ + { label: 'User prompt (redacted)', value: '[REDACTED]' }, + ]); + }); + it('orders Claude Code prompts, tool activity and responses as they happened', () => { + const [run] = attachHarnessConversation(claudeRuns(), 'claude-code', claudeLogs); + expect(conversations(run)).toEqual([ + [ + 'claude_code.interaction', + [ + ['User prompt', claudeLogs[0].prompt], + // A response whose model call is not loaded stays on the span that logged it. + ['Assistant response (redacted)', ''], + ], + ], + [ + 'claude_code.tool', + [ + ['Tool decision · Bash', 'accept (config)'], + ['Tool input · Bash', '{"command":"cat sample.txt"}'], + ['Tool output · Bash', 'hello'], + ], + ], + ['claude_code.llm_request', [['Assistant response', 'hello']]], + ]); + }); + it('shows Claude Code tool output from traces without a logs dataset', () => { + const [run] = attachHarnessConversation(claudeRuns(), 'claude-code'); + expect(conversations(run)).toEqual([['claude_code.tool', [['Tool output · Bash', 'hello']]]]); + }); + it('ignores records for runs that are not loaded, and other datasets', () => { + const runs = codexRuns(); + expect( + attachHarnessConversation(runs, 'codex', [codexLogs[2]]).flatMap((run) => + run.spans.flatMap((entry) => entry.conversation), + ), + ).toEqual([]); + const plain = codexRuns(); + expect(attachHarnessConversation(plain, undefined, codexLogs)).toBe(plain); + expect(plain[0].roots[0].conversation).toEqual([]); + }); +}); diff --git a/frontend/src/lib/agentHarnessLogs.ts b/frontend/src/lib/agentHarnessLogs.ts new file mode 100644 index 000000000..9372815c6 --- /dev/null +++ b/frontend/src/lib/agentHarnessLogs.ts @@ -0,0 +1,269 @@ +import { parseEventTimestamp } from '../components/explorer/timestamp'; +import { AGENT_HARNESSES, recordedFields, type AgentHarness } from './agentHarnesses'; +import type { AgentConversationEntry, AgentRun, AgentSpan } from './agentRuns'; +import { AGENT_RECORD_LIMIT } from './agents'; +import { ApiError } from './apiError'; +import { apmTransportBounds } from './apm'; +import { mapConcurrent } from './concurrency'; +import { quoteIdentifier, quoteLiteral, timeBounds } from './query'; +import type { LogRecord, ParseableClient, QueryRequest } from './types'; + +// Coding-agent CLIs export prompts, responses and tool results as log records +// stamped with the trace and span they happened in. Joining them by trace ID +// fills the Conversation tab for runs built from the harness's traces. +const LOG_PROFILES: Record = { + codex: { + signature: ['conversation.id', 'originator'], + events: ['codex.user_prompt', 'codex.tool_result', 'codex.tool_decision'], + }, + 'claude-code': { + signature: ['prompt.id', 'event.sequence'], + events: ['user_prompt', 'assistant_response', 'tool_result', 'tool_decision'], + }, +}; +// Claude Code records tool output as an event on the tool span, not as a log. +const CLAUDE_TOOL_OUTPUT = 'tool.output'; +const CODEX_PROMPT_LEAD_MS = 100; +const REDACTED = new Set(['[REDACTED]', '']); + +/** The harness whose logs this dataset holds; traces datasets are not logs. */ +export function detectHarnessLogs(fields: string[]): AgentHarness | undefined { + if (fields.includes('span_name') || !fields.includes('trace_id')) return; + if (!fields.includes('event.name')) return; + return (Object.keys(LOG_PROFILES) as AgentHarness[]).find((harness) => + LOG_PROFILES[harness].signature.every((field) => fields.includes(field)), + ); +} + +/** Recognized harness logs datasets, with operational schema failures reported separately. */ +export async function discoverHarnessLogs( + client: ParseableClient, + signal?: AbortSignal, +): Promise<{ + datasets: { name: string; harness: AgentHarness }[]; + unchecked: { name: string; error: string }[]; +}> { + const datasets = await client.listDatasets(signal); + const inspected = await mapConcurrent( + datasets, + 4, + async ({ name }): Promise<{ name: string; harness?: AgentHarness; error?: string }> => { + signal?.throwIfAborted(); + try { + return { name, harness: detectHarnessLogs(await client.schema(name, signal)) }; + } catch (error) { + if (signal?.aborted || (error instanceof ApiError && error.status === 401)) throw error; + if (error instanceof ApiError && [403, 404].includes(error.status)) return { name }; + return { name, error: error instanceof Error ? error.message : String(error) }; + } + }, + ); + return { + datasets: inspected.flatMap(({ name, harness }) => (harness ? [{ name, harness }] : [])), + unchecked: inspected.flatMap(({ name, error }) => + error !== undefined ? [{ name, error }] : [], + ), + }; +} + +/** Prefer the logs dataset named like the traces dataset, e.g. codex-traces → codex-logs. */ +export function pairedLogsDataset(traces: string, candidates: string[]) { + const sorted = [...candidates].sort(); + return ( + sorted.find((name) => name === traces.replace(/traces?/i, 'logs')) ?? + sorted.find((name) => name === traces.replace(/traces?/i, 'log')) ?? + sorted[0] + ); +} + +/** Conversation log records for the loaded runs' traces. */ +export function buildHarnessLogQuery( + dataset: string, + harness: AgentHarness, + traceIds: string[], + bounds: { startTime: string; endTime: string }, + timeColumn = 'p_timestamp', +): QueryRequest | undefined { + const ids = [...new Set(traceIds.filter((id) => id.trim()))]; + if (!ids.length) return; + const exact = timeBounds(bounds); + const time = quoteIdentifier(timeColumn); + const events = LOG_PROFILES[harness].events.map(quoteLiteral).join(', '); + return { + sql: `SELECT * FROM ${quoteIdentifier(dataset)} WHERE ${time} >= TIMESTAMP ${quoteLiteral(exact.startTime)} AND ${time} < TIMESTAMP ${quoteLiteral(exact.endTime)} AND ${quoteIdentifier('trace_id')} IN (${ids.map(quoteLiteral).join(', ')}) AND ${quoteIdentifier('event.name')} IN (${events}) ORDER BY ${time} DESC LIMIT ${AGENT_RECORD_LIMIT}`, + ...apmTransportBounds(exact), + }; +} + +function text(row: LogRecord, ...fields: string[]) { + for (const field of fields) { + const value = row[field]; + if (typeof value === 'string' && value.trim()) return value.trim(); + } +} +function entry(label: string, value: unknown): AgentConversationEntry[] { + if (value === undefined || value === null || value === '') return []; + return [ + { + label: + typeof value === 'string' && REDACTED.has(value.trim()) ? `${label} (redacted)` : label, + value, + }, + ]; +} +function withTool(label: string, row: LogRecord) { + const tool = text(row, 'tool_name'); + return tool ? `${label} · ${tool}` : label; +} + +type Occurrence = { + time: number; + sequence: number; + target?: AgentSpan; + entries: AgentConversationEntry[]; +}; + +/** + * Append harness conversation events to the spans they belong to, in the order + * they happened. Records whose run is not loaded are ignored, never guessed. + */ +export function attachHarnessConversation( + runs: AgentRun[], + harness: AgentHarness | undefined, + logs: LogRecord[] = [], +): AgentRun[] { + if (!harness) return runs; + const spans = new Map(); + const calls = new Map(); + const requests = new Map(); + const roots = new Map(); + const turns = new Map(); + // Codex logs a prompt on its turn's input span, which the turn records as its parent. + const inputs = new Map(); + for (const run of runs) { + if (run.traceId) { + roots.set(run.traceId, run.roots.find((span) => span.kind === 'agent') ?? run.roots[0]); + // The harness's own turns; an enclosing agent span of another service is not a turn. + const own = run.spans.filter( + (span) => span.kind === 'agent' && span.row.span_name === AGENT_HARNESSES[harness].runSpan, + ); + turns.set(run.traceId, own); + for (const span of own) { + const input = text( + recordedFields(span.rows[0] ?? span.row), + 'span_parent_span_id', + 'parent_span_id', + ); + const key = `${run.traceId}/${input}`; + // A parent shared by several turns is not one turn's input. + if (input) inputs.set(key, inputs.has(key) ? undefined : span); + } + } + for (const span of run.spans) { + if (run.traceId && span.spanId) spans.set(`${run.traceId}/${span.spanId}`, span); + const call = text(span.row, 'gen_ai.tool.call.id', 'tool_use_id', 'call_id'); + if (run.traceId && span.kind === 'tool' && call) calls.set(`${run.traceId}/${call}`, span); + const request = text(span.row, 'gen_ai.response.id', 'request_id'); + if (span.kind === 'model' && request) requests.set(request, span); + } + } + const occurrences: Occurrence[] = []; + const at = (row: LogRecord, ...fields: string[]) => { + for (const field of fields) { + const parsed = parseEventTimestamp(row[field]); + if (Number.isFinite(parsed)) return parsed; + } + return Number.POSITIVE_INFINITY; + }; + const turnAt = (trace: string, time: number, prompt: boolean) => { + const agents = turns.get(trace) ?? []; + // Without a time, only a trace with one loaded turn says where an event belongs. + if (!Number.isFinite(time)) return agents.length === 1 ? agents[0] : undefined; + const start = (span: AgentSpan) => Date.parse(span.startTime ?? ''); + const inside = agents.filter( + (span) => + span.durationMs !== undefined && + time >= start(span) && + time <= start(span) + span.durationMs, + ); + if (inside.length) return inside.length === 1 ? inside[0] : undefined; + // Codex logs a prompt just before its turn span starts (10 ms in the exported + // fixture), so a prompt belongs to a turn starting right after it. + if (!prompt || harness !== 'codex') return; + const next = agents + .filter((span) => start(span) >= time && start(span) - time <= CODEX_PROMPT_LEAD_MS) + .sort((a, b) => start(a) - start(b)); + // Turns starting together leave the prompt's turn unknown. + if (next.length === 1 || (next.length > 1 && start(next[0]) < start(next[1]))) return next[0]; + }; + for (const row of logs) { + const trace = text(row, 'trace_id'); + if (!trace || !roots.has(trace)) continue; + const event = String(row['event.name'] ?? '').replace(/^claude_code\./, ''); + const own = + spans.get(`${trace}/${text(row, 'span_id')}`) ?? + (event === 'codex.user_prompt' ? inputs.get(`${trace}/${text(row, 'span_id')}`) : undefined); + const time = at(row, 'event.timestamp', 'time_unix_nano', 'p_timestamp'); + const call = text(row, 'tool_use_id', 'call_id'); + const tool = own?.kind === 'tool' ? own : calls.get(`${trace}/${call ?? ''}`); + let target = + own ?? turnAt(trace, time, event === 'codex.user_prompt' || event === 'user_prompt'); + let entries: AgentConversationEntry[] = []; + if (event === 'codex.user_prompt' || event === 'user_prompt') { + entries = entry('User prompt', row.prompt ?? row.prompt_text); + } else if (event === 'assistant_response') { + // A response belongs to its model call, which may be a separate trace. + target = requests.get(text(row, 'request_id') ?? '') ?? target; + entries = entry('Assistant response', row.response); + } else if (event === 'codex.tool_result') { + target = tool ?? target; + entries = [ + ...entry(withTool('Tool arguments', row), row.arguments), + ...entry(withTool('Tool output', row), row.output), + ]; + } else if (event === 'tool_result') { + target = tool ?? target; + entries = entry(withTool('Tool input', row), row.tool_input ?? row.tool_parameters); + } else if (event === 'codex.tool_decision' || event === 'tool_decision') { + target = tool ?? target; + const source = text(row, 'source'); + entries = entry( + withTool('Tool decision', row), + row.decision === undefined || !source ? row.decision : `${row.decision} (${source})`, + ); + } + if (entries.length) + occurrences.push({ + // Claude Code logs a tool's input when it finishes; order it by the call's start. + time: time - (event === 'tool_result' ? Number(row.duration_ms) || 0 : 0), + sequence: Number(row['event.sequence'] ?? Number.POSITIVE_INFINITY), + target, + entries, + }); + } + if (harness === 'claude-code') { + const seen = new Set(); + for (const run of runs) + for (const span of run.spans) + for (const row of span.rows) + if (row.event_name === CLAUDE_TOOL_OUTPUT) { + // Flattened span rows repeat the same event, including its timestamp. + const key = JSON.stringify([ + span.id, + row.event_time_unix_nano_epoch ?? row.event_time_unix_nano ?? row.p_timestamp, + row.event_output, + ]); + if (seen.has(key)) continue; + seen.add(key); + occurrences.push({ + time: at(row, 'event_time_unix_nano', 'p_timestamp'), + sequence: Number.POSITIVE_INFINITY, + target: span, + entries: entry(withTool('Tool output', span.row), row.event_output), + }); + } + } + occurrences.sort((a, b) => a.time - b.time || a.sequence - b.sequence); + for (const { target, entries } of occurrences) if (target) target.conversation.push(...entries); + return runs; +}