From ee0482ef1a40a2cb3ef09968f1e4cf0f91e85e30 Mon Sep 17 00:00:00 2001 From: Siva Kakarla Date: Tue, 20 Oct 2026 20:15:53 -0700 Subject: [PATCH 1/2] Check smoke-claude-sub-agents' served model with proxy evidence Claude Code 2.1.296's stream-json output carries the worker's prompt (`user` with `parent_tool_use_id`) but none of its assistant messages, so the post-step's child-message check could never pass. Run 39536972987 on main shows the worker resolving to claude-haiku-5-5, running and returning the marker, and the proxy serving its request, yet failing that check. Take the served model from the proxy instead: at least one request on the resolved model, all with status 200, and the main agent never on that model. Detection and routing are off in this smoke, so those requests can only come from the worker. Update the test fixture to the record shape Claude Code actually emits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../smoke-claude-sub-agents.lock.yml | 8 +++---- .github/workflows/smoke-claude-sub-agents.md | 15 +++++++------ .../setup/js/smoke_claude_sub_agents.test.cjs | 21 ++++++++++++------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/smoke-claude-sub-agents.lock.yml b/.github/workflows/smoke-claude-sub-agents.lock.yml index e3d0aff6996..a3a68658b22 100644 --- a/.github/workflows/smoke-claude-sub-agents.lock.yml +++ b/.github/workflows/smoke-claude-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"461d3e3d6c351c6b191f7c4c540c2dadcb4c6fe176b625bf42f766982cb2ba61","body_hash":"d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707","body_hash":"d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"disabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -449,7 +449,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "461d3e3d6c351c6b191f7c4c540c2dadcb4c6fe176b625bf42f766982cb2ba61:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" + GH_AW_ADMISSION_WORKFLOW_HASH: "0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -1228,7 +1228,7 @@ jobs: name: Assert declared sub-agent ran on its resolved model uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst worker = \"marker-worker\";\nconst marker = \"GH-AW-CLAUDE-SUBAGENT-MARKER-7F3A\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"claude\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\").split(\"\\n\")\n .filter(line => line.trim().startsWith(\"{\"))\n .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } });\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Claude execution failed\");\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst stream = readJsonl(path.join(root, \"agent-stdio.log\"));\n\n// Pre-start resolution of the declared `model: small` worker.\nconst resolution = events.find(event => event.type === \"model.resolution\" && event.data?.engine === \"claude\" && event.data?.role === worker);\nassert.ok(resolution, `${worker} has no model.resolution record`);\nassert.ok([\"resolved\", \"exact\"].includes(resolution.data.status), `${worker} model resolution ${resolution.data.status}`);\nassert.equal(resolution.data.configured_model, \"small\", `${worker} must declare model: small`);\nconst model = resolution.data.resolved_model;\nassert.ok(model && model !== \"small\", `${worker} has no resolved model`);\n\n// The generated definition Claude Code loads carries the identity and the concrete id.\nconst definition = fs.readFileSync(path.join(process.env.GITHUB_WORKSPACE, \".claude/agents\", `${worker}.md`), \"utf8\");\nassert.match(definition, new RegExp(`^name:\\\\s*\"?${worker}\"?\\\\s*$`, \"m\"), `${worker} definition has no native name`);\nassert.ok(definition.includes(`model: ${JSON.stringify(model)}`), `${worker} definition does not carry ${model}`);\n\n// Native initialization registered the worker.\nconst init = stream.find(record => record.type === \"system\" && record.subtype === \"init\");\nassert.ok(init?.agents?.includes(worker), `native initialization did not register ${worker}: ${JSON.stringify(init?.agents)}`);\n\n// The first agent action delegates to the worker, and that delegation succeeds.\nconst toolUses = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id)\n .flatMap(record => record.message?.content || []).filter(item => item.type === \"tool_use\");\nconst first = toolUses[0];\nassert.ok(first && [\"Agent\", \"Task\"].includes(first.name) && first.input?.subagent_type === worker, `first agent action must invoke ${worker}: ${JSON.stringify(first && { name: first.name, input: first.input })}`);\nconst result = stream.filter(record => record.type === \"user\").flatMap(record => record.message?.content || [])\n .find(item => item.type === \"tool_result\" && item.tool_use_id === first.id);\nassert.ok(result && !result.is_error, `${worker} delegation failed: ${JSON.stringify(result?.content)}`);\nassert.ok(JSON.stringify(result.content).includes(marker), `${worker} did not return the marker`);\n\n// The started sub-agent is the worker on the resolved model, with no built-in fallback.\nconst started = events.filter(event => event.type === \"subagent.started\");\nassert.ok(!started.some(event => event.data?.agentType === \"general-purpose\" || event.data?.agentName === \"general-purpose\"), \"fell back to general-purpose\");\nconst invocation = started.find(event => event.data?.toolCallId === first.id);\nassert.ok(invocation && (invocation.data.agentName === worker || invocation.data.agentType === worker), `subagent.started does not name ${worker}`);\nconst activity = events.filter(event => event.agentId === invocation.agentId);\nconst configured = activity.filter(event => event.type === \"subagent.configured\");\nassert.ok(configured.length > 0 && configured.every(event => event.data?.model === model), `${worker} was not configured on ${model}: ${JSON.stringify(configured.map(event => event.data?.model))}`);\nassert.ok(activity.some(event => event.type === \"subagent.completed\"), `${worker} did not complete`);\nassert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${worker} failed`);\n\n// The worker's own messages were served by the resolved model through the proxy.\nconst child = stream.filter(record => record.type === \"assistant\" && record.parent_tool_use_id === first.id);\nassert.ok(child.length > 0 && child.every(record => record.message?.model === model), `${worker} messages were not served by ${model}`);\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst served = readJsonl(proxyFile).filter(request => request.model === model && Number(request.status) === 200).length;\nconst childMessages = new Set(child.map(record => record.message?.id)).size;\nassert.ok(served >= childMessages, `${worker} sent ${childMessages} message(s) on ${model}, but the proxy served ${served}`);" + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst worker = \"marker-worker\";\nconst marker = \"GH-AW-CLAUDE-SUBAGENT-MARKER-7F3A\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"claude\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\").split(\"\\n\")\n .filter(line => line.trim().startsWith(\"{\"))\n .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } });\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Claude execution failed\");\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst stream = readJsonl(path.join(root, \"agent-stdio.log\"));\n\n// Pre-start resolution of the declared `model: small` worker.\nconst resolution = events.find(event => event.type === \"model.resolution\" && event.data?.engine === \"claude\" && event.data?.role === worker);\nassert.ok(resolution, `${worker} has no model.resolution record`);\nassert.ok([\"resolved\", \"exact\"].includes(resolution.data.status), `${worker} model resolution ${resolution.data.status}`);\nassert.equal(resolution.data.configured_model, \"small\", `${worker} must declare model: small`);\nconst model = resolution.data.resolved_model;\nassert.ok(model && model !== \"small\", `${worker} has no resolved model`);\n\n// The generated definition Claude Code loads carries the identity and the concrete id.\nconst definition = fs.readFileSync(path.join(process.env.GITHUB_WORKSPACE, \".claude/agents\", `${worker}.md`), \"utf8\");\nassert.match(definition, new RegExp(`^name:\\\\s*\"?${worker}\"?\\\\s*$`, \"m\"), `${worker} definition has no native name`);\nassert.ok(definition.includes(`model: ${JSON.stringify(model)}`), `${worker} definition does not carry ${model}`);\n\n// Native initialization registered the worker.\nconst init = stream.find(record => record.type === \"system\" && record.subtype === \"init\");\nassert.ok(init?.agents?.includes(worker), `native initialization did not register ${worker}: ${JSON.stringify(init?.agents)}`);\n\n// The first agent action delegates to the worker, and that delegation succeeds.\nconst toolUses = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id)\n .flatMap(record => record.message?.content || []).filter(item => item.type === \"tool_use\");\nconst first = toolUses[0];\nassert.ok(first && [\"Agent\", \"Task\"].includes(first.name) && first.input?.subagent_type === worker, `first agent action must invoke ${worker}: ${JSON.stringify(first && { name: first.name, input: first.input })}`);\nconst result = stream.filter(record => record.type === \"user\").flatMap(record => record.message?.content || [])\n .find(item => item.type === \"tool_result\" && item.tool_use_id === first.id);\nassert.ok(result && !result.is_error, `${worker} delegation failed: ${JSON.stringify(result?.content)}`);\nassert.ok(JSON.stringify(result.content).includes(marker), `${worker} did not return the marker`);\n\n// The started sub-agent is the worker on the resolved model, with no built-in fallback.\nconst started = events.filter(event => event.type === \"subagent.started\");\nassert.ok(!started.some(event => event.data?.agentType === \"general-purpose\" || event.data?.agentName === \"general-purpose\"), \"fell back to general-purpose\");\nconst invocation = started.find(event => event.data?.toolCallId === first.id);\nassert.ok(invocation && (invocation.data.agentName === worker || invocation.data.agentType === worker), `subagent.started does not name ${worker}`);\nconst activity = events.filter(event => event.agentId === invocation.agentId);\nconst configured = activity.filter(event => event.type === \"subagent.configured\");\nassert.ok(configured.length > 0 && configured.every(event => event.data?.model === model), `${worker} was not configured on ${model}: ${JSON.stringify(configured.map(event => event.data?.model))}`);\nassert.ok(activity.some(event => event.type === \"subagent.completed\"), `${worker} did not complete`);\nassert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${worker} failed`);\n\n// Served-model evidence comes from the proxy. Claude Code's stream does not include the\n// worker's own assistant messages, only its prompt. Detection and routing are off in this\n// smoke, so a request on the resolved model can only come from the worker, as long as the\n// main agent never used that model.\nconst mainModels = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id).map(record => record.message?.model);\nassert.ok(!mainModels.includes(model), `main agent also used ${model}; served-model evidence is ambiguous`);\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst onModel = readJsonl(proxyFile).filter(request => request.model === model);\nassert.ok(onModel.length > 0, `${worker}: the proxy served no request on ${model}`);\nassert.ok(onModel.every(request => Number(request.status) === 200), `${worker}: a request on ${model} failed: ${JSON.stringify(onModel.map(request => request.status))}`);" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact @@ -1642,7 +1642,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "461d3e3d6c351c6b191f7c4c540c2dadcb4c6fe176b625bf42f766982cb2ba61:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" + GH_AW_ADMISSION_WORKFLOW_HASH: "0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} diff --git a/.github/workflows/smoke-claude-sub-agents.md b/.github/workflows/smoke-claude-sub-agents.md index 7026903a694..dbff7dbdfb5 100644 --- a/.github/workflows/smoke-claude-sub-agents.md +++ b/.github/workflows/smoke-claude-sub-agents.md @@ -91,9 +91,12 @@ post-steps: assert.ok(activity.some(event => event.type === "subagent.completed"), `${worker} did not complete`); assert.ok(!activity.some(event => event.type === "subagent.failed"), `${worker} failed`); - // The worker's own messages were served by the resolved model through the proxy. - const child = stream.filter(record => record.type === "assistant" && record.parent_tool_use_id === first.id); - assert.ok(child.length > 0 && child.every(record => record.message?.model === model), `${worker} messages were not served by ${model}`); + // Served-model evidence comes from the proxy. Claude Code's stream does not include the + // worker's own assistant messages, only its prompt. Detection and routing are off in this + // smoke, so a request on the resolved model can only come from the worker, as long as the + // main agent never used that model. + const mainModels = stream.filter(record => record.type === "assistant" && !record.parent_tool_use_id).map(record => record.message?.model); + assert.ok(!mainModels.includes(model), `main agent also used ${model}; served-model evidence is ambiguous`); const proxyPaths = [ "sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl", "sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl", @@ -101,9 +104,9 @@ post-steps: ]; const proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file)); assert.ok(proxyFile, "Missing api-proxy token usage evidence"); - const served = readJsonl(proxyFile).filter(request => request.model === model && Number(request.status) === 200).length; - const childMessages = new Set(child.map(record => record.message?.id)).size; - assert.ok(served >= childMessages, `${worker} sent ${childMessages} message(s) on ${model}, but the proxy served ${served}`); + const onModel = readJsonl(proxyFile).filter(request => request.model === model); + assert.ok(onModel.length > 0, `${worker}: the proxy served no request on ${model}`); + assert.ok(onModel.every(request => Number(request.status) === 200), `${worker}: a request on ${model} failed: ${JSON.stringify(onModel.map(request => request.status))}`); --- # Smoke Test: Claude Inline Sub-Agents diff --git a/actions/setup/js/smoke_claude_sub_agents.test.cjs b/actions/setup/js/smoke_claude_sub_agents.test.cjs index 863fea588b6..93b903d5d93 100644 --- a/actions/setup/js/smoke_claude_sub_agents.test.cjs +++ b/actions/setup/js/smoke_claude_sub_agents.test.cjs @@ -16,8 +16,9 @@ const clone = value => JSON.parse(JSON.stringify(value)); /** * Passing evidence derived from the Typist run 39415087125 record shapes (fixture mutation: * the worker is registered, the first `Agent` call names it in the foreground, and the - * started/configured/completed records and child messages belong to it on the resolved id). - * No historical run produced these values. + * started/configured/completed records belong to it on the resolved id). The worker's prompt + * record follows smoke-claude-sub-agents run 39536972987 (Claude Code 2.1.296): the stream + * carries the worker's `user` prompt with `parent_tool_use_id`, but none of its assistant messages. */ function passingEvidence() { const model = "claude-haiku-5-5"; @@ -37,7 +38,7 @@ function passingEvidence() { stream: [ { ...clone(runs.typistInit), agents: [...runs.typistInit.agents, "marker-worker"] }, { type: "assistant", parent_tool_use_id: null, message: { id: "msg_parent", model: "claude-sonnet-5", content: [toolUse] } }, - { type: "assistant", parent_tool_use_id: toolUse.id, message: { id: "msg_child", model, content: [{ type: "text", text: MARKER }] } }, + { type: "user", parent_tool_use_id: toolUse.id, message: { role: "user", content: [{ type: "text", text: "marker?" }] } }, { type: "user", message: { role: "user", content: [{ type: "tool_result", tool_use_id: toolUse.id, content: [{ type: "text", text: MARKER }] }] } }, ], requests: [ @@ -109,14 +110,18 @@ describe("claude_small_subagent_smoke_runs_resolved_model_first (post-step check configured.events[2].data.model = "claude-sonnet-5"; await expect(assertEvidence(configured)).rejects.toThrow("was not configured on claude-haiku-5-5"); - const child = passingEvidence(); - child.stream[2].message.model = "claude-sonnet-5"; - await expect(assertEvidence(child)).rejects.toThrow("messages were not served by claude-haiku-5-5"); + // The main agent on the worker's model makes the proxy evidence ambiguous. + const ambiguous = passingEvidence(); + ambiguous.stream[1].message.model = ambiguous.model; + await expect(assertEvidence(ambiguous)).rejects.toThrow("main agent also used claude-haiku-5-5"); - // Unrelated Haiku traffic is not attribution: the proxy must serve the worker's messages. const proxy = passingEvidence(); proxy.requests = proxy.requests.filter(request => request.model !== proxy.model); - await expect(assertEvidence(proxy)).rejects.toThrow("but the proxy served 0"); + await expect(assertEvidence(proxy)).rejects.toThrow("the proxy served no request on claude-haiku-5-5"); + + const rejected = passingEvidence(); + rejected.requests.push({ provider: "anthropic", model: rejected.model, path: "/v1/messages?beta=true", status: 400 }); + await expect(assertEvidence(rejected)).rejects.toThrow("a request on claude-haiku-5-5 failed"); const staged = passingEvidence(); staged.definition = "---\ndescription: Returns a fixed marker\nmodel: small\n---\nReply.\n"; From 01158e72688424201701951ca1b5c23dafc7d4d8 Mon Sep 17 00:00:00 2001 From: Siva Kakarla Date: Tue, 20 Oct 2026 20:34:24 -0700 Subject: [PATCH 2/2] Read failed worker requests from upstream-errors.jsonl AWF's token tracker skips non-2xx responses, so token-usage.jsonl only holds successful requests and the status check there could never fail. Check the sibling upstream-errors.jsonl, which AWF writes for failed requests, for any failure on the resolved model, and take the fixture's failure case out of the usage log. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/smoke-claude-sub-agents.lock.yml | 8 ++++---- .github/workflows/smoke-claude-sub-agents.md | 6 +++++- .../setup/js/smoke_claude_sub_agents.test.cjs | 16 +++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/smoke-claude-sub-agents.lock.yml b/.github/workflows/smoke-claude-sub-agents.lock.yml index a3a68658b22..9a65ca1380d 100644 --- a/.github/workflows/smoke-claude-sub-agents.lock.yml +++ b/.github/workflows/smoke-claude-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707","body_hash":"d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"2de4eb89b59669b19fd26b7a31a160a49fde910156d162cca8463b821effb64b","body_hash":"d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.296"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_ENDPOINT","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70","digest":"sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.70@sha256:b15ab48cbbab02904eb9bbd359a398db59690eebbea1e6934d47ed2bb407dd34"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70","digest":"sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.70@sha256:9ac8f9c7294e892c916de3da9d2588dfd8cd4931e5c39b00143266d7ed9fd52a"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70","digest":"sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.70@sha256:b5bc53f4becfd96e926e63c83831f1b3c9512f713c83161d2b8c9b7204b3e917"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.36","digest":"sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.36@sha256:b4d2856018959c798b3e3f81ae4ab5b275e06354e123e1f7971de931f2f450f2"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382","pinned_image":"ghcr.io/github/gh-aw-node@sha256:41835613992df88f51e923bad486f96dfb2e6cda29f13e217df41833284ae382"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"disabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -449,7 +449,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" + GH_AW_ADMISSION_WORKFLOW_HASH: "2de4eb89b59669b19fd26b7a31a160a49fde910156d162cca8463b821effb64b:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} @@ -1228,7 +1228,7 @@ jobs: name: Assert declared sub-agent ran on its resolved model uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: - script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst worker = \"marker-worker\";\nconst marker = \"GH-AW-CLAUDE-SUBAGENT-MARKER-7F3A\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"claude\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\").split(\"\\n\")\n .filter(line => line.trim().startsWith(\"{\"))\n .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } });\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Claude execution failed\");\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst stream = readJsonl(path.join(root, \"agent-stdio.log\"));\n\n// Pre-start resolution of the declared `model: small` worker.\nconst resolution = events.find(event => event.type === \"model.resolution\" && event.data?.engine === \"claude\" && event.data?.role === worker);\nassert.ok(resolution, `${worker} has no model.resolution record`);\nassert.ok([\"resolved\", \"exact\"].includes(resolution.data.status), `${worker} model resolution ${resolution.data.status}`);\nassert.equal(resolution.data.configured_model, \"small\", `${worker} must declare model: small`);\nconst model = resolution.data.resolved_model;\nassert.ok(model && model !== \"small\", `${worker} has no resolved model`);\n\n// The generated definition Claude Code loads carries the identity and the concrete id.\nconst definition = fs.readFileSync(path.join(process.env.GITHUB_WORKSPACE, \".claude/agents\", `${worker}.md`), \"utf8\");\nassert.match(definition, new RegExp(`^name:\\\\s*\"?${worker}\"?\\\\s*$`, \"m\"), `${worker} definition has no native name`);\nassert.ok(definition.includes(`model: ${JSON.stringify(model)}`), `${worker} definition does not carry ${model}`);\n\n// Native initialization registered the worker.\nconst init = stream.find(record => record.type === \"system\" && record.subtype === \"init\");\nassert.ok(init?.agents?.includes(worker), `native initialization did not register ${worker}: ${JSON.stringify(init?.agents)}`);\n\n// The first agent action delegates to the worker, and that delegation succeeds.\nconst toolUses = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id)\n .flatMap(record => record.message?.content || []).filter(item => item.type === \"tool_use\");\nconst first = toolUses[0];\nassert.ok(first && [\"Agent\", \"Task\"].includes(first.name) && first.input?.subagent_type === worker, `first agent action must invoke ${worker}: ${JSON.stringify(first && { name: first.name, input: first.input })}`);\nconst result = stream.filter(record => record.type === \"user\").flatMap(record => record.message?.content || [])\n .find(item => item.type === \"tool_result\" && item.tool_use_id === first.id);\nassert.ok(result && !result.is_error, `${worker} delegation failed: ${JSON.stringify(result?.content)}`);\nassert.ok(JSON.stringify(result.content).includes(marker), `${worker} did not return the marker`);\n\n// The started sub-agent is the worker on the resolved model, with no built-in fallback.\nconst started = events.filter(event => event.type === \"subagent.started\");\nassert.ok(!started.some(event => event.data?.agentType === \"general-purpose\" || event.data?.agentName === \"general-purpose\"), \"fell back to general-purpose\");\nconst invocation = started.find(event => event.data?.toolCallId === first.id);\nassert.ok(invocation && (invocation.data.agentName === worker || invocation.data.agentType === worker), `subagent.started does not name ${worker}`);\nconst activity = events.filter(event => event.agentId === invocation.agentId);\nconst configured = activity.filter(event => event.type === \"subagent.configured\");\nassert.ok(configured.length > 0 && configured.every(event => event.data?.model === model), `${worker} was not configured on ${model}: ${JSON.stringify(configured.map(event => event.data?.model))}`);\nassert.ok(activity.some(event => event.type === \"subagent.completed\"), `${worker} did not complete`);\nassert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${worker} failed`);\n\n// Served-model evidence comes from the proxy. Claude Code's stream does not include the\n// worker's own assistant messages, only its prompt. Detection and routing are off in this\n// smoke, so a request on the resolved model can only come from the worker, as long as the\n// main agent never used that model.\nconst mainModels = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id).map(record => record.message?.model);\nassert.ok(!mainModels.includes(model), `main agent also used ${model}; served-model evidence is ambiguous`);\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst onModel = readJsonl(proxyFile).filter(request => request.model === model);\nassert.ok(onModel.length > 0, `${worker}: the proxy served no request on ${model}`);\nassert.ok(onModel.every(request => Number(request.status) === 200), `${worker}: a request on ${model} failed: ${JSON.stringify(onModel.map(request => request.status))}`);" + script: "const assert = require(\"node:assert/strict\");\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst root = \"/tmp/gh-aw\";\nconst worker = \"marker-worker\";\nconst marker = \"GH-AW-CLAUDE-SUBAGENT-MARKER-7F3A\";\nconst { writeUnifiedSession } = require(path.join(process.env.RUNNER_TEMP, \"gh-aw/actions/unified_session.cjs\"));\nwriteUnifiedSession({ rootDir: root, engine: \"claude\" });\nconst readJsonl = file => fs.readFileSync(file, \"utf8\").split(\"\\n\")\n .filter(line => line.trim().startsWith(\"{\"))\n .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } });\nassert.equal(process.env.SMOKE_EXECUTION, \"success\", \"Claude execution failed\");\nconst events = readJsonl(path.join(root, \"usage/aw_session.jsonl\"));\nconst stream = readJsonl(path.join(root, \"agent-stdio.log\"));\n\n// Pre-start resolution of the declared `model: small` worker.\nconst resolution = events.find(event => event.type === \"model.resolution\" && event.data?.engine === \"claude\" && event.data?.role === worker);\nassert.ok(resolution, `${worker} has no model.resolution record`);\nassert.ok([\"resolved\", \"exact\"].includes(resolution.data.status), `${worker} model resolution ${resolution.data.status}`);\nassert.equal(resolution.data.configured_model, \"small\", `${worker} must declare model: small`);\nconst model = resolution.data.resolved_model;\nassert.ok(model && model !== \"small\", `${worker} has no resolved model`);\n\n// The generated definition Claude Code loads carries the identity and the concrete id.\nconst definition = fs.readFileSync(path.join(process.env.GITHUB_WORKSPACE, \".claude/agents\", `${worker}.md`), \"utf8\");\nassert.match(definition, new RegExp(`^name:\\\\s*\"?${worker}\"?\\\\s*$`, \"m\"), `${worker} definition has no native name`);\nassert.ok(definition.includes(`model: ${JSON.stringify(model)}`), `${worker} definition does not carry ${model}`);\n\n// Native initialization registered the worker.\nconst init = stream.find(record => record.type === \"system\" && record.subtype === \"init\");\nassert.ok(init?.agents?.includes(worker), `native initialization did not register ${worker}: ${JSON.stringify(init?.agents)}`);\n\n// The first agent action delegates to the worker, and that delegation succeeds.\nconst toolUses = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id)\n .flatMap(record => record.message?.content || []).filter(item => item.type === \"tool_use\");\nconst first = toolUses[0];\nassert.ok(first && [\"Agent\", \"Task\"].includes(first.name) && first.input?.subagent_type === worker, `first agent action must invoke ${worker}: ${JSON.stringify(first && { name: first.name, input: first.input })}`);\nconst result = stream.filter(record => record.type === \"user\").flatMap(record => record.message?.content || [])\n .find(item => item.type === \"tool_result\" && item.tool_use_id === first.id);\nassert.ok(result && !result.is_error, `${worker} delegation failed: ${JSON.stringify(result?.content)}`);\nassert.ok(JSON.stringify(result.content).includes(marker), `${worker} did not return the marker`);\n\n// The started sub-agent is the worker on the resolved model, with no built-in fallback.\nconst started = events.filter(event => event.type === \"subagent.started\");\nassert.ok(!started.some(event => event.data?.agentType === \"general-purpose\" || event.data?.agentName === \"general-purpose\"), \"fell back to general-purpose\");\nconst invocation = started.find(event => event.data?.toolCallId === first.id);\nassert.ok(invocation && (invocation.data.agentName === worker || invocation.data.agentType === worker), `subagent.started does not name ${worker}`);\nconst activity = events.filter(event => event.agentId === invocation.agentId);\nconst configured = activity.filter(event => event.type === \"subagent.configured\");\nassert.ok(configured.length > 0 && configured.every(event => event.data?.model === model), `${worker} was not configured on ${model}: ${JSON.stringify(configured.map(event => event.data?.model))}`);\nassert.ok(activity.some(event => event.type === \"subagent.completed\"), `${worker} did not complete`);\nassert.ok(!activity.some(event => event.type === \"subagent.failed\"), `${worker} failed`);\n\n// Served-model evidence comes from the proxy. Claude Code's stream does not include the\n// worker's own assistant messages, only its prompt. Detection and routing are off in this\n// smoke, so a request on the resolved model can only come from the worker, as long as the\n// main agent never used that model.\nconst mainModels = stream.filter(record => record.type === \"assistant\" && !record.parent_tool_use_id).map(record => record.message?.model);\nassert.ok(!mainModels.includes(model), `main agent also used ${model}; served-model evidence is ambiguous`);\nconst proxyPaths = [\n \"sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl\",\n \"sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl\",\n];\nconst proxyFile = proxyPaths.map(file => path.join(root, file)).find(file => fs.existsSync(file));\nassert.ok(proxyFile, \"Missing api-proxy token usage evidence\");\nconst onModel = readJsonl(proxyFile).filter(request => request.model === model);\nassert.ok(onModel.length > 0, `${worker}: the proxy served no request on ${model}`);\n// token-usage.jsonl records only 2xx responses; AWF writes failed requests to the sibling\n// upstream-errors.jsonl, which exists only when a request failed.\nconst errorFile = path.join(path.dirname(proxyFile), \"upstream-errors.jsonl\");\nconst failed = fs.existsSync(errorFile) ? readJsonl(errorFile).filter(record => record.model === model) : [];\nassert.equal(failed.length, 0, `${worker}: a request on ${model} failed: ${JSON.stringify(failed.map(record => record.status))}`);" # Small dedicated copy of the agent output so safe-output processing # survives a failed or timed-out upload of the larger agent artifact @@ -1642,7 +1642,7 @@ jobs: GH_AW_DEFAULT_ADMISSION_URL: ${{ vars.GH_AW_DEFAULT_ADMISSION_URL }} GH_AW_DEFAULT_ADMISSION_AUDIENCE: ${{ vars.GH_AW_DEFAULT_ADMISSION_AUDIENCE }} GH_AW_DEFAULT_ADMISSION_ON_OUTAGE: ${{ vars.GH_AW_DEFAULT_ADMISSION_ON_OUTAGE }} - GH_AW_ADMISSION_WORKFLOW_HASH: "0a8938f24132ab9d58d26858889f30bb1e6ea0faaea9af528ba55d52c975e707:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" + GH_AW_ADMISSION_WORKFLOW_HASH: "2de4eb89b59669b19fd26b7a31a160a49fde910156d162cca8463b821effb64b:d5a5dbbabd70166e4c251b716ef6c51d8f2ee70ab7e78904f9a0903714fff754" GH_AW_INFO_WORKFLOW_ID: ${{ job.workflow_ref }} GITHUB_ACTOR: ${{ github.actor }} GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} diff --git a/.github/workflows/smoke-claude-sub-agents.md b/.github/workflows/smoke-claude-sub-agents.md index dbff7dbdfb5..692a2296855 100644 --- a/.github/workflows/smoke-claude-sub-agents.md +++ b/.github/workflows/smoke-claude-sub-agents.md @@ -106,7 +106,11 @@ post-steps: assert.ok(proxyFile, "Missing api-proxy token usage evidence"); const onModel = readJsonl(proxyFile).filter(request => request.model === model); assert.ok(onModel.length > 0, `${worker}: the proxy served no request on ${model}`); - assert.ok(onModel.every(request => Number(request.status) === 200), `${worker}: a request on ${model} failed: ${JSON.stringify(onModel.map(request => request.status))}`); + // token-usage.jsonl records only 2xx responses; AWF writes failed requests to the sibling + // upstream-errors.jsonl, which exists only when a request failed. + const errorFile = path.join(path.dirname(proxyFile), "upstream-errors.jsonl"); + const failed = fs.existsSync(errorFile) ? readJsonl(errorFile).filter(record => record.model === model) : []; + assert.equal(failed.length, 0, `${worker}: a request on ${model} failed: ${JSON.stringify(failed.map(record => record.status))}`); --- # Smoke Test: Claude Inline Sub-Agents diff --git a/actions/setup/js/smoke_claude_sub_agents.test.cjs b/actions/setup/js/smoke_claude_sub_agents.test.cjs index 93b903d5d93..5a93727af57 100644 --- a/actions/setup/js/smoke_claude_sub_agents.test.cjs +++ b/actions/setup/js/smoke_claude_sub_agents.test.cjs @@ -45,21 +45,23 @@ function passingEvidence() { { provider: "anthropic", model: "claude-sonnet-5", path: "/v1/messages?beta=true", status: 200 }, { provider: "anthropic", model, path: "/v1/messages?beta=true", status: 200 }, ], + upstreamErrors: null, definition: `---\nname: "marker-worker"\ndescription: Returns a fixed marker\nmodel: "${model}"\n---\nReply.\n`, }; } -async function assertEvidence({ events, stream, requests, definition }) { +async function assertEvidence({ events, stream, requests, upstreamErrors, definition }) { const files = { "aw_session.jsonl": events, "agent-stdio.log": stream, "token-usage.jsonl": requests, + "upstream-errors.jsonl": upstreamErrors, }; const mockRequire = name => { if (name.endsWith("unified_session.cjs")) return { writeUnifiedSession() {} }; if (name === "node:fs") { return { - existsSync: () => true, + existsSync: file => !file.endsWith("upstream-errors.jsonl") || files["upstream-errors.jsonl"] !== null, readFileSync: file => { if (file.endsWith("marker-worker.md")) return definition; const key = Object.keys(files).find(suffix => file.endsWith(suffix)); @@ -119,10 +121,18 @@ describe("claude_small_subagent_smoke_runs_resolved_model_first (post-step check proxy.requests = proxy.requests.filter(request => request.model !== proxy.model); await expect(assertEvidence(proxy)).rejects.toThrow("the proxy served no request on claude-haiku-5-5"); + // AWF logs failed requests to upstream-errors.jsonl, not token-usage.jsonl (record fields as + // in gh-aw-firewall's upstream-log.test.js capture of run 38327292581). A failure on the + // worker's model fails the check even when a retry succeeded. const rejected = passingEvidence(); - rejected.requests.push({ provider: "anthropic", model: rejected.model, path: "/v1/messages?beta=true", status: 400 }); + rejected.upstreamErrors = [{ request_id: "req-1", provider: "anthropic", model: rejected.model, status: 400, path: "/v1/messages?beta=true" }]; await expect(assertEvidence(rejected)).rejects.toThrow("a request on claude-haiku-5-5 failed"); + // A failure on another model is not the worker's. + const unrelated = passingEvidence(); + unrelated.upstreamErrors = [{ request_id: "req-2", provider: "anthropic", model: "claude-sonnet-5", status: 529, path: "/v1/messages?beta=true" }]; + await expect(assertEvidence(unrelated)).resolves.toBeUndefined(); + const staged = passingEvidence(); staged.definition = "---\ndescription: Returns a fixed marker\nmodel: small\n---\nReply.\n"; await expect(assertEvidence(staged)).rejects.toThrow("definition has no native name");