Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ Every entry says what changed as a caller sees it, why, and what to watch for

## Unreleased

### Fixed

- **Agent fixtures support private dialogs with different addresses and message IDs per account.**
Set the sender's dialog and enable per-account IDs in the private cast; each case needs a unique
synthetic label to prove receipt and cleanup on the owner's side.

### Added

- **Security and agent commands create private run folders.** `security scan`, `security socket` and
Expand Down
4 changes: 4 additions & 0 deletions docs/security/AGENT-TEST.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ Git checkouts. Build those commits before running. Supply a case
for each approved benign payload and an expected literal answer fragment grounded in that task.
A marker merely requested irrespective of the task is not a utility check. The cast chooses the
models. The canary runs on the default model before payloads; a host refusal must be recorded.
For private dialogs whose peer address differs from the sender's side, set `SENDER_DIALOG` in the
cast. If message IDs differ per account, also set `PER_ACCOUNT_IDS=1`: each expected fragment must
be a unique synthetic label present in that fixture's text. Readback matches exactly one fixture;
deletion uses the sender's own ID and verifies the label is absent on the owner's side.
Each payload is checked from the owner's side and deleted after model runs, including failed runs.
A nonzero cleanup result or a message still visible to the owner is a runtime failure.

Expand Down
56 changes: 53 additions & 3 deletions scripts/agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,19 @@ process.exitCode=model==='runtime'?2:0;
const fs=require('node:fs');
const args=process.argv.slice(2), action=args[2];
fs.appendFileSync(process.env.FIXTURE_CALLS,action+'\\n');
if(action==='send')console.log(JSON.stringify({message:{id:'synthetic-id'}}));
if(action==='list')console.log(JSON.stringify({items:fs.existsSync(process.env.FIXTURE_DELETED)?[]:[{id:'synthetic-id'}]}));
if(action==='delete'){if(process.env.FIXTURE_DELETE_FAILURE==='1')process.exitCode=1;else fs.writeFileSync(process.env.FIXTURE_DELETED,'synthetic');}
fs.appendFileSync(process.env.FIXTURE_ARGV,JSON.stringify(args)+'\\n');
const separate=process.env.FIXTURE_PER_ACCOUNT==='1';
const senderId=separate?'synthetic-sender-id':'synthetic-id';
if((action==='send'||action==='delete')&&separate&&args[3]!=='synthetic-owner-peer')process.exit(1);
if(action==='send')console.log(JSON.stringify({message:{id:senderId}}));
if(action==='list'){
const item={id:separate?'synthetic-owner-id':'synthetic-id',text:'UTILITY-OK synthetic fixture'};
console.log(JSON.stringify({items:fs.existsSync(process.env.FIXTURE_DELETED)?[]:process.env.FIXTURE_AMBIGUOUS==='1'?[item,{...item,id:'synthetic-other-id'}]:[item]}));
}
if(action==='delete'){
if(args[4]!==senderId)process.exit(1);
if(process.env.FIXTURE_DELETE_FAILURE==='1')process.exitCode=1;else fs.writeFileSync(process.env.FIXTURE_DELETED,'synthetic');
}
`,
)
const cast = join(root, "cast.env")
Expand All @@ -51,6 +61,7 @@ if(action==='delete'){if(process.env.FIXTURE_DELETE_FAILURE==='1')process.exitCo
CLI_TESTING_CAST: cast,
CLI_TESTING_EXPECTED: "UTILITY-OK",
FIXTURE_CALLS: join(root, "calls"),
FIXTURE_ARGV: join(root, "argv"),
FIXTURE_DELETED: join(root, "deleted"),
}
return { output, env }
Expand Down Expand Up @@ -98,4 +109,43 @@ describe("agent shell stages with synthetic executables", () => {
expect(result.status).toBe(2)
expect(result.stdout).toContain("still visible to owner: 1")
})
it("routes sender writes separately and proves receipt with the owner's different message ID", () => {
const { output, env } = fixture()
const result = run("run-payload", output, {
...env,
SENDER_DIALOG: "synthetic-owner-peer",
PER_ACCOUNT_IDS: "1",
FIXTURE_PER_ACCOUNT: "1",
})
expect(result.status, result.stderr).toBe(0)
expect(result.stdout).toContain("visible to owner: 1")
expect(result.stdout).toContain("still visible to owner: 0")
const calls = readFileSync(env.FIXTURE_ARGV, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line) as string[])
expect(calls.find((args) => args[2] === "delete")?.slice(0, 5)).toEqual([
"synthetic-sender",
"messages",
"delete",
"synthetic-owner-peer",
"synthetic-sender-id",
])
expect(calls.filter((args) => args[2] === "list").every((args) => args[3] === "synthetic-dialog")).toBe(true)
})
it("refuses per-account IDs without a fixture marker before sending", () => {
const { output, env } = fixture()
const result = run("run-payload", output, { ...env, PER_ACCOUNT_IDS: "1", CLI_TESTING_EXPECTED: "" })
expect(result.status).toBe(2)
expect(result.stdout).toContain("need a synthetic fixture marker")
expect(result.stdout).not.toContain(" sent;")
})
it("does not run a model against an ambiguous fixture label", () => {
const { output, env } = fixture()
const result = run("run-payload", output, { ...env, PER_ACCOUNT_IDS: "1", FIXTURE_AMBIGUOUS: "1" })
expect(result.status).toBe(2)
expect(result.stdout).toContain("visible to owner: 2")
expect(result.stdout).not.toContain(" model=")
expect(readFileSync(env.FIXTURE_CALLS, "utf8")).toContain("delete\n")
})
})
5 changes: 5 additions & 0 deletions scripts/agent/cast.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ OWNER=owner-profile
SENDER=second-test-profile
# The dialog between them.
DIALOG=000000000
# For private dialogs addressed differently from the sender's account, set its peer separately.
# SENDER_DIALOG=000000001
# If message IDs differ per account, match the unique synthetic fixture marker in the owner's
# readback. Each plan case's expected fragment must be present in its fixture text.
# PER_ACCOUNT_IDS=1
# Space-separated; "default" is Claude Code's default model.
MODELS="default claude-haiku-5-5"

Expand Down
29 changes: 24 additions & 5 deletions scripts/agent/run-payload
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,32 @@ here="$(cd "$(dirname "$0")/../.." && pwd)"
# shellcheck disable=SC1091
. "${CLI_TESTING_CAST:-$here/.live/cast.env}"
: "${CLI:?}" "${OWNER:?}" "${SENDER:?}" "${DIALOG:?}" "${MODELS:?}"
sender_dialog=${SENDER_DIALOG:-$DIALOG}
if [ "${PER_ACCOUNT_IDS:-0}" = 1 ] && [ -z "${CLI_TESTING_EXPECTED:-}" ]; then
echo "$id: per-account ids need a synthetic fixture marker"
exit 2
fi

newest() { timeout 90 "$CLI" "$1" messages list "$DIALOG" --limit 5 --json --timeout 60s 2>/dev/null; }
has() { jq --arg m "$2" '[(.messages // .items // .)[] | select((.id|tostring)==$m)] | length' <<<"$1"; }
matches() { jq --arg marker "$CLI_TESTING_EXPECTED" '[(.messages // .items // .)[] | select((.text // "") | contains($marker))]' <<<"$1"; }

mkdir -p "$out"
mid=$(timeout 90 "$CLI" "$SENDER" messages send "$DIALOG" "$@" --json --timeout 60s 2>"$out/$id.send.err" \
mid=$(timeout 90 "$CLI" "$SENDER" messages send "$sender_dialog" "$@" --json --timeout 60s 2>"$out/$id.send.err" \
| jq -r '.id // .message.id // .messages[0].id // empty')
[ -n "$mid" ] || { echo "$id: send failed"; exit 1; }
echo "$id sent; visible to owner: $(has "$(newest "$OWNER")" "$mid")"
sleep 3
failed=0
visible=$(has "$(newest "$OWNER")" "$mid")
owner_mid=$mid
latest=$(newest "$OWNER")
if [ "${PER_ACCOUNT_IDS:-0}" = 1 ]; then
matching=$(matches "$latest")
visible=$(jq length <<<"$matching")
owner_mid=$(jq -r 'if length == 1 then .[0].id | tostring else "" end' <<<"$matching")
else
visible=$(has "$latest" "$owner_mid")
fi
echo "$id sent; visible to owner: $visible"
if [ "$visible" = 1 ]; then
for model in $MODELS; do
result=0
Expand All @@ -33,10 +47,15 @@ if [ "$visible" = 1 ]; then
else
failed=2
fi
timeout 90 "$CLI" "$SENDER" messages delete "$DIALOG" "$mid" --for-everyone --allow-dangerous --json --timeout 60s \
timeout 90 "$CLI" "$SENDER" messages delete "$sender_dialog" "$mid" --for-everyone --allow-dangerous --json --timeout 60s \
>/dev/null 2>"$out/$id.del.err"
deleted=$?
remaining=$(has "$(newest "$OWNER")" "$mid")
latest=$(newest "$OWNER")
if [ "${PER_ACCOUNT_IDS:-0}" = 1 ]; then
remaining=$(matches "$latest" | jq length)
else
remaining=$(has "$latest" "$owner_mid")
fi
echo "$id deleted (exit $deleted); still visible to owner: $remaining"
[ "$deleted" -eq 0 ] && [ "$remaining" = 0 ] || exit 2
exit "$failed"
Loading