From a7aea92d8f71f7d14d94ed1a20bc9dff155f3104 Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Mon, 12 Oct 2026 18:13:25 +0530 Subject: [PATCH 1/4] feat(skill): evidence-map script prints a pack's step sheet Joins every step folder of an evidence pack to the actions log, the authored steps and result.yaml by action_id, and lists the error lines of the run logs. Node, no dependencies, reads the .evidence zip itself. Co-Authored-By: Claude Opus 5.5 --- .../skills/kane-cli/scripts/evidence-map.mjs | 396 ++++++++++++++++++ .../skills/kane-cli/scripts/evidence-map.mjs | 396 ++++++++++++++++++ .github/workflows/skill-installer-tests.yml | 1 + skill-installer/TESTING.md | 1 + .../skills/scripts/evidence-map.mjs | 396 ++++++++++++++++++ skill-installer/test/evidence-map.test.mjs | 323 ++++++++++++++ 6 files changed, 1513 insertions(+) create mode 100644 .agents/skills/kane-cli/scripts/evidence-map.mjs create mode 100644 .claude/skills/kane-cli/scripts/evidence-map.mjs create mode 100644 skill-installer/skills/scripts/evidence-map.mjs create mode 100644 skill-installer/test/evidence-map.test.mjs diff --git a/.agents/skills/kane-cli/scripts/evidence-map.mjs b/.agents/skills/kane-cli/scripts/evidence-map.mjs new file mode 100644 index 0000000..b11f1f2 --- /dev/null +++ b/.agents/skills/kane-cli/scripts/evidence-map.mjs @@ -0,0 +1,396 @@ +#!/usr/bin/env node +// Prints the step sheet of a kane-cli evidence pack. +// +// Joins every step folder to the actions log, the authored execution tree and +// result.yaml by action_id, and lists the error lines of the run logs. +// +// node evidence-map.mjs --out +// node evidence-map.mjs [--test ] +// +// No dependencies. Works on Node 18 and later, on macOS, Linux and Windows. + +import fs from "node:fs"; +import path from "node:path"; +import zlib from "node:zlib"; +import { fileURLToPath } from "node:url"; + +const USAGE = `Usage: + node evidence-map.mjs --out + node evidence-map.mjs [--test ]`; + +const RUN_SIGNAL = /✗|Traceback|\bERROR\b|\bCRITICAL\b|API error|FAIL\]|Agent failed|\bstuck\b/i; +const PIPELINE_SIGNAL = /\b[A-Z][A-Z_]+_(FAILED|DENIED|INVALID|CONFLICT)\b|\bERROR\b/; +// [file name pattern, line pattern, lines shown per file] +const SIGNAL_SOURCES = [ + [/^\d+-run\.log$/, RUN_SIGNAL, 40], + [/^runner-stderr\.log$/, RUN_SIGNAL, 20], + [/^tui\.log$/, PIPELINE_SIGNAL, 12], +]; +const CLIP = 220; + +class UsageError extends Error {} + +function clip(value, width = CLIP) { + const text = String(value ?? "").split(/\s+/).filter(Boolean).join(" "); + return text.length <= width ? text : text.slice(0, width - 1) + "…"; +} + +function readText(file) { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return ""; + } +} + +function readJson(file) { + try { + return JSON.parse(readText(file)); + } catch { + return null; + } +} + +function lines(text) { + return text.split(/\r?\n/); +} + +function listDir(dir) { + try { + return fs.readdirSync(dir).sort(); + } catch { + return []; + } +} + +function isDir(target) { + try { + return fs.statSync(target).isDirectory(); + } catch { + return false; + } +} + +// The entries of a zip, read from its central directory. +function zipEntries(buffer) { + const END = 0x06054b50; + const CENTRAL = 0x02014b50; + let end = -1; + for (let i = buffer.length - 22; i >= Math.max(0, buffer.length - 22 - 0xffff); i--) { + if (buffer.readUInt32LE(i) === END) { + end = i; + break; + } + } + if (end < 0) throw new UsageError("not a zip file: no end-of-archive record found"); + const count = buffer.readUInt16LE(end + 10); + let offset = buffer.readUInt32LE(end + 16); + if (count === 0xffff || offset === 0xffffffff) { + throw new UsageError("this pack uses zip64, which this script does not read: unzip it yourself and pass the directory"); + } + const entries = []; + for (let i = 0; i < count; i++) { + if (buffer.readUInt32LE(offset) !== CENTRAL) throw new UsageError("damaged zip: bad central directory entry"); + const nameLength = buffer.readUInt16LE(offset + 28); + entries.push({ + name: buffer.toString("utf8", offset + 46, offset + 46 + nameLength), + method: buffer.readUInt16LE(offset + 10), + compressedSize: buffer.readUInt32LE(offset + 20), + localOffset: buffer.readUInt32LE(offset + 42), + }); + offset += 46 + nameLength + buffer.readUInt16LE(offset + 30) + buffer.readUInt16LE(offset + 32); + } + return entries; +} + +function zipEntryData(buffer, entry) { + const start = entry.localOffset + 30 + + buffer.readUInt16LE(entry.localOffset + 26) + buffer.readUInt16LE(entry.localOffset + 28); + const raw = buffer.subarray(start, start + entry.compressedSize); + if (entry.method === 0) return raw; + if (entry.method === 8) return zlib.inflateRawSync(raw); + throw new UsageError(`unsupported compression in pack entry: ${entry.name}`); +} + +// Unzips a pack into an empty directory, refusing entries that escape it. +function extract(pack, out) { + fs.mkdirSync(out, { recursive: true }); + if (fs.readdirSync(out).length > 0) throw new UsageError(`--out must be an empty directory: ${out}`); + const root = fs.realpathSync(out); + const buffer = fs.readFileSync(pack); + const entries = zipEntries(buffer); + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (target !== root && !target.startsWith(root + path.sep)) { + throw new UsageError(`refusing pack entry outside the output directory: ${entry.name}`); + } + } + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (entry.name.endsWith("/")) { + fs.mkdirSync(target, { recursive: true }); + } else { + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, zipEntryData(buffer, entry)); + } + } + return root; +} + +// The flat `steps:` list of result.yaml, one object per row. +function resultSteps(file) { + const rows = []; + let current = null; + let inside = false; + const unquote = (value) => value.trim().replace(/^['"]|['"]$/g, ""); + for (const line of lines(readText(file))) { + if (line.startsWith("steps:")) { + inside = true; + continue; + } + if (!inside) continue; + if (line && !line.startsWith(" ")) break; + const item = line.match(/^\s+-\s+(\w+):\s*(.*)$/); + const field = line.match(/^\s+(\w+):\s*(.*)$/); + if (item) { + current = { [item[1]]: unquote(item[2]) }; + rows.push(current); + } else if (field && current) { + current[field[1]] = unquote(field[2]); + } + } + return rows; +} + +// operation_id -> the authored instruction node that owns it. +function executionLeaves(node, found, container = null) { + if (Array.isArray(node)) { + for (const value of node) executionLeaves(value, found, container); + } else if (node && typeof node === "object") { + if (node.condition && String(node.content_type ?? "").endsWith("_block")) { + container = `${node.content_type} condition=${JSON.stringify(node.condition)} condition_met=${node.condition_met}`; + } + for (const op of Array.isArray(node.operations) ? node.operations : []) { + if (op && typeof op === "object" && op.operation_id) { + found.set(op.operation_id, { + instruction: node.instruction, + reasoning: node.reasoning, + intent: op.intent, + status: node.status, + error: node.error_message || op.error_message, + container, + }); + } + } + for (const value of Object.values(node)) executionLeaves(value, found, container); + } + return found; +} + +function actionsById(logsDir) { + const found = new Map(); + for (const name of listDir(logsDir).filter((file) => /-actions\.ndjson$/.test(file))) { + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + let row; + try { + row = JSON.parse(line); + } catch { + return; + } + if (row && typeof row === "object" && row.action_id) { + row._where = `${name}:${index + 1}`; + found.set(row.action_id, row); + } + }); + } + return found; +} + +function stepFolders(stepsDir) { + const folders = []; + for (const name of listDir(stepsDir)) { + const dir = path.join(stepsDir, name); + if (!isDir(dir)) continue; + const files = listDir(dir); + const lead = name.match(/^\d+/); + folders.push({ + name, + ordinal: lead ? Number(lead[0]) : 0, + step: readJson(path.join(dir, "step.json")) || {}, + screenshot: files.find((file) => file.startsWith("screenshot.")) || null, + annotated: files.find((file) => file.startsWith("annotated.")) || null, + failure: files.includes("failure.yaml"), + }); + } + return folders.sort((a, b) => a.ordinal - b.ordinal); +} + +function printSignals(logsDir, print) { + print("\nLOG SIGNALS (pattern matches only; open the log around each, and search it yourself for more)"); + const names = listDir(logsDir); + for (const [filePattern, signal, limit] of SIGNAL_SOURCES) { + for (const name of names.filter((file) => filePattern.test(file))) { + const hits = []; + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + if (signal.test(line)) hits.push([index + 1, line]); + }); + print(` ${name}: ${hits.length} matching line(s)`); + for (const [number, line] of hits.slice(0, limit)) print(` ${number}: ${clip(line)}`); + if (hits.length > limit) print(` … ${hits.length - limit} more, last at line ${hits[hits.length - 1][0]}`); + } + } +} + +function printTest(testDir, print) { + print("\n" + "=".repeat(78)); + print(`TEST ${path.basename(testDir)}`); + const logsDir = path.join(testDir, "logs"); + const folders = stepFolders(path.join(testDir, "steps")); + const actions = actionsById(logsDir); + const leaves = executionLeaves(readJson(path.join(testDir, "auteur", "execution.json")), new Map()); + const rows = resultSteps(path.join(testDir, "result.yaml")); + const rowByAction = new Map(rows.filter((row) => row.action_id).map((row) => [row.action_id, row])); + const rowById = new Map(rows.filter((row) => row.id).map((row) => [row.id, row])); + print(` step folders=${folders.length} action rows=${actions.size}` + + ` authored operations=${leaves.size} result.yaml rows=${rows.length}`); + + print("\nSTEP SHEET (screenshot = the screen BEFORE this step's action ran)"); + let carried = null; + const seenActions = new Set(); + for (const folder of folders) { + const step = folder.step; + const actionId = step.action_id; + const ids = actionId ? [actionId] : []; + for (const id of Array.isArray(step.analyze_action_ids) ? step.analyze_action_ids : []) { + if (!ids.includes(id)) ids.push(id); + } + for (const id of ids) seenActions.add(id); + const row = rowById.get(step.id) || rowByAction.get(actionId) || {}; + let shotNote; + if (folder.screenshot) { + carried = `steps/${folder.name}/${folder.screenshot}`; + shotNote = carried; + } else { + shotNote = `NONE — latest earlier image: ${carried || "none"}`; + } + const joined = [["A", actions], ["E", leaves], ["R", rowByAction]] + .filter(([, source]) => source.has(actionId)) + .map(([mark]) => mark) + .join(""); + print(`\n[${folder.name}] id=${step.id} kind=${step.kind}` + + ` step.json=${step.status} result.yaml=${row.status ?? "-"}` + + ` duration_ms=${step.duration_ms}${folder.failure ? " <-- FAILURE RECORD" : ""}`); + print(` action_id=${actionId || "NONE (join by id/ordinal only)"} joined=${joined || "-"}` + + " (A=actions log E=execution.json R=result.yaml)"); + print(` screenshot: ${shotNote}${folder.annotated ? ` annotated: ${folder.annotated}` : ""}`); + print(` summary: ${clip(step.summary)}`); + for (const id of ids) { + const leaf = leaves.get(id); + const action = actions.get(id); + if (leaf) { + print(` authored[${id}]: intent=${clip(leaf.intent, 120)} | instruction=${clip(leaf.instruction, 160)}` + + ` | status=${leaf.status}`); + if (leaf.container) print(` inside: ${clip(leaf.container, 200)}`); + if (leaf.reasoning) print(` reasoning: ${clip(leaf.reasoning, 400)}`); + if (leaf.error) print(` error: ${clip(leaf.error, 300)}`); + } + if (action) { + print(` action[${action._where}]: ${action.action_type} status=${action.status}` + + ` at=${action.timestamp} | ${clip(action.action_instruction, 160)}`); + if (action.action_params && Object.keys(action.action_params).length > 0) { + print(` params: ${clip(JSON.stringify(action.action_params), 300)}`); + } + if (action.rationale) print(` rationale: ${clip(action.rationale, 400)}`); + if (action.error) print(` error: ${clip(action.error, 300)}`); + } + } + } + + print("\nUNMAPPED"); + let unmapped = 0; + for (const [id, action] of actions) { + if (seenActions.has(id)) continue; + unmapped++; + print(` action with no step folder: ${action._where} ${action.action_type}` + + ` status=${action.status} | ${clip(action.action_instruction, 140)}`); + } + const folderIds = new Set(folders.map((folder) => folder.step.id)); + const skipped = []; + for (const row of rows) { + if (folderIds.has(row.id) || seenActions.has(row.action_id)) continue; + unmapped++; + if (row.status === "skipped") { + skipped.push(row); + continue; + } + const leaf = leaves.get(row.action_id) || {}; + print(` result.yaml row with no step folder: ordinal=${row.ordinal} id=${row.id}` + + ` status=${row.status} | authored: ${clip(leaf.instruction || leaf.intent, 140)}`); + } + if (skipped.length > 0) { + print(` ${skipped.length} skipped result.yaml row(s) with no step folder,` + + ` ordinals ${skipped[0].ordinal}-${skipped[skipped.length - 1].ordinal}: the run never reached them`); + } + if (unmapped === 0) print(" none"); + + for (const folder of folders.filter((one) => one.failure)) { + const text = readText(path.join(testDir, "steps", folder.name, "failure.yaml")); + if (!/^triage:/m.test(text)) continue; + const verified = text.match(/verification:\s*\r?\n\s*status:\s*(\S+)/); + print(`\nEMBEDDED TRIAGE in steps/${folder.name}/failure.yaml` + + ` (verification=${verified ? verified[1] : "unknown"}): a claim to check, not a finding`); + } + printSignals(logsDir, print); +} + +function parseArgs(argv) { + const args = { target: null, out: null, test: null }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--out" || arg === "--test") { + if (i + 1 >= argv.length) throw new UsageError(`${arg} needs a value`); + args[arg.slice(2)] = argv[++i]; + } else if (arg.startsWith("--")) { + throw new UsageError(`unknown option: ${arg}`); + } else if (args.target === null) { + args.target = arg; + } else { + throw new UsageError(`unexpected argument: ${arg}`); + } + } + if (args.target === null) throw new UsageError("pass a .evidence pack or an extracted pack directory"); + return args; +} + +export function main(argv, print = console.log) { + const args = parseArgs(argv); + let root = args.target; + if (!fs.existsSync(root)) throw new UsageError(`not found: ${root}`); + if (!isDir(root)) { + if (!args.out) throw new UsageError("--out is required for a .evidence file"); + root = extract(root, args.out); + } + print(`PACK ${root}`); + for (const name of ["run.yaml", "failure.yaml"]) { + print(`\n--- ${name}\n${readText(path.join(root, name)).trimEnd()}`); + } + const tests = listDir(path.join(root, "tests")) + .filter((name) => !args.test || name === args.test) + .map((name) => path.join(root, "tests", name)) + .filter(isDir); + if (tests.length === 0) { + throw new UsageError(args.test ? `no test named ${args.test} under tests/` : "no tests found under tests/"); + } + for (const testDir of tests) printTest(testDir, print); +} + +if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main(process.argv.slice(2)); + } catch (error) { + if (!(error instanceof UsageError)) throw error; + console.error(error.message); + console.error(USAGE); + process.exit(2); + } +} diff --git a/.claude/skills/kane-cli/scripts/evidence-map.mjs b/.claude/skills/kane-cli/scripts/evidence-map.mjs new file mode 100644 index 0000000..b11f1f2 --- /dev/null +++ b/.claude/skills/kane-cli/scripts/evidence-map.mjs @@ -0,0 +1,396 @@ +#!/usr/bin/env node +// Prints the step sheet of a kane-cli evidence pack. +// +// Joins every step folder to the actions log, the authored execution tree and +// result.yaml by action_id, and lists the error lines of the run logs. +// +// node evidence-map.mjs --out +// node evidence-map.mjs [--test ] +// +// No dependencies. Works on Node 18 and later, on macOS, Linux and Windows. + +import fs from "node:fs"; +import path from "node:path"; +import zlib from "node:zlib"; +import { fileURLToPath } from "node:url"; + +const USAGE = `Usage: + node evidence-map.mjs --out + node evidence-map.mjs [--test ]`; + +const RUN_SIGNAL = /✗|Traceback|\bERROR\b|\bCRITICAL\b|API error|FAIL\]|Agent failed|\bstuck\b/i; +const PIPELINE_SIGNAL = /\b[A-Z][A-Z_]+_(FAILED|DENIED|INVALID|CONFLICT)\b|\bERROR\b/; +// [file name pattern, line pattern, lines shown per file] +const SIGNAL_SOURCES = [ + [/^\d+-run\.log$/, RUN_SIGNAL, 40], + [/^runner-stderr\.log$/, RUN_SIGNAL, 20], + [/^tui\.log$/, PIPELINE_SIGNAL, 12], +]; +const CLIP = 220; + +class UsageError extends Error {} + +function clip(value, width = CLIP) { + const text = String(value ?? "").split(/\s+/).filter(Boolean).join(" "); + return text.length <= width ? text : text.slice(0, width - 1) + "…"; +} + +function readText(file) { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return ""; + } +} + +function readJson(file) { + try { + return JSON.parse(readText(file)); + } catch { + return null; + } +} + +function lines(text) { + return text.split(/\r?\n/); +} + +function listDir(dir) { + try { + return fs.readdirSync(dir).sort(); + } catch { + return []; + } +} + +function isDir(target) { + try { + return fs.statSync(target).isDirectory(); + } catch { + return false; + } +} + +// The entries of a zip, read from its central directory. +function zipEntries(buffer) { + const END = 0x06054b50; + const CENTRAL = 0x02014b50; + let end = -1; + for (let i = buffer.length - 22; i >= Math.max(0, buffer.length - 22 - 0xffff); i--) { + if (buffer.readUInt32LE(i) === END) { + end = i; + break; + } + } + if (end < 0) throw new UsageError("not a zip file: no end-of-archive record found"); + const count = buffer.readUInt16LE(end + 10); + let offset = buffer.readUInt32LE(end + 16); + if (count === 0xffff || offset === 0xffffffff) { + throw new UsageError("this pack uses zip64, which this script does not read: unzip it yourself and pass the directory"); + } + const entries = []; + for (let i = 0; i < count; i++) { + if (buffer.readUInt32LE(offset) !== CENTRAL) throw new UsageError("damaged zip: bad central directory entry"); + const nameLength = buffer.readUInt16LE(offset + 28); + entries.push({ + name: buffer.toString("utf8", offset + 46, offset + 46 + nameLength), + method: buffer.readUInt16LE(offset + 10), + compressedSize: buffer.readUInt32LE(offset + 20), + localOffset: buffer.readUInt32LE(offset + 42), + }); + offset += 46 + nameLength + buffer.readUInt16LE(offset + 30) + buffer.readUInt16LE(offset + 32); + } + return entries; +} + +function zipEntryData(buffer, entry) { + const start = entry.localOffset + 30 + + buffer.readUInt16LE(entry.localOffset + 26) + buffer.readUInt16LE(entry.localOffset + 28); + const raw = buffer.subarray(start, start + entry.compressedSize); + if (entry.method === 0) return raw; + if (entry.method === 8) return zlib.inflateRawSync(raw); + throw new UsageError(`unsupported compression in pack entry: ${entry.name}`); +} + +// Unzips a pack into an empty directory, refusing entries that escape it. +function extract(pack, out) { + fs.mkdirSync(out, { recursive: true }); + if (fs.readdirSync(out).length > 0) throw new UsageError(`--out must be an empty directory: ${out}`); + const root = fs.realpathSync(out); + const buffer = fs.readFileSync(pack); + const entries = zipEntries(buffer); + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (target !== root && !target.startsWith(root + path.sep)) { + throw new UsageError(`refusing pack entry outside the output directory: ${entry.name}`); + } + } + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (entry.name.endsWith("/")) { + fs.mkdirSync(target, { recursive: true }); + } else { + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, zipEntryData(buffer, entry)); + } + } + return root; +} + +// The flat `steps:` list of result.yaml, one object per row. +function resultSteps(file) { + const rows = []; + let current = null; + let inside = false; + const unquote = (value) => value.trim().replace(/^['"]|['"]$/g, ""); + for (const line of lines(readText(file))) { + if (line.startsWith("steps:")) { + inside = true; + continue; + } + if (!inside) continue; + if (line && !line.startsWith(" ")) break; + const item = line.match(/^\s+-\s+(\w+):\s*(.*)$/); + const field = line.match(/^\s+(\w+):\s*(.*)$/); + if (item) { + current = { [item[1]]: unquote(item[2]) }; + rows.push(current); + } else if (field && current) { + current[field[1]] = unquote(field[2]); + } + } + return rows; +} + +// operation_id -> the authored instruction node that owns it. +function executionLeaves(node, found, container = null) { + if (Array.isArray(node)) { + for (const value of node) executionLeaves(value, found, container); + } else if (node && typeof node === "object") { + if (node.condition && String(node.content_type ?? "").endsWith("_block")) { + container = `${node.content_type} condition=${JSON.stringify(node.condition)} condition_met=${node.condition_met}`; + } + for (const op of Array.isArray(node.operations) ? node.operations : []) { + if (op && typeof op === "object" && op.operation_id) { + found.set(op.operation_id, { + instruction: node.instruction, + reasoning: node.reasoning, + intent: op.intent, + status: node.status, + error: node.error_message || op.error_message, + container, + }); + } + } + for (const value of Object.values(node)) executionLeaves(value, found, container); + } + return found; +} + +function actionsById(logsDir) { + const found = new Map(); + for (const name of listDir(logsDir).filter((file) => /-actions\.ndjson$/.test(file))) { + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + let row; + try { + row = JSON.parse(line); + } catch { + return; + } + if (row && typeof row === "object" && row.action_id) { + row._where = `${name}:${index + 1}`; + found.set(row.action_id, row); + } + }); + } + return found; +} + +function stepFolders(stepsDir) { + const folders = []; + for (const name of listDir(stepsDir)) { + const dir = path.join(stepsDir, name); + if (!isDir(dir)) continue; + const files = listDir(dir); + const lead = name.match(/^\d+/); + folders.push({ + name, + ordinal: lead ? Number(lead[0]) : 0, + step: readJson(path.join(dir, "step.json")) || {}, + screenshot: files.find((file) => file.startsWith("screenshot.")) || null, + annotated: files.find((file) => file.startsWith("annotated.")) || null, + failure: files.includes("failure.yaml"), + }); + } + return folders.sort((a, b) => a.ordinal - b.ordinal); +} + +function printSignals(logsDir, print) { + print("\nLOG SIGNALS (pattern matches only; open the log around each, and search it yourself for more)"); + const names = listDir(logsDir); + for (const [filePattern, signal, limit] of SIGNAL_SOURCES) { + for (const name of names.filter((file) => filePattern.test(file))) { + const hits = []; + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + if (signal.test(line)) hits.push([index + 1, line]); + }); + print(` ${name}: ${hits.length} matching line(s)`); + for (const [number, line] of hits.slice(0, limit)) print(` ${number}: ${clip(line)}`); + if (hits.length > limit) print(` … ${hits.length - limit} more, last at line ${hits[hits.length - 1][0]}`); + } + } +} + +function printTest(testDir, print) { + print("\n" + "=".repeat(78)); + print(`TEST ${path.basename(testDir)}`); + const logsDir = path.join(testDir, "logs"); + const folders = stepFolders(path.join(testDir, "steps")); + const actions = actionsById(logsDir); + const leaves = executionLeaves(readJson(path.join(testDir, "auteur", "execution.json")), new Map()); + const rows = resultSteps(path.join(testDir, "result.yaml")); + const rowByAction = new Map(rows.filter((row) => row.action_id).map((row) => [row.action_id, row])); + const rowById = new Map(rows.filter((row) => row.id).map((row) => [row.id, row])); + print(` step folders=${folders.length} action rows=${actions.size}` + + ` authored operations=${leaves.size} result.yaml rows=${rows.length}`); + + print("\nSTEP SHEET (screenshot = the screen BEFORE this step's action ran)"); + let carried = null; + const seenActions = new Set(); + for (const folder of folders) { + const step = folder.step; + const actionId = step.action_id; + const ids = actionId ? [actionId] : []; + for (const id of Array.isArray(step.analyze_action_ids) ? step.analyze_action_ids : []) { + if (!ids.includes(id)) ids.push(id); + } + for (const id of ids) seenActions.add(id); + const row = rowById.get(step.id) || rowByAction.get(actionId) || {}; + let shotNote; + if (folder.screenshot) { + carried = `steps/${folder.name}/${folder.screenshot}`; + shotNote = carried; + } else { + shotNote = `NONE — latest earlier image: ${carried || "none"}`; + } + const joined = [["A", actions], ["E", leaves], ["R", rowByAction]] + .filter(([, source]) => source.has(actionId)) + .map(([mark]) => mark) + .join(""); + print(`\n[${folder.name}] id=${step.id} kind=${step.kind}` + + ` step.json=${step.status} result.yaml=${row.status ?? "-"}` + + ` duration_ms=${step.duration_ms}${folder.failure ? " <-- FAILURE RECORD" : ""}`); + print(` action_id=${actionId || "NONE (join by id/ordinal only)"} joined=${joined || "-"}` + + " (A=actions log E=execution.json R=result.yaml)"); + print(` screenshot: ${shotNote}${folder.annotated ? ` annotated: ${folder.annotated}` : ""}`); + print(` summary: ${clip(step.summary)}`); + for (const id of ids) { + const leaf = leaves.get(id); + const action = actions.get(id); + if (leaf) { + print(` authored[${id}]: intent=${clip(leaf.intent, 120)} | instruction=${clip(leaf.instruction, 160)}` + + ` | status=${leaf.status}`); + if (leaf.container) print(` inside: ${clip(leaf.container, 200)}`); + if (leaf.reasoning) print(` reasoning: ${clip(leaf.reasoning, 400)}`); + if (leaf.error) print(` error: ${clip(leaf.error, 300)}`); + } + if (action) { + print(` action[${action._where}]: ${action.action_type} status=${action.status}` + + ` at=${action.timestamp} | ${clip(action.action_instruction, 160)}`); + if (action.action_params && Object.keys(action.action_params).length > 0) { + print(` params: ${clip(JSON.stringify(action.action_params), 300)}`); + } + if (action.rationale) print(` rationale: ${clip(action.rationale, 400)}`); + if (action.error) print(` error: ${clip(action.error, 300)}`); + } + } + } + + print("\nUNMAPPED"); + let unmapped = 0; + for (const [id, action] of actions) { + if (seenActions.has(id)) continue; + unmapped++; + print(` action with no step folder: ${action._where} ${action.action_type}` + + ` status=${action.status} | ${clip(action.action_instruction, 140)}`); + } + const folderIds = new Set(folders.map((folder) => folder.step.id)); + const skipped = []; + for (const row of rows) { + if (folderIds.has(row.id) || seenActions.has(row.action_id)) continue; + unmapped++; + if (row.status === "skipped") { + skipped.push(row); + continue; + } + const leaf = leaves.get(row.action_id) || {}; + print(` result.yaml row with no step folder: ordinal=${row.ordinal} id=${row.id}` + + ` status=${row.status} | authored: ${clip(leaf.instruction || leaf.intent, 140)}`); + } + if (skipped.length > 0) { + print(` ${skipped.length} skipped result.yaml row(s) with no step folder,` + + ` ordinals ${skipped[0].ordinal}-${skipped[skipped.length - 1].ordinal}: the run never reached them`); + } + if (unmapped === 0) print(" none"); + + for (const folder of folders.filter((one) => one.failure)) { + const text = readText(path.join(testDir, "steps", folder.name, "failure.yaml")); + if (!/^triage:/m.test(text)) continue; + const verified = text.match(/verification:\s*\r?\n\s*status:\s*(\S+)/); + print(`\nEMBEDDED TRIAGE in steps/${folder.name}/failure.yaml` + + ` (verification=${verified ? verified[1] : "unknown"}): a claim to check, not a finding`); + } + printSignals(logsDir, print); +} + +function parseArgs(argv) { + const args = { target: null, out: null, test: null }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--out" || arg === "--test") { + if (i + 1 >= argv.length) throw new UsageError(`${arg} needs a value`); + args[arg.slice(2)] = argv[++i]; + } else if (arg.startsWith("--")) { + throw new UsageError(`unknown option: ${arg}`); + } else if (args.target === null) { + args.target = arg; + } else { + throw new UsageError(`unexpected argument: ${arg}`); + } + } + if (args.target === null) throw new UsageError("pass a .evidence pack or an extracted pack directory"); + return args; +} + +export function main(argv, print = console.log) { + const args = parseArgs(argv); + let root = args.target; + if (!fs.existsSync(root)) throw new UsageError(`not found: ${root}`); + if (!isDir(root)) { + if (!args.out) throw new UsageError("--out is required for a .evidence file"); + root = extract(root, args.out); + } + print(`PACK ${root}`); + for (const name of ["run.yaml", "failure.yaml"]) { + print(`\n--- ${name}\n${readText(path.join(root, name)).trimEnd()}`); + } + const tests = listDir(path.join(root, "tests")) + .filter((name) => !args.test || name === args.test) + .map((name) => path.join(root, "tests", name)) + .filter(isDir); + if (tests.length === 0) { + throw new UsageError(args.test ? `no test named ${args.test} under tests/` : "no tests found under tests/"); + } + for (const testDir of tests) printTest(testDir, print); +} + +if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main(process.argv.slice(2)); + } catch (error) { + if (!(error instanceof UsageError)) throw error; + console.error(error.message); + console.error(USAGE); + process.exit(2); + } +} diff --git a/.github/workflows/skill-installer-tests.yml b/.github/workflows/skill-installer-tests.yml index 6bbdf8d..88fe4c0 100644 --- a/.github/workflows/skill-installer-tests.yml +++ b/.github/workflows/skill-installer-tests.yml @@ -56,4 +56,5 @@ jobs: grep -q "lib/strip-install.mjs" pack.txt grep -q "skills/scripts/preflight.sh" pack.txt grep -q "skills/scripts/preflight.ps1" pack.txt + grep -q "skills/scripts/evidence-map.mjs" pack.txt if grep -q " test/" pack.txt; then echo "tests must not ship"; exit 1; fi diff --git a/skill-installer/TESTING.md b/skill-installer/TESTING.md index 2ea1ecb..f455149 100644 --- a/skill-installer/TESTING.md +++ b/skill-installer/TESTING.md @@ -18,6 +18,7 @@ The `Skill installer tests` workflow runs this on **ubuntu, macos and windows**, | `lib/agent-config.mjs` | Preferences merge, unknown keys kept, bad values rejected with the allowed list, seeding never overwrites | | `skills/scripts/preflight.sh` | Section order, a missing kane-cli, the mobile and grid flags, parallel calls, temp folder cleanup (Linux, macOS) | | `skills/scripts/preflight.ps1` | The same contract, run for real under Windows PowerShell against a stand-in `kane-cli.cmd` (Windows only) | +| `skills/scripts/evidence-map.mjs` | The step sheet from a hand-built pack: joins by `action_id`, the failure record mark, a step with no image, actions with no step folder, skipped steps, the triage flag, log lines. Reading a `.evidence` zip, and refusing a folder that is not empty or an entry that would land outside it | Refresh the fixtures when the kane-cli stream changes: capture `kane-cli run`, `testmd run` (twice) and `testrun run` with `--agent` or piped stdout, then replace home paths, internal hosts, share tokens and ids before committing. The repo is public. diff --git a/skill-installer/skills/scripts/evidence-map.mjs b/skill-installer/skills/scripts/evidence-map.mjs new file mode 100644 index 0000000..b11f1f2 --- /dev/null +++ b/skill-installer/skills/scripts/evidence-map.mjs @@ -0,0 +1,396 @@ +#!/usr/bin/env node +// Prints the step sheet of a kane-cli evidence pack. +// +// Joins every step folder to the actions log, the authored execution tree and +// result.yaml by action_id, and lists the error lines of the run logs. +// +// node evidence-map.mjs --out +// node evidence-map.mjs [--test ] +// +// No dependencies. Works on Node 18 and later, on macOS, Linux and Windows. + +import fs from "node:fs"; +import path from "node:path"; +import zlib from "node:zlib"; +import { fileURLToPath } from "node:url"; + +const USAGE = `Usage: + node evidence-map.mjs --out + node evidence-map.mjs [--test ]`; + +const RUN_SIGNAL = /✗|Traceback|\bERROR\b|\bCRITICAL\b|API error|FAIL\]|Agent failed|\bstuck\b/i; +const PIPELINE_SIGNAL = /\b[A-Z][A-Z_]+_(FAILED|DENIED|INVALID|CONFLICT)\b|\bERROR\b/; +// [file name pattern, line pattern, lines shown per file] +const SIGNAL_SOURCES = [ + [/^\d+-run\.log$/, RUN_SIGNAL, 40], + [/^runner-stderr\.log$/, RUN_SIGNAL, 20], + [/^tui\.log$/, PIPELINE_SIGNAL, 12], +]; +const CLIP = 220; + +class UsageError extends Error {} + +function clip(value, width = CLIP) { + const text = String(value ?? "").split(/\s+/).filter(Boolean).join(" "); + return text.length <= width ? text : text.slice(0, width - 1) + "…"; +} + +function readText(file) { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return ""; + } +} + +function readJson(file) { + try { + return JSON.parse(readText(file)); + } catch { + return null; + } +} + +function lines(text) { + return text.split(/\r?\n/); +} + +function listDir(dir) { + try { + return fs.readdirSync(dir).sort(); + } catch { + return []; + } +} + +function isDir(target) { + try { + return fs.statSync(target).isDirectory(); + } catch { + return false; + } +} + +// The entries of a zip, read from its central directory. +function zipEntries(buffer) { + const END = 0x06054b50; + const CENTRAL = 0x02014b50; + let end = -1; + for (let i = buffer.length - 22; i >= Math.max(0, buffer.length - 22 - 0xffff); i--) { + if (buffer.readUInt32LE(i) === END) { + end = i; + break; + } + } + if (end < 0) throw new UsageError("not a zip file: no end-of-archive record found"); + const count = buffer.readUInt16LE(end + 10); + let offset = buffer.readUInt32LE(end + 16); + if (count === 0xffff || offset === 0xffffffff) { + throw new UsageError("this pack uses zip64, which this script does not read: unzip it yourself and pass the directory"); + } + const entries = []; + for (let i = 0; i < count; i++) { + if (buffer.readUInt32LE(offset) !== CENTRAL) throw new UsageError("damaged zip: bad central directory entry"); + const nameLength = buffer.readUInt16LE(offset + 28); + entries.push({ + name: buffer.toString("utf8", offset + 46, offset + 46 + nameLength), + method: buffer.readUInt16LE(offset + 10), + compressedSize: buffer.readUInt32LE(offset + 20), + localOffset: buffer.readUInt32LE(offset + 42), + }); + offset += 46 + nameLength + buffer.readUInt16LE(offset + 30) + buffer.readUInt16LE(offset + 32); + } + return entries; +} + +function zipEntryData(buffer, entry) { + const start = entry.localOffset + 30 + + buffer.readUInt16LE(entry.localOffset + 26) + buffer.readUInt16LE(entry.localOffset + 28); + const raw = buffer.subarray(start, start + entry.compressedSize); + if (entry.method === 0) return raw; + if (entry.method === 8) return zlib.inflateRawSync(raw); + throw new UsageError(`unsupported compression in pack entry: ${entry.name}`); +} + +// Unzips a pack into an empty directory, refusing entries that escape it. +function extract(pack, out) { + fs.mkdirSync(out, { recursive: true }); + if (fs.readdirSync(out).length > 0) throw new UsageError(`--out must be an empty directory: ${out}`); + const root = fs.realpathSync(out); + const buffer = fs.readFileSync(pack); + const entries = zipEntries(buffer); + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (target !== root && !target.startsWith(root + path.sep)) { + throw new UsageError(`refusing pack entry outside the output directory: ${entry.name}`); + } + } + for (const entry of entries) { + const target = path.resolve(root, entry.name); + if (entry.name.endsWith("/")) { + fs.mkdirSync(target, { recursive: true }); + } else { + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, zipEntryData(buffer, entry)); + } + } + return root; +} + +// The flat `steps:` list of result.yaml, one object per row. +function resultSteps(file) { + const rows = []; + let current = null; + let inside = false; + const unquote = (value) => value.trim().replace(/^['"]|['"]$/g, ""); + for (const line of lines(readText(file))) { + if (line.startsWith("steps:")) { + inside = true; + continue; + } + if (!inside) continue; + if (line && !line.startsWith(" ")) break; + const item = line.match(/^\s+-\s+(\w+):\s*(.*)$/); + const field = line.match(/^\s+(\w+):\s*(.*)$/); + if (item) { + current = { [item[1]]: unquote(item[2]) }; + rows.push(current); + } else if (field && current) { + current[field[1]] = unquote(field[2]); + } + } + return rows; +} + +// operation_id -> the authored instruction node that owns it. +function executionLeaves(node, found, container = null) { + if (Array.isArray(node)) { + for (const value of node) executionLeaves(value, found, container); + } else if (node && typeof node === "object") { + if (node.condition && String(node.content_type ?? "").endsWith("_block")) { + container = `${node.content_type} condition=${JSON.stringify(node.condition)} condition_met=${node.condition_met}`; + } + for (const op of Array.isArray(node.operations) ? node.operations : []) { + if (op && typeof op === "object" && op.operation_id) { + found.set(op.operation_id, { + instruction: node.instruction, + reasoning: node.reasoning, + intent: op.intent, + status: node.status, + error: node.error_message || op.error_message, + container, + }); + } + } + for (const value of Object.values(node)) executionLeaves(value, found, container); + } + return found; +} + +function actionsById(logsDir) { + const found = new Map(); + for (const name of listDir(logsDir).filter((file) => /-actions\.ndjson$/.test(file))) { + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + let row; + try { + row = JSON.parse(line); + } catch { + return; + } + if (row && typeof row === "object" && row.action_id) { + row._where = `${name}:${index + 1}`; + found.set(row.action_id, row); + } + }); + } + return found; +} + +function stepFolders(stepsDir) { + const folders = []; + for (const name of listDir(stepsDir)) { + const dir = path.join(stepsDir, name); + if (!isDir(dir)) continue; + const files = listDir(dir); + const lead = name.match(/^\d+/); + folders.push({ + name, + ordinal: lead ? Number(lead[0]) : 0, + step: readJson(path.join(dir, "step.json")) || {}, + screenshot: files.find((file) => file.startsWith("screenshot.")) || null, + annotated: files.find((file) => file.startsWith("annotated.")) || null, + failure: files.includes("failure.yaml"), + }); + } + return folders.sort((a, b) => a.ordinal - b.ordinal); +} + +function printSignals(logsDir, print) { + print("\nLOG SIGNALS (pattern matches only; open the log around each, and search it yourself for more)"); + const names = listDir(logsDir); + for (const [filePattern, signal, limit] of SIGNAL_SOURCES) { + for (const name of names.filter((file) => filePattern.test(file))) { + const hits = []; + lines(readText(path.join(logsDir, name))).forEach((line, index) => { + if (signal.test(line)) hits.push([index + 1, line]); + }); + print(` ${name}: ${hits.length} matching line(s)`); + for (const [number, line] of hits.slice(0, limit)) print(` ${number}: ${clip(line)}`); + if (hits.length > limit) print(` … ${hits.length - limit} more, last at line ${hits[hits.length - 1][0]}`); + } + } +} + +function printTest(testDir, print) { + print("\n" + "=".repeat(78)); + print(`TEST ${path.basename(testDir)}`); + const logsDir = path.join(testDir, "logs"); + const folders = stepFolders(path.join(testDir, "steps")); + const actions = actionsById(logsDir); + const leaves = executionLeaves(readJson(path.join(testDir, "auteur", "execution.json")), new Map()); + const rows = resultSteps(path.join(testDir, "result.yaml")); + const rowByAction = new Map(rows.filter((row) => row.action_id).map((row) => [row.action_id, row])); + const rowById = new Map(rows.filter((row) => row.id).map((row) => [row.id, row])); + print(` step folders=${folders.length} action rows=${actions.size}` + + ` authored operations=${leaves.size} result.yaml rows=${rows.length}`); + + print("\nSTEP SHEET (screenshot = the screen BEFORE this step's action ran)"); + let carried = null; + const seenActions = new Set(); + for (const folder of folders) { + const step = folder.step; + const actionId = step.action_id; + const ids = actionId ? [actionId] : []; + for (const id of Array.isArray(step.analyze_action_ids) ? step.analyze_action_ids : []) { + if (!ids.includes(id)) ids.push(id); + } + for (const id of ids) seenActions.add(id); + const row = rowById.get(step.id) || rowByAction.get(actionId) || {}; + let shotNote; + if (folder.screenshot) { + carried = `steps/${folder.name}/${folder.screenshot}`; + shotNote = carried; + } else { + shotNote = `NONE — latest earlier image: ${carried || "none"}`; + } + const joined = [["A", actions], ["E", leaves], ["R", rowByAction]] + .filter(([, source]) => source.has(actionId)) + .map(([mark]) => mark) + .join(""); + print(`\n[${folder.name}] id=${step.id} kind=${step.kind}` + + ` step.json=${step.status} result.yaml=${row.status ?? "-"}` + + ` duration_ms=${step.duration_ms}${folder.failure ? " <-- FAILURE RECORD" : ""}`); + print(` action_id=${actionId || "NONE (join by id/ordinal only)"} joined=${joined || "-"}` + + " (A=actions log E=execution.json R=result.yaml)"); + print(` screenshot: ${shotNote}${folder.annotated ? ` annotated: ${folder.annotated}` : ""}`); + print(` summary: ${clip(step.summary)}`); + for (const id of ids) { + const leaf = leaves.get(id); + const action = actions.get(id); + if (leaf) { + print(` authored[${id}]: intent=${clip(leaf.intent, 120)} | instruction=${clip(leaf.instruction, 160)}` + + ` | status=${leaf.status}`); + if (leaf.container) print(` inside: ${clip(leaf.container, 200)}`); + if (leaf.reasoning) print(` reasoning: ${clip(leaf.reasoning, 400)}`); + if (leaf.error) print(` error: ${clip(leaf.error, 300)}`); + } + if (action) { + print(` action[${action._where}]: ${action.action_type} status=${action.status}` + + ` at=${action.timestamp} | ${clip(action.action_instruction, 160)}`); + if (action.action_params && Object.keys(action.action_params).length > 0) { + print(` params: ${clip(JSON.stringify(action.action_params), 300)}`); + } + if (action.rationale) print(` rationale: ${clip(action.rationale, 400)}`); + if (action.error) print(` error: ${clip(action.error, 300)}`); + } + } + } + + print("\nUNMAPPED"); + let unmapped = 0; + for (const [id, action] of actions) { + if (seenActions.has(id)) continue; + unmapped++; + print(` action with no step folder: ${action._where} ${action.action_type}` + + ` status=${action.status} | ${clip(action.action_instruction, 140)}`); + } + const folderIds = new Set(folders.map((folder) => folder.step.id)); + const skipped = []; + for (const row of rows) { + if (folderIds.has(row.id) || seenActions.has(row.action_id)) continue; + unmapped++; + if (row.status === "skipped") { + skipped.push(row); + continue; + } + const leaf = leaves.get(row.action_id) || {}; + print(` result.yaml row with no step folder: ordinal=${row.ordinal} id=${row.id}` + + ` status=${row.status} | authored: ${clip(leaf.instruction || leaf.intent, 140)}`); + } + if (skipped.length > 0) { + print(` ${skipped.length} skipped result.yaml row(s) with no step folder,` + + ` ordinals ${skipped[0].ordinal}-${skipped[skipped.length - 1].ordinal}: the run never reached them`); + } + if (unmapped === 0) print(" none"); + + for (const folder of folders.filter((one) => one.failure)) { + const text = readText(path.join(testDir, "steps", folder.name, "failure.yaml")); + if (!/^triage:/m.test(text)) continue; + const verified = text.match(/verification:\s*\r?\n\s*status:\s*(\S+)/); + print(`\nEMBEDDED TRIAGE in steps/${folder.name}/failure.yaml` + + ` (verification=${verified ? verified[1] : "unknown"}): a claim to check, not a finding`); + } + printSignals(logsDir, print); +} + +function parseArgs(argv) { + const args = { target: null, out: null, test: null }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--out" || arg === "--test") { + if (i + 1 >= argv.length) throw new UsageError(`${arg} needs a value`); + args[arg.slice(2)] = argv[++i]; + } else if (arg.startsWith("--")) { + throw new UsageError(`unknown option: ${arg}`); + } else if (args.target === null) { + args.target = arg; + } else { + throw new UsageError(`unexpected argument: ${arg}`); + } + } + if (args.target === null) throw new UsageError("pass a .evidence pack or an extracted pack directory"); + return args; +} + +export function main(argv, print = console.log) { + const args = parseArgs(argv); + let root = args.target; + if (!fs.existsSync(root)) throw new UsageError(`not found: ${root}`); + if (!isDir(root)) { + if (!args.out) throw new UsageError("--out is required for a .evidence file"); + root = extract(root, args.out); + } + print(`PACK ${root}`); + for (const name of ["run.yaml", "failure.yaml"]) { + print(`\n--- ${name}\n${readText(path.join(root, name)).trimEnd()}`); + } + const tests = listDir(path.join(root, "tests")) + .filter((name) => !args.test || name === args.test) + .map((name) => path.join(root, "tests", name)) + .filter(isDir); + if (tests.length === 0) { + throw new UsageError(args.test ? `no test named ${args.test} under tests/` : "no tests found under tests/"); + } + for (const testDir of tests) printTest(testDir, print); +} + +if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main(process.argv.slice(2)); + } catch (error) { + if (!(error instanceof UsageError)) throw error; + console.error(error.message); + console.error(USAGE); + process.exit(2); + } +} diff --git a/skill-installer/test/evidence-map.test.mjs b/skill-installer/test/evidence-map.test.mjs new file mode 100644 index 0000000..ea32fd4 --- /dev/null +++ b/skill-installer/test/evidence-map.test.mjs @@ -0,0 +1,323 @@ +// Tests for skills/scripts/evidence-map.mjs. Every pack here is built by hand +// in a temp folder, so nothing comes from a real run or a real account. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.resolve(here, '..', 'skills', 'scripts', 'evidence-map.mjs'); + +const RESULT_YAML = `evidence: '0.1' +test: checkout +status: broken +steps: + - id: 0-1 + ordinal: 1 + status: passed + kind: navigate + action_id: act-1 + - id: 0-2 + ordinal: 2 + status: passed + kind: click + action_id: act-2 + - id: 0-3 + ordinal: 3 + status: broken + kind: click + action_id: act-3 + - id: later-1 + ordinal: 4 + status: skipped + kind: click + action_id: act-5 + - id: later-2 + ordinal: 5 + status: skipped + kind: type + action_id: act-6 +environment: + browser: Chrome +`; + +const STEP_FAILURE_YAML = `step: 0-3 +status: broken +error: + message: 'action_failed: click @ step 3' +triage: + status: triaged + rca: + root_cause: A dialog covers the button. + verification: + status: not_verified +`; + +const EXECUTION = { + instructions: [ + { instruction: 'Open the shop', reasoning: 'Start at the home page', status: 'success', + operations: [{ operation_id: 'act-1', intent: 'Navigate to the shop' }] }, + { instruction: 'Click Shop', status: 'success', + operations: [{ operation_id: 'act-2', intent: 'Open the Shop link' }] }, + { content_type: 'if_block', condition: 'a banner is shown', condition_met: true, + children: [ + { instruction: 'Click Pay', reasoning: 'Finish the order', status: 'success', + operations: [{ operation_id: 'act-3', intent: 'Press the Pay button' }] }, + ] }, + { instruction: 'Click Confirm', status: 'success', + operations: [{ operation_id: 'act-5', intent: 'Confirm the order' }] }, + { instruction: 'Type a note', status: 'success', + operations: [{ operation_id: 'act-6', intent: 'Leave a note' }] }, + ], +}; + +const ACTIONS = [ + { action_id: 'act-1', step: '1', action_type: 'navigate', action_instruction: 'Navigate to the shop', + action_params: { url: 'http://shop.test/' }, status: 'success', timestamp: 'T1', rationale: 'Begin the flow' }, + { action_id: 'act-2', step: '2', action_type: 'click', action_instruction: 'Open the Shop link', + action_params: {}, status: 'success', timestamp: 'T2' }, + { action_id: 'act-3', step: '3', action_type: 'click', action_instruction: 'Press the Pay button', + action_params: {}, status: 'error', timestamp: 'T3', error: 'Timeout 5000ms exceeded.' }, + { action_id: 'act-4', step: '4', action_type: 'wait', action_instruction: 'Wait for the page', + action_params: {}, status: 'success', timestamp: 'T4' }, +]; + +function packFiles(testId = 'checkout') { + const base = `tests/${testId}`; + return { + 'run.yaml': "evidence: '0.1'\nrun_kind: replay\n", + 'failure.yaml': `failures:\n - test: ${testId}\n step: 0-3\n`, + [`${base}/result.yaml`]: RESULT_YAML, + [`${base}/test.md`]: 'Buy a plant\n', + [`${base}/auteur/execution.json`]: JSON.stringify(EXECUTION), + [`${base}/logs/0-actions.ndjson`]: ACTIONS.map((row) => JSON.stringify(row)).join('\n') + '\n', + [`${base}/logs/0-run.log`]: 'starting\nall good\n[STEP 3 FAIL] Timeout 5000ms exceeded.\ndone\n', + [`${base}/logs/tui.log`]: 'session started\nUPLOAD_FAILED could not publish\n', + [`${base}/steps/1-0-1/step.json`]: JSON.stringify( + { id: '0-1', kind: 'navigate', status: 'passed', summary: 'navigate: shop', action_id: 'act-1' }), + [`${base}/steps/1-0-1/screenshot.jpg`]: 'image-1', + [`${base}/steps/2-0-2/step.json`]: JSON.stringify( + { id: '0-2', kind: 'click', status: 'passed', summary: 'click: Shop', action_id: 'act-2' }), + [`${base}/steps/3-0-3/step.json`]: JSON.stringify( + { id: '0-3', kind: 'click', status: 'broken', summary: 'click: Pay', action_id: 'act-3' }), + [`${base}/steps/3-0-3/screenshot.png`]: 'image-3', + [`${base}/steps/3-0-3/annotated.png`]: 'image-3-marked', + [`${base}/steps/3-0-3/failure.yaml`]: STEP_FAILURE_YAML, + }; +} + +function tempDir(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'evidence-map-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + return dir; +} + +function writeTree(root, files) { + for (const [name, content] of Object.entries(files)) { + const target = path.join(root, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content); + } + return root; +} + +const CRC_TABLE = Array.from({ length: 256 }, (_, n) => { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + return c >>> 0; +}); + +function crc32(data) { + let crc = 0xffffffff; + for (const byte of data) crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +// A real zip: the first entry is stored, the rest are deflated. +function writeZip(file, files) { + const locals = []; + const centrals = []; + let offset = 0; + Object.entries(files).forEach(([name, content], index) => { + const data = Buffer.from(content); + const method = index === 0 ? 0 : 8; + const body = method === 0 ? data : zlib.deflateRawSync(data); + const nameBytes = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(method, 8); + local.writeUInt32LE(crc32(data), 14); + local.writeUInt32LE(body.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(nameBytes.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(method, 10); + central.writeUInt32LE(crc32(data), 16); + central.writeUInt32LE(body.length, 20); + central.writeUInt32LE(data.length, 24); + central.writeUInt16LE(nameBytes.length, 28); + central.writeUInt32LE(offset, 42); + locals.push(local, nameBytes, body); + centrals.push(central, nameBytes); + offset += 30 + nameBytes.length + body.length; + }); + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(Object.keys(files).length, 8); + end.writeUInt16LE(Object.keys(files).length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + fs.writeFileSync(file, Buffer.concat([...locals, directory, end])); + return file; +} + +function run(...args) { + const result = spawnSync(process.execPath, [SCRIPT, ...args], { encoding: 'utf8' }); + return { code: result.status, out: result.stdout, err: result.stderr }; +} + +// The lines of one step's block in the sheet, from its header to the next blank line. +function stepBlock(out, folder) { + const all = out.split('\n'); + const start = all.findIndex((line) => line.startsWith(`[${folder}]`)); + assert.notEqual(start, -1, `no block for ${folder}`); + const end = all.findIndex((line, index) => index > start && line === ''); + return all.slice(start, end === -1 ? undefined : end).join('\n'); +} + +test('step sheet', async (t) => { + const pack = writeTree(tempDir(t), packFiles()); + const { code, out } = run(pack); + assert.equal(code, 0); + + const cases = [ + { name: 'prints run.yaml and the failure index first', + check: () => { + assert.match(out, /--- run\.yaml\nevidence: '0\.1'\nrun_kind: replay/); + assert.match(out, /--- failure\.yaml\nfailures:/); + } }, + { name: 'counts what it found', + check: () => assert.match(out, /step folders=3 action rows=4 authored operations=5 result\.yaml rows=5/) }, + { name: 'joins a folder to the actions log, the authored tree and result.yaml', + check: () => { + const block = stepBlock(out, '1-0-1'); + assert.match(block, /id=0-1 kind=navigate step\.json=passed result\.yaml=passed/); + assert.match(block, /action_id=act-1 joined=AER/); + assert.match(block, /authored\[act-1\]: intent=Navigate to the shop \| instruction=Open the shop \| status=success/); + assert.match(block, /reasoning: Start at the home page/); + assert.match(block, /action\[0-actions\.ndjson:1\]: navigate status=success at=T1/); + assert.match(block, /params: \{"url":"http:\/\/shop\.test\/"\}/); + assert.match(block, /rationale: Begin the flow/); + } }, + { name: 'names the screenshot of a step', + check: () => assert.match(stepBlock(out, '1-0-1'), /screenshot: steps\/1-0-1\/screenshot\.jpg/) }, + { name: 'a folder with no image points at the latest earlier one', + check: () => assert.match(stepBlock(out, '2-0-2'), /screenshot: NONE — latest earlier image: steps\/1-0-1\/screenshot\.jpg/) }, + { name: 'leaves out empty params', + check: () => assert.doesNotMatch(stepBlock(out, '2-0-2'), /params:/) }, + { name: 'marks the step that carries the failure record', + check: () => { + const block = stepBlock(out, '3-0-3'); + assert.match(block, /result\.yaml=broken .*<-- FAILURE RECORD/); + assert.match(block, /annotated: annotated\.png/); + assert.match(block, /error: Timeout 5000ms exceeded\./); + } }, + { name: 'shows the block an authored step sits inside', + check: () => assert.match(stepBlock(out, '3-0-3'), /inside: if_block condition="a banner is shown" condition_met=true/) }, + { name: 'only the failing step is marked', + check: () => assert.equal(out.match(/<-- FAILURE RECORD/g).length, 1) }, + { name: 'lists an action that has no step folder', + check: () => assert.match(out, /action with no step folder: 0-actions\.ndjson:4 wait status=success \| Wait for the page/) }, + { name: 'sums up the steps the run never reached', + check: () => assert.match(out, /2 skipped result\.yaml row\(s\) with no step folder, ordinals 4-5/) }, + { name: 'flags the triage inside the pack as a claim', + check: () => assert.match(out, /EMBEDDED TRIAGE in steps\/3-0-3\/failure\.yaml \(verification=not_verified\): a claim to check/) }, + { name: 'lists matching log lines with their line numbers', + check: () => { + assert.match(out, /0-run\.log: 1 matching line\(s\)\n {4}3: \[STEP 3 FAIL\] Timeout 5000ms exceeded\./); + assert.match(out, /tui\.log: 1 matching line\(s\)\n {4}2: UPLOAD_FAILED could not publish/); + } }, + ]; + for (const { name, check } of cases) await t.test(name, check); +}); + +test('a pack where every step has a folder', (t) => { + const files = packFiles(); + files['tests/checkout/result.yaml'] = RESULT_YAML.replace(/ {2}- id: later-1[\s\S]*?(?=environment:)/, ''); + files['tests/checkout/logs/0-actions.ndjson'] = ACTIONS.slice(0, 3).map((row) => JSON.stringify(row)).join('\n'); + const { code, out } = run(writeTree(tempDir(t), files)); + assert.equal(code, 0); + assert.match(out, /UNMAPPED\n {2}none/); +}); + +test('test selection', async (t) => { + const pack = writeTree(tempDir(t), { ...packFiles('alpha'), ...packFiles('beta') }); + const cases = [ + { name: 'prints every test by default', args: [pack], code: 0, has: ['TEST alpha', 'TEST beta'], lacks: [] }, + { name: 'prints one test with --test', args: [pack, '--test', 'beta'], code: 0, has: ['TEST beta'], lacks: ['TEST alpha'] }, + { name: 'an unknown test is a usage error', args: [pack, '--test', 'gamma'], code: 2, err: /no test named gamma/ }, + ]; + for (const one of cases) { + await t.test(one.name, () => { + const { code, out, err } = run(...one.args); + assert.equal(code, one.code); + for (const text of one.has || []) assert.ok(out.includes(text), `missing ${text}`); + for (const text of one.lacks || []) assert.ok(!out.includes(text), `unexpected ${text}`); + if (one.err) assert.match(err, one.err); + }); + } +}); + +test('reading a .evidence file', async (t) => { + const dir = tempDir(t); + const pack = writeZip(path.join(dir, 'run.evidence'), packFiles()); + + await t.test('unzips into an empty folder and prints the sheet', () => { + const out = path.join(dir, 'out'); + const result = run(pack, '--out', out); + assert.equal(result.code, 0); + assert.match(result.out, /action_id=act-3 joined=AER/); + assert.equal(fs.readFileSync(path.join(out, 'tests/checkout/steps/3-0-3/screenshot.png'), 'utf8'), 'image-3'); + assert.equal(fs.readFileSync(path.join(out, 'run.yaml'), 'utf8'), packFiles()['run.yaml']); + }); + + const full = path.join(dir, 'full'); + writeTree(full, { 'keep.txt': 'mine' }); + const escaping = writeZip(path.join(dir, 'escape.evidence'), { 'run.yaml': 'x', '../outside.txt': 'bad' }); + const notZip = path.join(dir, 'broken.evidence'); + fs.writeFileSync(notZip, 'this is not a zip'); + + const cases = [ + { name: 'needs --out for a file', args: [pack], err: /--out is required/ }, + { name: 'refuses a folder that already holds files', args: [pack, '--out', full], err: /--out must be an empty directory/ }, + { name: 'refuses an entry that would land outside the folder', + args: [escaping, '--out', path.join(dir, 'escape-out')], err: /refusing pack entry outside the output directory/ }, + { name: 'says so when the file is not a zip', args: [notZip, '--out', path.join(dir, 'broken-out')], err: /not a zip file/ }, + { name: 'says so when the pack is missing', args: [path.join(dir, 'missing.evidence')], err: /not found/ }, + { name: 'needs a pack', args: [], err: /pass a \.evidence pack/ }, + { name: 'rejects an unknown option', args: [pack, '--verbose'], err: /unknown option: --verbose/ }, + ]; + for (const one of cases) { + await t.test(one.name, () => { + const { code, err } = run(...one.args); + assert.equal(code, 2); + assert.match(err, one.err); + assert.match(err, /Usage:/); + }); + } + + await t.test('leaves a refused folder untouched', () => { + assert.deepEqual(fs.readdirSync(full), ['keep.txt']); + assert.ok(!fs.existsSync(path.join(dir, 'outside.txt'))); + assert.deepEqual(fs.readdirSync(path.join(dir, 'escape-out')), []); + }); +}); From 674d830279c38d79d73b4bd999a23168b4e7668a Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Mon, 12 Oct 2026 18:13:25 +0530 Subject: [PATCH 2/4] docs, skill: triage a run from its evidence pack New references/triage.md: the triage steps, the screen table, the report and the four causes. debug.md gains step identity, the join rules and what a step folder holds, and says a step's screenshot is taken before its action runs. Screenshot paths read screenshot.* (png or jpg). Co-Authored-By: Claude Opus 5.5 --- .agents/skills/kane-cli/SKILL.md | 4 +- .agents/skills/kane-cli/references/debug.md | 39 ++++++++- .../skills/kane-cli/references/evidence.md | 4 +- .agents/skills/kane-cli/references/triage.md | 79 +++++++++++++++++++ .claude/skills/kane-cli/SKILL.md | 4 +- .claude/skills/kane-cli/references/debug.md | 39 ++++++++- .../skills/kane-cli/references/evidence.md | 4 +- .claude/skills/kane-cli/references/triage.md | 79 +++++++++++++++++++ skill-installer/skills/SKILL.md | 4 +- skill-installer/skills/references/debug.md | 39 ++++++++- skill-installer/skills/references/evidence.md | 4 +- skill-installer/skills/references/triage.md | 79 +++++++++++++++++++ 12 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 .agents/skills/kane-cli/references/triage.md create mode 100644 .claude/skills/kane-cli/references/triage.md create mode 100644 skill-installer/skills/references/triage.md diff --git a/.agents/skills/kane-cli/SKILL.md b/.agents/skills/kane-cli/SKILL.md index f6a3dd0..2b0ae03 100644 --- a/.agents/skills/kane-cli/SKILL.md +++ b/.agents/skills/kane-cli/SKILL.md @@ -136,7 +136,7 @@ For exit code 1 (or `status: "failed"` in `run_end`), present the failure card. | ➡️ **Next** | · | ``` -The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.png" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. +The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.*" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. Exit code 2 means nothing ran: that is a `🟡 Didn't start` card, not a failure (`references/cards.md` §4). @@ -168,6 +168,7 @@ When the user's request involves a browser — or writing test cases: - Multiple independent browser tasks → Read `references/parallel.md` first - View, share, or validate run evidence (`.evidence` packs) → Read `references/evidence.md` - Debug a failed run → Read `references/debug.md` +- The quick debug did not explain a failure, a run passed but looks wrong, or the person asks whose fault a failure is or whether the triage in a pack is right → Read `references/triage.md` - Configure kane-cli or check directory layout → Read `references/setup-and-config.md` - Browse / create / pick a Test Manager project or folder, or interpret the auto-default event → Read `references/test-manager.md` - The person wants results saved somewhere else ("change project") → Read `references/test-manager.md` §6. The change is global, and the question must say so @@ -350,6 +351,7 @@ A draft in chat cannot be run, and a hand-written `_test.md` carries no requirem | Need the assurance NDJSON event schema (`--mode agent`) | `references/assurance-parsing.md` | | Share the context store with a team, join one, keep stores level, or resolve a sync conflict (kane-cli 0.8.14+) | `references/context-sync.md` | | Run failed, need to diagnose | `references/debug.md` | +| Full triage of a pack: the step sheet, screenshots set against intent, a report with one cause | `references/triage.md` | | View, share, validate, or merge evidence packs | `references/evidence.md` | | Multiple independent browser tasks | `references/parallel.md` | | Need full NDJSON event schema (`run`) | `references/parsing.md` | diff --git a/.agents/skills/kane-cli/references/debug.md b/.agents/skills/kane-cli/references/debug.md index 372d188..c3a5e6e 100644 --- a/.agents/skills/kane-cli/references/debug.md +++ b/.agents/skills/kane-cli/references/debug.md @@ -1,4 +1,4 @@ - + # Failure Handling & Log Inspection @@ -17,7 +17,7 @@ unzip -l # list entries unzip -p "tests/*/result.yaml" # verdict + per-step outcomes unzip -p "tests/*/steps/*/failure.yaml" # failure records (failed steps only) unzip -p "tests/*/logs/0-console.ndjson" # browser console, run 0 -unzip "tests/*/steps/*/screenshot.png" -d /tmp/ev # extract screenshots to view +unzip "tests/*/steps/*/screenshot.*" -d /tmp/ev # extract screenshots to view ``` Pack layout (per test): @@ -33,18 +33,49 @@ tests// │ ├── -actions.ndjson # agent actions per run — the old runs// actions log, now in-pack │ ├── -console.ndjson # browser console, per-step attributed │ └── -network.har # network traffic, per-step attributed -├── steps/-/ # screenshot.png, annotated.png, step.json, +├── steps/-/ # step.json, screenshot.* (png or jpg), annotated.png (some steps), │ # failure.yaml (failed steps only) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries ``` +### Step identity and joins + +One step has a different name in each file. Join on `action_id`, never on a step number. + +| Name | Where | Meaning | +|---|---|---| +| ordinal | leading number of the step folder; `result.yaml` `ordinal` | order in the pack | +| id | rest of the folder name; `step.json` `id`; `result.yaml` `id` | `-` for a step that ran | +| action_id | `step.json`, the `result.yaml` row, the `-actions.ndjson` line, `auteur/execution.json` `operations[].operation_id` | the join key across files | +| step | `-actions.ndjson` `step` | counted per file; never a join key | + +Join a step folder to the other files by `id`, then `action_id`, then ordinal. The run index `` of a step id `-` is the prefix of its logs: `logs/-run.log`, `logs/-actions.ndjson`. + +Where the intent of a step lives: + +| Source | Fields | Notes | +|---|---|---| +| `auteur/execution.json` | `instruction`, `reasoning`, `status`, `error_message`; `operations[].intent` | The authored steps. In a replay pack this describes the run that authored the test, so its `status` is the authored outcome | +| `logs/-actions.ndjson` | `action_type`, `action_instruction`, `rationale`, `action_params`, `selectors`, `status`, `error`, `timestamp` | What this run did | +| `step.json` | `kind`, `summary`, `status`, `duration_ms`, `url` | One-line view of the step | + +### What is and is not in a step folder + +- **The screenshot is taken before the step's action runs**, in an authoring run and in a replay. The effect of a step is in the next step's screenshot. +- `annotated.png` draws the action's marker on that same before-the-action screenshot. Not every step has one. +- Steps the run never reached are `skipped` rows in `result.yaml` and have no folder. +- An action can have a line in `-actions.ndjson` and no folder: it has no screenshot of its own. + +`scripts/evidence-map.mjs` prints all of this as one step sheet (`references/triage.md` §1). + ## Debugging Flow 1. **Parse the `run_end` event** from stdout — `status`, `reason`, `summary`, plus `session_dir`. 2. **Open the pack**: the failed step's `failure.yaml` (error + page state), then that step's slice of `-console.ndjson` / `-network.har` — a 4xx/5xx or JS error usually explains the failure; cite it in plain language. -3. **Look at the failing step's `annotated.png`** — it highlights the element the agent acted on; makes "clicked the wrong thing" / "element missing" obvious. Render it inline. +3. **Look at the failing step's `screenshot.*` and `annotated.png`.** They show the page **before** the step's action ran, with a marker on what the agent was about to act on; that makes "aimed at the wrong thing", "element missing" and "something covers it" obvious. Whether an action took effect is in the **next** step's screenshot. Render the image inline. 4. **Check `tui.log`** (in the pack's `logs/`, or `{session_dir}/tui.log`) for session-level issues (Chrome launch, auth, upload). +5. **Still unexplained, a pass that looks wrong, or the person asks whose fault it is?** Do the full triage: `references/triage.md`. Offer the user the visual route too: `kane-cli evidence serve ` opens it in the hosted viewer. If a pack won't open, `kane-cli evidence validate ` tells you whether it's truncated/unsealed. diff --git a/.agents/skills/kane-cli/references/evidence.md b/.agents/skills/kane-cli/references/evidence.md index 34125fe..c90a022 100644 --- a/.agents/skills/kane-cli/references/evidence.md +++ b/.agents/skills/kane-cli/references/evidence.md @@ -18,7 +18,7 @@ A `.evidence` file is a standard zip: `unzip -l ` lists it, `unzip -p -run.log, -actions.ndjson, │ # -console.ndjson, -network.har - ├── steps/-/ # screenshot.png, annotated.png, step.json + ├── steps/-/ # screenshot.* (png or jpg), annotated.png, step.json │ # (+ failure.yaml on failed steps) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries @@ -74,7 +74,7 @@ Combines packs (execution ids or paths, order-significant) into one, sealed by d ## Debugging with a pack -The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent actually acted on). Full failure workflow: `references/debug.md`. +The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent was about to act on; a step's screenshot is taken before its action runs). Full failure workflow: `references/debug.md`. A step-by-step triage with a report: `references/triage.md`. ## Evidence merge identity diff --git a/.agents/skills/kane-cli/references/triage.md b/.agents/skills/kane-cli/references/triage.md new file mode 100644 index 0000000..7e018b3 --- /dev/null +++ b/.agents/skills/kane-cli/references/triage.md @@ -0,0 +1,79 @@ + + +# Triaging a run from its evidence pack + +A pack records three views of every step: what the screen showed, what the agent meant to do, and what the logs say happened. A triage is a claim that holds in all three. **The screen can hold facts no log mentions** (a dialog, a toast, a spinner, an error banner, a keyboard), so the screenshots are read every time. + +Pack layout, step identity and join rules: `references/debug.md`. + +## 1. Steps + +1. **Map the pack.** Run the step sheet. It joins step folders, the actions log, `auteur/execution.json` and `result.yaml` by `action_id`, prints `run.yaml` and the failure index, and lists the error lines of the run logs. + + ```bash + node "/scripts/evidence-map.mjs" --out + node "/scripts/evidence-map.mjs" [--test ] + ``` + + The first form unzips the pack into the folder you name, which must be empty. A merged or suite pack holds many tests: take them one at a time from the failure index, with `--test`. + +2. **Frame the run** from `run.yaml` and `result.yaml`: `run_kind` (`author` or `replay`), `origin`, the kane-cli version, and the environment. A replay repeats recorded actions, so a wrong choice of action belongs to the run that authored the test, not to this one. + +3. **Read every log signal** the sheet printed, in the log itself, with the lines around it. A sign-in, network or browser error here outranks any step-level story. The sheet only matches patterns: search the logs yourself for the failing step, by the action's `at=` time. + +4. **Pick the window.** The step carrying the failure record, the step before it, and the step after it when one exists. + - Check the sheet's `UNMAPPED` list first. An action with no step folder ran without a screenshot of its own, and it can be what ended the run. + - Add an earlier step of the same kind as the failing one, with the step after it. A before and after pair of the same action that worked is the cheapest comparison. + +5. **Fill one screen row per window step** (the table in §2). For each: + - Open `screenshot.*` with your image-reading tool. Open `annotated.*` too when present. + - Take the intent from the sheet: the authored `intent`, `instruction`, `reasoning` and `inside`, then the action's instruction, `params` and `rationale`. When the instruction text disagrees with the action type or the rationale, say so and trust the screenshot. + - Compare the screen to the intent. + - Prove the effect from the **next** step's screenshot: a step's own screenshot is taken before its action runs. If no later screenshot exists, write "not provable from the screen" and name the log line you used in its place. + +6. **Check the triage inside the pack.** A step `failure.yaml` with a `triage:` block is a claim written by an automatic investigation. State whether the screen table and the logs support it. + +7. **Name one cause** (§3) and report. + +## 2. Report + +The report has these six parts, in this order: + +1. **Verdict.** One or two sentences: what failed, and the cause. +2. **Screen table.** Required, one row per window step: + + | Step folder | Mapped by | Screenshot opened | What the screen shows | What the agent was doing | Screen vs intent | Effect seen in next screenshot | + |---|---|---|---|---|---|---| + + `Mapped by` is `id`, `action_id` or `ordinal`. A step folder with no image names the earlier image it borrows. An action with no step folder gets one row with "no image": the last folder's screenshot was taken before it and is not its screen. +3. **Log evidence.** Each claim with the file and line inside the pack. +4. **Triage inside the pack.** Agree or disagree, and why. "None in pack" when absent. +5. **Other findings.** Defects the pack shows that did not cause this failure, such as a step that passed without changing the screen. "None" when absent. +6. **Not determined.** What the pack cannot show, and what would settle it (a re-run, a look at the app, a bug report). + +Label anything you have not tested as a hypothesis. + +Give the person the verdict in plain words first. Show the table and the rest when they ask for the detail, or when the cause is not the obvious one. + +## 3. Causes + +Pick the earliest fault the pack proves. + +| Cause | Typical sign | What to do next | +|---|---|---| +| The app under test | The screenshot shows the product misbehaving: an error banner, a wrong value, a button that does nothing | Tell the person it is a product bug and show the screenshot. Do not change the test to get around it | +| The test or the objective | `test.md` asks for something ambiguous, impossible, or no longer true of the app | Fix the wording. For a designed test, read `references/assurance.md` §6.1 first | +| The environment | Sign-in, network, browser or device errors in the logs around the failing step | Fix the setup and run again (`references/setup-and-config.md`) | +| kane-cli | The screen was right and the step was clear, and the action was still wrong or did not take effect; or a replay fails a step that the authoring run passed on the same screen | Offer a bug report (`references/debug.md` §Filing a bug report) | + +## 4. Common mistakes + +| Mistake | Fix | +|---|---| +| Matching a screenshot to a step by step number | Step numbers are counted per file. Use the sheet's join | +| Reading a screenshot as the result of its step | It is the screen before the action. Look one step later | +| Trusting a `passed` step | Passed means the action returned, not that the screen changed | +| Blaming the step that carries the failure record | Check the `UNMAPPED` list for actions after it first | +| Repeating the triage inside the pack | Check it against the log signals and the screen table first | +| Reading an off-target marker as a wrong click | The marker is drawn on a screenshot taken before the action. Confirm the click in the actions log | +| Quoting `*.har` content | It can carry credentials. Cite the request, never the secret | diff --git a/.claude/skills/kane-cli/SKILL.md b/.claude/skills/kane-cli/SKILL.md index f6a3dd0..2b0ae03 100644 --- a/.claude/skills/kane-cli/SKILL.md +++ b/.claude/skills/kane-cli/SKILL.md @@ -136,7 +136,7 @@ For exit code 1 (or `status: "failed"` in `run_end`), present the failure card. | ➡️ **Next** | · | ``` -The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.png" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. +The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.*" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. Exit code 2 means nothing ran: that is a `🟡 Didn't start` card, not a failure (`references/cards.md` §4). @@ -168,6 +168,7 @@ When the user's request involves a browser — or writing test cases: - Multiple independent browser tasks → Read `references/parallel.md` first - View, share, or validate run evidence (`.evidence` packs) → Read `references/evidence.md` - Debug a failed run → Read `references/debug.md` +- The quick debug did not explain a failure, a run passed but looks wrong, or the person asks whose fault a failure is or whether the triage in a pack is right → Read `references/triage.md` - Configure kane-cli or check directory layout → Read `references/setup-and-config.md` - Browse / create / pick a Test Manager project or folder, or interpret the auto-default event → Read `references/test-manager.md` - The person wants results saved somewhere else ("change project") → Read `references/test-manager.md` §6. The change is global, and the question must say so @@ -350,6 +351,7 @@ A draft in chat cannot be run, and a hand-written `_test.md` carries no requirem | Need the assurance NDJSON event schema (`--mode agent`) | `references/assurance-parsing.md` | | Share the context store with a team, join one, keep stores level, or resolve a sync conflict (kane-cli 0.8.14+) | `references/context-sync.md` | | Run failed, need to diagnose | `references/debug.md` | +| Full triage of a pack: the step sheet, screenshots set against intent, a report with one cause | `references/triage.md` | | View, share, validate, or merge evidence packs | `references/evidence.md` | | Multiple independent browser tasks | `references/parallel.md` | | Need full NDJSON event schema (`run`) | `references/parsing.md` | diff --git a/.claude/skills/kane-cli/references/debug.md b/.claude/skills/kane-cli/references/debug.md index 372d188..c3a5e6e 100644 --- a/.claude/skills/kane-cli/references/debug.md +++ b/.claude/skills/kane-cli/references/debug.md @@ -1,4 +1,4 @@ - + # Failure Handling & Log Inspection @@ -17,7 +17,7 @@ unzip -l # list entries unzip -p "tests/*/result.yaml" # verdict + per-step outcomes unzip -p "tests/*/steps/*/failure.yaml" # failure records (failed steps only) unzip -p "tests/*/logs/0-console.ndjson" # browser console, run 0 -unzip "tests/*/steps/*/screenshot.png" -d /tmp/ev # extract screenshots to view +unzip "tests/*/steps/*/screenshot.*" -d /tmp/ev # extract screenshots to view ``` Pack layout (per test): @@ -33,18 +33,49 @@ tests// │ ├── -actions.ndjson # agent actions per run — the old runs// actions log, now in-pack │ ├── -console.ndjson # browser console, per-step attributed │ └── -network.har # network traffic, per-step attributed -├── steps/-/ # screenshot.png, annotated.png, step.json, +├── steps/-/ # step.json, screenshot.* (png or jpg), annotated.png (some steps), │ # failure.yaml (failed steps only) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries ``` +### Step identity and joins + +One step has a different name in each file. Join on `action_id`, never on a step number. + +| Name | Where | Meaning | +|---|---|---| +| ordinal | leading number of the step folder; `result.yaml` `ordinal` | order in the pack | +| id | rest of the folder name; `step.json` `id`; `result.yaml` `id` | `-` for a step that ran | +| action_id | `step.json`, the `result.yaml` row, the `-actions.ndjson` line, `auteur/execution.json` `operations[].operation_id` | the join key across files | +| step | `-actions.ndjson` `step` | counted per file; never a join key | + +Join a step folder to the other files by `id`, then `action_id`, then ordinal. The run index `` of a step id `-` is the prefix of its logs: `logs/-run.log`, `logs/-actions.ndjson`. + +Where the intent of a step lives: + +| Source | Fields | Notes | +|---|---|---| +| `auteur/execution.json` | `instruction`, `reasoning`, `status`, `error_message`; `operations[].intent` | The authored steps. In a replay pack this describes the run that authored the test, so its `status` is the authored outcome | +| `logs/-actions.ndjson` | `action_type`, `action_instruction`, `rationale`, `action_params`, `selectors`, `status`, `error`, `timestamp` | What this run did | +| `step.json` | `kind`, `summary`, `status`, `duration_ms`, `url` | One-line view of the step | + +### What is and is not in a step folder + +- **The screenshot is taken before the step's action runs**, in an authoring run and in a replay. The effect of a step is in the next step's screenshot. +- `annotated.png` draws the action's marker on that same before-the-action screenshot. Not every step has one. +- Steps the run never reached are `skipped` rows in `result.yaml` and have no folder. +- An action can have a line in `-actions.ndjson` and no folder: it has no screenshot of its own. + +`scripts/evidence-map.mjs` prints all of this as one step sheet (`references/triage.md` §1). + ## Debugging Flow 1. **Parse the `run_end` event** from stdout — `status`, `reason`, `summary`, plus `session_dir`. 2. **Open the pack**: the failed step's `failure.yaml` (error + page state), then that step's slice of `-console.ndjson` / `-network.har` — a 4xx/5xx or JS error usually explains the failure; cite it in plain language. -3. **Look at the failing step's `annotated.png`** — it highlights the element the agent acted on; makes "clicked the wrong thing" / "element missing" obvious. Render it inline. +3. **Look at the failing step's `screenshot.*` and `annotated.png`.** They show the page **before** the step's action ran, with a marker on what the agent was about to act on; that makes "aimed at the wrong thing", "element missing" and "something covers it" obvious. Whether an action took effect is in the **next** step's screenshot. Render the image inline. 4. **Check `tui.log`** (in the pack's `logs/`, or `{session_dir}/tui.log`) for session-level issues (Chrome launch, auth, upload). +5. **Still unexplained, a pass that looks wrong, or the person asks whose fault it is?** Do the full triage: `references/triage.md`. Offer the user the visual route too: `kane-cli evidence serve ` opens it in the hosted viewer. If a pack won't open, `kane-cli evidence validate ` tells you whether it's truncated/unsealed. diff --git a/.claude/skills/kane-cli/references/evidence.md b/.claude/skills/kane-cli/references/evidence.md index 34125fe..c90a022 100644 --- a/.claude/skills/kane-cli/references/evidence.md +++ b/.claude/skills/kane-cli/references/evidence.md @@ -18,7 +18,7 @@ A `.evidence` file is a standard zip: `unzip -l ` lists it, `unzip -p -run.log, -actions.ndjson, │ # -console.ndjson, -network.har - ├── steps/-/ # screenshot.png, annotated.png, step.json + ├── steps/-/ # screenshot.* (png or jpg), annotated.png, step.json │ # (+ failure.yaml on failed steps) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries @@ -74,7 +74,7 @@ Combines packs (execution ids or paths, order-significant) into one, sealed by d ## Debugging with a pack -The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent actually acted on). Full failure workflow: `references/debug.md`. +The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent was about to act on; a step's screenshot is taken before its action runs). Full failure workflow: `references/debug.md`. A step-by-step triage with a report: `references/triage.md`. ## Evidence merge identity diff --git a/.claude/skills/kane-cli/references/triage.md b/.claude/skills/kane-cli/references/triage.md new file mode 100644 index 0000000..7e018b3 --- /dev/null +++ b/.claude/skills/kane-cli/references/triage.md @@ -0,0 +1,79 @@ + + +# Triaging a run from its evidence pack + +A pack records three views of every step: what the screen showed, what the agent meant to do, and what the logs say happened. A triage is a claim that holds in all three. **The screen can hold facts no log mentions** (a dialog, a toast, a spinner, an error banner, a keyboard), so the screenshots are read every time. + +Pack layout, step identity and join rules: `references/debug.md`. + +## 1. Steps + +1. **Map the pack.** Run the step sheet. It joins step folders, the actions log, `auteur/execution.json` and `result.yaml` by `action_id`, prints `run.yaml` and the failure index, and lists the error lines of the run logs. + + ```bash + node "/scripts/evidence-map.mjs" --out + node "/scripts/evidence-map.mjs" [--test ] + ``` + + The first form unzips the pack into the folder you name, which must be empty. A merged or suite pack holds many tests: take them one at a time from the failure index, with `--test`. + +2. **Frame the run** from `run.yaml` and `result.yaml`: `run_kind` (`author` or `replay`), `origin`, the kane-cli version, and the environment. A replay repeats recorded actions, so a wrong choice of action belongs to the run that authored the test, not to this one. + +3. **Read every log signal** the sheet printed, in the log itself, with the lines around it. A sign-in, network or browser error here outranks any step-level story. The sheet only matches patterns: search the logs yourself for the failing step, by the action's `at=` time. + +4. **Pick the window.** The step carrying the failure record, the step before it, and the step after it when one exists. + - Check the sheet's `UNMAPPED` list first. An action with no step folder ran without a screenshot of its own, and it can be what ended the run. + - Add an earlier step of the same kind as the failing one, with the step after it. A before and after pair of the same action that worked is the cheapest comparison. + +5. **Fill one screen row per window step** (the table in §2). For each: + - Open `screenshot.*` with your image-reading tool. Open `annotated.*` too when present. + - Take the intent from the sheet: the authored `intent`, `instruction`, `reasoning` and `inside`, then the action's instruction, `params` and `rationale`. When the instruction text disagrees with the action type or the rationale, say so and trust the screenshot. + - Compare the screen to the intent. + - Prove the effect from the **next** step's screenshot: a step's own screenshot is taken before its action runs. If no later screenshot exists, write "not provable from the screen" and name the log line you used in its place. + +6. **Check the triage inside the pack.** A step `failure.yaml` with a `triage:` block is a claim written by an automatic investigation. State whether the screen table and the logs support it. + +7. **Name one cause** (§3) and report. + +## 2. Report + +The report has these six parts, in this order: + +1. **Verdict.** One or two sentences: what failed, and the cause. +2. **Screen table.** Required, one row per window step: + + | Step folder | Mapped by | Screenshot opened | What the screen shows | What the agent was doing | Screen vs intent | Effect seen in next screenshot | + |---|---|---|---|---|---|---| + + `Mapped by` is `id`, `action_id` or `ordinal`. A step folder with no image names the earlier image it borrows. An action with no step folder gets one row with "no image": the last folder's screenshot was taken before it and is not its screen. +3. **Log evidence.** Each claim with the file and line inside the pack. +4. **Triage inside the pack.** Agree or disagree, and why. "None in pack" when absent. +5. **Other findings.** Defects the pack shows that did not cause this failure, such as a step that passed without changing the screen. "None" when absent. +6. **Not determined.** What the pack cannot show, and what would settle it (a re-run, a look at the app, a bug report). + +Label anything you have not tested as a hypothesis. + +Give the person the verdict in plain words first. Show the table and the rest when they ask for the detail, or when the cause is not the obvious one. + +## 3. Causes + +Pick the earliest fault the pack proves. + +| Cause | Typical sign | What to do next | +|---|---|---| +| The app under test | The screenshot shows the product misbehaving: an error banner, a wrong value, a button that does nothing | Tell the person it is a product bug and show the screenshot. Do not change the test to get around it | +| The test or the objective | `test.md` asks for something ambiguous, impossible, or no longer true of the app | Fix the wording. For a designed test, read `references/assurance.md` §6.1 first | +| The environment | Sign-in, network, browser or device errors in the logs around the failing step | Fix the setup and run again (`references/setup-and-config.md`) | +| kane-cli | The screen was right and the step was clear, and the action was still wrong or did not take effect; or a replay fails a step that the authoring run passed on the same screen | Offer a bug report (`references/debug.md` §Filing a bug report) | + +## 4. Common mistakes + +| Mistake | Fix | +|---|---| +| Matching a screenshot to a step by step number | Step numbers are counted per file. Use the sheet's join | +| Reading a screenshot as the result of its step | It is the screen before the action. Look one step later | +| Trusting a `passed` step | Passed means the action returned, not that the screen changed | +| Blaming the step that carries the failure record | Check the `UNMAPPED` list for actions after it first | +| Repeating the triage inside the pack | Check it against the log signals and the screen table first | +| Reading an off-target marker as a wrong click | The marker is drawn on a screenshot taken before the action. Confirm the click in the actions log | +| Quoting `*.har` content | It can carry credentials. Cite the request, never the secret | diff --git a/skill-installer/skills/SKILL.md b/skill-installer/skills/SKILL.md index f6a3dd0..2b0ae03 100644 --- a/skill-installer/skills/SKILL.md +++ b/skill-installer/skills/SKILL.md @@ -136,7 +136,7 @@ For exit code 1 (or `status: "failed"` in `run_end`), present the failure card. | ➡️ **Next** | · | ``` -The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.png" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. +The failing step's screenshot lives inside the run's evidence pack (the stderr hint names the pack path): extract it with `unzip "tests/*/steps/*/screenshot.*" -d `, Read it, and show it **under** the card. For the pack layout and deeper diagnosis, see `references/debug.md`. Exit code 2 means nothing ran: that is a `🟡 Didn't start` card, not a failure (`references/cards.md` §4). @@ -168,6 +168,7 @@ When the user's request involves a browser — or writing test cases: - Multiple independent browser tasks → Read `references/parallel.md` first - View, share, or validate run evidence (`.evidence` packs) → Read `references/evidence.md` - Debug a failed run → Read `references/debug.md` +- The quick debug did not explain a failure, a run passed but looks wrong, or the person asks whose fault a failure is or whether the triage in a pack is right → Read `references/triage.md` - Configure kane-cli or check directory layout → Read `references/setup-and-config.md` - Browse / create / pick a Test Manager project or folder, or interpret the auto-default event → Read `references/test-manager.md` - The person wants results saved somewhere else ("change project") → Read `references/test-manager.md` §6. The change is global, and the question must say so @@ -350,6 +351,7 @@ A draft in chat cannot be run, and a hand-written `_test.md` carries no requirem | Need the assurance NDJSON event schema (`--mode agent`) | `references/assurance-parsing.md` | | Share the context store with a team, join one, keep stores level, or resolve a sync conflict (kane-cli 0.8.14+) | `references/context-sync.md` | | Run failed, need to diagnose | `references/debug.md` | +| Full triage of a pack: the step sheet, screenshots set against intent, a report with one cause | `references/triage.md` | | View, share, validate, or merge evidence packs | `references/evidence.md` | | Multiple independent browser tasks | `references/parallel.md` | | Need full NDJSON event schema (`run`) | `references/parsing.md` | diff --git a/skill-installer/skills/references/debug.md b/skill-installer/skills/references/debug.md index 372d188..c3a5e6e 100644 --- a/skill-installer/skills/references/debug.md +++ b/skill-installer/skills/references/debug.md @@ -1,4 +1,4 @@ - + # Failure Handling & Log Inspection @@ -17,7 +17,7 @@ unzip -l # list entries unzip -p "tests/*/result.yaml" # verdict + per-step outcomes unzip -p "tests/*/steps/*/failure.yaml" # failure records (failed steps only) unzip -p "tests/*/logs/0-console.ndjson" # browser console, run 0 -unzip "tests/*/steps/*/screenshot.png" -d /tmp/ev # extract screenshots to view +unzip "tests/*/steps/*/screenshot.*" -d /tmp/ev # extract screenshots to view ``` Pack layout (per test): @@ -33,18 +33,49 @@ tests// │ ├── -actions.ndjson # agent actions per run — the old runs// actions log, now in-pack │ ├── -console.ndjson # browser console, per-step attributed │ └── -network.har # network traffic, per-step attributed -├── steps/-/ # screenshot.png, annotated.png, step.json, +├── steps/-/ # step.json, screenshot.* (png or jpg), annotated.png (some steps), │ # failure.yaml (failed steps only) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries ``` +### Step identity and joins + +One step has a different name in each file. Join on `action_id`, never on a step number. + +| Name | Where | Meaning | +|---|---|---| +| ordinal | leading number of the step folder; `result.yaml` `ordinal` | order in the pack | +| id | rest of the folder name; `step.json` `id`; `result.yaml` `id` | `-` for a step that ran | +| action_id | `step.json`, the `result.yaml` row, the `-actions.ndjson` line, `auteur/execution.json` `operations[].operation_id` | the join key across files | +| step | `-actions.ndjson` `step` | counted per file; never a join key | + +Join a step folder to the other files by `id`, then `action_id`, then ordinal. The run index `` of a step id `-` is the prefix of its logs: `logs/-run.log`, `logs/-actions.ndjson`. + +Where the intent of a step lives: + +| Source | Fields | Notes | +|---|---|---| +| `auteur/execution.json` | `instruction`, `reasoning`, `status`, `error_message`; `operations[].intent` | The authored steps. In a replay pack this describes the run that authored the test, so its `status` is the authored outcome | +| `logs/-actions.ndjson` | `action_type`, `action_instruction`, `rationale`, `action_params`, `selectors`, `status`, `error`, `timestamp` | What this run did | +| `step.json` | `kind`, `summary`, `status`, `duration_ms`, `url` | One-line view of the step | + +### What is and is not in a step folder + +- **The screenshot is taken before the step's action runs**, in an authoring run and in a replay. The effect of a step is in the next step's screenshot. +- `annotated.png` draws the action's marker on that same before-the-action screenshot. Not every step has one. +- Steps the run never reached are `skipped` rows in `result.yaml` and have no folder. +- An action can have a line in `-actions.ndjson` and no folder: it has no screenshot of its own. + +`scripts/evidence-map.mjs` prints all of this as one step sheet (`references/triage.md` §1). + ## Debugging Flow 1. **Parse the `run_end` event** from stdout — `status`, `reason`, `summary`, plus `session_dir`. 2. **Open the pack**: the failed step's `failure.yaml` (error + page state), then that step's slice of `-console.ndjson` / `-network.har` — a 4xx/5xx or JS error usually explains the failure; cite it in plain language. -3. **Look at the failing step's `annotated.png`** — it highlights the element the agent acted on; makes "clicked the wrong thing" / "element missing" obvious. Render it inline. +3. **Look at the failing step's `screenshot.*` and `annotated.png`.** They show the page **before** the step's action ran, with a marker on what the agent was about to act on; that makes "aimed at the wrong thing", "element missing" and "something covers it" obvious. Whether an action took effect is in the **next** step's screenshot. Render the image inline. 4. **Check `tui.log`** (in the pack's `logs/`, or `{session_dir}/tui.log`) for session-level issues (Chrome launch, auth, upload). +5. **Still unexplained, a pass that looks wrong, or the person asks whose fault it is?** Do the full triage: `references/triage.md`. Offer the user the visual route too: `kane-cli evidence serve ` opens it in the hosted viewer. If a pack won't open, `kane-cli evidence validate ` tells you whether it's truncated/unsealed. diff --git a/skill-installer/skills/references/evidence.md b/skill-installer/skills/references/evidence.md index 34125fe..c90a022 100644 --- a/skill-installer/skills/references/evidence.md +++ b/skill-installer/skills/references/evidence.md @@ -18,7 +18,7 @@ A `.evidence` file is a standard zip: `unzip -l ` lists it, `unzip -p -run.log, -actions.ndjson, │ # -console.ndjson, -network.har - ├── steps/-/ # screenshot.png, annotated.png, step.json + ├── steps/-/ # screenshot.* (png or jpg), annotated.png, step.json │ # (+ failure.yaml on failed steps) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries @@ -74,7 +74,7 @@ Combines packs (execution ids or paths, order-significant) into one, sealed by d ## Debugging with a pack -The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent actually acted on). Full failure workflow: `references/debug.md`. +The pack is the first place to look on a failure: the failed step's **failure record** (error + page state), its **console/network slice** (4xx/5xx or JS errors usually explain it), and the **annotated screenshot** (what the agent was about to act on; a step's screenshot is taken before its action runs). Full failure workflow: `references/debug.md`. A step-by-step triage with a report: `references/triage.md`. ## Evidence merge identity diff --git a/skill-installer/skills/references/triage.md b/skill-installer/skills/references/triage.md new file mode 100644 index 0000000..7e018b3 --- /dev/null +++ b/skill-installer/skills/references/triage.md @@ -0,0 +1,79 @@ + + +# Triaging a run from its evidence pack + +A pack records three views of every step: what the screen showed, what the agent meant to do, and what the logs say happened. A triage is a claim that holds in all three. **The screen can hold facts no log mentions** (a dialog, a toast, a spinner, an error banner, a keyboard), so the screenshots are read every time. + +Pack layout, step identity and join rules: `references/debug.md`. + +## 1. Steps + +1. **Map the pack.** Run the step sheet. It joins step folders, the actions log, `auteur/execution.json` and `result.yaml` by `action_id`, prints `run.yaml` and the failure index, and lists the error lines of the run logs. + + ```bash + node "/scripts/evidence-map.mjs" --out + node "/scripts/evidence-map.mjs" [--test ] + ``` + + The first form unzips the pack into the folder you name, which must be empty. A merged or suite pack holds many tests: take them one at a time from the failure index, with `--test`. + +2. **Frame the run** from `run.yaml` and `result.yaml`: `run_kind` (`author` or `replay`), `origin`, the kane-cli version, and the environment. A replay repeats recorded actions, so a wrong choice of action belongs to the run that authored the test, not to this one. + +3. **Read every log signal** the sheet printed, in the log itself, with the lines around it. A sign-in, network or browser error here outranks any step-level story. The sheet only matches patterns: search the logs yourself for the failing step, by the action's `at=` time. + +4. **Pick the window.** The step carrying the failure record, the step before it, and the step after it when one exists. + - Check the sheet's `UNMAPPED` list first. An action with no step folder ran without a screenshot of its own, and it can be what ended the run. + - Add an earlier step of the same kind as the failing one, with the step after it. A before and after pair of the same action that worked is the cheapest comparison. + +5. **Fill one screen row per window step** (the table in §2). For each: + - Open `screenshot.*` with your image-reading tool. Open `annotated.*` too when present. + - Take the intent from the sheet: the authored `intent`, `instruction`, `reasoning` and `inside`, then the action's instruction, `params` and `rationale`. When the instruction text disagrees with the action type or the rationale, say so and trust the screenshot. + - Compare the screen to the intent. + - Prove the effect from the **next** step's screenshot: a step's own screenshot is taken before its action runs. If no later screenshot exists, write "not provable from the screen" and name the log line you used in its place. + +6. **Check the triage inside the pack.** A step `failure.yaml` with a `triage:` block is a claim written by an automatic investigation. State whether the screen table and the logs support it. + +7. **Name one cause** (§3) and report. + +## 2. Report + +The report has these six parts, in this order: + +1. **Verdict.** One or two sentences: what failed, and the cause. +2. **Screen table.** Required, one row per window step: + + | Step folder | Mapped by | Screenshot opened | What the screen shows | What the agent was doing | Screen vs intent | Effect seen in next screenshot | + |---|---|---|---|---|---|---| + + `Mapped by` is `id`, `action_id` or `ordinal`. A step folder with no image names the earlier image it borrows. An action with no step folder gets one row with "no image": the last folder's screenshot was taken before it and is not its screen. +3. **Log evidence.** Each claim with the file and line inside the pack. +4. **Triage inside the pack.** Agree or disagree, and why. "None in pack" when absent. +5. **Other findings.** Defects the pack shows that did not cause this failure, such as a step that passed without changing the screen. "None" when absent. +6. **Not determined.** What the pack cannot show, and what would settle it (a re-run, a look at the app, a bug report). + +Label anything you have not tested as a hypothesis. + +Give the person the verdict in plain words first. Show the table and the rest when they ask for the detail, or when the cause is not the obvious one. + +## 3. Causes + +Pick the earliest fault the pack proves. + +| Cause | Typical sign | What to do next | +|---|---|---| +| The app under test | The screenshot shows the product misbehaving: an error banner, a wrong value, a button that does nothing | Tell the person it is a product bug and show the screenshot. Do not change the test to get around it | +| The test or the objective | `test.md` asks for something ambiguous, impossible, or no longer true of the app | Fix the wording. For a designed test, read `references/assurance.md` §6.1 first | +| The environment | Sign-in, network, browser or device errors in the logs around the failing step | Fix the setup and run again (`references/setup-and-config.md`) | +| kane-cli | The screen was right and the step was clear, and the action was still wrong or did not take effect; or a replay fails a step that the authoring run passed on the same screen | Offer a bug report (`references/debug.md` §Filing a bug report) | + +## 4. Common mistakes + +| Mistake | Fix | +|---|---| +| Matching a screenshot to a step by step number | Step numbers are counted per file. Use the sheet's join | +| Reading a screenshot as the result of its step | It is the screen before the action. Look one step later | +| Trusting a `passed` step | Passed means the action returned, not that the screen changed | +| Blaming the step that carries the failure record | Check the `UNMAPPED` list for actions after it first | +| Repeating the triage inside the pack | Check it against the log signals and the screen table first | +| Reading an off-target marker as a wrong click | The marker is drawn on a screenshot taken before the action. Confirm the click in the actions log | +| Quoting `*.har` content | It can carry credentials. Cite the request, never the secret | From be43ceb3cc7bb94eb56949c04477670e19a5ecb8 Mon Sep 17 00:00:00 2001 From: Zeeshan Date: Mon, 12 Oct 2026 18:13:25 +0530 Subject: [PATCH 3/4] kiro-powers: mirror the evidence triage reference Co-Authored-By: Claude Opus 5.5 --- integrations/docs/kiro-powers.md | 5 ++-- .../kiro-powers/steering/kane-cli-run.md | 30 ++++++++++++++++--- .../kiro-powers/steering/kane-cli-testrun.md | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/integrations/docs/kiro-powers.md b/integrations/docs/kiro-powers.md index c4592ae..81b2f9b 100644 --- a/integrations/docs/kiro-powers.md +++ b/integrations/docs/kiro-powers.md @@ -15,7 +15,7 @@ The `integrations/kiro-powers/` folder is a [Kiro power](https://kiro.dev/docs/p | Artifact | Path | Purpose | |---|---|---| | **Canonical skill** | `skill-installer/skills/SKILL.md` | Source of every CLI fact: command shapes, flags, exit codes, NDJSON essentials, decision tree, results presentation. **Edit this first.** Every other integration mirrors from here. | -| **Canonical references** | `skill-installer/skills/references/*.md` | On-demand reference content: `ready-check.md` (preflight, the ready card, problems that stop a run, sign-in), `first-run.md` (run first, the verbatim tour, three choices asked once), `agent-config.md` (the saved preferences file: location, schema, shell read and write, hard-case rules), `cards.md` (every result card and the rules for all cards), `live-strip.md` (the Claude Code status bar strip, not offered in Kiro), `objectives-cookbook.md` (pattern catalog + checkpoint analyze methods), `testmd.md` (file format, replay, the saved-test stream), `generate.md` + `generate-parsing.md` (AI test-case authoring), `test-manager.md` (project/folder agent surface, auto-default event, the global "changing where results go" flow), `parsing.md` (full NDJSON schema and the 0.8.17+ stream contract), `debug.md` (log layout), `parallel.md`, `setup-and-config.md`. Equally authoritative: facts in any of these must mirror through. | +| **Canonical references** | `skill-installer/skills/references/*.md` | On-demand reference content: `ready-check.md` (preflight, the ready card, problems that stop a run, sign-in), `first-run.md` (run first, the verbatim tour, three choices asked once), `agent-config.md` (the saved preferences file: location, schema, shell read and write, hard-case rules), `cards.md` (every result card and the rules for all cards), `live-strip.md` (the Claude Code status bar strip, not offered in Kiro), `objectives-cookbook.md` (pattern catalog + checkpoint analyze methods), `testmd.md` (file format, replay, the saved-test stream), `generate.md` + `generate-parsing.md` (AI test-case authoring), `test-manager.md` (project/folder agent surface, auto-default event, the global "changing where results go" flow), `parsing.md` (full NDJSON schema and the 0.8.17+ stream contract), `debug.md` (log layout, step identity and join rules), `triage.md` (full triage of a pack), `parallel.md`, `setup-and-config.md`. Equally authoritative: facts in any of these must mirror through. | | Kiro power root | `integrations/kiro-powers/POWER.md` | Frontmatter (name/displayName/keywords/author), onboarding (install, sign-in rules, project/folder), the **Every session** protocol (ready check commands, ready card, problems table, no-human rule, launch line, watch mode), condensed command reference, steering-file mapping. | | First session + saved preferences steering | `integrations/kiro-powers/steering/kane-cli-first-run.md` | Loaded on a user's first session, on "kane tour" and on "kane preferences": the run-first rule and its defaults, the first-run tour (verbatim from canonical `first-run.md`), the first result's extra rows, the three choices asked once after the first result, and the saved preferences file (location, schema, shell read and write, hard-case rules; host key `kiro`). | | `kane-cli run` steering | `integrations/kiro-powers/steering/kane-cli-run.md` | Full reference for one-shot `kane-cli run`: objective patterns + checkpoint analyze methods (Visual / Textual-DOM / URL / Title / DevTools→Network/Console/Performance/Cookies/localStorage), full flag table, NDJSON parsing (including `project_folder_auto_defaulted`), results presentation, failure diagnosis, parallel execution, project/folder management (`projects`/`folders list|create`, auto-default gate). | @@ -50,7 +50,8 @@ The canonical skill is a thin `SKILL.md` (7 sections, ~300 lines) plus on-demand | `references/testmd.md`: testmd file format, replay & cascade, `@import`, commands, parse errors, gate-fires-before-launch note, the saved-test stream table (`test_md_step_start` through `test_md_done`) | All of `steering/kane-cli-testmd.md` (the stream table lives under Replay policy and completion → The saved-test stream); authoring from a description points at the assurance steering | | `references/parsing.md`: the 0.8.17+ stream contract (`stream_start`, `v`, `ts`, ignore unknown fields and event types, typeless step lines, completion event last, `events.ndjson` and the active-run pointer file), full NDJSON event schemas (`project_folder_auto_defaulted`, `bifurcation`, `child_agent_*`, `ask_user`, complete `run_end` fields incl. `credits_consumed`) | `steering/kane-cli-run.md` → Parsing the NDJSON output (The stream contract + full event-type list + Terminal `run_end` event) | | `references/test-manager.md`: project/folder agent surface (`projects list|create`, `folders list|create` with the **required** `--project `, NDJSON wire shape, pagination), run-startup auto-default gate, `project_folder_auto_defaulted` event, self-healing for stale IDs, the global "changing where results go" flow | `POWER.md` → Step 3 (project/folder onboarding) **and** `steering/kane-cli-run.md` → Browsing / creating projects and folders + The run-startup auto-default gate + Changing where results go **and** `steering/kane-cli-testmd.md` → Quick start (gate note) | -| `references/debug.md` — log layout, debugging flow, common failure patterns (incl. "did you mean" subcommand and self-healing rows), bug-report heuristic | `steering/kane-cli-run.md` → Failure handling & log inspection + Bug-report heuristic | +| `references/debug.md` — log layout, step identity and join rules, what a step folder holds (the screenshot is taken before the action), debugging flow, common failure patterns (incl. "did you mean" subcommand and self-healing rows), bug-report heuristic | `steering/kane-cli-run.md` → Failure handling & log inspection + Full triage from the pack (join rule) + Bug-report heuristic | +| `references/triage.md`: the triage steps, the window of steps to read, proving an effect from the next screenshot, checking the triage inside a pack, the four causes. **Harness difference:** the canonical skill runs `scripts/evidence-map.mjs` to print a step sheet; the Kiro power has no script, so the steering describes the joins and the steps to do by hand | `steering/kane-cli-run.md` → Full triage from the pack | | `references/parallel.md` — when to split, agent prompt template, batch summary | `steering/kane-cli-run.md` → Parallel execution | | `references/testrun.md`: batch runs: selection (paths/`--match`/`--tags`), preflight reasons, `--remote` (HyperExecute jobs, `remote_*` events, remote preflight codes, the 0.8.17+ remote additions), mobile members, `testrun_*` event schema (incl. `session_id`, `log_path`, `failure`, the authored member pair, `testrun_progress`), each test's own log and the `--stream-members` rule, suite card presentation, exit codes | All of `steering/kane-cli-testrun.md` | | `references/evidence.md`: pack locations, the stderr view hint, `evidence serve/validate/merge`, debugging with a pack, the first-run rule (serve the pack and link the viewer instead of offering) | `steering/kane-cli-testrun.md` → Evidence packs **and** `steering/kane-cli-run.md` → Failure handling (evidence-first flow) **and** `steering/kane-cli-first-run.md` → The first result | diff --git a/integrations/kiro-powers/steering/kane-cli-run.md b/integrations/kiro-powers/steering/kane-cli-run.md index cb2bfa9..cc695b3 100644 --- a/integrations/kiro-powers/steering/kane-cli-run.md +++ b/integrations/kiro-powers/steering/kane-cli-run.md @@ -472,7 +472,7 @@ For exit code `1` (or a failed status), present the failure card. Explain what w 🔍 Likely cause is Kiro's own diagnosis: a missing element, a popup over the button, a slow page, an ambiguous objective, an auth wall. ➡️ Next pairs a retry Kiro can run now with an offer to walk through the failing step. -Then extract the failing-step screenshot from the run's evidence pack (`unzip "tests/*/steps/*/screenshot.png" -d `) and show it **under** the card. +Then extract the failing-step screenshot from the run's evidence pack (`unzip "tests/*/steps/*/screenshot.*" -d `) and show it **under** the card. ## Didn't start (exit `2`) @@ -541,7 +541,7 @@ unzip -l # list entries unzip -p "tests/*/result.yaml" # verdict + per-step outcomes unzip -p "tests/*/steps/*/failure.yaml" # failure records (failed steps only) unzip -p "tests/*/logs/0-console.ndjson" # browser console, run 0 -unzip "tests/*/steps/*/screenshot.png" -d /tmp/ev # extract screenshots to view +unzip "tests/*/steps/*/screenshot.*" -d /tmp/ev # extract screenshots to view ``` Pack layout (per test): @@ -553,7 +553,7 @@ tests// ├── logs/ # meta.yaml, tui.log, and per run index n: │ # -run.log, -actions.ndjson, │ # -console.ndjson, -network.har -├── steps/-/ # screenshot.png, annotated.png, step.json +├── steps/-/ # step.json, screenshot.* (png or jpg), annotated.png (some steps) │ # (+ failure.yaml on failed steps) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries @@ -563,11 +563,33 @@ tests// 1. Parse `run_end` from stdout — `status`, `reason`, `summary`, `session_dir`. 2. Open the pack: the failed step's `failure.yaml` (error + page state), then that step's slice of `-console.ndjson` / `-network.har` — a 4xx/5xx or JS error usually explains the failure; cite it in plain language. -3. Look at the failing step's `annotated.png` — it highlights the element the agent acted on. Render it inline. +3. Look at the failing step's `screenshot.*` and `annotated.png`. They show the page **before** the step's action ran, with a marker on what the agent was about to act on. Whether an action took effect is in the **next** step's screenshot. Render the image inline. 4. Check `tui.log` (in the pack's `logs/`, or `{session_dir}/tui.log`) for session-level issues (Chrome launch, auth, upload). Offer the visual route: `kane-cli evidence serve ` starts a local-only server and prints a hosted-viewer link. After agent-mode runs, Kane CLI prints a stderr hint line (`` evidence: view locally with `kane-cli evidence serve ` ``) — it is plain text on stderr, never a stdout event. If a pack won't open, `kane-cli evidence validate ` reports whether it's truncated/unsealed. Full evidence + testrun surface: load **`kane-cli-testrun`** steering. +## Full triage from the pack + +Use this when the flow above did not explain a failure, when a run passed but looks wrong, or when the person asks whose fault a failure is or whether the triage inside a pack is right. + +One step has a different name in each file. Join on `action_id`, never on a step number: it is in `step.json`, the `result.yaml` row, the `-actions.ndjson` line, and `auteur/execution.json` under `operations[].operation_id`. In a replay pack `auteur/execution.json` describes the run that authored the test. + +1. **Frame the run** from `run.yaml` and `result.yaml`: `run_kind` (`author` or `replay`), `origin`, the Kane CLI version, the environment. A replay repeats recorded actions, so a wrong choice of action belongs to the run that authored the test. +2. **Read the error lines** in `logs/-run.log`, `logs/runner-stderr.log` and `logs/tui.log`, with the lines around them. A sign-in, network or browser error outranks any step-level story. +3. **Pick the window**: the step carrying `failure.yaml`, the step before it, and the step after it when one exists. Steps the run never reached are `skipped` rows in `result.yaml` with no folder. An action with a line in `-actions.ndjson` and no step folder has no screenshot of its own, and can be what ended the run. +4. **For each window step**, open `screenshot.*` (and `annotated.png` when present), take the intent from the authored step and the actions log line, and compare the two. Prove the effect from the **next** step's screenshot. +5. **Check the `triage:` block** in the step's `failure.yaml` against the screenshots and the logs before repeating it. +6. **Name one cause**, the earliest fault the pack proves, and report it with the screenshots and the log lines that show it. + +| Cause | Typical sign | What to do next | +|---|---|---| +| The app under test | The screenshot shows the product misbehaving | Tell the person it is a product bug and show the screenshot. Do not change the test to get around it | +| The test or the objective | `test.md` asks for something ambiguous, impossible, or no longer true of the app | Fix the wording. For a designed test, classify the failure first (see the assurance steering) | +| The environment | Sign-in, network, browser or device errors in the logs around the failing step | Fix the setup and run again | +| Kane CLI | The screen was right and the step was clear, and the action was still wrong or did not take effect | Offer a bug report | + +Passed means the action returned, not that the screen changed. A `*.har` file can carry credentials: cite the request, never the secret. + ## Common failure patterns | Symptom | Likely cause | Fix | diff --git a/integrations/kiro-powers/steering/kane-cli-testrun.md b/integrations/kiro-powers/steering/kane-cli-testrun.md index 2fd44ca..8bdcdaa 100644 --- a/integrations/kiro-powers/steering/kane-cli-testrun.md +++ b/integrations/kiro-powers/steering/kane-cli-testrun.md @@ -159,7 +159,7 @@ Every Kane CLI run seals an **evidence pack**: one `.evidence` file (a zip) hold ├── logs/ # meta.yaml, tui.log, and per run index n: │ # -run.log, -actions.ndjson, │ # -console.ndjson, -network.har - ├── steps/-/ # screenshot.png, annotated.png, step.json + ├── steps/-/ # screenshot.* (png or jpg), annotated.png, step.json │ # (+ failure.yaml on failed steps) ├── auteur/execution.json # full execution trajectory └── v16-trajectory/ # per-run planning summaries From d5245e44238d7163d3dceeba9e8a0e25fa904bca Mon Sep 17 00:00:00 2001 From: Zeeshan Siddiqui Date: Mon, 12 Oct 2026 18:25:01 +0530 Subject: [PATCH 4/4] Update triage.md with mistake and fix clarifications Clarified mistakes and fixes in triage process. --- .claude/skills/kane-cli/references/triage.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.claude/skills/kane-cli/references/triage.md b/.claude/skills/kane-cli/references/triage.md index 7e018b3..9a7a06c 100644 --- a/.claude/skills/kane-cli/references/triage.md +++ b/.claude/skills/kane-cli/references/triage.md @@ -72,7 +72,6 @@ Pick the earliest fault the pack proves. |---|---| | Matching a screenshot to a step by step number | Step numbers are counted per file. Use the sheet's join | | Reading a screenshot as the result of its step | It is the screen before the action. Look one step later | -| Trusting a `passed` step | Passed means the action returned, not that the screen changed | | Blaming the step that carries the failure record | Check the `UNMAPPED` list for actions after it first | | Repeating the triage inside the pack | Check it against the log signals and the screen table first | | Reading an off-target marker as a wrong click | The marker is drawn on a screenshot taken before the action. Confirm the click in the actions log |