From ebe5228bf7245743a976cc39997a1c55173dc3ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:50:00 +0000 Subject: [PATCH] Add model-based Copilot SDK shell parser fuzz generator Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/cgo.yml | 1 + Makefile | 1 + actions/setup/js/fuzz_bash_command_model.cjs | 104 ++++++++++++++++++ .../setup/js/fuzz_bash_command_model.test.cjs | 67 +++++++++++ .../js/fuzz_bash_command_parser_harness.cjs | 10 +- pkg/workflow/bash_command_parser_fuzz_test.go | 70 ++++++++++++ 6 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 actions/setup/js/fuzz_bash_command_model.cjs create mode 100644 actions/setup/js/fuzz_bash_command_model.test.cjs create mode 100644 pkg/workflow/bash_command_parser_fuzz_test.go diff --git a/.github/workflows/cgo.yml b/.github/workflows/cgo.yml index 9aa3a6d7795..6afce76aeb5 100644 --- a/.github/workflows/cgo.yml +++ b/.github/workflows/cgo.yml @@ -1832,6 +1832,7 @@ jobs: - group: Workflow-Core tests: >- FuzzExpressionParser:./pkg/workflow/ + FuzzBashCommandParserModel:./pkg/workflow/ FuzzMentionsFiltering:./pkg/workflow/ FuzzSanitizeOutput:./pkg/workflow/ FuzzSanitizeIncomingText:./pkg/workflow/ diff --git a/Makefile b/Makefile index 6659765300a..cfd1d380cbc 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,7 @@ fuzz: @echo "Running fuzz tests for 30 seconds..." go test -fuzz=FuzzParseFrontmatter -fuzztime=30s ./pkg/parser/ go test -fuzz=FuzzExpressionParser -fuzztime=30s ./pkg/workflow/ + go test -run='^$$' -fuzz=FuzzBashCommandParserModel -fuzztime=30s ./pkg/workflow/ # Run security regression tests .PHONY: test-security diff --git a/actions/setup/js/fuzz_bash_command_model.cjs b/actions/setup/js/fuzz_bash_command_model.cjs new file mode 100644 index 00000000000..eb71f813959 --- /dev/null +++ b/actions/setup/js/fuzz_bash_command_model.cjs @@ -0,0 +1,104 @@ +// @ts-check +"use strict"; + +const assert = require("node:assert/strict"); +const { splitOnPipelineOperators, extractCommandName, extractCommandNamesFromPipeline } = require("./bash_command_parser.cjs"); +const { buildCopilotSDKPermissionHandler } = require("./copilot_sdk_permissions.cjs"); + +const COMMANDS = ["echo", "cat", "jq", "wc", "git", "python3"]; +const OPERATORS = [" && ", " || ", " | ", " |& ", "; ", " & ", "\n", "\r\n"]; +const ARGUMENTS = ["plain#hash", "'# && || | ; \" $()'", '"# && || | ; \\" literal"', "\\#escaped\\;word", "'first\n# still quoted'", "${value:-#fallback}", '"${outer:-${inner:-#fallback}}"']; +const ASSIGNMENTS = ["", "A=1 ", "A='# literal' B=two ", 'A="# literal" ']; +const REDIRECTIONS = ["", " 2>/dev/null", " >output 2>&1", " &>output", " (bytes[cursor++] ?? 0) & 255; + const pick = values => values[next() % values.length]; + // LF alone ends Bash comments; retain CR, quotes, operators and backslashes. + const comment = `# ${payload.slice(0, 256).replace(/\n/g, " ")} ' " $( ; && | \r python3`; + const literal = payload.slice(0, 128).replace(/\0/g, ""); + const argumentChoices = [...ARGUMENTS, `'${literal.replace(/'/g, "'\\''")}'`, `"${literal.replace(/[\\"$`]/g, "\\$&")}"`]; + const count = next() % 9; + const commands = Array.from({ length: count }, () => ({ + name: pick(COMMANDS), + assignment: pick(ASSIGNMENTS), + blank: pick(BLANKS), + args: Array.from({ length: next() % 4 }, () => pick(argumentChoices)), + redirection: pick(REDIRECTIONS), + operator: pick(OPERATORS), + comment: next() % 2 === 0, + })); + const render = (command, blank) => `${command.assignment}${command.name}${command.args.length ? blank + command.args.join(" ") : ""}${command.redirection}`; + const segments = commands.map(command => render(command, command.blank === BLANKS[2] ? " " : command.blank)); + let source = `${comment}\n`; + for (let i = 0; i < commands.length; i++) { + source += render(commands[i], commands[i].blank); + if (i + 1 < commands.length) { + source += commands[i].operator; + if (commands[i].comment) source += `${comment}\n`; + } + } + source += ` # ${comment}`; + return { + source, + segments, + commandNames: commands.map(command => command.name), + names: [...new Set(commands.map(command => command.name))], + unsupported: pick(UNSUPPORTED), + }; +} + +/** + * Assert exact extraction for the supported grammar, fail-closed behaviour for + * unsupported constructs, and robustness for arbitrary malformed shell text. + * SDK allow/deny expectations come from the model, not from parser output. + * + * @param {number[]} bytes + * @param {string} payload + */ +async function checkShellProgramModel(bytes, payload) { + const model = generateShellProgram(bytes, payload); + const context = JSON.stringify(model.source); + const expectedSegments = model.segments; + assert.deepEqual(splitOnPipelineOperators(model.source), expectedSegments, context); + assert.deepEqual(extractCommandNamesFromPipeline(model.source), model.names, context); + for (let i = 0; i < expectedSegments.length; i++) { + assert.equal(extractCommandName(expectedSegments[i]), model.commandNames[i], context); + } + const allowed = COMMANDS.filter(name => name !== "python3"); + const handler = buildCopilotSDKPermissionHandler({ allowedTools: allowed.map(name => `shell(${name})`) }, () => ({ kind: "approve-once" })); + /** @type {import("@github/copilot-sdk").PermissionRequest} */ + const request = { kind: "shell", fullCommandText: model.source, commands: [], canOfferSessionApproval: false, hasWriteFileRedirection: false, intention: "Parse only", possiblePaths: [], possibleUrls: [] }; + const permission = await handler(request, { sessionId: "shell-model-fuzz" }); + assert.equal(permission.kind, model.names.length > 0 && !model.names.includes("python3") ? "approve-once" : "reject", context); + + // Every unsupported construct must invalidate the whole parse, even after a + // valid prefix; never accept only the commands preceding an ambiguous suffix. + const unsupported = `${model.source}\necho ${model.unsupported}`; + assert.deepEqual(splitOnPipelineOperators(unsupported), [], JSON.stringify(unsupported)); + assert.deepEqual(extractCommandNamesFromPipeline(unsupported), [], JSON.stringify(unsupported)); + assert.equal((await handler({ ...request, fullCommandText: unsupported }, { sessionId: "shell-model-fuzz" })).kind, "reject", JSON.stringify(unsupported)); + + const cut = Math.floor(((bytes.at(-1) ?? 0) * model.source.length) / 256); + for (const text of [payload.slice(0, 4096), model.source.slice(0, cut)]) { + assert.ok(splitOnPipelineOperators(text).every(segment => typeof segment === "string")); + assert.ok(extractCommandNamesFromPipeline(text).every(name => typeof name === "string")); + const name = extractCommandName(text); + assert.ok(name === null || typeof name === "string"); + } + return model; +} + +module.exports = { generateShellProgram, checkShellProgramModel }; diff --git a/actions/setup/js/fuzz_bash_command_model.test.cjs b/actions/setup/js/fuzz_bash_command_model.test.cjs new file mode 100644 index 00000000000..efbd47aa4d5 --- /dev/null +++ b/actions/setup/js/fuzz_bash_command_model.test.cjs @@ -0,0 +1,67 @@ +// @ts-check + +import { describe, it, expect } from "vitest"; +import { createRequire } from "module"; + +const require = createRequire(import.meta.url); +const { generateShellProgram, checkShellProgramModel } = require("./fuzz_bash_command_model.cjs"); + +describe("model-based Bash parser fuzz generator", () => { + it("is deterministic and bounded on empty and oversized inputs", async () => { + expect(generateShellProgram([], "")).toEqual(generateShellProgram([], "")); + expect((await checkShellProgramModel([], "")).names).toEqual([]); + const bytes = Array(4096).fill(255); + const payload = "'\"#\n\r\\; && $(python3) ".repeat(1000); + const model = await checkShellProgramModel(bytes, payload); + expect(model).toEqual(generateShellProgram(bytes, payload)); + expect(model.commandNames.length).toBeLessThanOrEqual(8); + expect(model.source.length).toBeLessThan(5000); + }); + + it("checks all byte choices with reproducible generated programs", async () => { + const names = new Set(); + const segments = new Set(); + for (let seed = 0; seed < 256; seed++) { + // Fixed LCG: no Math.random, clock, shell execution or external services. + let state = seed; + const bytes = Array.from({ length: 256 }, () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state >>> 24; + }); + const model = await checkShellProgramModel(bytes, `seed=${seed} ' " \r # ; | & \\`); + for (const name of model.names) names.add(name); + for (const segment of model.segments) segments.add(segment); + } + expect([...names].sort()).toEqual(["cat", "echo", "git", "jq", "python3", "wc"]); + expect(segments.size).toBeGreaterThan(500); + }); + + it("does not treat command text inside arbitrary comments as executable", async () => { + for (const payload of ['"unclosed', "'unclosed", "$(python3)", "\rpython3; #", "\\\npython3", "\0\u00a0\v"]) { + const model = await checkShellProgramModel([1], payload); + expect(model.names).toEqual(["echo"]); + } + }); + + it("deduplicates in first-occurrence order and denies a forbidden command at every position", async () => { + for (let position = 0; position < 3; position++) { + const bytes = [3]; + for (let i = 0; i < 3; i++) bytes.push(i === position ? 5 : 0, 0, 0, 0, 0, 4, 0); + const model = await checkShellProgramModel(bytes, "python3 is only a comment here"); + expect(model.commandNames).toEqual(Array.from({ length: 3 }, (_, i) => (i === position ? "python3" : "echo"))); + expect(model.names).toEqual(position === 0 ? ["python3", "echo"] : ["echo", "python3"]); + } + }); + + it("exercises each supported operator and unsupported suffix", async () => { + const operators = [" && ", " || ", " | ", " |& ", "; ", " & ", "\n", "\r\n"]; + for (let i = 0; i < 11; i++) { + // Two commands, no arguments; the final byte selects the unsafe suffix. + const bytes = [2, 0, 0, 0, 0, 0, i % 8, 0, 1, 0, 0, 0, 0, 0, 0, i]; + const model = await checkShellProgramModel(bytes, "ignored"); + expect(model.names).toEqual(["echo", "cat"]); + expect(model.segments).toEqual(["echo", "cat"]); + expect(model.source).toContain(`echo${operators[i % 8]}#`); + } + }); +}); diff --git a/actions/setup/js/fuzz_bash_command_parser_harness.cjs b/actions/setup/js/fuzz_bash_command_parser_harness.cjs index 516380a6f8b..b73045e886a 100644 --- a/actions/setup/js/fuzz_bash_command_parser_harness.cjs +++ b/actions/setup/js/fuzz_bash_command_parser_harness.cjs @@ -126,14 +126,18 @@ if (require.main === module) { input += chunk; }); - process.stdin.on("end", () => { + process.stdin.on("end", async () => { try { - // Expected JSON: { commandText: string, mode?: "split" | "name" | "pipeline" } - const { commandText, mode } = JSON.parse(input); + // Expected JSON: { commandText: string, mode?: "split" | "name" | "pipeline" | "model", bytes?: number[] } + const { commandText, mode, bytes } = JSON.parse(input); const text = commandText ?? ""; let result; switch (mode) { + case "model": + await require("./fuzz_bash_command_model.cjs").checkShellProgramModel(bytes ?? [], text); + result = { ok: true }; + break; case "split": result = testSplitOnPipelineOperators(text); break; diff --git a/pkg/workflow/bash_command_parser_fuzz_test.go b/pkg/workflow/bash_command_parser_fuzz_test.go new file mode 100644 index 00000000000..173a0857559 --- /dev/null +++ b/pkg/workflow/bash_command_parser_fuzz_test.go @@ -0,0 +1,70 @@ +//go:build !integration + +package workflow + +import ( + "bytes" + "context" + "encoding/json" + "os/exec" + "path/filepath" + "testing" + "time" +) + +// FuzzBashCommandParserModel generates shell programs from a grammar with an +// independent executable-name oracle, then checks the Copilot SDK parser and +// permission handler. Shell programs are only parsed, never executed. +// +// go test -run='^$' -fuzz=FuzzBashCommandParserModel -fuzztime=30s ./pkg/workflow +func FuzzBashCommandParserModel(f *testing.F) { + node, err := exec.LookPath("node") + if err != nil { + f.Skip("Node.js is required for the Bash parser fuzz harness") + } + harness, err := filepath.Abs("../../actions/setup/js/fuzz_bash_command_parser_harness.cjs") + if err != nil { + f.Fatal(err) + } + f.Add([]byte{}, "") + f.Add([]byte{1}, "' \" # ; && || | $() \r \\\n") + f.Add([]byte{8, 5, 3, 2, 3, 6, 5, 4, 7, 1}, "# comments must not conceal python3") + f.Add(bytes.Repeat([]byte{255}, 128), "\x00\r\n\u00a0") + for seed := range byte(16) { + input := make([]byte, 128) + for i := range input { + input[i] = seed + byte(i*17) + } + f.Add(input, "quoted ' \" # ; | & \r text") + } + f.Fuzz(func(t *testing.T, input []byte, payload string) { + if len(input) > 4096 || len(payload) > 4096 { + t.Skip("Input too large") + } + // JSON encodes []byte as base64; use integers for the JS choice stream. + choices := make([]int, len(input)) + for i, value := range input { + choices[i] = int(value) + } + request, err := json.Marshal(map[string]any{ + "mode": "model", "bytes": choices, "commandText": payload, + }) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, node, harness) + cmd.Stdin = bytes.NewReader(request) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("shell model failed: %v\ninput=%x payload=%q\n%s", err, input, payload, output) + } + var result struct { + OK bool `json:"ok"` + } + if err := json.Unmarshal(output, &result); err != nil || !result.OK { + t.Fatalf("invalid harness response: %s (error: %v)", output, err) + } + }) +}