Add model-based fuzzing for the Copilot SDK shell parser

.github/workflows/cgo.yml

@@ -1832,6 +1832,7 @@ jobs:
         - group: Workflow-Core
           tests: >-
             FuzzExpressionParser:./pkg/workflow/
+            FuzzBashCommandParserModel:./pkg/workflow/
             FuzzMentionsFiltering:./pkg/workflow/
             FuzzSanitizeOutput:./pkg/workflow/
             FuzzSanitizeIncomingText:./pkg/workflow/

Makefile

@@ -233,6 +233,7 @@ fuzz:
 	@echo "Running fuzz tests for 30 seconds..."
 	go test -fuzz=FuzzParseFrontmatter -fuzztime=30s ./pkg/parser/
 	go test -fuzz=FuzzExpressionParser -fuzztime=30s ./pkg/workflow/
+	go test -run='^$$' -fuzz=FuzzBashCommandParserModel -fuzztime=30s ./pkg/workflow/
 
 # Run security regression tests
 .PHONY: test-security

actions/setup/js/fuzz_bash_command_model.cjs

@@ -0,0 +1,104 @@
+// @ts-check
+"use strict";
+
+const assert = require("node:assert/strict");
+const { splitOnPipelineOperators, extractCommandName, extractCommandNamesFromPipeline } = require("./bash_command_parser.cjs");
+const { buildCopilotSDKPermissionHandler } = require("./copilot_sdk_permissions.cjs");
+
+const COMMANDS = ["echo", "cat", "jq", "wc", "git", "python3"];
+const OPERATORS = [" && ", " || ", " | ", " |& ", "; ", " & ", "\n", "\r\n"];
+const ARGUMENTS = ["plain#hash", "'# && || | ; \" $()'", '"# && || | ; \\" literal"', "\\#escaped\\;word", "'first\n# still quoted'", "${value:-#fallback}", '"${outer:-${inner:-#fallback}}"'];
+const ASSIGNMENTS = ["", "A=1 ", "A='# literal' B=two ", 'A="# literal" '];
+const REDIRECTIONS = ["", " 2>/dev/null", " >output 2>&1", " &>output", " <input"];
+const BLANKS = [" ", "\t", " \\\n  "];
+const UNSUPPORTED = ["'unclosed", '"unclosed', "${value:-unclosed", "$(echo unclosed", '"$(echo "#")"', "$'escaped\\' #'; python3", "@(x #); python3", "?(x #); python3", "*(x #); python3", "+(x #); python3", "!(x #); python3"];
+
+/**
+ * Build a bounded shell-program model from fuzzer-controlled bytes. Choices are
+ * consumed directly (not hashed), so Go can shrink failures to small byte inputs.
+ * The model records executable names before rendering; it never parses its own
+ * output to compute the oracle. Generated programs are data, never executed.
+ *
+ * @param {number[]} bytes
+ * @param {string} payload Arbitrary text, quoted as an argument and kept in comments.
+ */
+function generateShellProgram(bytes, payload) {
+  let cursor = 0;
+  const next = () => (bytes[cursor++] ?? 0) & 255;
+  const pick = values => values[next() % values.length];
+  // LF alone ends Bash comments; retain CR, quotes, operators and backslashes.
+  const comment = `# ${payload.slice(0, 256).replace(/\n/g, " ")} ' " $( ; && | \r python3`;
+  const literal = payload.slice(0, 128).replace(/\0/g, "");
+  const argumentChoices = [...ARGUMENTS, `'${literal.replace(/'/g, "'\\''")}'`, `"${literal.replace(/[\\"$`]/g, "\\$&")}"`];
+  const count = next() % 9;
+  const commands = Array.from({ length: count }, () => ({
+    name: pick(COMMANDS),
+    assignment: pick(ASSIGNMENTS),
+    blank: pick(BLANKS),
+    args: Array.from({ length: next() % 4 }, () => pick(argumentChoices)),
+    redirection: pick(REDIRECTIONS),
+    operator: pick(OPERATORS),
+    comment: next() % 2 === 0,
+  }));
+  const render = (command, blank) => `${command.assignment}${command.name}${command.args.length ? blank + command.args.join(" ") : ""}${command.redirection}`;
+  const segments = commands.map(command => render(command, command.blank === BLANKS[2] ? " " : command.blank));
+  let source = `${comment}\n`;
+  for (let i = 0; i < commands.length; i++) {
+    source += render(commands[i], commands[i].blank);
+    if (i + 1 < commands.length) {
+      source += commands[i].operator;
+      if (commands[i].comment) source += `${comment}\n`;
+    }
+  }
+  source += ` # ${comment}`;
+  return {
+    source,
+    segments,
+    commandNames: commands.map(command => command.name),
+    names: [...new Set(commands.map(command => command.name))],
+    unsupported: pick(UNSUPPORTED),
+  };
+}
+
+/**
+ * Assert exact extraction for the supported grammar, fail-closed behaviour for
+ * unsupported constructs, and robustness for arbitrary malformed shell text.
+ * SDK allow/deny expectations come from the model, not from parser output.
+ *
+ * @param {number[]} bytes
+ * @param {string} payload
+ */
+async function checkShellProgramModel(bytes, payload) {
+  const model = generateShellProgram(bytes, payload);
+  const context = JSON.stringify(model.source);
+  const expectedSegments = model.segments;
+  assert.deepEqual(splitOnPipelineOperators(model.source), expectedSegments, context);
+  assert.deepEqual(extractCommandNamesFromPipeline(model.source), model.names, context);
+  for (let i = 0; i < expectedSegments.length; i++) {
+    assert.equal(extractCommandName(expectedSegments[i]), model.commandNames[i], context);
+  }
+  const allowed = COMMANDS.filter(name => name !== "python3");
+  const handler = buildCopilotSDKPermissionHandler({ allowedTools: allowed.map(name => `shell(${name})`) }, () => ({ kind: "approve-once" }));
+  /** @type {import("@github/copilot-sdk").PermissionRequest} */
+  const request = { kind: "shell", fullCommandText: model.source, commands: [], canOfferSessionApproval: false, hasWriteFileRedirection: false, intention: "Parse only", possiblePaths: [], possibleUrls: [] };
+  const permission = await handler(request, { sessionId: "shell-model-fuzz" });
+  assert.equal(permission.kind, model.names.length > 0 && !model.names.includes("python3") ? "approve-once" : "reject", context);
+
+  // Every unsupported construct must invalidate the whole parse, even after a
+  // valid prefix; never accept only the commands preceding an ambiguous suffix.
+  const unsupported = `${model.source}\necho ${model.unsupported}`;
+  assert.deepEqual(splitOnPipelineOperators(unsupported), [], JSON.stringify(unsupported));
+  assert.deepEqual(extractCommandNamesFromPipeline(unsupported), [], JSON.stringify(unsupported));
+  assert.equal((await handler({ ...request, fullCommandText: unsupported }, { sessionId: "shell-model-fuzz" })).kind, "reject", JSON.stringify(unsupported));
+
+  const cut = Math.floor(((bytes.at(-1) ?? 0) * model.source.length) / 256);
+  for (const text of [payload.slice(0, 4096), model.source.slice(0, cut)]) {
+    assert.ok(splitOnPipelineOperators(text).every(segment => typeof segment === "string"));
+    assert.ok(extractCommandNamesFromPipeline(text).every(name => typeof name === "string"));
+    const name = extractCommandName(text);
+    assert.ok(name === null || typeof name === "string");
+  }
+  return model;
+}
+
+module.exports = { generateShellProgram, checkShellProgramModel };

actions/setup/js/fuzz_bash_command_model.test.cjs

@@ -0,0 +1,67 @@
+// @ts-check
+
+import { describe, it, expect } from "vitest";
+import { createRequire } from "module";
+
+const require = createRequire(import.meta.url);
+const { generateShellProgram, checkShellProgramModel } = require("./fuzz_bash_command_model.cjs");
+
+describe("model-based Bash parser fuzz generator", () => {
+  it("is deterministic and bounded on empty and oversized inputs", async () => {
+    expect(generateShellProgram([], "")).toEqual(generateShellProgram([], ""));
+    expect((await checkShellProgramModel([], "")).names).toEqual([]);
+    const bytes = Array(4096).fill(255);
+    const payload = "'\"#\n\r\\; && $(python3) ".repeat(1000);
+    const model = await checkShellProgramModel(bytes, payload);
+    expect(model).toEqual(generateShellProgram(bytes, payload));
+    expect(model.commandNames.length).toBeLessThanOrEqual(8);
+    expect(model.source.length).toBeLessThan(5000);
+  });
+
+  it("checks all byte choices with reproducible generated programs", async () => {
+    const names = new Set();
+    const segments = new Set();
+    for (let seed = 0; seed < 256; seed++) {
+      // Fixed LCG: no Math.random, clock, shell execution or external services.
+      let state = seed;
+      const bytes = Array.from({ length: 256 }, () => {
+        state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
+        return state >>> 24;
+      });
+      const model = await checkShellProgramModel(bytes, `seed=${seed} ' " \r # ; | & \\`);
+      for (const name of model.names) names.add(name);
+      for (const segment of model.segments) segments.add(segment);
+    }
+    expect([...names].sort()).toEqual(["cat", "echo", "git", "jq", "python3", "wc"]);
+    expect(segments.size).toBeGreaterThan(500);
+  });
+
+  it("does not treat command text inside arbitrary comments as executable", async () => {
+    for (const payload of ['"unclosed', "'unclosed", "$(python3)", "\rpython3; #", "\\\npython3", "\0\u00a0\v"]) {
+      const model = await checkShellProgramModel([1], payload);
+      expect(model.names).toEqual(["echo"]);
+    }
+  });
+
+  it("deduplicates in first-occurrence order and denies a forbidden command at every position", async () => {
+    for (let position = 0; position < 3; position++) {
+      const bytes = [3];
+      for (let i = 0; i < 3; i++) bytes.push(i === position ? 5 : 0, 0, 0, 0, 0, 4, 0);
+      const model = await checkShellProgramModel(bytes, "python3 is only a comment here");
+      expect(model.commandNames).toEqual(Array.from({ length: 3 }, (_, i) => (i === position ? "python3" : "echo")));
+      expect(model.names).toEqual(position === 0 ? ["python3", "echo"] : ["echo", "python3"]);
+    }
+  });
+
+  it("exercises each supported operator and unsupported suffix", async () => {
+    const operators = [" && ", " || ", " | ", " |& ", "; ", " & ", "\n", "\r\n"];
+    for (let i = 0; i < 11; i++) {
+      // Two commands, no arguments; the final byte selects the unsafe suffix.
+      const bytes = [2, 0, 0, 0, 0, 0, i % 8, 0, 1, 0, 0, 0, 0, 0, 0, i];
+      const model = await checkShellProgramModel(bytes, "ignored");
+      expect(model.names).toEqual(["echo", "cat"]);
+      expect(model.segments).toEqual(["echo", "cat"]);
+      expect(model.source).toContain(`echo${operators[i % 8]}#`);
+    }
+  });
+});

actions/setup/js/fuzz_bash_command_parser_harness.cjs

@@ -126,14 +126,18 @@ if (require.main === module) {
     input += chunk;
   });
 
-  process.stdin.on("end", () => {
+  process.stdin.on("end", async () => {
     try {
-      // Expected JSON: { commandText: string, mode?: "split" | "name" | "pipeline" }
-      const { commandText, mode } = JSON.parse(input);
+      // Expected JSON: { commandText: string, mode?: "split" | "name" | "pipeline" | "model", bytes?: number[] }
+      const { commandText, mode, bytes } = JSON.parse(input);
       const text = commandText ?? "";
 
       let result;
       switch (mode) {
+        case "model":
+          await require("./fuzz_bash_command_model.cjs").checkShellProgramModel(bytes ?? [], text);
+          result = { ok: true };
+          break;
         case "split":
           result = testSplitOnPipelineOperators(text);
           break;

pkg/workflow/bash_command_parser_fuzz_test.go

@@ -0,0 +1,70 @@
+//go:build !integration
+
+package workflow
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// FuzzBashCommandParserModel generates shell programs from a grammar with an
+// independent executable-name oracle, then checks the Copilot SDK parser and
+// permission handler. Shell programs are only parsed, never executed.
+//
+//	go test -run='^$' -fuzz=FuzzBashCommandParserModel -fuzztime=30s ./pkg/workflow
+func FuzzBashCommandParserModel(f *testing.F) {
+	node, err := exec.LookPath("node")
+	if err != nil {
+		f.Skip("Node.js is required for the Bash parser fuzz harness")
+	}
+	harness, err := filepath.Abs("../../actions/setup/js/fuzz_bash_command_parser_harness.cjs")
+	if err != nil {
+		f.Fatal(err)
+	}
+	f.Add([]byte{}, "")
+	f.Add([]byte{1}, "' \" # ; && || | $() \r \\\n")
+	f.Add([]byte{8, 5, 3, 2, 3, 6, 5, 4, 7, 1}, "# comments must not conceal python3")
+	f.Add(bytes.Repeat([]byte{255}, 128), "\x00\r\n\u00a0")
+	for seed := range byte(16) {
+		input := make([]byte, 128)
+		for i := range input {
+			input[i] = seed + byte(i*17)
+		}
+		f.Add(input, "quoted ' \" # ; | & \r text")
+	}
+	f.Fuzz(func(t *testing.T, input []byte, payload string) {
+		if len(input) > 4096 || len(payload) > 4096 {
+			t.Skip("Input too large")
+		}
+		// JSON encodes []byte as base64; use integers for the JS choice stream.
+		choices := make([]int, len(input))
+		for i, value := range input {
+			choices[i] = int(value)
+		}
+		request, err := json.Marshal(map[string]any{
+			"mode": "model", "bytes": choices, "commandText": payload,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+		defer cancel()
+		cmd := exec.CommandContext(ctx, node, harness)
+		cmd.Stdin = bytes.NewReader(request)
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("shell model failed: %v\ninput=%x payload=%q\n%s", err, input, payload, output)
+		}
+		var result struct {
+			OK bool `json:"ok"`
+		}
+		if err := json.Unmarshal(output, &result); err != nil || !result.OK {
+			t.Fatalf("invalid harness response: %s (error: %v)", output, err)
+		}
+	})
+}