From ed8bd1f27d4963dff2fd04ff8d29f9f56df46406 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 21 Aug 2026 17:47:07 +0200 Subject: [PATCH 1/3] fix(skill-harness): make pnpm 11 frozen installs work pnpm 11 no longer reads the `pnpm` key in package.json, so the esbuild override that lived there was silently ignored and `pnpm install --frozen-lockfile` refused the committed lockfile. Move the override to skill-harness/pnpm-workspace.yaml, which is where pnpm 11 looks for it. While there, declare `allowBuilds: esbuild: false`. esbuild's postinstall is only a fallback for platforms without a prebuilt optional dependency; on every platform we run on the binary comes from the optional package, so the build script is not needed and the install stays free of arbitrary postinstall execution. Pin the toolchain in package.json so CI and local runs agree: `packageManager: pnpm@11.18.0` (read by pnpm/action-setup) and `engines.node: 24` (read by actions/setup-node). Add a `typecheck` script (`tsc --noEmit`) so CI can run it by name. The lockfile is unchanged: a frozen install accepts it byte-for-byte once the override is read from the right place. Closes #30 Signed-off-by: Otto van der Schaaf --- skill-harness/package.json | 10 +++++----- skill-harness/pnpm-workspace.yaml | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 skill-harness/pnpm-workspace.yaml diff --git a/skill-harness/package.json b/skill-harness/package.json index 6f3eccb..cdcbf52 100644 --- a/skill-harness/package.json +++ b/skill-harness/package.json @@ -4,6 +4,10 @@ "private": true, "description": "Benchmark harness for the dtwo-gateway-config Claude skill. Offline rubrics + live bench (claude-cli / Anthropic API) with pass@k + Wilson CI aggregation and a committed regression baseline.", "type": "module", + "packageManager": "pnpm@11.18.0", + "engines": { + "node": "24" + }, "exports": { "./*": [ "./src/*" @@ -13,6 +17,7 @@ "test": "node --import tsx --test 'src/__tests__/**/*.test.ts'", "biome:check": "biome check --error-on-warnings", "biome:fix": "biome check --write .", + "typecheck": "tsc --noEmit", "bench": "tsx src/bin/bench.ts" }, "dependencies": { @@ -26,10 +31,5 @@ "@types/node": "25.6.0", "tsx": "4.21.0", "typescript": "5.9.3" - }, - "pnpm": { - "overrides": { - "esbuild": ">=0.28.1" - } } } diff --git a/skill-harness/pnpm-workspace.yaml b/skill-harness/pnpm-workspace.yaml new file mode 100644 index 0000000..39f3ddd --- /dev/null +++ b/skill-harness/pnpm-workspace.yaml @@ -0,0 +1,4 @@ +overrides: + esbuild: '>=0.28.1' +allowBuilds: + esbuild: false From c2ee7ee20f7f55f98ea4c4f0c5d5000ed289f7f4 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 21 Aug 2026 17:47:43 +0200 Subject: [PATCH 2/3] ci: run the skill-harness suite on push and PR Add .github/workflows/skill-harness.yml. On every pull request and every push to main it runs the offline checks from skill-harness/: the schema digest `--check` (first, on the runner's default Node, since the script needs no dependencies), a frozen-lockfile install, a lockfile-stability guard (`git diff --exit-code` on the lockfile and workspace file), the test suite, `biome check`, and `tsc --noEmit`. The checks after install are gated on the install step rather than on the previous step, so one run reports every failure. The live bench (`pnpm bench`) is deliberately not run: it calls a model API and needs credentials. No paths filter, so the job can later be made a required check. Token is read-only and checkout does not persist credentials; nothing in the job needs more. Actions are pinned to commit SHAs with the tag in a comment. The dependabot config keeps those pins current (weekly, grouped) and bumps the harness npm dependencies in one weekly PR. zod majors are ignored: the harness asserts against a vendored validator bundle built for the current zod major, so that bump is a deliberate re-vendor. Closes #33 Signed-off-by: Otto van der Schaaf --- .github/dependabot.yml | 25 +++++++++ .github/workflows/skill-harness.yml | 80 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/skill-harness.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1449925 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +version: 2 +updates: + # Keep the commit-SHA pins in .github/workflows current. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + groups: + actions: + patterns: ["*"] + + - package-ecosystem: npm + directory: /skill-harness + schedule: + interval: weekly + groups: + # One PR per week for all harness dependencies; a failing suite is + # the signal that a bump needs a human. + harness: + patterns: ["*"] + ignore: + # The harness asserts against a byte-pinned validator bundle built for + # this zod major; a major bump is a deliberate re-vendor, not a bot PR. + - dependency-name: zod + update-types: ["version-update:semver-major"] diff --git a/.github/workflows/skill-harness.yml b/.github/workflows/skill-harness.yml new file mode 100644 index 0000000..8b4a47f --- /dev/null +++ b/.github/workflows/skill-harness.yml @@ -0,0 +1,80 @@ +# Skill-harness CI (see #33). Offline checks only: the live bench +# (`pnpm bench`) calls a model API and is deliberately not run here. +name: skill-harness + +on: + pull_request: + push: + branches: [main] + +# Read-only token. No step needs secrets or write access, so fork PRs run +# with the exact same grants as same-repo PRs. +permissions: + contents: read + +# One run per ref. A newer push to a PR cancels that PR's older run; pushes +# to main are never cancelled, so every main commit keeps its own verdict. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + working-directory: skill-harness + +jobs: + check: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The digest generator imports only node: builtins, so it runs on the + # runner's default Node before any toolchain setup. Putting it first + # means a stale digest is reported even if the install below fails. + - name: Schema digest is in sync + working-directory: . + run: node scripts/generate-schema-digest.mjs --check + + # pnpm version comes from `packageManager` in skill-harness/package.json. + # Do not add a `version:` input here: the action errors when both are + # set and disagree, and one source of truth is the point. + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + with: + package_json_file: skill-harness/package.json + + # Node version comes from `engines.node` in the same package.json. + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: skill-harness/package.json + cache: pnpm + cache-dependency-path: skill-harness/pnpm-lock.yaml + + - name: Install (frozen lockfile) + id: install + run: pnpm install --frozen-lockfile + + # The checks below run independently once install succeeds, so a + # single run reports every failure rather than stopping at the first. + + # A frozen install must not want to rewrite the lockfile or the + # workspace file; if it does, the committed files are out of date. + - name: Lockfile is stable + if: ${{ !cancelled() && steps.install.conclusion == 'success' }} + working-directory: . + run: git diff --exit-code -- skill-harness/pnpm-lock.yaml skill-harness/pnpm-workspace.yaml + + - name: Test + if: ${{ !cancelled() && steps.install.conclusion == 'success' }} + run: pnpm test + + - name: Biome check + if: ${{ !cancelled() && steps.install.conclusion == 'success' }} + run: pnpm biome:check + + - name: Typecheck + if: ${{ !cancelled() && steps.install.conclusion == 'success' }} + run: pnpm typecheck From e2d68c23bff9e9256a92bb50d2fd5868c192cc62 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Fri, 21 Aug 2026 17:48:00 +0200 Subject: [PATCH 3/3] docs: point at the CI workflow as the enforcing gate The schema-digest `--check` and the harness suite are now run by .github/workflows/skill-harness.yml on every PR and push to main, so the docs that described `pnpm test` as the only enforcement point now name the workflow. Also list `pnpm typecheck` with the other offline commands and note that the live bench is not run in CI. Signed-off-by: Otto van der Schaaf --- CLAUDE.md | 4 +++- README.md | 2 +- skill-harness/README.md | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 59a6465..1cf7e17 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ If you add a second plugin to the marketplace later, the same rule applies: vers Production Claude Code caches only each skill's `SKILL.md` in the system prompt; a skill's `references/` directory is **not** auto-loaded for real users. Anything a policy/skill author needs at authoring time must live inline in `SKILL.md`, not in a sidecar file. -The `dtwo-gateway-config` skill's `### Schema Digest` (between `` / ``) is **generated** from `dtwo/skills/dtwo-gateway-config/schema-reference.json` — do not hand-edit it. Regenerate with `node scripts/generate-schema-digest.mjs`; `--check` fails if it's stale, and the `skill-harness` suite runs exactly that, so `pnpm test` is what enforces it. The vendored schema is a verbatim copy of the artifact the product repo's schema generator emits; refresh it by copying, never by editing in place. +The `dtwo-gateway-config` skill's `### Schema Digest` (between `` / ``) is **generated** from `dtwo/skills/dtwo-gateway-config/schema-reference.json` — do not hand-edit it. Regenerate with `node scripts/generate-schema-digest.mjs`; `--check` fails if it's stale; the `skill-harness` suite runs exactly that, and CI (`.github/workflows/skill-harness.yml`) runs both the `--check` and the suite on every push to `main` and every PR. The vendored schema is a verbatim copy of the artifact the product repo's schema generator emits; refresh it by copying, never by editing in place. ## Test locally before pushing @@ -35,6 +35,8 @@ The `dtwo-gateway-config` skill's `### Schema Digest` (between `