diff --git a/.github/actions/conformance/expected-failures.2026-07-28.yml b/.github/actions/conformance/expected-failures.2026-07-28.yml index 02a3ef81af..a35edba3e2 100644 --- a/.github/actions/conformance/expected-failures.2026-07-28.yml +++ b/.github/actions/conformance/expected-failures.2026-07-28.yml @@ -10,10 +10,10 @@ # 2026 leg reads the `server:` section. Both burn down independently of the # 2025 legs. # -# Baseline established against @modelcontextprotocol/conformance pinned in -# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4). -# New conformance releases are adopted by deliberately bumping that pin and -# reconciling both this file and expected-failures.yml in the same change. +# Baseline established against the harness pinned via CONFORMANCE_PKG in +# .github/workflows/conformance.yml. New conformance releases are adopted by +# deliberately bumping that pin and reconciling both this file and +# expected-failures.yml in the same change. # # Entries are grouped by what unblocks them. As each gap closes the # corresponding scenarios start passing and MUST be removed from this list diff --git a/.github/actions/conformance/expected-failures.yml b/.github/actions/conformance/expected-failures.yml index cc6071205b..c518384e69 100644 --- a/.github/actions/conformance/expected-failures.yml +++ b/.github/actions/conformance/expected-failures.yml @@ -1,11 +1,10 @@ # Conformance scenarios not yet passing against the Python SDK on main. # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. # -# Baseline established against @modelcontextprotocol/conformance pinned in -# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.4). -# New conformance releases are adopted by deliberately bumping that pin and -# reconciling both this file and expected-failures.2026-07-28.yml in the same -# change. +# Baseline established against the harness pinned via CONFORMANCE_PKG in +# .github/workflows/conformance.yml. New conformance releases are adopted by +# deliberately bumping that pin and reconciling both this file and +# expected-failures.2026-07-28.yml in the same change. # # Entries are grouped by SEP. As each SEP lands in the SDK the corresponding # scenarios start passing and MUST be removed from this list (the runner fails @@ -40,9 +39,6 @@ client: - auth/offline-access-not-supported # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- - # SEP-2350 (scope step-up): WARNING-only; the expected-failures evaluator - # counts WARNINGs as failures. - - auth/scope-step-up # SEP-990 (enterprise-managed authorization extension): no fixture handler / # client support for the token-exchange + JWT bearer flow. - auth/enterprise-managed-authorization diff --git a/.github/actions/conformance/run-server.sh b/.github/actions/conformance/run-server.sh index 30068c18c8..c026f4a02e 100755 --- a/.github/actions/conformance/run-server.sh +++ b/.github/actions/conformance/run-server.sh @@ -47,5 +47,5 @@ done echo "Server ready at $SERVER_URL" -npx --yes @modelcontextprotocol/conformance@"${CONFORMANCE_VERSION:?set CONFORMANCE_VERSION (pinned in .github/workflows/conformance.yml)}" \ +npx --yes "${CONFORMANCE_PKG:?set CONFORMANCE_PKG (pinned in .github/workflows/conformance.yml)}" \ server --url "$SERVER_URL" "$@" diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 24486151cb..e985a52f6b 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -14,10 +14,16 @@ permissions: contents: read env: - # Pinned conformance harness version. Bump deliberately and reconcile - # both .github/actions/conformance/expected-failures*.yml files in the - # same change. - CONFORMANCE_VERSION: "0.2.0-alpha.4" + # Pinned conformance harness package spec (passed verbatim to `npx --yes`). + # Use a published version, e.g. @modelcontextprotocol/conformance@0.2.0-alpha.5. + # Bump deliberately and reconcile both + # .github/actions/conformance/expected-failures*.yml files in the same change. + # + # TODO: replace with @modelcontextprotocol/conformance@0.2.0-alpha.5 once + # https://github.com/modelcontextprotocol/conformance/pull/357 publishes, and + # drop CONFORMANCE_PKG_SHA256 plus the fetch-and-verify step below. + CONFORMANCE_PKG: "https://pkg.pr.new/@modelcontextprotocol/conformance@65fcd39" + CONFORMANCE_PKG_SHA256: "9a381d7083f8be2fe7ae44efeca54530f18c61425805ddaf9cd88915efcc1574" jobs: server-conformance: @@ -33,6 +39,19 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 + - name: Fetch and verify conformance harness + # Only when CONFORMANCE_PKG is a URL: download, check the recorded + # sha256, and re-point CONFORMANCE_PKG at the verified local tarball. + # When CONFORMANCE_PKG is a registry spec, this step is a no-op (npm's + # own integrity check applies). + run: | + case "$CONFORMANCE_PKG" in + https://*) + curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz + echo "$CONFORMANCE_PKG_SHA256 /tmp/conformance.tgz" | sha256sum -c - + echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV" + ;; + esac - run: uv sync --frozen --all-extras --package mcp-everything-server - name: Run server conformance (active suite) run: >- @@ -64,16 +83,25 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 + - name: Fetch and verify conformance harness + run: | + case "$CONFORMANCE_PKG" in + https://*) + curl -fsSL "$CONFORMANCE_PKG" -o /tmp/conformance.tgz + echo "$CONFORMANCE_PKG_SHA256 /tmp/conformance.tgz" | sha256sum -c - + echo "CONFORMANCE_PKG=file:/tmp/conformance.tgz" >> "$GITHUB_ENV" + ;; + esac - run: uv sync --frozen --all-extras --package mcp - name: Run client conformance (all suite) run: >- - npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client + npx --yes "$CONFORMANCE_PKG" client --command 'uv run --frozen python .github/actions/conformance/client.py' --suite all --expected-failures ./.github/actions/conformance/expected-failures.yml - name: Run client conformance (2026-07-28 wire, all suite) run: >- - npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client + npx --yes "$CONFORMANCE_PKG" client --command 'uv run --frozen python .github/actions/conformance/client.py' --suite all --spec-version 2026-07-28