diff --git a/README.md b/README.md index 71a8cb2c..f664b65d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. +The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. Clients under test must derive the lifecycle from the protocol version they are asked to run: dated versions through `2025-11-25` use the stateful lifecycle (initialize handshake), while the 2026 draft (`DRAFT-2026-v1`) uses the stateless lifecycle (per-request `_meta`). ### Server Testing diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 63ca051b..ef272e4a 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -21,6 +21,8 @@ import { } from '@modelcontextprotocol/sdk/client/auth-extensions.js'; import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import { ClientConformanceContextSchema } from '../../../src/schemas/context.js'; +import { DRAFT_PROTOCOL_VERSION } from '../../../src/types.js'; +import { STATELESS_SPEC_VERSIONS } from '../../../src/connection/select.js'; import { auth, extractWWWAuthenticateParams @@ -70,10 +72,97 @@ export function getHandler(scenarioName: string): ScenarioHandler | undefined { } // ============================================================================ -// Basic scenarios (initialize, tools-call) +// Stateless requester (SEP-2575 / 2026-x lifecycle) +// +// Shim for the fact that the SDK Client doesn't support stateless mode yet. +// Carry-forward handlers below pick this when the runner says the resolved +// spec version is stateless, so the same handler exercises both lifecycles. +// ============================================================================ + +const PROTOCOL_VERSION = process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; + +// Lifecycle decision: derived from the runner-provided protocol version. +// The version→lifecycle mapping is spec knowledge a client must own; this +// in-repo client imports the stateless version set from src/ so it cannot +// drift from the runner's mapping. +const USE_STATELESS_LIFECYCLE = PROTOCOL_VERSION + ? (STATELESS_SPEC_VERSIONS as readonly string[]).includes(PROTOCOL_VERSION) + : false; + +// Wire protocolVersion for stateless requests: the runner-resolved version +// when available (so a dated stateless release is exercised under its own +// identifier), the current draft otherwise. +const STATELESS_PROTOCOL_VERSION = PROTOCOL_VERSION ?? DRAFT_PROTOCOL_VERSION; + +const STATELESS_META_BASE = { + 'io.modelcontextprotocol/clientInfo': { + name: 'conformance-test-client', + version: '1.0.0' + }, + 'io.modelcontextprotocol/clientCapabilities': { + tools: {}, + roots: {}, + sampling: {}, + elicitation: {} + } +}; + +let _nextStatelessId = 1; +async function statelessRequest( + serverUrl: string, + method: string, + params: Record = {} +): Promise { + const _meta = { + 'io.modelcontextprotocol/protocolVersion': STATELESS_PROTOCOL_VERSION, + ...STATELESS_META_BASE, + ...((params._meta as object | undefined) ?? {}) + }; + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + // Servers built on the SDK's StreamableHTTPServerTransport reject + // requests that don't accept both JSON and SSE responses. + Accept: 'application/json, text/event-stream', + 'MCP-Protocol-Version': STATELESS_PROTOCOL_VERSION + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: _nextStatelessId++, + method, + params: { ...params, _meta } + }) + }); + const body = await response.json(); + if (body.error) { + throw new Error( + `${method} failed: ${body.error.code} ${body.error.message}` + ); + } + return body.result; +} + +// ============================================================================ +// Basic scenarios (initialize, tools_call) // ============================================================================ async function runBasicClient(serverUrl: string): Promise { + if (USE_STATELESS_LIFECYCLE) { + logger.debug('Stateless lifecycle: calling tools/list + tools/call'); + const list = await statelessRequest(serverUrl, 'tools/list'); + logger.debug('Successfully listed tools:', JSON.stringify(list)); + const tool = list?.tools?.[0]; + if (tool) { + const result = await statelessRequest(serverUrl, 'tools/call', { + name: tool.name, + arguments: { a: 2, b: 3 } + }); + logger.debug('Successfully called tool:', JSON.stringify(result)); + } + return; + } + const client = new Client( { name: 'test-client', version: '1.0.0' }, { capabilities: {} } @@ -84,20 +173,52 @@ async function runBasicClient(serverUrl: string): Promise { await client.connect(transport); logger.debug('Successfully connected to MCP server'); - await client.listTools(); + const list = await client.listTools(); logger.debug('Successfully listed tools'); + const tool = list.tools[0]; + if (tool) { + await client.callTool({ name: tool.name, arguments: { a: 2, b: 3 } }); + logger.debug('Successfully called tool'); + } + await transport.close(); logger.debug('Connection closed successfully'); } -registerScenarios(['initialize', 'tools-call'], runBasicClient); +registerScenarios(['initialize', 'tools_call', 'tools-call'], runBasicClient); // SEP-2106: json-schema-ref-no-deref advertises a tool whose inputSchema // contains a network-URI $ref. A conformant client lists tools normally and -// simply never fetches that URI, so the basic connect+listTools flow is the -// correct behavior here. -registerScenario('json-schema-ref-no-deref', runBasicClient); +// simply never fetches that URI. The scenario's mock only serves tools/list, +// so this handler stops after listing instead of reusing runBasicClient +// (whose tools/call would get -32601 and fail the run). +async function runListToolsOnlyClient(serverUrl: string): Promise { + if (USE_STATELESS_LIFECYCLE) { + logger.debug('Stateless lifecycle: calling tools/list'); + const list = await statelessRequest(serverUrl, 'tools/list'); + logger.debug('Successfully listed tools:', JSON.stringify(list)); + return; + } + + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('json-schema-ref-no-deref', runListToolsOnlyClient); // ============================================================================ // request-metadata scenario (SEP-2575) @@ -106,20 +227,9 @@ registerScenario('json-schema-ref-no-deref', runBasicClient); async function runRequestMetadataClient(serverUrl: string): Promise { logger.debug('Starting request-metadata client flow...'); - const meta = { - 'io.modelcontextprotocol/clientInfo': { - name: 'conformance-test-client', - version: '1.0.0' - }, - 'io.modelcontextprotocol/clientCapabilities': { - tools: {}, - roots: {}, - sampling: {}, - elicitation: {} - } - }; + const meta = STATELESS_META_BASE; - let activeVersion = 'DRAFT-2026-v1'; + let activeVersion = STATELESS_PROTOCOL_VERSION; const sendRequestWithNegotiation = async ( method: string, @@ -166,7 +276,9 @@ async function runRequestMetadataClient(serverUrl: string): Promise { ); const serverSupported: string[] = errorResult.error.data?.supported || []; - const clientSupported = ['DRAFT-2026-v1']; + const clientSupported = [ + ...new Set([STATELESS_PROTOCOL_VERSION, DRAFT_PROTOCOL_VERSION]) + ]; const mutuallySupported = clientSupported.filter((v) => serverSupported.includes(v) ); diff --git a/examples/servers/typescript/everything-server.ts b/examples/servers/typescript/everything-server.ts index 7a587ffa..ce439451 100644 --- a/examples/servers/typescript/everything-server.ts +++ b/examples/servers/typescript/everything-server.ts @@ -1283,7 +1283,10 @@ app.post('/mcp', async (req, res) => { error: { code: -32004, message: 'UnsupportedProtocolVersionError', - data: { supported: ['DRAFT-2026-v1'] } + data: { + supported: ['DRAFT-2026-v1'], + requested: String(metaVersion) + } } }); } @@ -1436,6 +1439,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1465,6 +1474,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1549,6 +1564,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } @@ -1570,6 +1591,12 @@ app.post('/mcp', async (req, res) => { cacheScope: 'public' } }); + } catch (e: any) { + return res.json({ + jsonrpc: '2.0', + id, + error: { code: e.code ?? -32603, message: e.message, data: e.data } + }); } finally { await dispatch.close(); } diff --git a/src/connection/connection.test.ts b/src/connection/connection.test.ts index b308284b..be093417 100644 --- a/src/connection/connection.test.ts +++ b/src/connection/connection.test.ts @@ -1,8 +1,13 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import { connectFor } from './select'; +import { + connectFor, + isStatefulVersion, + STATELESS_SPEC_VERSIONS +} from './select'; import { connectStateful } from './stateful'; import { connectStateless } from './stateless'; import { JsonRpcError } from './index'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; describe('connectFor', () => { it('returns stateful for dated 2025-x versions', () => { @@ -20,6 +25,18 @@ describe('connectFor', () => { }); }); +describe('STATELESS_SPEC_VERSIONS', () => { + it('contains exactly the versions isStatefulVersion rejects', () => { + expect(STATELESS_SPEC_VERSIONS.length).toBeGreaterThan(0); + for (const v of STATELESS_SPEC_VERSIONS) { + expect(isStatefulVersion(v)).toBe(false); + } + }); + it('currently contains only the draft version', () => { + expect(STATELESS_SPEC_VERSIONS).toEqual([DRAFT_PROTOCOL_VERSION]); + }); +}); + describe('connectStateless', () => { const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); diff --git a/src/connection/select.ts b/src/connection/select.ts index ca33b4e6..aa6fe171 100644 --- a/src/connection/select.ts +++ b/src/connection/select.ts @@ -1,4 +1,8 @@ -import type { SpecVersion } from '../types'; +import { + DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION, + type SpecVersion +} from '../types'; import type { Connection } from './index'; import { connectStateful } from './stateful'; import { connectStateless } from './stateless'; @@ -14,10 +18,30 @@ const STATEFUL_VERSIONS: ReadonlySet = new Set([ '2025-11-25' ]); +/** Every spec version the suite can target, in timeline order. */ +const ALL_SPEC_VERSIONS: readonly SpecVersion[] = [ + ...DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION +]; + +export function isStatefulVersion(v: SpecVersion): boolean { + return STATEFUL_VERSIONS.has(v); +} + +/** + * Spec versions that use the stateless lifecycle, derived from + * {@link isStatefulVersion} so there is a single source of truth for the + * version→lifecycle mapping. The list grows automatically when the draft is + * dated (added to `DATED_SPEC_VERSIONS` without joining `STATEFUL_VERSIONS`) + * or a second stateless version appears. + */ +export const STATELESS_SPEC_VERSIONS: readonly SpecVersion[] = + ALL_SPEC_VERSIONS.filter((v) => !isStatefulVersion(v)); + export function connectFor( specVersion: SpecVersion ): (serverUrl: string) => Promise { - return STATEFUL_VERSIONS.has(specVersion) + return isStatefulVersion(specVersion) ? connectStateful : // Pass the version through so stateless requests declare the spec // version the run was invoked with (matters under --force). diff --git a/src/index.ts b/src/index.ts index 637a3577..ba507bb4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -255,7 +255,12 @@ program // If no command provided, run in interactive mode if (!validated.command) { - await runInteractiveMode(validated.scenario, verbose, outputDir); + await runInteractiveMode( + validated.scenario, + verbose, + outputDir, + specVersionFilter + ); process.exit(0); } diff --git a/src/mock-server/index.ts b/src/mock-server/index.ts new file mode 100644 index 00000000..7303cb83 --- /dev/null +++ b/src/mock-server/index.ts @@ -0,0 +1,60 @@ +/** + * Version-aware mock-server abstraction for client-conformance scenarios. + * + * A `MockServer` is the HTTP server a client-under-test connects to. The + * lifecycle scaffold (initialize handshake vs per-request `_meta` validation) + * is supplied by the runner based on `--spec-version`; the scenario only + * provides per-method handlers and asserts on the recorded requests. + * + * This is the client-conformance mirror of `Connection` in `../connection`. + */ + +import type { SpecVersion } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; + +/** + * Per-method response handlers. Called with the request `params` object; + * return value becomes the JSON-RPC `result`. Throw to produce an error + * response. + */ +export type RequestHandlers = Record< + string, + ( + params: Record, + request: JSONRPCRequest + ) => unknown | Promise +>; + +export interface MockServer { + /** Full URL of the `/mcp` endpoint. */ + url: string; + /** Base URL (no `/mcp` suffix), for scenarios that serve sibling routes. */ + baseUrl: string; + /** + * Every JSON-RPC request the client sent, in arrival order, excluding the + * lifecycle preamble (`initialize` / `notifications/initialized` under the + * stateful impl; `server/discover` under stateless). Recording happens + * before validation, so requests the server rejects (e.g. missing header + * or `_meta`) still appear here. + */ + readonly recorded: JSONRPCRequest[]; + close(): Promise; +} + +/** + * Per-run context handed to `Scenario.start()`. The runner constructs this + * from the resolved `--spec-version`. + */ +export interface ScenarioContext { + specVersion: SpecVersion; + /** + * Create a version-appropriate mock server. Scenarios that test the + * lifecycle itself (initialize, SSE-retry) bypass this and build a raw + * `http.createServer`. + */ + createServer(handlers: RequestHandlers): Promise; +} + +export { createServerStateful } from './stateful'; +export { createServerStateless, validateStatelessRequest } from './stateless'; +export { createServerFor } from './select'; diff --git a/src/mock-server/mock-server.test.ts b/src/mock-server/mock-server.test.ts new file mode 100644 index 00000000..c0247f88 --- /dev/null +++ b/src/mock-server/mock-server.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect } from 'vitest'; +import { createServerFor } from './select'; +import { createServerStateful } from './stateful'; +import { createServerStateless, validateStatelessRequest } from './stateless'; +import { STATELESS_SPEC_VERSIONS } from '../connection/select'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; + +const meta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientInfo': { name: 't', version: '1' }, + 'io.modelcontextprotocol/clientCapabilities': {} +}; +const headers = { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION }; + +async function post(url: string, body: object, headers: object = {}) { + const r = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: JSON.stringify(body) + }); + return { status: r.status, body: await r.json() }; +} + +describe('validateStatelessRequest', () => { + it('returns reject for invalid requests', () => { + const v = validateStatelessRequest( + { + headers: {}, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'reject', status: 400 }); + }); + + it('returns handled for server/discover', () => { + const v = validateStatelessRequest( + { + headers, + body: { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'handled', status: 200 }); + }); + + it('returns route for valid non-discover requests', () => { + const v = validateStatelessRequest( + { + headers, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ kind: 'route', id: 1, method: 'tools/list' }); + }); + + it('rejects versions outside the supported list with -32004 and echoes it', () => { + const v = validateStatelessRequest( + { + headers: { 'mcp-protocol-version': '2099-01-01' }, + body: { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { + _meta: { + ...meta, + 'io.modelcontextprotocol/protocolVersion': '2099-01-01' + } + } + } + }, + {}, + [DRAFT_PROTOCOL_VERSION] + ); + expect(v).toMatchObject({ + kind: 'reject', + status: 400, + body: { + error: { + code: -32004, + data: { supported: [DRAFT_PROTOCOL_VERSION], requested: '2099-01-01' } + } + } + }); + }); +}); + +describe('createServerFor', () => { + it('returns stateful for dated 2025-x versions', () => { + expect(createServerFor('2025-06-18')).toBe(createServerStateful); + expect(createServerFor('2025-11-25')).toBe(createServerStateful); + }); + it('returns a stateless factory bound to the requested version', async () => { + const srv = await createServerFor(DRAFT_PROTOCOL_VERSION)({}); + try { + const { status, body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + headers + ); + expect(status).toBe(200); + expect(body.result.supportedVersions).toEqual([DRAFT_PROTOCOL_VERSION]); + } finally { + await srv.close(); + } + }); +}); + +describe('createServerStateless', () => { + it('rejects requests missing the version header', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post(srv.url, { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }); + expect(status).toBe(400); + expect(body.error.code).toBe(-32001); + } finally { + await srv.close(); + } + }); + + it('rejects requests missing required _meta keys', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(400); + expect(body.error.code).toBe(-32602); + } finally { + await srv.close(); + } + }); + + it('serves server/discover, defaulting to every known stateless version', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(200); + expect(body.result.supportedVersions).toEqual(STATELESS_SPEC_VERSIONS); + expect(body.result.serverInfo.name).toBe('conformance-mock-server'); + } finally { + await srv.close(); + } + }); + + it('accepts the version it was created for and rejects others with -32004', async () => { + const srv = await createServerStateless( + { 'tools/list': () => ({ tools: [] }) }, + DRAFT_PROTOCOL_VERSION + ); + try { + const accepted = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }, + headers + ); + expect(accepted.status).toBe(200); + + const rejected = await post( + srv.url, + { + jsonrpc: '2.0', + id: 2, + method: 'tools/list', + params: { + _meta: { + ...meta, + 'io.modelcontextprotocol/protocolVersion': '2099-01-01' + } + } + }, + { 'mcp-protocol-version': '2099-01-01' } + ); + expect(rejected.status).toBe(400); + expect(rejected.body.error.code).toBe(-32004); + expect(rejected.body.error.data.supported).toEqual([ + DRAFT_PROTOCOL_VERSION + ]); + expect(rejected.body.error.data.requested).toBe('2099-01-01'); + } finally { + await srv.close(); + } + }); + + it('routes to handlers and records requests', async () => { + const srv = await createServerStateless({ + 'tools/list': () => ({ tools: [{ name: 'x' }] }) + }); + try { + const { body } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(body.result.tools[0].name).toBe('x'); + expect(srv.recorded).toHaveLength(1); + expect(srv.recorded[0].method).toBe('tools/list'); + } finally { + await srv.close(); + } + }); + + it('records requests rejected by validation (missing _meta)', async () => { + const srv = await createServerStateless({}); + try { + const { status } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'tools/list', params: {} }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(400); + expect(srv.recorded.map((r) => r.method)).toEqual(['tools/list']); + } finally { + await srv.close(); + } + }); + + it('does not record the server/discover preamble', async () => { + const srv = await createServerStateless({}); + try { + const { status } = await post( + srv.url, + { + jsonrpc: '2.0', + id: 1, + method: 'server/discover', + params: { _meta: meta } + }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(200); + expect(srv.recorded).toHaveLength(0); + } finally { + await srv.close(); + } + }); + + it('returns -32601 for unknown methods', async () => { + const srv = await createServerStateless({}); + try { + const { status, body } = await post( + srv.url, + { jsonrpc: '2.0', id: 1, method: 'nope', params: { _meta: meta } }, + { 'mcp-protocol-version': DRAFT_PROTOCOL_VERSION } + ); + expect(status).toBe(404); + expect(body.error.code).toBe(-32601); + } finally { + await srv.close(); + } + }); +}); + +describe('createServerStateful', () => { + async function postInit(url: string) { + const r = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-11-25', + capabilities: {}, + clientInfo: { name: 't', version: '1' } + } + }) + }); + return { + status: r.status, + contentType: r.headers.get('content-type') ?? '' + }; + } + + it('accepts initialize and routes to handlers, recording non-preamble', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + // SDK transport in sessionless mode handles initialize internally; we + // can drive it via the SDK Client. + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client.listTools(); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toEqual(['tools/list']); + } finally { + await srv.close(); + } + }); + + it('derives capabilities from handler keys; non-tools handler does not 500 initialize', async () => { + const srv = await createServerStateful({ + 'prompts/list': () => ({ prompts: [] }) + }); + try { + const { status, contentType } = await postInit(srv.url); + expect(status).toBe(200); + expect(contentType).not.toContain('text/html'); + } finally { + await srv.close(); + } + }); + + it('records requests for unregistered methods (parity with stateless)', async () => { + const srv = await createServerStateful({ + 'tools/list': () => ({ tools: [] }) + }); + try { + const { Client } = + await import('@modelcontextprotocol/sdk/client/index.js'); + const { StreamableHTTPClientTransport } = + await import('@modelcontextprotocol/sdk/client/streamableHttp.js'); + const { ResultSchema } = + await import('@modelcontextprotocol/sdk/types.js'); + const client = new Client( + { name: 't', version: '1' }, + { capabilities: {} } + ); + await client.connect(new StreamableHTTPClientTransport(new URL(srv.url))); + await client + .request( + { method: 'tools/call', params: { name: 'nope' } }, + ResultSchema + ) + .catch(() => {}); + await client.close(); + expect(srv.recorded.map((r) => r.method)).toContain('tools/call'); + } finally { + await srv.close(); + } + }); +}); diff --git a/src/mock-server/select.ts b/src/mock-server/select.ts new file mode 100644 index 00000000..34586478 --- /dev/null +++ b/src/mock-server/select.ts @@ -0,0 +1,13 @@ +import type { SpecVersion } from '../types'; +import type { MockServer, RequestHandlers } from './index'; +import { isStatefulVersion } from '../connection/select'; +import { createServerStateful } from './stateful'; +import { createServerStateless } from './stateless'; + +export function createServerFor( + specVersion: SpecVersion +): (handlers: RequestHandlers) => Promise { + return isStatefulVersion(specVersion) + ? createServerStateful + : (handlers) => createServerStateless(handlers, specVersion); +} diff --git a/src/mock-server/stateful.ts b/src/mock-server/stateful.ts new file mode 100644 index 00000000..8dab5414 --- /dev/null +++ b/src/mock-server/stateful.ts @@ -0,0 +1,145 @@ +/** + * Stateful mock server: 2025-x lifecycle (initialize handshake). + * + * Backed by the SDK's `Server` + `StreamableHTTPServerTransport` so we don't + * reimplement the handshake or SSE response framing. The SDK is the scaffold + * here, not the system-under-test; the client-under-test connecting to this + * mock is what's being verified. + */ + +import express from 'express'; +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; +import { z } from 'zod'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; + +const CAPABILITY_BY_PREFIX: Record = { + tools: 'tools', + prompts: 'prompts', + resources: 'resources', + completion: 'completions', + logging: 'logging' +}; + +/** + * Derive the server `capabilities` object from the registered handler method + * names so the SDK's `assertRequestHandlerCapability` gate is always satisfied. + * Shared with the stateless impl for `server/discover`. + */ +export function capabilitiesFromHandlers( + handlers: RequestHandlers +): Record { + const out: Record = {}; + for (const method of Object.keys(handlers)) { + const cap = CAPABILITY_BY_PREFIX[method.split('/')[0]]; + if (cap) out[cap] = {}; + } + return out; +} + +export async function createServerStateful( + handlers: RequestHandlers +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + + // Fresh SDK Server per HTTP request (the SDK transport is single-shot in + // sessionless mode after GHSA-345p-7cg4-v4c7). + function newServer(): Server { + const server = new Server( + { name: 'conformance-mock-server', version: '1.0.0' }, + { capabilities } + ); + for (const [method, handler] of Object.entries(handlers)) { + // The SDK's setRequestHandler matches by parsing against the schema's + // method literal; build a minimal schema so any method string works. + const schema = z.object({ + method: z.literal(method), + params: z.unknown().optional() + }); + server.setRequestHandler(schema, async (request) => { + try { + return (await handler( + (request.params ?? {}) as Record, + request as JSONRPCRequest + )) as Record; + } catch (e) { + if (e instanceof McpError) throw e; + throw new McpError( + ErrorCode.InternalError, + e instanceof Error ? e.message : String(e) + ); + } + }); + } + return server; + } + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + // Record every JSON-RPC request the client sends (excluding the lifecycle + // preamble) at the HTTP layer so unregistered methods are captured too, + // matching the stateless impl and the MockServer.recorded contract. + const body = req.body; + if ( + body?.method && + body.method !== 'initialize' && + body.method !== 'notifications/initialized' + ) { + recorded.push(body as JSONRPCRequest); + } + try { + const server = newServer(); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined + }); + await server.connect(transport); + // Register cleanup before handing the request to the transport so the + // pair is torn down even when handleRequest throws. + res.on('close', () => { + transport.close(); + server.close(); + }); + await transport.handleRequest(req, res, req.body); + } catch (e) { + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + id: req.body?.id ?? null, + error: { code: -32603, message: String(e) } + }); + } + } + }); + + return listen(app, recorded); +} + +function listen( + app: express.Application, + recorded: JSONRPCRequest[] +): Promise { + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((res) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => res()); + }) + }); + }); + }); +} diff --git a/src/mock-server/stateless.ts b/src/mock-server/stateless.ts new file mode 100644 index 00000000..78fd6384 --- /dev/null +++ b/src/mock-server/stateless.ts @@ -0,0 +1,194 @@ +/** + * Stateless mock server: 2026-x lifecycle (SEP-2575). + * + * No initialize handshake. Validates `_meta` (protocolVersion / clientInfo / + * clientCapabilities) and the `MCP-Protocol-Version` header on every request, + * serves `server/discover`, and routes other methods to the supplied handlers. + * Implemented with raw express so it can front-run SDK support. + */ + +import express from 'express'; +import type { SpecVersion } from '../types'; +import type { JSONRPCRequest } from '../spec-types/2025-11-25'; +import type { MockServer, RequestHandlers } from './index'; +import { STATELESS_SPEC_VERSIONS } from '../connection/select'; +import { capabilitiesFromHandlers } from './stateful'; + +const META_KEYS = [ + 'io.modelcontextprotocol/protocolVersion', + 'io.modelcontextprotocol/clientInfo', + 'io.modelcontextprotocol/clientCapabilities' +] as const; + +type IncomingHeaders = Record; + +export type StatelessValidation = + | { kind: 'reject'; status: number; body: object } + | { kind: 'handled'; status: number; body: object } + | { + kind: 'route'; + id: string | number | null; + method: string; + params: Record; + }; + +/** + * Shared SEP-2575 request validation: header presence, `_meta` 3-key check, + * header/`_meta` version match, version-supported check, and `server/discover` + * handling. Returns `reject` when validation failed, `handled` when the + * request was valid and already answered (`server/discover`), and `route` + * when the caller should dispatch to its own handlers. Consumers write + * `res.status(v.status).json(v.body)` for `reject` and `handled` alike and + * route only on `route`. + * + * Exported so any mock server that needs a stateless `/mcp` route (e.g. + * `auth/helpers/createServer.ts`) uses the same validation as this module. + * + * `supportedVersions` is the list of wire protocolVersion strings this + * endpoint accepts; anything else is rejected with -32004 carrying + * `{ supported, requested }` in the error data, and the list is echoed in + * the `server/discover` result. + */ +export function validateStatelessRequest( + req: { headers: IncomingHeaders; body: unknown }, + capabilities: Record, + supportedVersions: readonly string[] +): StatelessValidation { + const body = (req.body ?? {}) as Record; + const id = (body.id ?? null) as string | number | null; + const method = body.method as string; + const params = (body.params ?? {}) as Record; + const meta = params._meta as Record | undefined; + + const reject = (status: number, code: number, message: string) => + ({ + kind: 'reject', + status, + body: { jsonrpc: '2.0', id, error: { code, message } } + }) as const; + + const headerVersion = req.headers['mcp-protocol-version']; + if (!headerVersion) { + return reject(400, -32001, 'Missing MCP-Protocol-Version header'); + } + const missing = META_KEYS.filter((k) => meta?.[k] === undefined); + if (missing.length > 0) { + return reject( + 400, + -32602, + `Invalid params: missing _meta keys: ${missing.join(', ')}` + ); + } + if (meta?.[META_KEYS[0]] !== headerVersion) { + return reject( + 400, + -32001, + 'MCP-Protocol-Version header does not match _meta.protocolVersion' + ); + } + if ( + typeof headerVersion !== 'string' || + !supportedVersions.includes(headerVersion) + ) { + return { + kind: 'reject', + status: 400, + body: { + jsonrpc: '2.0', + id, + error: { + code: -32004, + message: 'Unsupported protocol version', + data: { + supported: supportedVersions, + requested: String(headerVersion) + } + } + } + }; + } + if (method === 'server/discover') { + return { + kind: 'handled', + status: 200, + body: { + jsonrpc: '2.0', + id, + result: { + supportedVersions, + capabilities, + serverInfo: { name: 'conformance-mock-server', version: '1.0.0' } + } + } + }; + } + return { kind: 'route', id, method, params }; +} + +/** + * When `specVersion` is given (the runner's resolved `--spec-version`), the + * server accepts exactly that version. Without it, every known stateless + * version is accepted. + */ +export async function createServerStateless( + handlers: RequestHandlers, + specVersion?: SpecVersion +): Promise { + const recorded: JSONRPCRequest[] = []; + const capabilities = capabilitiesFromHandlers(handlers); + const supportedVersions: readonly string[] = specVersion + ? [specVersion] + : STATELESS_SPEC_VERSIONS; + + const app = express(); + app.use(express.json()); + + app.post('/mcp', async (req, res) => { + // Record every JSON-RPC request the client sends (excluding the + // `server/discover` lifecycle preamble) before validation so rejected + // requests are captured too, matching the stateful impl and the + // MockServer.recorded contract. + const body = req.body as Record | undefined; + if (body?.method && body.method !== 'server/discover') { + recorded.push(req.body as JSONRPCRequest); + } + const v = validateStatelessRequest(req, capabilities, supportedVersions); + if (v.kind !== 'route') { + return res.status(v.status).json(v.body); + } + const { id, method, params } = v; + const error = (status: number, code: number, message: string) => + res.status(status).json({ jsonrpc: '2.0', id, error: { code, message } }); + + const handler = handlers[method]; + if (!handler) { + return error(404, -32601, `Method not found: ${method}`); + } + try { + const result = await handler(params, req.body as JSONRPCRequest); + return res.json({ jsonrpc: '2.0', id, result }); + } catch (e) { + return error(500, -32603, e instanceof Error ? e.message : String(e)); + } + }); + + return new Promise((resolve, reject) => { + const httpServer = app.listen(0); + httpServer.on('error', reject); + httpServer.on('listening', () => { + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + const baseUrl = `http://localhost:${port}`; + resolve({ + url: `${baseUrl}/mcp`, + baseUrl, + recorded, + close: () => + new Promise((r) => { + httpServer.closeAllConnections?.(); + httpServer.close(() => r()); + }) + }); + }); + }); +} diff --git a/src/mock-server/testing.ts b/src/mock-server/testing.ts new file mode 100644 index 00000000..1e307b8d --- /dev/null +++ b/src/mock-server/testing.ts @@ -0,0 +1,17 @@ +import { LATEST_SPEC_VERSION, type SpecVersion } from '../types'; +import { createServerFor } from './select'; +import type { ScenarioContext } from './index'; + +/** + * Build a ScenarioContext for unit tests that drive a Scenario directly. + * Defaults to the latest dated spec version (stateful lifecycle) so existing + * tests keep their pre-ScenarioContext behaviour. + */ +export function testScenarioContext( + specVersion: SpecVersion = LATEST_SPEC_VERSION +): ScenarioContext { + return { + specVersion, + createServer: (handlers) => createServerFor(specVersion)(handlers) + }; +} diff --git a/src/runner/client.ts b/src/runner/client.ts index 1bf8c9f6..1521912b 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,8 +1,9 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck, SpecVersion } from '../types'; +import { ConformanceCheck, SpecVersion, LATEST_SPEC_VERSION } from '../types'; import { getScenario } from '../scenarios'; +import { createServerFor, type ScenarioContext } from '../mock-server'; import { createResultDir, formatPrettyChecks } from './utils'; export interface ClientExecutionResult { @@ -34,10 +35,9 @@ async function executeClient( // 2. Simpler to read a string vs parsing JSON just to get the scenario name // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; + const resolvedVersion = specVersion ?? LATEST_SPEC_VERSION; env.MCP_CONFORMANCE_SCENARIO = scenarioName; - if (specVersion) { - env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion; - } + env.MCP_CONFORMANCE_PROTOCOL_VERSION = resolvedVersion; if (context) { // Include scenario name in context for discriminated union parsing env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ @@ -114,8 +114,14 @@ export async function runConformanceTest( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + const resolvedVersion = specVersion ?? LATEST_SPEC_VERSION; + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.error(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.error(`Executing client: ${clientCommand} ${urls.serverUrl}`); if (urls.context) { @@ -129,7 +135,7 @@ export async function runConformanceTest( urls.serverUrl, timeout, urls.context, - specVersion + resolvedVersion ); // Print stdout/stderr if client exited with nonzero code @@ -269,7 +275,8 @@ export function printClientResults( export async function runInteractiveMode( scenarioName: string, verbose: boolean = false, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion ): Promise { let resultDir: string | undefined; @@ -281,8 +288,14 @@ export async function runInteractiveMode( // Scenario is guaranteed to exist by CLI validation const scenario = getScenario(scenarioName)!; + const resolvedVersion = specVersion ?? LATEST_SPEC_VERSION; + const ctx: ScenarioContext = { + specVersion: resolvedVersion, + createServer: (handlers) => createServerFor(resolvedVersion)(handlers) + }; + console.log(`Starting scenario: ${scenarioName}`); - const urls = await scenario.start(); + const urls = await scenario.start(ctx); console.log(`Server URL: ${urls.serverUrl}`); console.log('Press Ctrl+C to stop...'); diff --git a/src/scenarios/client/auth/authorization-server-migration.ts b/src/scenarios/client/auth/authorization-server-migration.ts index 858bddb5..9fe8add2 100644 --- a/src/scenarios/client/auth/authorization-server-migration.ts +++ b/src/scenarios/client/auth/authorization-server-migration.ts @@ -6,6 +6,7 @@ * to AS₂. On the next 401 the client re-discovers PRM, sees a new issuer, and * MUST re-register with AS₂ rather than reuse AS₁'s client credentials. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Request, Response, NextFunction } from 'express'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; @@ -25,7 +26,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, ['mcp:basic']); @@ -125,6 +126,7 @@ export class AuthorizationServerMigrationScenario implements Scenario { }; const app = createServer( + ctx, this.checks, this.server.getUrl, currentAuthServerUrl, diff --git a/src/scenarios/client/auth/basic-cimd.ts b/src/scenarios/client/auth/basic-cimd.ts index a99b4e35..7a9969fc 100644 --- a/src/scenarios/client/auth/basic-cimd.ts +++ b/src/scenarios/client/auth/basic-cimd.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -29,7 +30,7 @@ export class AuthBasicCIMDScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const authApp = createAuthServer(this.checks, this.authServer.getUrl, { @@ -60,6 +61,7 @@ export class AuthBasicCIMDScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/client-credentials.ts b/src/scenarios/client/auth/client-credentials.ts index eaebcb01..1171e5f0 100644 --- a/src/scenarios/client/auth/client-credentials.ts +++ b/src/scenarios/client/auth/client-credentials.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; @@ -42,7 +43,7 @@ export class ClientCredentialsJwtScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate a fresh keypair for this test run @@ -201,6 +202,7 @@ export class ClientCredentialsJwtScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl @@ -263,7 +265,7 @@ export class ClientCredentialsBasicScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const authApp = createAuthServer(this.checks, this.authServer.getUrl, { @@ -364,6 +366,7 @@ export class ClientCredentialsBasicScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl diff --git a/src/scenarios/client/auth/discovery-metadata.ts b/src/scenarios/client/auth/discovery-metadata.ts index 7387ead4..16b3dd9f 100644 --- a/src/scenarios/client/auth/discovery-metadata.ts +++ b/src/scenarios/client/auth/discovery-metadata.ts @@ -6,6 +6,7 @@ * generated from them. */ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -94,7 +95,7 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { **OAuth metadata:** ${config.oauthMetadataLocation} `, - async start(): Promise { + async start(ctx: ScenarioContext): Promise { checks = []; const authApp = createAuthServer(checks, authServer.getUrl, { @@ -131,7 +132,7 @@ function createMetadataScenario(config: MetadataScenarioConfig): Scenario { ? () => `${authServer.getUrl()}${routePrefix}` : authServer.getUrl; - const app = createServer(checks, server.getUrl, getAuthServerUrl, { + const app = createServer(ctx, checks, server.getUrl, getAuthServerUrl, { prmPath: config.prmLocation, includePrmInWwwAuth: config.inWwwAuth }); diff --git a/src/scenarios/client/auth/enterprise-managed-authorization.ts b/src/scenarios/client/auth/enterprise-managed-authorization.ts index ddadfbf5..b89a2474 100644 --- a/src/scenarios/client/auth/enterprise-managed-authorization.ts +++ b/src/scenarios/client/auth/enterprise-managed-authorization.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import * as jose from 'jose'; import type { CryptoKey } from 'jose'; import express, { type Request, type Response } from 'express'; @@ -69,7 +70,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { private idpPrivateKey?: CryptoKey; private grantKeypairs: Map = new Map(); - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Generate IDP keypair @@ -120,6 +121,7 @@ export class EnterpriseManagedAuthorizationScenario implements Scenario { // Start MCP server with shared token verifier const mcpApp = createServer( + ctx, this.checks, this.mcpServer.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/helpers/createServer.ts b/src/scenarios/client/auth/helpers/createServer.ts index c836630d..8cf04d97 100644 --- a/src/scenarios/client/auth/helpers/createServer.ts +++ b/src/scenarios/client/auth/helpers/createServer.ts @@ -10,6 +10,11 @@ import { import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js'; import express, { Request, Response, NextFunction } from 'express'; import type { ConformanceCheck } from '../../../../types'; +import { + validateStatelessRequest, + type ScenarioContext +} from '../../../../mock-server'; +import { isStatefulVersion } from '../../../../connection/select'; import { createRequestLogger } from '../../../request-logger'; import { MockTokenVerifier } from './mockTokenVerifier'; import { SpecReferences } from '../spec-references'; @@ -27,6 +32,7 @@ export interface ServerOptions { } export function createServer( + ctx: ScenarioContext, checks: ConformanceCheck[], getBaseUrl: () => string, getAuthServerUrl: () => string, @@ -157,6 +163,9 @@ export function createServer( authMiddleware(req, res, async (err?: any) => { if (err) return next(err); + if (!isStatefulVersion(ctx.specVersion)) { + return handleStateless(req, res); + } const server = createMcpServer(); const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined @@ -164,12 +173,13 @@ export function createServer( try { await server.connect(transport); - - await transport.handleRequest(req, res, req.body); + // Register cleanup before handing the request to the transport so the + // pair is torn down even when handleRequest throws. res.on('close', () => { transport.close(); server.close(); }); + await transport.handleRequest(req, res, req.body); } catch (error) { console.error('Error handling MCP request:', error); if (!res.headersSent) { @@ -186,5 +196,38 @@ export function createServer( }); }); + // Stateless lifecycle for the /mcp route: shared SEP-2575 validation + + // server/discover from mock-server/stateless, then the same tools handlers + // as createMcpServer. Bearer-auth middleware and PRM route above are + // version-independent. + function handleStateless(req: Request, res: Response) { + const v = validateStatelessRequest(req, { tools: {} }, [ctx.specVersion]); + if (v.kind !== 'route') { + return res.status(v.status).json(v.body); + } + const { id, method } = v; + if (method === 'tools/list') { + return res.json({ + jsonrpc: '2.0', + id, + result: { + tools: [{ name: 'test-tool', inputSchema: { type: 'object' } }] + } + }); + } + if (method === 'tools/call') { + return res.json({ + jsonrpc: '2.0', + id, + result: { content: [{ type: 'text', text: 'test' }] } + }); + } + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: 'Method not found' } + }); + } + return app; } diff --git a/src/scenarios/client/auth/issuer-parameter.ts b/src/scenarios/client/auth/issuer-parameter.ts index ecb2bc2b..81dac383 100644 --- a/src/scenarios/client/auth/issuer-parameter.ts +++ b/src/scenarios/client/auth/issuer-parameter.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -30,7 +31,7 @@ export class IssParameterSupportedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; @@ -48,6 +49,7 @@ export class IssParameterSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -99,7 +101,7 @@ export class IssParameterNotAdvertisedScenario implements Scenario { private checks: ConformanceCheck[] = []; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.tokenRequestMade = false; @@ -117,6 +119,7 @@ export class IssParameterNotAdvertisedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -170,7 +173,7 @@ export class IssParameterSupportedMissingScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -192,6 +195,7 @@ export class IssParameterSupportedMissingScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -256,7 +260,7 @@ export class IssParameterWrongIssuerScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -278,6 +282,7 @@ export class IssParameterWrongIssuerScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -343,7 +348,7 @@ export class IssParameterUnexpectedScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -365,6 +370,7 @@ export class IssParameterUnexpectedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -435,7 +441,7 @@ export class IssParameterNormalizedVariantScenario implements Scenario { private authReached = false; private tokenRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authReached = false; this.tokenRequestMade = false; @@ -457,6 +463,7 @@ export class IssParameterNormalizedVariantScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -522,7 +529,7 @@ export class MetadataIssuerMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private metadataEndpointsUsed = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.metadataEndpointsUsed = false; @@ -555,6 +562,7 @@ export class MetadataIssuerMismatchScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/march-spec-backcompat.ts b/src/scenarios/client/auth/march-spec-backcompat.ts index b2502092..68868b03 100644 --- a/src/scenarios/client/auth/march-spec-backcompat.ts +++ b/src/scenarios/client/auth/march-spec-backcompat.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -17,7 +18,7 @@ export class Auth20250326OAuthMetadataBackcompatScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; // Legacy server, so we create the auth server endpoints on the // same URL as the main server (rather than separating AS / RS). @@ -28,6 +29,7 @@ export class Auth20250326OAuthMetadataBackcompatScenario implements Scenario { routePrefix: '/oauth' }); const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, @@ -81,10 +83,11 @@ export class Auth20250326OEndpointFallbackScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const app = createServer( + ctx, this.checks, this.server.getUrl, this.server.getUrl, diff --git a/src/scenarios/client/auth/offline-access.ts b/src/scenarios/client/auth/offline-access.ts index afcd593f..2f927c69 100644 --- a/src/scenarios/client/auth/offline-access.ts +++ b/src/scenarios/client/auth/offline-access.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -33,7 +34,7 @@ export class OfflineAccessScopeScenario implements Scenario { private grantTypesChecked = false; private capturedCimdUrl: string | undefined; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.grantTypesChecked = false; this.capturedCimdUrl = undefined; @@ -101,6 +102,7 @@ export class OfflineAccessScopeScenario implements Scenario { // PRM does NOT include offline_access (per SEP-2207 server guidance: // servers SHOULD NOT include offline_access in PRM scopes_supported) const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -235,7 +237,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, [ @@ -268,6 +270,7 @@ export class OfflineAccessNotSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/pre-registration.ts b/src/scenarios/client/auth/pre-registration.ts index b482e4f3..eefdd033 100644 --- a/src/scenarios/client/auth/pre-registration.ts +++ b/src/scenarios/client/auth/pre-registration.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; import { createServer } from './helpers/createServer'; @@ -27,7 +28,7 @@ export class PreRegistrationScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); @@ -105,6 +106,7 @@ export class PreRegistrationScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts index 9c15d4d6..e59ebcc1 100644 --- a/src/scenarios/client/auth/resource-mismatch.ts +++ b/src/scenarios/client/auth/resource-mismatch.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -37,7 +38,7 @@ export class ResourceMismatchScenario implements Scenario { private checks: ConformanceCheck[] = []; private authorizationRequestMade = false; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationRequestMade = false; @@ -60,6 +61,7 @@ export class ResourceMismatchScenario implements Scenario { // Create server that returns a mismatched resource in PRM const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index c24d51d6..c1154ca2 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types'; import { ScenarioUrls } from '../../../types'; import { createAuthServer } from './helpers/createAuthServer'; @@ -22,7 +23,7 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const expectedScope = 'mcp:basic'; @@ -53,6 +54,7 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -108,7 +110,7 @@ export class ScopeFromScopesSupportedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const scopesSupported = ['mcp:basic', 'mcp:read', 'mcp:write']; @@ -148,6 +150,7 @@ export class ScopeFromScopesSupportedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -204,7 +207,7 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const tokenVerifier = new MockTokenVerifier(this.checks, []); @@ -232,6 +235,7 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -291,7 +295,7 @@ export class ScopeStepUpAuthScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const initialScope = 'mcp:basic'; @@ -437,6 +441,7 @@ export class ScopeStepUpAuthScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, @@ -528,7 +533,7 @@ export class ScopeRetryLimitScenario implements Scenario { private server = new ServerLifecycle(); private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; const requiredScope = 'mcp:admin'; @@ -613,6 +618,7 @@ export class ScopeRetryLimitScenario implements Scenario { }; const baseApp = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/auth/test_helpers/testClient.ts b/src/scenarios/client/auth/test_helpers/testClient.ts index 0dbc8d8d..c28e0811 100644 --- a/src/scenarios/client/auth/test_helpers/testClient.ts +++ b/src/scenarios/client/auth/test_helpers/testClient.ts @@ -1,4 +1,5 @@ import { getScenario } from '../../../index'; +import { testScenarioContext } from '../../../../mock-server/testing'; import { spawn } from 'child_process'; const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete @@ -103,13 +104,15 @@ export async function runClientAgainstScenario( } // Start the scenario server - const urls = await scenario.start(); + const ctx = testScenarioContext(); + const urls = await scenario.start(ctx); const serverUrl = urls.serverUrl; try { // Set environment variables for inline clients // These mirror what src/runner/client.ts does for spawned processes process.env.MCP_CONFORMANCE_SCENARIO = scenarioName; + process.env.MCP_CONFORMANCE_PROTOCOL_VERSION = ctx.specVersion; if (urls.context) { process.env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ name: scenarioName, @@ -179,6 +182,7 @@ export async function runClientAgainstScenario( // Clean up environment variables delete process.env.MCP_CONFORMANCE_SCENARIO; delete process.env.MCP_CONFORMANCE_CONTEXT; + delete process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; // Stop the scenario server await scenario.stop(); diff --git a/src/scenarios/client/auth/token-endpoint-auth.ts b/src/scenarios/client/auth/token-endpoint-auth.ts index 79d62968..7b7616ff 100644 --- a/src/scenarios/client/auth/token-endpoint-auth.ts +++ b/src/scenarios/client/auth/token-endpoint-auth.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../../mock-server'; import type { Scenario, ConformanceCheck } from '../../../types.js'; import { ScenarioUrls } from '../../../types.js'; import { createAuthServer } from './helpers/createAuthServer.js'; @@ -62,7 +63,7 @@ class TokenEndpointAuthScenario implements Scenario { this.description = `Tests that client uses ${AUTH_METHOD_NAMES[expectedAuthMethod]} when server only supports ${expectedAuthMethod}`; } - async start(): Promise { + async start(ctx: ScenarioContext): Promise { this.checks = []; this.authorizationResource = undefined; this.tokenResource = undefined; @@ -137,6 +138,7 @@ class TokenEndpointAuthScenario implements Scenario { await this.authServer.start(authApp); const app = createServer( + ctx, this.checks, this.server.getUrl, this.authServer.getUrl, diff --git a/src/scenarios/client/elicitation-defaults.ts b/src/scenarios/client/elicitation-defaults.ts index c78f3495..d4170295 100644 --- a/src/scenarios/client/elicitation-defaults.ts +++ b/src/scenarios/client/elicitation-defaults.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-1034: Elicitation defaults test * Validates that clients properly apply default values for omitted fields @@ -13,7 +14,7 @@ import { } from '@modelcontextprotocol/sdk/types.js'; import type { Scenario, ConformanceCheck } from '../../types'; import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; +import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../types'; import { createRequestLogger } from '../request-logger'; import { randomUUID } from 'crypto'; @@ -474,7 +475,10 @@ function createServer(checks: ConformanceCheck[]): { export class ElicitationClientDefaultsScenario implements Scenario { name = 'elicitation-sep1034-client-defaults'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests client applies default values for omitted elicitation fields (SEP-1034)'; private app: express.Application | null = null; @@ -482,7 +486,7 @@ export class ElicitationClientDefaultsScenario implements Scenario { private checks: ConformanceCheck[] = []; private cleanup: (() => void) | null = null; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; const { app, cleanup } = createServer(this.checks); this.app = app; diff --git a/src/scenarios/client/http-base.ts b/src/scenarios/client/http-base.ts index 06c2afaf..9f6187d3 100644 --- a/src/scenarios/client/http-base.ts +++ b/src/scenarios/client/http-base.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * Shared HTTP test-server scaffold for client-under-test SEP-2243 scenarios. * @@ -27,7 +28,7 @@ export abstract class BaseHttpScenario implements Scenario { protected port: number = 0; protected sessionId: string = `session-${Date.now()}`; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/http-custom-headers.test.ts b/src/scenarios/client/http-custom-headers.test.ts index 88714eea..8a68f22a 100644 --- a/src/scenarios/client/http-custom-headers.test.ts +++ b/src/scenarios/client/http-custom-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpCustomHeadersScenario, @@ -43,7 +44,7 @@ function statusesFor( describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('emits exactly the declared requirement IDs as FAILURE when the client never connects', async () => { const scenario = new HttpCustomHeadersScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); expect(idsOf(checks)).toEqual(new Set(CUSTOM_HEADERS_DECLARED_CHECK_IDS)); @@ -57,7 +58,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('maps each parameter kind to its requirement ID on a conforming tool call', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { const nonAscii = 'Hello, 世界'; const nonAsciiB64 = Buffer.from(nonAscii, 'utf-8').toString('base64'); @@ -123,7 +124,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { it('FAILs client-mirrors-designated-params when an annotated header is missing', async () => { const scenario = new HttpCustomHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post( serverUrl, @@ -154,7 +155,7 @@ describe('HttpCustomHeadersScenario (SEP-2243) check IDs', () => { describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('emits every x-mcp-header constraint ID, SUCCESS when only valid_tool is called', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { @@ -176,7 +177,7 @@ describe('HttpInvalidToolHeadersScenario (SEP-2243) check IDs', () => { it('FAILs the violated constraint ID when the client calls an invalid tool', async () => { const scenario = new HttpInvalidToolHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await post(serverUrl, { jsonrpc: '2.0', id: 1, method: 'tools/list' }); await post(serverUrl, { diff --git a/src/scenarios/client/http-custom-headers.ts b/src/scenarios/client/http-custom-headers.ts index e91cf2c0..25e36610 100644 --- a/src/scenarios/client/http-custom-headers.ts +++ b/src/scenarios/client/http-custom-headers.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * HTTP Custom Headers conformance test scenario for MCP clients (SEP-2243) * @@ -186,8 +187,8 @@ export class HttpCustomHeadersScenario extends BaseHttpScenario { private toolCallReceived: boolean = false; private nullToolCallReceived: boolean = false; - async start(): Promise { - const urls = await super.start(); + async start(_ctx: ScenarioContext): Promise { + const urls = await super.start(_ctx); // Pass test values via context for encoding edge cases. // The conformance client should use these values when calling test_custom_headers. urls.context = { diff --git a/src/scenarios/client/http-standard-headers.test.ts b/src/scenarios/client/http-standard-headers.test.ts index 8c5ff6ad..3478e060 100644 --- a/src/scenarios/client/http-standard-headers.test.ts +++ b/src/scenarios/client/http-standard-headers.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, it, expect } from 'vitest'; import { HttpStandardHeadersScenario } from './http-standard-headers'; @@ -38,7 +39,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('FAILs the initialize Mcp-Method emission when Mcp-Method is missing', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, {}); // no Mcp-Method header const checks = scenario.getChecks(); @@ -53,7 +54,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('SUCCEEDs the initialize Mcp-Method emission when Mcp-Method matches', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const checks = scenario.getChecks(); @@ -68,7 +69,7 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { it('getChecks() is idempotent', async () => { const scenario = new HttpStandardHeadersScenario(); - const { serverUrl } = await scenario.start(); + const { serverUrl } = await scenario.start(testScenarioContext()); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const first = scenario.getChecks(); diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index b5e2aef2..be8d7082 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,23 +1,28 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, ScenarioUrls, ConformanceCheck, LATEST_SPEC_VERSION, - NEGOTIABLE_PROTOCOL_VERSIONS + NEGOTIABLE_PROTOCOL_VERSIONS, + DRAFT_PROTOCOL_VERSION } from '../../types'; import { clientChecks } from '../../checks/index'; export class InitializeScenario implements Scenario { name = 'initialize'; - readonly source = { introducedIn: '2025-06-18' } as const; + readonly source = { + introducedIn: '2025-06-18', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests MCP client initialization handshake'; private server: http.Server | null = null; private checks: ConformanceCheck[] = []; private port: number = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/json-schema-ref-deref.test.ts b/src/scenarios/client/json-schema-ref-deref.test.ts index b11d2d25..a3091933 100644 --- a/src/scenarios/client/json-schema-ref-deref.test.ts +++ b/src/scenarios/client/json-schema-ref-deref.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; @@ -92,7 +93,7 @@ describe('json-schema-ref-no-deref (SEP-2106)', () => { test('client that never lists tools fails: requirement cannot be evaluated', async () => { const scenario = new JsonSchemaRefDerefScenario(); - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const check = checks.find( diff --git a/src/scenarios/client/json-schema-ref-deref.ts b/src/scenarios/client/json-schema-ref-deref.ts index 91ccc9e0..a7b315e0 100644 --- a/src/scenarios/client/json-schema-ref-deref.ts +++ b/src/scenarios/client/json-schema-ref-deref.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; @@ -82,7 +83,7 @@ The scenario advertises a tool whose inputSchema contains a \`$ref\` pointing at private canaryRequests: Array<{ method: string; userAgent?: string }> = []; private toolsListed = false; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.canaryRequests = []; this.toolsListed = false; diff --git a/src/scenarios/client/mrtr-client.ts b/src/scenarios/client/mrtr-client.ts index 431fafe6..4521ca04 100644 --- a/src/scenarios/client/mrtr-client.ts +++ b/src/scenarios/client/mrtr-client.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SEP-2322: MRTR Client Conformance Tests * @@ -440,7 +441,7 @@ export class MRTRClientScenario implements Scenario { private httpServer: ReturnType | null = null; private checks: ConformanceCheck[] = []; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.checks = []; this.app = createMRTRServer(this.checks); this.httpServer = this.app.listen(0); diff --git a/src/scenarios/client/request-metadata.test.ts b/src/scenarios/client/request-metadata.test.ts index b2f9b637..d1ef6759 100644 --- a/src/scenarios/client/request-metadata.test.ts +++ b/src/scenarios/client/request-metadata.test.ts @@ -1,3 +1,4 @@ +import { testScenarioContext } from '../../mock-server/testing'; import { describe, test, expect } from 'vitest'; import { runClientAgainstScenario, @@ -104,10 +105,9 @@ async function incompatibleVersionClient(serverUrl: string) { if (response.status === 400) { const body = await response.json(); - if (body.error?.code === -32004 || body.error?.code === -32001) { + if (body.error?.code === -32004) { return body; // Abort cleanly } - return body; } return response.json(); } @@ -207,7 +207,7 @@ describe('request-metadata client scenario — client never connects', () => { throw new Error('Scenario not found'); } - await scenario.start(); + await scenario.start(testScenarioContext()); try { const checks = scenario.getChecks(); const byId = new Map(checks.map((c) => [c.id, c])); diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts index 9d9fd69d..38576b5d 100644 --- a/src/scenarios/client/request-metadata.ts +++ b/src/scenarios/client/request-metadata.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; import http from 'http'; import { Scenario, @@ -46,7 +47,7 @@ export class RequestMetadataScenario implements Scenario { private hasSimulatedRejection = false; private requestsObserved = 0; - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { this.hasSimulatedRejection = false; this.checks = []; this.requestsObserved = 0; @@ -281,7 +282,8 @@ export class RequestMetadataScenario implements Scenario { code: -32004, message: 'Unsupported protocol version', data: { - supported: [DRAFT_PROTOCOL_VERSION] + supported: [DRAFT_PROTOCOL_VERSION], + requested: String(headerVersion ?? metaVersion ?? '') } } }) diff --git a/src/scenarios/client/sse-retry.ts b/src/scenarios/client/sse-retry.ts index b90bf40a..e89b23eb 100644 --- a/src/scenarios/client/sse-retry.ts +++ b/src/scenarios/client/sse-retry.ts @@ -1,3 +1,4 @@ +import type { ScenarioContext } from '../../mock-server'; /** * SSE Retry conformance test scenarios for MCP clients (SEP-1699) * @@ -8,11 +9,19 @@ */ import http from 'http'; -import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; +import { + Scenario, + ScenarioUrls, + ConformanceCheck, + DRAFT_PROTOCOL_VERSION +} from '../../types.js'; export class SSERetryScenario implements Scenario { name = 'sse-retry'; - readonly source = { introducedIn: '2025-11-25' } as const; + readonly source = { + introducedIn: '2025-11-25', + removedIn: DRAFT_PROTOCOL_VERSION + } as const; description = 'Tests that client respects SSE retry field timing and reconnects properly (SEP-1699)'; @@ -38,7 +47,7 @@ export class SSERetryScenario implements Scenario { private readonly LATE_TOLERANCE = 200; // Allow 200ms late for network/event loop private readonly VERY_LATE_MULTIPLIER = 2; // If >2x retry value, client is likely ignoring it - async start(): Promise { + async start(_ctx: ScenarioContext): Promise { return new Promise((resolve, reject) => { this.server = http.createServer((req, res) => { this.handleRequest(req, res); diff --git a/src/scenarios/client/tools_call.test.ts b/src/scenarios/client/tools_call.test.ts new file mode 100644 index 00000000..b9fc42d1 --- /dev/null +++ b/src/scenarios/client/tools_call.test.ts @@ -0,0 +1,48 @@ +import { testScenarioContext } from '../../mock-server/testing'; +import { describe, it, expect } from 'vitest'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { ToolsCallScenario } from './tools_call'; + +describe('tools_call scenario', () => { + it('emits a single FAILURE check when the tool was never called', async () => { + const scenario = new ToolsCallScenario(); + await scenario.start(testScenarioContext()); + try { + const checks = scenario.getChecks(); + expect(checks).toHaveLength(1); + expect(checks[0]).toMatchObject({ + id: 'tool-add-numbers', + status: 'FAILURE' + }); + } finally { + await scenario.stop(); + } + }); + + it('emits SUCCESS after a valid tools/call and getChecks() is idempotent', async () => { + const scenario = new ToolsCallScenario(); + const { serverUrl } = await scenario.start(testScenarioContext()); + try { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + await client.connect(transport); + await client.callTool({ name: 'add_numbers', arguments: { a: 2, b: 3 } }); + await transport.close(); + + const first = scenario.getChecks(); + expect(first).toHaveLength(1); + expect(first[0].status).toBe('SUCCESS'); + + // Repeated calls must not accumulate duplicate checks. + const second = scenario.getChecks(); + expect(second).toHaveLength(1); + expect(second[0].status).toBe('SUCCESS'); + } finally { + await scenario.stop(); + } + }); +}); diff --git a/src/scenarios/client/tools_call.ts b/src/scenarios/client/tools_call.ts index 59470f37..b3dcb74c 100644 --- a/src/scenarios/client/tools_call.ts +++ b/src/scenarios/client/tools_call.ts @@ -1,164 +1,83 @@ -import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; -import { - CallToolRequestSchema, - ListToolsRequestSchema -} from '@modelcontextprotocol/sdk/types.js'; -import type { Scenario, ConformanceCheck } from '../../types'; -import express, { Request, Response } from 'express'; -import { ScenarioUrls } from '../../types'; -import { createRequestLogger } from '../request-logger'; +import type { ScenarioContext, MockServer } from '../../mock-server'; +import type { Scenario, ConformanceCheck, ScenarioUrls } from '../../types'; +import type { CallToolRequest } from '../../spec-types/2025-06-18'; -function createMcpServer(checks: ConformanceCheck[]): Server { - const server = new Server( - { - name: 'add-numbers-server', - version: '1.0.0' - }, - { - capabilities: { - tools: {} - } - } - ); - - server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [ - { - name: 'add_numbers', - description: 'Add two numbers together', - inputSchema: { - type: 'object', - properties: { - a: { - type: 'number', - description: 'First number' - }, - b: { - type: 'number', - description: 'Second number' - } - }, - required: ['a', 'b'] - } - } - ] - }; - }); - - server.setRequestHandler(CallToolRequestSchema, async (request) => { - if (request.params.name === 'add_numbers') { - const { a, b } = request.params.arguments as { a: number; b: number }; - const result = a + b; - - checks.push({ - id: 'tool-add-numbers', - name: 'ToolAddNumbers', - description: 'Validates that the add_numbers tool works correctly', - status: 'SUCCESS', - timestamp: new Date().toISOString(), - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' - } - ], - details: { - a, - b, - result - } - }); - - return { - content: [ - { - type: 'text', - text: `The sum of ${a} and ${b} is ${result}` - } - ] - }; - } - - throw new Error(`Unknown tool: ${request.params.name}`); - }); - - return server; -} - -function createServerApp(checks: ConformanceCheck[]): express.Application { - const app = express(); - app.use(express.json()); - - app.use( - createRequestLogger(checks, { - incomingId: 'incoming-request', - outgoingId: 'outgoing-response', - mcpRoute: '/mcp' - }) - ); - - app.post('/mcp', async (req: Request, res: Response) => { - // Stateless: create a fresh server and transport per request - const server = createMcpServer(checks); - const transport = new StreamableHTTPServerTransport({ - sessionIdGenerator: undefined - }); - await server.connect(transport); - - await transport.handleRequest(req, res, req.body); - }); - - return app; -} +const SPEC_REF = { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' +}; export class ToolsCallScenario implements Scenario { name = 'tools_call'; readonly source = { introducedIn: '2025-06-18' } as const; description = 'Tests calling tools with various parameter types'; - private app: express.Application | null = null; - private httpServer: any = null; - private checks: ConformanceCheck[] = []; + private srv: MockServer | null = null; - async start(): Promise { - this.checks = []; - this.app = createServerApp(this.checks); - this.httpServer = this.app.listen(0); - const port = this.httpServer.address().port; - return { serverUrl: `http://localhost:${port}/mcp` }; + async start(ctx: ScenarioContext): Promise { + this.srv = await ctx.createServer({ + 'tools/list': () => ({ + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers together', + inputSchema: { + type: 'object', + properties: { + a: { type: 'number', description: 'First number' }, + b: { type: 'number', description: 'Second number' } + }, + required: ['a', 'b'] + } + } + ] + }), + 'tools/call': (params) => { + const p = params as CallToolRequest['params']; + if (p.name !== 'add_numbers') { + throw new Error(`Unknown tool: ${p.name}`); + } + const { a, b } = p.arguments as { a: number; b: number }; + return { + content: [ + { type: 'text', text: `The sum of ${a} and ${b} is ${a + b}` } + ] + }; + } + }); + return { serverUrl: this.srv.url }; } async stop() { - if (this.httpServer) { - await new Promise((resolve) => this.httpServer.close(resolve)); - this.httpServer = null; - } - this.app = null; + await this.srv?.close(); + this.srv = null; } getChecks(): ConformanceCheck[] { - const expectedSlugs = ['tool-add-numbers']; - // add a failure if not in there already - for (const slug of expectedSlugs) { - if (!this.checks.find((c) => c.id === slug)) { - // TODO: this is duplicated from above, refactor - this.checks.push({ - id: slug, - name: `ToolAddNumbers`, - description: `Validates that the add_numbers tool works correctly`, - status: 'FAILURE', - timestamp: new Date().toISOString(), - details: { message: 'Tool was not called by client' }, - specReferences: [ - { - id: 'MCP-Tools', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' + // Built fresh on every call so getChecks() is idempotent — the runner may + // call it more than once and we must not accumulate duplicates. + const call = this.srv?.recorded.find((r) => r.method === 'tools/call'); + const args = (call?.params as CallToolRequest['params'] | undefined) + ?.arguments as { a?: unknown; b?: unknown } | undefined; + const ok = + call !== undefined && + typeof args?.a === 'number' && + typeof args?.b === 'number'; + return [ + { + id: 'tool-add-numbers', + name: 'ToolAddNumbers', + description: 'Validates that the add_numbers tool works correctly', + status: ok ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SPEC_REF], + details: ok + ? { + a: args!.a, + b: args!.b, + result: (args!.a as number) + (args!.b as number) } - ] - }); + : { message: 'Tool was not called by client' } } - } - return this.checks; + ]; } } diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index 93b46620..dc84b0e9 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -6,6 +6,7 @@ import { listDraftClientScenarios, listActiveClientScenarios, listExtensionScenarios, + getScenario, getScenarioSpecVersions, resolveSpecVersion, ALL_SPEC_VERSIONS, @@ -57,11 +58,17 @@ describe('specVersions helpers', () => { expect(current.length).toBeGreaterThan(overlap.length); }); - it('the draft spec version is a superset of the latest dated release', () => { + it('every scenario in latest but not in draft is explicitly removedIn: DRAFT', () => { const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); const draft = new Set(listScenariosForSpec(DRAFT_PROTOCOL_VERSION)); for (const name of latest) { - expect(draft.has(name)).toBe(true); + if (!draft.has(name)) { + const s = getScenario(name)!; + expect( + 'removedIn' in s.source && s.source.removedIn, + `"${name}" is in ${LATEST_SPEC_VERSION} but not in draft without removedIn` + ).toBe(DRAFT_PROTOCOL_VERSION); + } } for (const name of listDraftScenarios()) { expect(draft.has(name)).toBe(true); diff --git a/src/types.ts b/src/types.ts index 6c6376e5..2e9dd22a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,5 @@ import type { RunContext } from './connection'; +import type { ScenarioContext } from './mock-server'; export type CheckStatus = | 'SUCCESS' @@ -102,7 +103,7 @@ export interface Scenario { * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth). */ allowClientError?: boolean; - start(): Promise; + start(ctx: ScenarioContext): Promise; stop(): Promise; getChecks(): ConformanceCheck[]; }