diff --git a/src/scenarios/server/client-helper.test.ts b/src/scenarios/server/client-helper.test.ts new file mode 100644 index 00000000..3d681575 --- /dev/null +++ b/src/scenarios/server/client-helper.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect } from 'vitest'; +import { reportSetupFailure } from './client-helper'; + +describe('reportSetupFailure', () => { + it('emits a single FAILURE check id-d "-setup"', () => { + const checks = reportSetupFailure( + 'tools-list', + new Error('connect ECONNREFUSED') + ); + + expect(checks).toHaveLength(1); + expect(checks[0]).toMatchObject({ + id: 'tools-list-setup', + status: 'FAILURE', + errorMessage: 'Setup failed: connect ECONNREFUSED' + }); + }); + + it('stringifies a non-Error thrown value', () => { + const checks = reportSetupFailure('prompts-list', 'boom'); + + expect(checks[0]?.errorMessage).toBe('Setup failed: boom'); + }); + + it('attaches spec references when provided and omits the field otherwise', () => { + const withRefs = reportSetupFailure('resources-list', new Error('nope'), [ + { id: 'MCP-Resources-List' } + ]); + expect(withRefs[0]?.specReferences).toEqual([{ id: 'MCP-Resources-List' }]); + + const withoutRefs = reportSetupFailure('resources-list', new Error('nope')); + expect(withoutRefs[0]).not.toHaveProperty('specReferences'); + }); + + it('sets a timestamp', () => { + const checks = reportSetupFailure('server-initialize', new Error('x')); + expect(typeof checks[0]?.timestamp).toBe('string'); + expect(checks[0]?.timestamp.length).toBeGreaterThan(0); + }); +}); diff --git a/src/scenarios/server/client-helper.ts b/src/scenarios/server/client-helper.ts index eebbd9b5..db550c19 100644 --- a/src/scenarios/server/client-helper.ts +++ b/src/scenarios/server/client-helper.ts @@ -8,12 +8,55 @@ import { LoggingMessageNotificationSchema, ProgressNotificationSchema } from '@modelcontextprotocol/sdk/types.js'; +import { ConformanceCheck } from '../../types'; export interface MCPClientConnection { client: Client; close: () => Promise; } +/** + * Emit a single `-setup` check as FAILURE for a scenario that + * could not get far enough to evaluate its real checks (connect failure, + * missing fixture, capability not advertised, etc.). + * + * See #248: previously each scenario hand-rolled a try/catch around connect + * and pinned the setup error onto whichever check ID happened to be first. + * That mislabels the failure — the error ends up under a check that has + * nothing to do with the actual problem, and any *other* checks the scenario + * would have emitted silently disappear. Routing setup failures through this + * helper gives them a dedicated, semantically honest ID and a consistent + * output shape across scenarios. + * + * The convention is that a scenario that cannot execute counts as a FAILURE; + * the escape hatches are scenario filtering (`--suite`/`--scenario`) and the + * expected-failures baseline, not in-scenario skipping or silent passes. + * + * @param scenarioName The scenario's `name`; the emitted check id is + * `-setup`. + * @param error The thrown setup error. + * @param specReferences Optional spec references to attach to the check. + * @returns A one-element array, so a scenario can `return reportSetupFailure(...)`. + */ +export function reportSetupFailure( + scenarioName: string, + error: unknown, + specReferences?: ConformanceCheck['specReferences'] +): ConformanceCheck[] { + const message = error instanceof Error ? error.message : String(error); + return [ + { + id: `${scenarioName}-setup`, + name: `${scenarioName} setup`, + description: `Scenario "${scenarioName}" could not be set up (connect/fixture/capability)`, + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Setup failed: ${message}`, + ...(specReferences ? { specReferences } : {}) + } + ]; +} + /** * Create and connect an MCP client to a server */ diff --git a/src/scenarios/server/lifecycle.test.ts b/src/scenarios/server/lifecycle.test.ts index 15ee6d80..4e85b32d 100644 --- a/src/scenarios/server/lifecycle.test.ts +++ b/src/scenarios/server/lifecycle.test.ts @@ -1,9 +1,15 @@ import { ServerInitializeScenario } from './lifecycle'; import { connectToServer } from './client-helper'; -vi.mock('./client-helper', () => ({ - connectToServer: vi.fn() -})); +vi.mock(import('./client-helper'), async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // Only the connection factory is mocked; reportSetupFailure stays real so + // the scenario's setup-failure path is exercised end-to-end. + connectToServer: vi.fn() + }; +}); describe('ServerInitializeScenario', () => { const serverUrl = 'http://localhost:3000/mcp'; @@ -91,4 +97,23 @@ describe('ServerInitializeScenario', () => { } }); }); + + it('reports a single setup FAILURE when the connection cannot be established', async () => { + vi.mocked(connectToServer).mockRejectedValueOnce( + new Error('connect ECONNREFUSED 127.0.0.1:3000') + ); + + const checks = await new ServerInitializeScenario().run(serverUrl); + + // A connect failure should not be mislabeled as the initialize or + // session-id check failing (#248): a single dedicated setup check instead. + expect(checks).toHaveLength(1); + expect(checks[0]).toMatchObject({ + id: 'server-initialize-setup', + status: 'FAILURE', + errorMessage: 'Setup failed: connect ECONNREFUSED 127.0.0.1:3000' + }); + // The session-id check is never reached, so no raw fetch is attempted. + expect(fetchMock).not.toHaveBeenCalled(); + }); }); diff --git a/src/scenarios/server/lifecycle.ts b/src/scenarios/server/lifecycle.ts index bb55869e..a076aa6d 100644 --- a/src/scenarios/server/lifecycle.ts +++ b/src/scenarios/server/lifecycle.ts @@ -3,7 +3,7 @@ */ import { ClientScenario, ConformanceCheck } from '../../types'; -import { connectToServer } from './client-helper'; +import { connectToServer, reportSetupFailure } from './client-helper'; const VISIBLE_ASCII_REGEX = /^[\x21-\x7E]+$/; @@ -61,22 +61,10 @@ and validates session ID format if one is assigned.`; await connection.close(); } catch (error) { - checks.push({ - id: 'server-initialize', - name: 'ServerInitialize', - description: - 'Server responds to initialize request with valid structure', - status: 'FAILURE', - timestamp: new Date().toISOString(), - errorMessage: `Failed to initialize: ${error instanceof Error ? error.message : String(error)}`, - specReferences: [ - { - id: 'MCP-Initialize', - url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization' - } - ] - }); - return checks; + // The handshake never completed, so neither the initialize check nor the + // session-id check below can be evaluated. Report a single setup failure + // rather than mislabeling it as one specific check failing (#248). + return reportSetupFailure(this.name, error); } // Check: Session ID visible ASCII validation diff --git a/src/scenarios/server/prompts.ts b/src/scenarios/server/prompts.ts index 2f65723f..f9c53f51 100644 --- a/src/scenarios/server/prompts.ts +++ b/src/scenarios/server/prompts.ts @@ -3,7 +3,7 @@ */ import { ClientScenario, ConformanceCheck } from '../../types'; -import { connectToServer } from './client-helper'; +import { connectToServer, reportSetupFailure } from './client-helper'; export class PromptsListScenario implements ClientScenario { name = 'prompts-list'; @@ -24,9 +24,15 @@ export class PromptsListScenario implements ClientScenario { async run(serverUrl: string): Promise { const checks: ConformanceCheck[] = []; + let connection; try { - const connection = await connectToServer(serverUrl); + connection = await connectToServer(serverUrl); + } catch (error) { + // Couldn't connect, so prompts/list never ran; report as setup (#248). + return reportSetupFailure(this.name, error); + } + try { const result = await connection.client.listPrompts(); // Validate response structure diff --git a/src/scenarios/server/resources.ts b/src/scenarios/server/resources.ts index 36f56600..06cb7110 100644 --- a/src/scenarios/server/resources.ts +++ b/src/scenarios/server/resources.ts @@ -7,7 +7,7 @@ import { ConformanceCheck, DRAFT_PROTOCOL_VERSION } from '../../types'; -import { connectToServer } from './client-helper'; +import { connectToServer, reportSetupFailure } from './client-helper'; import { sendStatelessRequest } from './stateless-client'; import { TextResourceContents, @@ -34,9 +34,15 @@ export class ResourcesListScenario implements ClientScenario { async run(serverUrl: string): Promise { const checks: ConformanceCheck[] = []; + let connection; try { - const connection = await connectToServer(serverUrl); + connection = await connectToServer(serverUrl); + } catch (error) { + // Couldn't connect, so resources/list never ran; report as setup (#248). + return reportSetupFailure(this.name, error); + } + try { const result = await connection.client.listResources(); // Validate response structure diff --git a/src/scenarios/server/tools.ts b/src/scenarios/server/tools.ts index 5875e725..99fe7953 100644 --- a/src/scenarios/server/tools.ts +++ b/src/scenarios/server/tools.ts @@ -3,7 +3,11 @@ */ import { ClientScenario, ConformanceCheck } from '../../types'; -import { connectToServer, NotificationCollector } from './client-helper'; +import { + connectToServer, + NotificationCollector, + reportSetupFailure +} from './client-helper'; import { CallToolResultSchema, CreateMessageRequestSchema, @@ -107,9 +111,17 @@ export class ToolsListScenario implements ClientScenario { async run(serverUrl: string): Promise { const checks: ConformanceCheck[] = []; + let connection; try { - const connection = await connectToServer(serverUrl); + connection = await connectToServer(serverUrl); + } catch (error) { + // A connect failure isn't a `tools-list` failure; pinning it there would + // also drop the `tools-name-format` check entirely. Report it as setup + // (#248). + return reportSetupFailure(this.name, error); + } + try { const result = await connection.client.listTools(); // Validate response structure