diff --git a/package.json b/package.json index 9a1e72ef..0780bfe1 100644 --- a/package.json +++ b/package.json @@ -153,6 +153,7 @@ "test:native-agent-task-interruption": "node tests/execute-native-agent-task-interruption.test.mjs", "test:native-agent-task-playground-e2e": "tsx tests/execute-native-agent-task-playground-e2e.test.ts", "test:bench-command-step-behavior": "tsx tests/bench-command-step-behavior.test.ts", + "test:external-http-load-integration": "npm run build && tsx tests/external-http-load.integration.test.ts", "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:browser-canonical-preview-origin && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", "test:temp-runtime-cleanup": "tsx tests/temp-runtime-cleanup.test.ts", "smoke:cli-version": "tsx scripts/cli-version-smoke.ts", diff --git a/packages/runtime-core/src/benchmark-contracts.ts b/packages/runtime-core/src/benchmark-contracts.ts index 558cd25a..ef2db90f 100644 --- a/packages/runtime-core/src/benchmark-contracts.ts +++ b/packages/runtime-core/src/benchmark-contracts.ts @@ -105,7 +105,7 @@ export interface BenchResults { } export interface BenchmarkDefinitionWorkloadStep { - type: "php" | "wp-cli" | "rest-request" | "rest-db-query-profiler" | "db-inventory" | "external-http-guardrail" | "artifact-postprocess" | "ability" | (string & {}) + type: "php" | "wp-cli" | "rest-request" | "rest-db-query-profiler" | "db-inventory" | "external-http-guardrail" | "external-http-load" | "artifact-postprocess" | "ability" | (string & {}) action?: "install" | "collect" | "reset" | (string & {}) code?: string file?: string @@ -115,6 +115,14 @@ export interface BenchmarkDefinitionWorkloadStep { blockNetwork?: boolean redactUrls?: boolean blockResponse?: { code?: number; message?: string; body?: string } + url?: string + method?: string + headers?: Record + body?: unknown + requestCount?: number + concurrency?: number + expectedStatus?: number + expectedStatuses?: number[] sampleLimit?: number queryLengthLimit?: number helper?: string @@ -303,6 +311,7 @@ function benchmarkSchemaDefs(): Record { enum: [ "wp-codebox/wordpress-db-inventory/v1", "wp-codebox/wordpress-external-http-guardrail/v1", + "wp-codebox/wordpress-external-http-load/v1", "wp-codebox/wordpress-rest-db-query-profile/v1", ], }, @@ -458,6 +467,14 @@ function benchmarkSchemaDefs(): Record { body: { type: "string" }, }, }, + url: { type: "string", minLength: 1 }, + method: { type: "string", minLength: 1 }, + headers: { type: "object", additionalProperties: true }, + body: true, + requestCount: { type: "integer", minimum: 1, maximum: 100 }, + concurrency: { type: "integer", minimum: 1, maximum: 20 }, + expectedStatus: { type: "integer", minimum: 100, maximum: 599 }, + expectedStatuses: { type: "array", minItems: 1, uniqueItems: true, items: { type: "integer", minimum: 100, maximum: 599 } }, sampleLimit: { type: "integer", minimum: 0 }, queryLengthLimit: { type: "integer", minimum: 80 }, helper: { type: "string", minLength: 1 }, diff --git a/packages/runtime-playground/src/external-http-load.ts b/packages/runtime-playground/src/external-http-load.ts new file mode 100644 index 00000000..32e49ab9 --- /dev/null +++ b/packages/runtime-playground/src/external-http-load.ts @@ -0,0 +1,159 @@ +export interface RuntimeExternalHttpLoadResult { + schema: "wp-codebox/wordpress-external-http-load/v1" + success: boolean + requestCount: number + concurrency: number + maxObservedConcurrency: number + completedCount: number + successCount: number + failureCount: number + statusDistribution: Record + durationMs: number + latenciesMs: number[] + latency: Record + diagnostics: Array> + provenance: { + source: "host-side-external-http" + transport: "runtime-preview-http" + runtimeScope: "single-runtime" + target: string + method: string + } +} + +export async function runRuntimeExternalHttpLoad(action: Record, runtimeBaseUrl?: string): Promise { + if (!runtimeBaseUrl) { + throw new Error("external_http_load requires an active runtime preview origin") + } + + const requestCount = boundedInteger(action.requestCount, "requestCount", 100) + const concurrency = boundedInteger(action.concurrency, "concurrency", 20) + if (concurrency > requestCount) { + throw new Error("external_http_load concurrency must not exceed requestCount") + } + + const baseUrl = new URL(runtimeBaseUrl) + const inputUrl = typeof action.url === "string" && action.url.trim() !== "" ? action.url.trim() : "/" + const resolvedUrl = new URL(inputUrl, baseUrl) + if (resolvedUrl.origin !== baseUrl.origin) { + throw new Error("external_http_load url must resolve to the active runtime preview origin") + } + + const method = typeof action.method === "string" && action.method.trim() !== "" ? action.method.trim().toUpperCase() : "GET" + const headers = normalizeHttpHeaders(action.headers) + const body = action.body === undefined || action.body === null ? undefined : String(action.body) + const expectedStatuses = normalizeExpectedStatuses(action.expectedStatuses ?? (action.expectedStatus === undefined ? undefined : [action.expectedStatus])) + const statusDistribution: Record = {} + const latenciesMs: number[] = [] + const diagnostics: Array> = [] + let nextRequest = 0 + let activeRequests = 0 + let maxObservedConcurrency = 0 + let completedCount = 0 + let successCount = 0 + let failureCount = 0 + const loadStarted = performance.now() + + const worker = async (): Promise => { + while (true) { + const requestIndex = nextRequest++ + if (requestIndex >= requestCount) { + return + } + activeRequests++ + maxObservedConcurrency = Math.max(maxObservedConcurrency, activeRequests) + const started = performance.now() + try { + const response = await fetch(resolvedUrl, { method, headers, body }) + statusDistribution[String(response.status)] = (statusDistribution[String(response.status)] ?? 0) + 1 + await response.arrayBuffer() + latenciesMs.push(performance.now() - started) + if (expectedStatuses.includes(response.status)) { + successCount++ + } else { + failureCount++ + diagnostics.push({ code: "unexpected_status", requestIndex, expectedStatuses, actualStatus: response.status }) + } + } catch (error) { + latenciesMs.push(performance.now() - started) + failureCount++ + diagnostics.push({ code: "request_failed", requestIndex, message: errorMessage(error) }) + } finally { + completedCount++ + activeRequests-- + } + } + } + + await Promise.all(Array.from({ length: concurrency }, () => worker())) + if (completedCount !== requestCount) { + diagnostics.push({ code: "incomplete_execution", expected: requestCount, actual: completedCount }) + } + + return { + schema: "wp-codebox/wordpress-external-http-load/v1", + success: failureCount === 0 && completedCount === requestCount, + requestCount, + concurrency, + maxObservedConcurrency, + completedCount, + successCount, + failureCount, + statusDistribution, + durationMs: performance.now() - loadStarted, + latenciesMs, + latency: numericSummary(latenciesMs), + diagnostics, + provenance: { + source: "host-side-external-http", + transport: "runtime-preview-http", + runtimeScope: "single-runtime", + target: inputUrl, + method, + }, + } +} + +function boundedInteger(value: unknown, name: string, maximum: number): number { + if (!Number.isInteger(value) || (value as number) < 1 || (value as number) > maximum) { + throw new Error(`external_http_load ${name} must be an integer between 1 and ${maximum}`) + } + return value as number +} + +function normalizeExpectedStatuses(value: unknown): number[] { + if (value === undefined) { + return [200] + } + if (!Array.isArray(value) || value.length === 0 || value.some((status) => !Number.isInteger(status) || status < 100 || status > 599)) { + throw new Error("external_http_load expectedStatuses must contain one or more HTTP status codes") + } + return [...new Set(value as number[])] +} + +function normalizeHttpHeaders(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return {} + } + return Object.fromEntries(Object.entries(value).map(([name, headerValue]) => [name, String(headerValue)])) +} + +function numericSummary(values: number[]): Record { + const sorted = [...values].sort((left, right) => left - right) + const count = sorted.length + const mean = count > 0 ? sorted.reduce((sum, value) => sum + value, 0) / count : 0 + const percentile = (fraction: number): number => count > 0 ? sorted[Math.max(0, Math.ceil(fraction * count) - 1)] : 0 + return { + count, + mean, + p50: percentile(0.5), + p95: percentile(0.95), + p99: percentile(0.99), + min: count > 0 ? sorted[0] : 0, + max: count > 0 ? sorted[count - 1] : 0, + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/packages/runtime-playground/src/wordpress-command-runners.ts b/packages/runtime-playground/src/wordpress-command-runners.ts index b5e7f735..568dce6c 100644 --- a/packages/runtime-playground/src/wordpress-command-runners.ts +++ b/packages/runtime-playground/src/wordpress-command-runners.ts @@ -56,6 +56,7 @@ import type { PlaygroundCliServer } from "./preview-server.js" import { persistCorePhpunitResult, persistPluginPhpunitCompletedResult, persistPluginPhpunitResult, persistVfsDiagnosticFileToHost, readCorePhpunitDiagnostic, readPluginPhpunitCompletedResult, readPluginPhpunitDiagnostic, readPluginPhpunitDiscoveryResult } from "./runtime-diagnostics.js" import { phpunitExecutionSemantics, requiresManagedMysqlMultisitePreinstall } from "./phpunit-command-semantics.js" import { parsePhpunitOutput } from "./phpunit-test-results.js" +import { runRuntimeExternalHttpLoad, type RuntimeExternalHttpLoadResult } from "./external-http-load.js" import type { RuntimeWpCliBridge } from "./runtime-wp-cli-bridge.js" import { COMMAND_DIAGNOSTICS_ARTIFACT_SCHEMA, PERFORMANCE_OBSERVATION_SCHEMA, commandDiagnosticsCaptureArgs, commandDiagnosticsCaptureSpecFromArgs, createRuntimeCommandResultEnvelope, redactJsonValue, type ExecutionSpec, type MountSpec, type PerformanceObservation, type RuntimeCommandResultEnvelope, type RuntimeCreateSpec, type RuntimeEpisodeTraceRef } from "@automattic/wp-codebox-core" import { wordpressUserSessionFromCommandArgs } from "./wordpress-user-sessions.js" @@ -897,11 +898,14 @@ export async function runBenchCommand({ const scenarioIds = jsonArrayArg(args, "scenario-ids-json").filter((id): id is string => typeof id === "string" && id.trim() !== "").map((id) => id.trim()) const lifecycle = jsonObjectArg(args, "lifecycle-json") const resetPolicy = jsonObjectArg(args, "reset-policy-json") - const bridge = benchWorkloadsUseWpCli([workloads, lifecycle]) ? await createRuntimeWpCliBridge(server) : undefined + const externalHttpLoadPlans = benchExternalHttpLoadPlans(workloads) + .filter((plan) => scenarioIds.length === 0 || scenarioIds.includes(plan.scenarioId)) + const runtimeWorkloads = benchReplaceExternalHttpLoadSteps(workloads) + const bridge = benchWorkloadsUseWpCli([runtimeWorkloads, lifecycle]) ? await createRuntimeWpCliBridge(server) : undefined let response: PlaygroundRunResponse try { response = await runPlaygroundCommand("wordpress.bench", server, { - code: bootstrapPhpCode(runtimeSpec, benchRunCode({ componentId, pluginSlug, iterations, warmupIterations, dependencySlugs, env, bootstrapFiles, workloads, scenarioIds, lifecycle, resetPolicy, wpCliBridge: bridge }), []), + code: bootstrapPhpCode(runtimeSpec, benchRunCode({ componentId, pluginSlug, iterations, warmupIterations, dependencySlugs, env, bootstrapFiles, workloads: runtimeWorkloads, scenarioIds, lifecycle, resetPolicy, wpCliBridge: bridge }), []), }) assertPlaygroundResponseOk("wordpress.bench", response) } finally { @@ -910,7 +914,13 @@ export async function runBenchCommand({ } } - return promoteBrowserMetricsToBenchResults(response.text, browserProbes) + const withExternalHttpLoad = await benchMergeExternalHttpLoadResults(response.text, externalHttpLoadPlans, { + baseUrl: runtimeSpec.preview?.publicUrl ?? server.serverUrl, + iterations, + warmupIterations, + workloads, + }) + return promoteBrowserMetricsToBenchResults(withExternalHttpLoad, browserProbes) } export async function runPhpunitCommand({ @@ -1114,6 +1124,196 @@ export async function runCorePhpunitCommand({ return response.text } +interface BenchExternalHttpLoadPlan { + scenarioId: string + stepIndex: number + step: Record +} + +function benchExternalHttpLoadPlans(workloads: unknown[]): BenchExternalHttpLoadPlan[] { + const plans: BenchExternalHttpLoadPlan[] = [] + workloads.forEach((workload, workloadIndex) => { + if (!workload || typeof workload !== "object" || Array.isArray(workload)) { + return + } + const record = workload as Record + const scenarioId = typeof record.id === "string" && record.id.trim() !== "" ? record.id : `configured-${workloadIndex}` + const steps = Array.isArray(record.run) ? record.run : (Array.isArray(record.steps) ? record.steps : []) + steps.forEach((step, stepIndex) => { + if (step && typeof step === "object" && !Array.isArray(step) && (step as Record).type === "external-http-load") { + plans.push({ scenarioId, stepIndex, step: step as Record }) + } + }) + }) + return plans +} + +function benchReplaceExternalHttpLoadSteps(workloads: unknown[]): unknown[] { + return workloads.map((workload) => { + if (!workload || typeof workload !== "object" || Array.isArray(workload)) { + return workload + } + const record = workload as Record + const field = Array.isArray(record.run) ? "run" : (Array.isArray(record.steps) ? "steps" : undefined) + if (!field) { + return workload + } + return { + ...record, + [field]: (record[field] as unknown[]).map((step) => step && typeof step === "object" && !Array.isArray(step) && (step as Record).type === "external-http-load" + ? { type: "php", code: "return array();" } + : step), + } + }) +} + +async function benchMergeExternalHttpLoadResults( + text: string, + plans: BenchExternalHttpLoadPlan[], + options: { baseUrl: string; iterations: number; warmupIterations: number; workloads: unknown[] }, +): Promise { + if (plans.length === 0) { + return text + } + + const results = JSON.parse(text) as { schema?: string; scenarios?: Array>; provenance?: Record } + if (results.schema !== "wp-codebox/bench-results/v1" || !Array.isArray(results.scenarios)) { + throw new Error("external-http-load could not merge into an invalid wordpress.bench result") + } + if (results.provenance?.definition && typeof results.provenance.definition === "object") { + results.provenance.definition.workloads = options.workloads + } + + for (const plan of plans) { + const scenario = results.scenarios.find((candidate) => candidate.id === plan.scenarioId) + if (!scenario) { + throw new Error(`external-http-load scenario result is missing: ${plan.scenarioId}`) + } + + const measured: RuntimeExternalHttpLoadResult[] = [] + for (let iteration = 0; iteration < options.warmupIterations + options.iterations; iteration++) { + const result = await runRuntimeExternalHttpLoad(plan.step, options.baseUrl) + if (!result.success) { + throw new Error(`external-http-load assertions failed. diagnostic=${JSON.stringify({ + schema: "wp-codebox/bench-external-http-load-diagnostic/v1", + scenarioId: plan.scenarioId, + stepIndex: plan.stepIndex, + requestCount: result.requestCount, + completedCount: result.completedCount, + failureCount: result.failureCount, + statusDistribution: result.statusDistribution, + diagnostics: result.diagnostics, + })}`) + } + if (iteration >= options.warmupIterations) { + measured.push(result) + } + } + + const prefix = benchExternalHttpMetricPrefix(plan.step) + scenario.metrics ??= {} + const metricValues: Record = { + [`${prefix}_request_count`]: { unit: "count", values: measured.map((result) => result.requestCount) }, + [`${prefix}_completed_count`]: { unit: "count", values: measured.map((result) => result.completedCount) }, + [`${prefix}_success_count`]: { unit: "count", values: measured.map((result) => result.successCount) }, + [`${prefix}_failure_count`]: { unit: "count", values: measured.map((result) => result.failureCount) }, + [`${prefix}_max_observed_concurrency_count`]: { unit: "count", values: measured.map((result) => result.maxObservedConcurrency) }, + [`${prefix}_latency_mean_ms`]: { unit: "ms", values: measured.map((result) => result.latency.mean) }, + [`${prefix}_latency_p50_ms`]: { unit: "ms", values: measured.map((result) => result.latency.p50) }, + [`${prefix}_latency_p95_ms`]: { unit: "ms", values: measured.map((result) => result.latency.p95) }, + [`${prefix}_latency_p99_ms`]: { unit: "ms", values: measured.map((result) => result.latency.p99) }, + } + for (const [name, metric] of Object.entries(metricValues)) { + scenario.metrics[name] = benchExternalHttpMetric(metric.values, metric.unit) + } + scenario.metrics.duration = benchExternalHttpMetric(measured.map((result) => result.durationMs), "ms") + + const artifact = benchAggregateExternalHttpLoadResults(measured) + const artifactName = plans.filter((candidate) => candidate.scenarioId === plan.scenarioId).length === 1 + ? "external-http-load" + : `external-http-load-${plan.stepIndex}` + scenario.artifacts ??= {} + scenario.artifacts[artifactName] = artifact + scenario.steps ??= [] + scenario.steps.push({ + schema: "wp-codebox/bench-command-step/v1", + type: "external-http-load", + requestCount: artifact.requestCount, + concurrency: artifact.concurrency, + maxObservedConcurrency: artifact.maxObservedConcurrency, + iterations: measured.length, + provenance: artifact.provenance, + }) + scenario.provenance = { + ...(scenario.provenance ?? {}), + external_http_load: [...(Array.isArray(scenario.provenance?.external_http_load) ? scenario.provenance.external_http_load : []), { + artifact: artifactName, + ...artifact.provenance, + }], + } + } + + return `${JSON.stringify(results, null, 2)}\n` +} + +function benchExternalHttpMetricPrefix(step: Record): string { + const value = typeof step["metric-prefix"] === "string" ? step["metric-prefix"] : "external_http_load" + return value.trim().replace(/[^A-Za-z0-9_]+/g, "_").replace(/^_+|_+$/g, "") || "external_http_load" +} + +function benchExternalHttpMetric(values: number[], unit: "count" | "ms"): Record { + return { unit, samples: benchNumericSummary(values) } +} + +function benchNumericSummary(values: number[]): Record { + const sorted = [...values].sort((left, right) => left - right) + const count = sorted.length + const mean = count > 0 ? sorted.reduce((sum, value) => sum + value, 0) / count : 0 + const variance = count > 0 ? sorted.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / count : 0 + const standardDeviation = Math.sqrt(variance) + const percentile = (fraction: number): number => count > 0 ? sorted[Math.max(0, Math.ceil(fraction * count) - 1)] : 0 + return { + count, + mean, + p50: percentile(0.5), + p95: percentile(0.95), + p99: percentile(0.99), + min: count > 0 ? sorted[0] : 0, + max: count > 0 ? sorted[count - 1] : 0, + standard_deviation: standardDeviation, + relative_standard_deviation: mean !== 0 ? standardDeviation / Math.abs(mean) : 0, + values: sorted, + } +} + +function benchAggregateExternalHttpLoadResults(results: RuntimeExternalHttpLoadResult[]): Record { + const statusDistribution: Record = {} + const latenciesMs: number[] = [] + for (const result of results) { + for (const [status, count] of Object.entries(result.statusDistribution as Record)) { + statusDistribution[status] = (statusDistribution[status] ?? 0) + count + } + latenciesMs.push(...result.latenciesMs) + } + const first = results[0] ?? {} + return { + schema: "wp-codebox/wordpress-external-http-load/v1", + requestCount: first.requestCount ?? 0, + concurrency: first.concurrency ?? 0, + maxObservedConcurrency: Math.max(0, ...results.map((result) => result.maxObservedConcurrency)), + completedCount: results.reduce((sum, result) => sum + result.completedCount, 0), + successCount: results.reduce((sum, result) => sum + result.successCount, 0), + failureCount: results.reduce((sum, result) => sum + result.failureCount, 0), + statusDistribution, + durationMs: results.reduce((sum, result) => sum + result.durationMs, 0), + latenciesMs, + latency: benchNumericSummary(latenciesMs), + runs: results, + diagnostics: results.flatMap((result) => result.diagnostics), + provenance: first.provenance ?? {}, + } +} + function benchWorkloadsUseWpCli(value: unknown): boolean { if (Array.isArray(value)) { return value.some(benchWorkloadsUseWpCli) diff --git a/tests/benchmark-contracts.test.ts b/tests/benchmark-contracts.test.ts index 8acc4a1d..e7145b9c 100644 --- a/tests/benchmark-contracts.test.ts +++ b/tests/benchmark-contracts.test.ts @@ -60,6 +60,9 @@ assert.ok(workloadStepDefinition.properties?.helperPath, "workload steps should assert.ok(workloadStepDefinition.properties?.inputArtifactRoot, "workload steps should expose artifact-postprocess inputArtifactRoot") assert.ok(workloadStepDefinition.properties?.outputArtifactPath, "workload steps should expose artifact-postprocess outputArtifactPath") assert.ok(workloadStepDefinition.properties?.expectedOutputSchema, "workload steps should expose artifact-postprocess expectedOutputSchema") +assert.ok(workloadStepDefinition.properties?.requestCount, "workload steps should expose external HTTP requestCount") +assert.ok(workloadStepDefinition.properties?.concurrency, "workload steps should expose external HTTP concurrency") +assert.ok(workloadStepDefinition.properties?.expectedStatuses, "workload steps should expose external HTTP expectedStatuses") assert.deepEqual(routeDefinition.anyOf, [{ required: ["path"] }, { required: ["route"] }]) assert.deepEqual(restCaseDefinition.anyOf, [{ required: ["path"] }, { required: ["route"] }]) diff --git a/tests/external-http-load.integration.test.ts b/tests/external-http-load.integration.test.ts new file mode 100644 index 00000000..c67a7c40 --- /dev/null +++ b/tests/external-http-load.integration.test.ts @@ -0,0 +1,82 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { promisify } from "node:util" + +import { withTempDir } from "../scripts/test-kit.js" + +const execute = promisify(execFile) + +await withTempDir("wp-codebox-external-http-load-integration-", async (artifactRoot) => { + const workload = [{ + id: "external-load", + run: [{ + type: "external-http-load", + url: "/", + requestCount: 4, + concurrency: 2, + expectedStatuses: [200], + }], + }] + const { stdout } = await execute(process.execPath, [ + "packages/cli/dist/index.js", + "run", + "--mount", "tests/fixtures/fuzz-relative-plugin:/wordpress/wp-content/plugins/fuzz-relative-plugin", + "--command", "wordpress.bench", + "--arg", "plugin-slug=fuzz-relative-plugin", + "--arg", "iterations=1", + "--arg", "warmup=0", + "--arg", `workloads-json=${JSON.stringify(workload)}`, + "--artifacts", artifactRoot, + "--json", + ], { + cwd: process.cwd(), + maxBuffer: 20 * 1024 * 1024, + timeout: 180_000, + }) + + const command = JSON.parse(stdout) as { + success: boolean + execution: { + result: { + json: { + schema: string + provenance: { definition: { workloads: typeof workload } } + scenarios: Array<{ + metrics: Record + artifacts: Record + }> + } + } + } + } + assert.equal(command.success, true) + const results = command.execution.result.json + assert.equal(results.schema, "wp-codebox/bench-results/v1") + assert.deepEqual(results.provenance.definition.workloads, workload) + const scenario = results.scenarios[0] + const load = scenario.artifacts["external-http-load"] + assert.equal(load.schema, "wp-codebox/wordpress-external-http-load/v1") + assert.equal(load.completedCount, 4) + assert.equal(load.successCount, 4) + assert.equal(load.failureCount, 0) + assert.equal(load.maxObservedConcurrency, 2) + assert.deepEqual(load.provenance, { + source: "host-side-external-http", + transport: "runtime-preview-http", + runtimeScope: "single-runtime", + target: "/", + method: "GET", + }) + assert.equal(scenario.metrics.external_http_load_completed_count.samples.mean, 4) + assert.equal(scenario.metrics.external_http_load_max_observed_concurrency_count.samples.mean, 2) + assert.ok(scenario.metrics.duration.samples.mean > 0) +}) + +console.log("external HTTP load runs against one WordPress preview runtime") diff --git a/tests/runtime-wp-cli-bridge.test.ts b/tests/runtime-wp-cli-bridge.test.ts index 6524d16c..73be4bc4 100644 --- a/tests/runtime-wp-cli-bridge.test.ts +++ b/tests/runtime-wp-cli-bridge.test.ts @@ -1,7 +1,10 @@ import assert from "node:assert/strict" import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { createServer } from "node:http" import { tmpdir } from "node:os" import { delimiter, join } from "node:path" +import { setTimeout as delay } from "node:timers/promises" +import { runRuntimeExternalHttpLoad } from "../packages/runtime-playground/src/external-http-load.js" import { createRuntimeWpCliBridge } from "../packages/runtime-playground/src/runtime-wp-cli-bridge.js" const bridge = await createRuntimeWpCliBridge(async () => ({ exitCode: 0, text: "", errors: "" })) @@ -55,6 +58,66 @@ try { await bridge.close() } +let activeRequests = 0 +let maxActiveRequests = 0 +const target = createServer(async (_request, response) => { + activeRequests++ + maxActiveRequests = Math.max(maxActiveRequests, activeRequests) + await delay(30) + activeRequests-- + response.writeHead(204).end() +}) +await new Promise((resolve) => target.listen(0, "127.0.0.1", resolve)) +const address = target.address() +assert.ok(address && typeof address === "object") +const targetUrl = `http://127.0.0.1:${address.port}` +try { + await assert.rejects( + runRuntimeExternalHttpLoad({ requestCount: 101, concurrency: 1, expectedStatuses: [200] }, targetUrl), + /requestCount must be an integer between 1 and 100/, + ) + await assert.rejects( + runRuntimeExternalHttpLoad({ requestCount: 2, concurrency: 3, expectedStatuses: [200] }, targetUrl), + /concurrency must not exceed requestCount/, + ) + await assert.rejects( + runRuntimeExternalHttpLoad({ url: "https://example.com/", requestCount: 1, concurrency: 1, expectedStatuses: [200] }, targetUrl), + /must resolve to the active runtime preview origin/, + ) + + const loadResponse = await runRuntimeExternalHttpLoad({ + url: "/load", + method: "POST", + body: "payload", + requestCount: 8, + concurrency: 3, + expectedStatuses: [204], + }, targetUrl) + assert.equal(loadResponse.success, true) + assert.equal(loadResponse.schema, "wp-codebox/wordpress-external-http-load/v1") + assert.equal(loadResponse.completedCount, 8) + assert.equal(loadResponse.successCount, 8) + assert.equal(loadResponse.failureCount, 0) + assert.equal(loadResponse.maxObservedConcurrency, 3) + assert.equal(maxActiveRequests, 3) + assert.deepEqual(loadResponse.statusDistribution, { 204: 8 }) + assert.equal(loadResponse.latenciesMs.length, 8) + assert.equal(loadResponse.provenance.source, "host-side-external-http") + assert.equal(loadResponse.provenance.runtimeScope, "single-runtime") + + const statusFailure = await runRuntimeExternalHttpLoad({ + url: "/load", + requestCount: 2, + concurrency: 1, + expectedStatuses: [200], + }, targetUrl) + assert.equal(statusFailure.success, false) + assert.equal(statusFailure.failureCount, 2) + assert.equal(statusFailure.diagnostics[0].code, "unexpected_status") +} finally { + await new Promise((resolve, reject) => target.close((error) => error ? reject(error) : resolve())) +} + async function postBridgeAction(url: string, token: string, action: Record): Promise> { const response = await fetch(`${url}/execute`, { method: "POST",