From 9bdb00e52f447dc0b94e252c07594e3261ab9eef Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 22 Oct 2026 16:47:02 +0800 Subject: [PATCH 1/4] feat(spend): meter, monthly budget, burn ceiling, warning and /api/spend (ADR-0006) Admission reads month-to-date spend before it enqueues and before each claim; at or over the monthly budget the run ends RunSkipped (neutral check, skipped row). The claim admits only while admitted hourly rates across every pool plus the claimant's fit under the burn ceiling. Fan-out parents read spend before spawning. The meter prices admission to completion on the provisioned list rate, from one sandbox-to-instance table checked against wrangler.jsonc. --- apps/dispatcher/src/dashboard.ts | 5 +- apps/dispatcher/src/env.ts | 30 +++ apps/dispatcher/src/http-app.ts | 17 +- apps/dispatcher/src/routes/spend.test.ts | 129 ++++++++++++ apps/dispatcher/src/routes/spend.ts | 92 ++++++++ apps/dispatcher/src/workflow.ts | 116 ++++++++-- apps/docs/src/data/benchmarks.ts | 10 +- infra/migrations/0011_spend_guard.sql | 27 +++ packages/core/src/cost.test.ts | 39 ++-- packages/core/src/cost.ts | 72 ++++--- packages/core/src/fakes/child-runs-fake.ts | 8 +- packages/core/src/index.ts | 29 +++ packages/core/src/primitives/fan-out.test.ts | 21 ++ packages/core/src/primitives/fan-out.ts | 14 +- packages/core/src/run-admission.test.ts | 11 + packages/core/src/run-admission.ts | 9 + packages/core/src/services/child-runs.ts | 10 +- packages/core/src/spend.test.ts | 149 +++++++++++++ packages/core/src/spend.ts | 198 ++++++++++++++++++ packages/runtime-cf/package.json | 1 + packages/runtime-cf/src/child-runs-cf.ts | 21 ++ packages/runtime-cf/src/deferred.ts | 2 + .../runtime-cf/src/execution-cost.test.ts | 49 +++++ packages/runtime-cf/src/execution-cost.ts | 70 +++++-- .../src/execution-cost.workers.test.ts | 62 +++++- packages/runtime-cf/src/index.ts | 17 +- packages/runtime-cf/src/run-admission-d1.ts | 106 ++++++++-- .../src/run-admission-d1.workers.test.ts | 119 ++++++++++- packages/runtime-cf/src/runtime.ts | 6 + packages/runtime-cf/src/spend-d1.ts | 179 ++++++++++++++++ .../runtime-cf/src/spend-d1.workers.test.ts | 118 +++++++++++ runs/contextful-gate.test.ts | 17 ++ runs/contextful-gate.ts | 25 ++- 33 files changed, 1658 insertions(+), 120 deletions(-) create mode 100644 apps/dispatcher/src/routes/spend.test.ts create mode 100644 apps/dispatcher/src/routes/spend.ts create mode 100644 infra/migrations/0011_spend_guard.sql create mode 100644 packages/core/src/spend.test.ts create mode 100644 packages/core/src/spend.ts create mode 100644 packages/runtime-cf/src/execution-cost.test.ts create mode 100644 packages/runtime-cf/src/spend-d1.ts create mode 100644 packages/runtime-cf/src/spend-d1.workers.test.ts diff --git a/apps/dispatcher/src/dashboard.ts b/apps/dispatcher/src/dashboard.ts index 0648ca79..ce53a7b2 100644 --- a/apps/dispatcher/src/dashboard.ts +++ b/apps/dispatcher/src/dashboard.ts @@ -52,6 +52,8 @@ export interface DashboardData { readonly nowMs: number; /** `owner/repo` slug for the source link. */ readonly repoSlug: string; + /** The spend guard's month-to-date line (`GET /api/spend`); absent when unreadable. */ + readonly spendLine?: string; } /** Server-side filters, parsed from the `run`/`repo`/`status` query params. */ @@ -196,6 +198,7 @@ const STYLE = ` header { display: flex; flex-wrap: wrap; align-items: baseline; gap: .75rem; margin-bottom: .25rem; } h1 { font-size: 1.4rem; margin: 0; letter-spacing: -0.01em; } .tagline { color: #9aa3b2; margin: 0 0 1.75rem; } + .spend { color: #9aa3b2; margin: -1.25rem 0 1.75rem; font-size: 0.9rem; } h2 { font-size: .82rem; text-transform: uppercase; letter-spacing: .08em; color: #9aa3b2; margin: 2rem 0 .6rem; } table { width: 100%; border-collapse: collapse; font-size: 14px; } th { text-align: left; font-weight: 600; color: #9aa3b2; padding: .4rem .5rem; border-bottom: 1px solid #232838; } @@ -375,7 +378,7 @@ export const renderDashboard = (data: DashboardData): string => {

FlareDispatch

Dispatch CI/CD runs to Cloudflare — inspect logs, executions, and product demos.

- +${data.spendLine !== undefined ? `

${esc(data.spendLine)}

\n` : ""}

Latest executions

${renderFilters(data.filters)} diff --git a/apps/dispatcher/src/env.ts b/apps/dispatcher/src/env.ts index a80de656..ab54d364 100644 --- a/apps/dispatcher/src/env.ts +++ b/apps/dispatcher/src/env.ts @@ -464,6 +464,36 @@ export interface Env { */ readonly ADMISSION_CAP_AGENT?: string; + /** + * The monthly spend budget in USD (specs/adr/0006-spend-guard.md) — a var. + * At or over it, admission skips every run with a neutral check. Unset or + * unparsable → 90; `unlimited` disables it. CONFIG_KV + * `spend.budgetMonthlyUsd` overrides it without a deploy. The default lives + * in code (core spend.ts), because an operator overlay replaces the + * committed wrangler.jsonc. + */ + readonly SPEND_BUDGET_MONTHLY_USD?: string; + + /** + * The burn ceiling in USD per hour — a var. The summed hourly rate of every + * admitted execution, across all pools, stays at or under it. Unset or + * unparsable → 1.50; `unlimited` disables it. CONFIG_KV + * `spend.burnMaxUsdPerHour` overrides it. + */ + readonly SPEND_BURN_MAX_USD_PER_HOUR?: string; + + /** + * Multiplier on metered container + model spend for Workflow steps and + * Durable Objects — a var. Unset or outside [1, 10] → 1.10. + */ + readonly SPEND_OVERHEAD_FACTOR?: string; + + /** + * Fraction of the monthly budget whose crossing sends the month's one + * warning on the notice path — a var. Unset or outside (0, 1] → 0.8. + */ + readonly SPEND_WARN_RATIO?: string; + /** * Cloudflare account id (the 32-hex id in the dashboard URL) — a var, not a * secret. When set, `RunWorkflow` builds a `details_url` on the GitHub diff --git a/apps/dispatcher/src/http-app.ts b/apps/dispatcher/src/http-app.ts index bdf32d75..f617b086 100644 --- a/apps/dispatcher/src/http-app.ts +++ b/apps/dispatcher/src/http-app.ts @@ -45,6 +45,7 @@ import { handleProductDemo } from "./routes/product-demos"; import { handleReplay } from "./routes/replay"; import { handleGithubWebhook } from "./routes/webhook"; import { handleSignalsWebhook } from "./routes/signals-webhook"; +import { handleSpend, readSpendNow, spendLine, spendView } from "./routes/spend"; /** `owner/repo` slug for the dashboard source link. */ const REPO_SLUG = "fractalbox/flare-dispatch"; @@ -368,6 +369,8 @@ const appShellRoute = Effect.gen(function* () { } const { origin, rows, filters, pagination } = yield* dashboardData; + // The spend line is a read beside the list, never a reason the page fails. + const spend = yield* readSpendNow(env).pipe(Effect.option); const html = renderDashboard({ origin, rows, @@ -375,6 +378,7 @@ const appShellRoute = Effect.gen(function* () { pagination, nowMs: Date.now(), repoSlug: REPO_SLUG, + ...(Option.isSome(spend) ? { spendLine: spendLine(spend.value) } : {}), }); return HttpServerResponse.raw(html, { status: 200, @@ -396,8 +400,13 @@ const dashboardJsonRoute = Effect.gen(function* () { if (request.method !== "GET") return methodNotAllowed; const { origin, rows, filters, pagination } = yield* dashboardData; + const env = yield* CurrentEnv; + const spend = yield* readSpendNow(env).pipe( + Effect.map((status) => ({ ...spendView(status), line: spendLine(status) })), + Effect.orElseSucceed(() => null), + ); return HttpServerResponse.raw( - JSON.stringify({ origin, repoSlug: REPO_SLUG, rows, filters, pagination }), + JSON.stringify({ origin, repoSlug: REPO_SLUG, rows, filters, pagination, spend }), { status: 200, headers: { "content-type": "application/json; charset=utf-8" }, @@ -585,6 +594,12 @@ const router = baseRouter.pipe( ), HttpRouter.all("/v1/dashboard.json", dashboardJsonRoute), HttpRouter.all("/v1/analytics.json", analyticsJsonRoute), + // The spend guard's month-to-date view (ADR-0006) — Access-gated like the + // executions and analytics reads. + HttpRouter.all( + "/api/spend", + route("GET", ({ env }) => handleSpend(env), { viewer: true }), + ), // Container-side OTP read — TOKEN-ONLY (no `viewer: true`): a sandbox can't // carry an Access JWT, so the expiring capability token + burn-after-read are // the gate. See routes/mailbox.ts for the posture. diff --git a/apps/dispatcher/src/routes/spend.test.ts b/apps/dispatcher/src/routes/spend.test.ts new file mode 100644 index 00000000..067ad006 --- /dev/null +++ b/apps/dispatcher/src/routes/spend.test.ts @@ -0,0 +1,129 @@ +// The spend guard's reader-facing surfaces: a budget or ceiling skip concludes +// the check neutral, and `GET /api/spend` / the dashboard line project one +// spend read (specs/adr/0006-spend-guard.md). + +import { Exit } from "effect"; +import { describe, expect, it } from "vitest"; +import { + decideBudget, + resolveSpendLimits, + RunSkipped, + spendCeilingReason, + spendMonth, +} from "@fractalboxdev/flare-dispatch-core"; +import type { SpendStatus } from "@fractalboxdev/flare-dispatch-runtime-cf/spend"; +import { runSkippedReason } from "../failure-summary"; +import { handleRequest } from "../router"; +import { makeFakeD1, makeFakeEnv, makeFakeR2, makeFakeWorkflow } from "../test-helpers"; +import { checkConclusion } from "../terminal-verdict"; +import { spendLine, spendView } from "./spend"; + +const NOW = Date.UTC(2026, 9, 22, 12, 0); + +const status = (monthToDateMicroUsd: number, admittedMicroUsdPerHour = 401_040): SpendStatus => { + const limits = resolveSpendLimits({}); + return { + month: spendMonth(NOW), + limits, + reading: { + finishedMicroUsd: monthToDateMicroUsd, + inFlightMicroUsd: 0, + admittedMicroUsdPerHour, + admittedCount: 1, + }, + monthToDateMicroUsd, + decision: decideBudget(monthToDateMicroUsd, limits), + }; +}; + +describe("a spend skip concludes neutral", () => { + it("records a budget skip as skipped with its reason, and a neutral check", () => { + const decision = decideBudget(90_500_000, resolveSpendLimits({})); + if (decision._kind !== "closed") throw new Error("expected a closed budget"); + const exit = Exit.fail(new RunSkipped({ reason: decision.reason })); + expect(runSkippedReason(exit)).toBe("spend budget reached: $90.50 of $90.00 this month"); + expect(checkConclusion("skipped", true)).toBe("neutral"); + }); + + it("records a burn-ceiling queue timeout as a skip", () => { + const reason = spendCeilingReason({ + admittedMicroUsdPerHour: 1_203_120, + ownMicroUsdPerHour: 401_040, + burnMaxMicroUsdPerHour: 1_500_000, + }); + expect(runSkippedReason(Exit.fail(new RunSkipped({ reason })))).toMatch(/^spend ceiling: /); + }); +}); + +describe("spendView", () => { + it("returns month-to-date spend, the limits, the admitted burn and the reset", () => { + expect(spendView(status(12_340_000))).toEqual({ + month: "2026-10", + monthToDateUsd: 12.34, + budgetUsd: 90, + burnMaxUsdPerHour: 1.5, + admittedUsdPerHour: 0.40104, + admittedCount: 1, + overheadFactor: 1.1, + warnRatio: 0.8, + budgetReached: false, + resetsAt: "2026-11-01T00:00:00.000Z", + }); + }); + + it("reports an unlimited limit as null", () => { + const open = status(1_000_000); + const view = spendView({ + ...open, + limits: { ...open.limits, budgetMicroUsd: null, burnMaxMicroUsdPerHour: null }, + }); + expect(view).toMatchObject({ budgetUsd: null, burnMaxUsdPerHour: null }); + }); +}); + +describe("GET /api/spend", () => { + const env = (viewerAccessMode?: "required") => + makeFakeEnv({ + hmacSecret: "s", + workflow: makeFakeWorkflow(), + storage: makeFakeR2(), + metadata: makeFakeD1({ executions: [] }), + ...(viewerAccessMode !== undefined ? { viewerAccessMode } : {}), + }); + + it("returns the deploy's spend view under the code defaults", async () => { + const res = await handleRequest(new Request("https://dispatcher.example.com/api/spend"), env()); + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ + monthToDateUsd: 0, + budgetUsd: 90, + burnMaxUsdPerHour: 1.5, + budgetReached: false, + }); + }); + + it("is a GET viewer surface: Access gates it first, then the method", async () => { + const gated = await handleRequest( + new Request("https://dispatcher.example.com/api/spend"), + env("required"), + ); + expect(gated.status).toBe(503); + const posted = await handleRequest( + new Request("https://dispatcher.example.com/api/spend", { method: "POST" }), + env(), + ); + expect(posted.status).toBe(405); + }); +}); + +describe("spendLine", () => { + it("is the dashboard's one-line summary", () => { + expect(spendLine(status(12_340_000))).toBe( + "Spend 2026-10: $12.34 of $90.00 · burn $0.40/h of $1.50/h · resets 2026-11-01", + ); + }); + + it("says when the budget closed", () => { + expect(spendLine(status(95_000_000))).toContain("$95.00 of $90.00 — budget reached, runs skip"); + }); +}); diff --git a/apps/dispatcher/src/routes/spend.ts b/apps/dispatcher/src/routes/spend.ts new file mode 100644 index 00000000..c4f8cf19 --- /dev/null +++ b/apps/dispatcher/src/routes/spend.ts @@ -0,0 +1,92 @@ +// FlareDispatch Dispatcher — `GET /api/spend`: the spend guard's month-to-date view. +// +// Returns the deploy's month-to-date spend, the monthly budget, the burn +// ceiling, the admitted hourly rate and when the month resets — the same +// numbers admission decides with (runtime-cf spend-d1.ts). The dashboard +// renders `spendLine` of the same view. Access-gated like the executions and +// analytics reads. +// +// Spec: specs/adr/0006-spend-guard.md § Warning and visibility. + +import { Effect } from "effect"; +import { MICRO_USD_PER_USD, formatUsd } from "@fractalboxdev/flare-dispatch-core"; +import { readSpendLimits, readSpendStatus, type SpendStatus } from "@fractalboxdev/flare-dispatch-runtime-cf/spend"; +import type { Env } from "../env"; + +/** The JSON body of `GET /api/spend`. USD figures; `null` = unlimited. */ +export type SpendView = { + /** The UTC calendar month, `YYYY-MM`. */ + readonly month: string; + readonly monthToDateUsd: number; + readonly budgetUsd: number | null; + readonly burnMaxUsdPerHour: number | null; + /** The summed hourly rate of every admitted execution, across all pools. */ + readonly admittedUsdPerHour: number; + readonly admittedCount: number; + readonly overheadFactor: number; + readonly warnRatio: number; + /** `true` once month-to-date spend reaches the budget: runs skip. */ + readonly budgetReached: boolean; + /** The first instant of the next UTC month, ISO 8601. */ + readonly resetsAt: string; +}; + +const usd = (microUsd: number): number => microUsd / MICRO_USD_PER_USD; + +/** Project a spend read onto the API view. Pure. */ +export const spendView = (status: SpendStatus): SpendView => ({ + month: status.month.key, + monthToDateUsd: usd(status.monthToDateMicroUsd), + budgetUsd: status.limits.budgetMicroUsd === null ? null : usd(status.limits.budgetMicroUsd), + burnMaxUsdPerHour: + status.limits.burnMaxMicroUsdPerHour === null ? null : usd(status.limits.burnMaxMicroUsdPerHour), + admittedUsdPerHour: usd(status.reading.admittedMicroUsdPerHour), + admittedCount: status.reading.admittedCount, + overheadFactor: status.limits.overheadFactor, + warnRatio: status.limits.warnRatio, + budgetReached: status.decision._kind === "closed", + resetsAt: new Date(status.month.resetsAtMs).toISOString(), +}); + +/** The one-line summary the dashboard shows. Pure. */ +export const spendLine = (status: SpendStatus): string => { + const budget = status.limits.budgetMicroUsd; + const ceiling = status.limits.burnMaxMicroUsdPerHour; + return [ + `Spend ${status.month.key}: ${formatUsd(status.monthToDateMicroUsd)} of ${ + budget === null ? "unlimited" : formatUsd(budget) + }${status.decision._kind === "closed" ? " — budget reached, runs skip" : ""}`, + `burn ${formatUsd(status.reading.admittedMicroUsdPerHour)}/h of ${ + ceiling === null ? "unlimited" : `${formatUsd(ceiling)}/h` + }`, + `resets ${new Date(status.month.resetsAtMs).toISOString().slice(0, 10)}`, + ].join(" · "); +}; + +/** Read the deploy's spend status now. */ +export const readSpendNow = (env: Env, nowMs: number = Date.now()): Effect.Effect => + readSpendLimits(env.CONFIG_KV, env).pipe( + Effect.flatMap((limits) => readSpendStatus(env.RUNS_METADATA, limits, nowMs)), + ); + +/** `GET /api/spend`. A D1 read failure answers 503 with the reason. */ +export const handleSpend = (env: Env, nowMs: number = Date.now()): Promise => + Effect.runPromise( + readSpendNow(env, nowMs).pipe( + Effect.map( + (status) => + new Response(JSON.stringify(spendView(status)), { + status: 200, + headers: { "content-type": "application/json; charset=utf-8" }, + }), + ), + Effect.catchAll((error) => + Effect.succeed( + new Response(JSON.stringify({ error: "spend_unreadable", detail: error.message }), { + status: 503, + headers: { "content-type": "application/json" }, + }), + ), + ), + ), + ); diff --git a/apps/dispatcher/src/workflow.ts b/apps/dispatcher/src/workflow.ts index f4fd7c81..59af0331 100644 --- a/apps/dispatcher/src/workflow.ts +++ b/apps/dispatcher/src/workflow.ts @@ -72,7 +72,10 @@ import { isEngineInterruptedCause, isEngineInterruption, withRunCeiling, - type RunSkipped, + RunSkipped, + spendCeilingReason, + type AdmissionObservation, + type SpendLimits, type RunContext, type RunError, } from "@fractalboxdev/flare-dispatch-core"; @@ -84,6 +87,7 @@ import { type ChecksGithubConfig, describeOutcome, type EmailCloudflareConfig, + hourlyRateForSandboxImage, instanceForSandboxImage, LEASE_HEARTBEAT_EVERY_MS, makeCFRuntimeLive, @@ -94,9 +98,13 @@ import { makeSerialQueueD1, makeWritebackTokenMinter, previewSafeSandboxId, + readSpendLimits, + readSpendStatus, recordExecutionCost, runSerialGate, runWriteback, + type SpendStatus, + warnSpendOnce, type WritebackOutcome, } from "@fractalboxdev/flare-dispatch-runtime-cf"; import { WRITEBACK_ARTIFACT } from "@fractalboxdev/flare-dispatch-core"; @@ -614,6 +622,17 @@ export class RunWorkflow extends WorkflowEntrypoint { // The `Workflow` binding backs the `childRuns` capability — a run can // `spawnChildRun` / `fanOut` to independent child `RunWorkflow` instances. runsWorkflow: this.env.RUNS_WORKFLOW, + // A fan-out parent reads spend before it spawns children: over the + // monthly budget it skips and spawns none (ADR-0006). `fanOut` calls + // this inside its spawn step, so a replay reads the checkpoint. + spendGate: async () => { + const status = await Effect.runPromise( + readSpendLimits(this.env.CONFIG_KV, this.env).pipe( + Effect.flatMap((limits) => readSpendStatus(this.env.RUNS_METADATA, limits, Date.now())), + ), + ); + return status.decision._kind === "closed" ? status.decision.reason : undefined; + }, executionId: payload.executionId, execution: { repo: payload.github.repo, @@ -878,12 +897,59 @@ export class RunWorkflow extends WorkflowEntrypoint { }); return yield* Effect.fail(error); }); + // --- Spend guard (specs/adr/0006-spend-guard.md) ----------------------- + // + // Admission reads month-to-date spend before it enqueues and again + // before each claim; at or over the monthly budget the run ends + // `RunSkipped` — a neutral check, a `skipped` row. The claim itself + // enforces the burn ceiling across every pool, at this run's container + // class rate. Every read sits in a `step.do`, so a replay reads the + // checkpointed answer. + const hourlyRateMicroUsd = hourlyRateForSandboxImage(admissionPool); + const spendCheck = (name: string, limits: SpendLimits) => + Effect.gen(function* () { + const status: SpendStatus = yield* stepDo(name, () => + Effect.runPromise(readSpendStatus(db, limits, Date.now())), + ); + if (status.decision._kind === "closed") { + yield* Effect.logWarning(`spend.closed ${payload.executionId} — ${status.decision.reason}`); + return yield* Effect.fail(new RunSkipped({ reason: status.decision.reason })); + } + return status; + }); const admissionGate: Effect.Effect = Effect.gen(function* () { + const limits = yield* stepDo("spend-limits", () => + Effect.runPromise(readSpendLimits(env.CONFIG_KV, env)), + ); + let spend = yield* spendCheck("spend-check-enqueue", limits); + const burnMaxMicroUsdPerHour = limits.burnMaxMicroUsdPerHour; + const claimer = makeRunAdmissionD1(db, Date.now, admissionCap, { + hourlyRateMicroUsd, + burnMaxMicroUsdPerHour, + }); + // A queue timeout while the burn ceiling, not the pool cap, held the + // run back is a spend skip; under the cap alone it stays a failure. + const queueTimeout = ( + observed: AdmissionObservation & { readonly now: number }, + timedOut: AdmissionTimedOut, + ): Effect.Effect => + observed.ceilingHeld === true && burnMaxMicroUsdPerHour !== null + ? Effect.fail( + new RunSkipped({ + reason: spendCeilingReason({ + admittedMicroUsdPerHour: spend.reading.admittedMicroUsdPerHour, + ownMicroUsdPerHour: hourlyRateMicroUsd, + burnMaxMicroUsdPerHour, + }), + }), + ) + : admissionTimedOut(timedOut, enqueuedAt, observed.now); + // Enqueue — the row's `enqueued_at` (FIFO key + dispatch-age basis) // is read inside the step, so a replay sees the SAME timestamp. const { enqueuedAt } = yield* stepDo("admission-enqueue", () => Effect.runPromise( - admissions.enqueue(payload.executionId, admissionPool, payload.parentExecutionId), + claimer.enqueue(payload.executionId, admissionPool, payload.parentExecutionId), ), ); @@ -899,12 +965,13 @@ export class RunWorkflow extends WorkflowEntrypoint { // A child reads its parent's row before each claim: once a parent // that joins its children has ended, the child leaves the queue // instead (admission-parent.ts). + spend = yield* spendCheck(`spend-check-${i}`, limits); const observed = yield* stepDo(`admission-claim-${i}`, () => claimUnlessParentEnded({ parentExecutionId: payload.parentExecutionId, parentRow: (id) => Effect.runPromise(supersessions.row(id)), joinsChildren: (name) => lookupRun(name)?.joinsChildren === true, - claim: () => Effect.runPromise(admissions.attempt(payload.executionId, admissionPool, enqueuedAt)), + claim: () => Effect.runPromise(claimer.attempt(payload.executionId, admissionPool, enqueuedAt)), }), ); if (isParentEnded(observed)) { @@ -941,16 +1008,37 @@ export class RunWorkflow extends WorkflowEntrypoint { admissionMaxQueueAgeMs, ); - if (decision._kind === "admit") return; + if (decision._kind === "admit") { + // The month's one warning, past `SPEND_WARN_RATIO` of the budget. + const status = spend; + yield* stepDo("spend-warn", () => + Effect.runPromise( + warnSpendOnce({ + kv: env.CONFIG_KV, + status, + notify: async (text, monthKey) => { + const sent = await emitSlackNotice(env, { + useCase: "spend-warn", + deliveryId: `spend-warn-${monthKey}`, + text, + run: payload.run, + executionId: payload.executionId, + }); + if (sent.outcome === "failed") throw new Error(sent.reason); + }, + }), + ), + ); + return; + } if (decision._kind === "timeout") { - return yield* admissionTimedOut( + return yield* queueTimeout( + observed, new AdmissionTimedOut({ queuedForMs: decision.queuedForMs, position: decision.position, poolBusy: decision.poolBusy, }), - enqueuedAt, - observed.now, ); } @@ -987,14 +1075,13 @@ export class RunWorkflow extends WorkflowEntrypoint { if (i < maxAttempts - 1) { yield* stepSleep(`admission-wait-${i}`, ADMISSION_POLL_EVERY_MS); } else { - return yield* admissionTimedOut( + return yield* queueTimeout( + observed, new AdmissionTimedOut({ queuedForMs: Math.max(0, observed.now - enqueuedAt), position: decision.position, poolBusy: decision.poolBusy, }), - enqueuedAt, - observed.now, ); } } @@ -1343,13 +1430,14 @@ export class RunWorkflow extends WorkflowEntrypoint { // Per-execution cost rollup — METERED model tokens (summed from // `execution_model_usage`, written by the modelGateway wrapper) + MODELED - // container compute (instance × wall-time). Runs AFTER finishExecution so - // the wall-time it reads is complete. Best-effort: a cost failure logs and - // never flips the verdict (recordExecutionCost absorbs its own errors). + // container compute (the admission pool's instance list rate × admitted + // seconds, ADR-0006 § Meter). Runs AFTER finishExecution so the window + // it reads is closed. Best-effort: a cost failure logs and never flips + // the verdict (recordExecutionCost absorbs its own errors). yield* recordExecutionCost({ db, executionId: payload.executionId, - instance: instanceForSandboxImage(run.sandboxImage), + instance: instanceForSandboxImage(admissionPool), }); // --- Post-run writeback (the Worker proposes the diff as a PR) -------- diff --git a/apps/docs/src/data/benchmarks.ts b/apps/docs/src/data/benchmarks.ts index 414c9ca9..83b50484 100644 --- a/apps/docs/src/data/benchmarks.ts +++ b/apps/docs/src/data/benchmarks.ts @@ -11,10 +11,9 @@ // and diffs. Model-calling runs are priced on the Anthropic backend as a representative // metered figure; the default Workers AI catalog backend bills as account-level Neurons. import { - CONTAINER_GIB_MICRO_USD_PER_SEC, - CONTAINER_VCPU_MICRO_USD_PER_SEC, INSTANCE_SPECS, containerCostMicroUsd, + containerMicroUsdPerSec, modelCost, type InstanceType, } from "@fractalboxdev/flare-dispatch-core/cost"; @@ -200,9 +199,4 @@ export const benchmarks: readonly BenchmarkRow[] = PROFILES.map((p) => { }); /** Container rate for an instance, in micro-USD per active second. */ -export const containerMicroUsdPerSec = (instance: InstanceType): number => { - const spec = INSTANCE_SPECS[instance]; - return spec.vcpu * CONTAINER_VCPU_MICRO_USD_PER_SEC + spec.gib * CONTAINER_GIB_MICRO_USD_PER_SEC; -}; - -export { INSTANCE_SPECS }; +export { containerMicroUsdPerSec, INSTANCE_SPECS }; diff --git a/infra/migrations/0011_spend_guard.sql b/infra/migrations/0011_spend_guard.sql new file mode 100644 index 00000000..e74462b4 --- /dev/null +++ b/infra/migrations/0011_spend_guard.sql @@ -0,0 +1,27 @@ +-- Migration number: 0011 spend guard +-- +-- The meter, the monthly budget and the burn ceiling of +-- specs/adr/0006-spend-guard.md: +-- +-- * `executions.admitted_at` — when the run's admission claim landed. A +-- container is active from admission to completion, so the finish path +-- prices that window (runtime-cf execution-cost.ts) and queue wait costs +-- nothing. NULL for a run never admitted. +-- * `executions.cost_meter` — which rule priced `cost_micro_usd`: +-- `admitted-active` from here on; rows priced on wall time before this +-- migration keep their value and read `modeled-wall`. +-- * `run_admissions.hourly_rate_micro_usd` — the list rate of the row's +-- container class per hour. The claim admits only while the live admitted +-- rates plus the claimant's own fit under the burn ceiling, across every +-- pool (runtime-cf run-admission-d1.ts). +-- +-- The month-to-date spend read (runtime-cf spend-d1.ts) sums `cost_micro_usd` +-- over executions admitted this month; `executions_admitted_at` serves that +-- range scan. + +ALTER TABLE executions ADD COLUMN admitted_at INTEGER; -- ms epoch; NULL = never admitted +ALTER TABLE executions ADD COLUMN cost_meter TEXT; -- admitted-active | modeled-wall +UPDATE executions SET cost_meter = 'modeled-wall' WHERE cost_micro_usd IS NOT NULL; +CREATE INDEX IF NOT EXISTS executions_admitted_at ON executions(admitted_at); + +ALTER TABLE run_admissions ADD COLUMN hourly_rate_micro_usd INTEGER; -- NULL rows (pre-0011) count as 0 diff --git a/packages/core/src/cost.test.ts b/packages/core/src/cost.test.ts index 1d079f8d..9b21d5db 100644 --- a/packages/core/src/cost.test.ts +++ b/packages/core/src/cost.test.ts @@ -5,11 +5,13 @@ import { describe, expect, it } from "vitest"; import { + CONTAINER_DISK_MICRO_USD_PER_SEC, CONTAINER_GIB_MICRO_USD_PER_SEC, CONTAINER_VCPU_MICRO_USD_PER_SEC, INSTANCE_SPECS, MICRO_USD_PER_USD, containerCostMicroUsd, + containerMicroUsdPerHour, estimateExecutionCost, formatMicroUsd, microUsdToUsd, @@ -18,13 +20,25 @@ import { } from "./cost.js"; describe("rate card (pinned to specs/06-cost.md)", () => { - it("standard-2 is 1 vCPU / 6 GiB", () => { - expect(INSTANCE_SPECS["standard-2"]).toEqual({ vcpu: 1, gib: 6 }); + it("standard-2 is 1 vCPU / 6 GiB / 12 GB disk", () => { + expect(INSTANCE_SPECS["standard-2"]).toEqual({ vcpu: 1, gib: 6, diskGb: 12 }); }); - it("basic is 1/4 vCPU / 1 GiB and standard-3 is 2 vCPU / 8 GiB", () => { - expect(INSTANCE_SPECS.basic).toEqual({ vcpu: 1 / 4, gib: 1 }); - expect(INSTANCE_SPECS["standard-3"]).toEqual({ vcpu: 2, gib: 8 }); + it("basic, standard-3 and standard-4 carry their provisioned disk", () => { + expect(INSTANCE_SPECS.basic).toEqual({ vcpu: 1 / 4, gib: 1, diskGb: 4 }); + expect(INSTANCE_SPECS["standard-3"]).toEqual({ vcpu: 2, gib: 8, diskGb: 16 }); + expect(INSTANCE_SPECS["standard-4"]).toEqual({ vcpu: 4, gib: 12, diskGb: 20 }); + }); + + it("disk is $0.00000007/GB-s in micro-USD", () => { + expect(CONTAINER_DISK_MICRO_USD_PER_SEC).toBe(0.07); + }); + + it("prices an hour on the provisioned size", () => { + // standard-4: 4 × 0.072 + 12 × 0.009 + 20 × 0.000252 = $0.40104/h. + expect(containerMicroUsdPerHour("standard-4")).toBe(401_040); + // standard-2: 0.072 + 6 × 0.009 + 12 × 0.000252 = $0.129024/h. + expect(containerMicroUsdPerHour("standard-2")).toBe(129_024); }); it("container rates are $0.000020/vCPU-s and $0.0000025/GiB-s in micro-USD", () => { @@ -35,17 +49,18 @@ describe("rate card (pinned to specs/06-cost.md)", () => { }); describe("containerCostMicroUsd — spec-06 per-execution anatomy", () => { - it("standard-2 for an 8-minute run ≈ $0.0168 (the spec's ~$0.017 example)", () => { + it("standard-2 for an 8-minute run ≈ $0.0172 (the spec's ~$0.017 example)", () => { // specs/06-cost.md § Per-execution cost anatomy: a standard-2 (1 vCPU, - // 6 GiB) running 8 min (480 s) = ~$0.0096 vCPU + ~$0.0072 mem ≈ $0.0168. + // 6 GiB, 12 GB) running 8 min (480 s) = $0.0096 vCPU + $0.0072 mem + + // $0.000403 disk ≈ $0.0172. const micro = containerCostMicroUsd({ instance: "standard-2", activeSeconds: 480 }); - expect(micro).toBe(16_800); - expect(microUsdToUsd(micro)).toBeCloseTo(0.0168, 6); + expect(micro).toBe(17_203); + expect(microUsdToUsd(micro)).toBeCloseTo(0.0172, 4); }); it("scales linearly with active seconds", () => { - const a = containerCostMicroUsd({ instance: "standard-2", activeSeconds: 60 }); - const b = containerCostMicroUsd({ instance: "standard-2", activeSeconds: 120 }); + const a = containerCostMicroUsd({ instance: "standard-2", activeSeconds: 100 }); + const b = containerCostMicroUsd({ instance: "standard-2", activeSeconds: 200 }); expect(b).toBe(a * 2); }); @@ -144,7 +159,7 @@ describe("estimateExecutionCost — honesty basis", () => { }); expect(cost.basis).toBe("modeled"); expect(cost.modelMicroUsd).toBeNull(); - expect(cost.totalMicroUsd).toBe(16_800); + expect(cost.totalMicroUsd).toBe(17_203); }); it("metered model with no container → metered", () => { diff --git a/packages/core/src/cost.ts b/packages/core/src/cost.ts index ba99be5b..79964767 100644 --- a/packages/core/src/cost.ts +++ b/packages/core/src/cost.ts @@ -39,10 +39,11 @@ export const MICRO_USD_PER_USD = 1_000_000; // --- Container instance types --------------------------------------------------- // -// vCPU + memory per Cloudflare Containers instance type (specs/05-byoc.md § -// Wrangler config, "Instance types (2026-05)"). A run's instance type is fixed -// by the Durable Object class its `sandboxImage` selects (wrangler.jsonc): -// `lean`/`browser` → `standard-2`, `agent` → `standard-3`. +// Provisioned vCPU, memory and disk per Cloudflare Containers instance type +// (specs/05-byoc.md § Wrangler config, "Instance types (2026-05)"). A run's +// instance type is fixed by the container class its `sandboxImage` selects in +// wrangler.jsonc — the one table is `SANDBOX_INSTANCE` in runtime-cf +// execution-cost.ts, checked against that file by its test. export type InstanceType = | "lite" @@ -57,30 +58,54 @@ export type InstanceSpec = { readonly vcpu: number; /** Allotted memory in GiB. */ readonly gib: number; + /** Provisioned disk in GB. */ + readonly diskGb: number; }; -/** vCPU / memory by instance type (specs/05-byoc.md § Wrangler config). */ +/** vCPU / memory / disk by instance type (specs/05-byoc.md § Wrangler config). */ export const INSTANCE_SPECS: Record = { - lite: { vcpu: 1 / 16, gib: 0.25 }, - basic: { vcpu: 1 / 4, gib: 1 }, - "standard-1": { vcpu: 1 / 2, gib: 4 }, - "standard-2": { vcpu: 1, gib: 6 }, - "standard-3": { vcpu: 2, gib: 8 }, - "standard-4": { vcpu: 4, gib: 12 }, + lite: { vcpu: 1 / 16, gib: 0.25, diskGb: 2 }, + basic: { vcpu: 1 / 4, gib: 1, diskGb: 4 }, + "standard-1": { vcpu: 1 / 2, gib: 4, diskGb: 8 }, + "standard-2": { vcpu: 1, gib: 6, diskGb: 12 }, + "standard-3": { vcpu: 2, gib: 8, diskGb: 16 }, + "standard-4": { vcpu: 4, gib: 12, diskGb: 20 }, } as const; // --- Container rates (specs/06-cost.md § Pricing model) ------------------------- // -// $0.000020 / vCPU-s and $0.0000025 / GiB-s. In µ$: multiply USD by 1e6. +// List prices on the PROVISIONED size: $0.000020 / vCPU-s, $0.0000025 / GiB-s +// and $0.00000007 / GB-s of disk. In µ$: multiply USD by 1e6. // $0.000020 × 1e6 = 20 µ$ per vCPU-second // $0.0000025 × 1e6 = 2.5 µ$ per GiB-second -// (Disk GB-s is billed too but rounds to zero per-execution and we don't track -// it — see specs/06-cost.md § Per-execution cost anatomy.) +// $0.00000007 × 1e6 = 0.07 µ$ per disk GB-second +// Cloudflare bills active vCPU, so a provisioned-size rate is an upper bound on +// the billed rate — the direction a spend cap wants (ADR-0006 § Meter). /** Container vCPU-second rate, in micro-USD (`$0.000020/vCPU-s`). */ export const CONTAINER_VCPU_MICRO_USD_PER_SEC = 20; /** Container memory GiB-second rate, in micro-USD (`$0.0000025/GiB-s`). */ export const CONTAINER_GIB_MICRO_USD_PER_SEC = 2.5; +/** Container disk GB-second rate, in micro-USD (`$0.00000007/GB-s`). */ +export const CONTAINER_DISK_MICRO_USD_PER_SEC = 0.07; + +/** One instance's list rate per active second, in (fractional) micro-USD. */ +export const containerMicroUsdPerSec = (instance: InstanceType): number => { + const spec = INSTANCE_SPECS[instance]; + return ( + spec.vcpu * CONTAINER_VCPU_MICRO_USD_PER_SEC + + spec.gib * CONTAINER_GIB_MICRO_USD_PER_SEC + + spec.diskGb * CONTAINER_DISK_MICRO_USD_PER_SEC + ); +}; + +/** + * One instance's list rate per hour, in whole micro-USD — the unit the burn + * ceiling sums across admitted executions (`run_admissions.hourly_rate_micro_usd`). + * `standard-4` ≈ $0.401/h, `standard-2` ≈ $0.129/h. + */ +export const containerMicroUsdPerHour = (instance: InstanceType): number => + Math.round(containerMicroUsdPerSec(instance) * 3600); // --- Model token rates ---------------------------------------------------------- // @@ -209,23 +234,16 @@ export const modelCost = (opts: { }; /** - * Modeled container-compute cost: `(vCPU-s × vCPU-rate) + (GiB-s × GiB-rate)`. - * GiB-seconds derive from the instance's allotted memory × active seconds. This - * is ALWAYS modeled — Cloudflare meters container vCPU-s at the account level - * only, never per-execution. + * Modeled container-compute cost: `(vCPU-s × vCPU-rate) + (GiB-s × GiB-rate) + + * (disk GB-s × disk rate)`, each on the instance's provisioned size × active + * seconds. This is ALWAYS modeled — Cloudflare meters container usage at the + * account level only, never per-execution. */ export const containerCostMicroUsd = (opts: { readonly instance: InstanceType; - /** Seconds the container was active (boot → destroy, or summed exec time). */ + /** Seconds the container was active (admission → completion). */ readonly activeSeconds: number; -}): number => { - const spec = INSTANCE_SPECS[opts.instance]; - const vcpuSeconds = spec.vcpu * opts.activeSeconds; - const gibSeconds = spec.gib * opts.activeSeconds; - return Math.round( - vcpuSeconds * CONTAINER_VCPU_MICRO_USD_PER_SEC + gibSeconds * CONTAINER_GIB_MICRO_USD_PER_SEC, - ); -}; +}): number => Math.round(containerMicroUsdPerSec(opts.instance) * opts.activeSeconds); // --- Whole-execution estimate --------------------------------------------------- diff --git a/packages/core/src/fakes/child-runs-fake.ts b/packages/core/src/fakes/child-runs-fake.ts index 45117653..7bde7070 100644 --- a/packages/core/src/fakes/child-runs-fake.ts +++ b/packages/core/src/fakes/child-runs-fake.ts @@ -8,7 +8,7 @@ // Spec: specs/03-dsl.md § spawnChildRun, specs/pm/plan.md § 3 (fakes/). import { Effect, Layer } from "effect"; -import { ChildSpawnFailed } from "../errors"; +import { ChildSpawnFailed, RunSkipped } from "../errors"; import { type ChildRunStatus, ChildRuns, @@ -49,6 +49,8 @@ export const makeChildRunsFake = (opts?: { existing?: Iterable; /** Reject selected spawns before recording them, for partial fan-out tests. */ failSpawn?: (opts: SpawnChildRunOpts) => boolean; + /** When set, `spendGate` fails `RunSkipped` with this reason (budget reached). */ + spendClosed?: string; /** * Static per-id poll statuses. An id absent from the map polls as `missing`. * For a loop test that flips `running → success`, supply `pollFn` instead. @@ -77,6 +79,10 @@ export const makeChildRunsFake = (opts?: { }; const service: ChildRunsService = { + spendGate: () => + opts?.spendClosed !== undefined + ? Effect.fail(new RunSkipped({ reason: opts.spendClosed })) + : Effect.void, handoffAdmission: () => Effect.sync(() => { state.admissionHandoffs += 1; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4512ebec..11eb6731 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -87,6 +87,32 @@ export { type AdmissionDecision, } from "./run-admission"; +// --- Spend guard (pure budget + burn-ceiling math, ADR-0006) ----------------- +export { + SPEND_BUDGET_MONTHLY_USD_DEFAULT, + SPEND_BURN_MAX_USD_PER_HOUR_DEFAULT, + SPEND_OVERHEAD_FACTOR_DEFAULT, + SPEND_WARN_RATIO_DEFAULT, + SPEND_UNLIMITED, + SPEND_BUDGET_KV_KEY, + SPEND_BURN_KV_KEY, + spendWarnedKvKey, + parseUsdLimit, + resolveSpendLimits, + spendMonth, + monthToDateMicroUsd, + formatUsd, + decideBudget, + fitsBurnCeiling, + spendCeilingReason, + type SpendLimits, + type SpendVars, + type SpendOverrides, + type SpendMonth, + type SpendReading, + type BudgetDecision, +} from "./spend"; + // --- Run ceiling (limits.maxDurationSec, replay-stable) ---------------------- export { withRunCeiling, type RunCeiling } from "./run-ceiling"; export { @@ -303,6 +329,9 @@ export { INSTANCE_SPECS, CONTAINER_VCPU_MICRO_USD_PER_SEC, CONTAINER_GIB_MICRO_USD_PER_SEC, + CONTAINER_DISK_MICRO_USD_PER_SEC, + containerMicroUsdPerSec, + containerMicroUsdPerHour, modelRate, modelCost, containerCostMicroUsd, diff --git a/packages/core/src/primitives/fan-out.test.ts b/packages/core/src/primitives/fan-out.test.ts index f7365961..fb1ee87e 100644 --- a/packages/core/src/primitives/fan-out.test.ts +++ b/packages/core/src/primitives/fan-out.test.ts @@ -82,3 +82,24 @@ describe("fanOut", () => { expect(spawned).toEqual([]); }); }); + +describe("fanOut — the spend gate (ADR-0006)", () => { + const reason = "spend budget reached: $90.00 of $90.00 this month"; + + it("skips and spawns no child once the monthly budget is reached", async () => { + const child = makeChildRunsFake({ spendClosed: reason }); + const error = await Effect.runPromise( + fanOut({ run: "check", items: ["a", "b"], toInput: (item) => ({ item }) }).pipe( + Effect.flip, + Effect.provide(child.layer), + ), + ); + expect(error).toMatchObject({ _tag: "RunSkipped", reason }); + expect(child.state.spawned).toEqual([]); + }); + + it("spawns under the budget", async () => { + const { spawned } = await runFanOut({ run: "check", items: ["a"], toInput: (item) => ({ item }) }); + expect(spawned).toHaveLength(1); + }); +}); diff --git a/packages/core/src/primitives/fan-out.ts b/packages/core/src/primitives/fan-out.ts index ebf30b9f..8732058a 100644 --- a/packages/core/src/primitives/fan-out.ts +++ b/packages/core/src/primitives/fan-out.ts @@ -20,10 +20,14 @@ // checkpoint; the deterministic child instance ids make the underlying // `create` idempotent on replay regardless. // +// Before it spawns anything, `fanOut` reads the deploy's spend through the +// capability's `spendGate`: at or over the monthly budget it fails +// `RunSkipped` and spawns no child (specs/adr/0006-spend-guard.md). +// // Spec: specs/03-dsl.md § Primitives + § spawnChildRun. import { Effect } from "effect"; -import type { ChildSpawnFailed } from "../errors"; +import type { ChildSpawnFailed, RunSkipped } from "../errors"; import { type ChildRunHandle, ChildRuns, spawnChildRun } from "../services/child-runs"; import type { Shard } from "./sharded"; @@ -65,9 +69,10 @@ export const fanOut = (opts: { readonly toInstanceId?: (item: T, shard: Shard) => string; /** Max concurrent spawns. Default: all at once (spawn is a cheap create). */ readonly concurrency?: number; -}): Effect.Effect => { +}): Effect.Effect => { const total = opts.items.length; - return Effect.forEach( + if (total === 0) return Effect.succeed([]); + const spawnAll = Effect.forEach( opts.items.map((item, i): FanOutShard => ({ item, shard: { index: i + 1, total }, @@ -78,6 +83,7 @@ export const fanOut = (opts: { input: opts.toInput(item, shard), ...(opts.toInstanceId !== undefined ? { instanceId: opts.toInstanceId(item, shard) } : {}), }), - { concurrency: opts.concurrency ?? (total === 0 ? 1 : total) }, + { concurrency: opts.concurrency ?? total }, ); + return Effect.flatMap(ChildRuns, (c) => c.spendGate(opts.run)).pipe(Effect.andThen(spawnAll)); }; diff --git a/packages/core/src/run-admission.test.ts b/packages/core/src/run-admission.test.ts index a66fd92e..1016daaf 100644 --- a/packages/core/src/run-admission.test.ts +++ b/packages/core/src/run-admission.test.ts @@ -69,9 +69,20 @@ describe("decideAdmission", () => { queuedForMs: MAX_QUEUE_AGE, position: 2, poolBusy: 16, + ceilingHeld: false, }); }); + it("carries the burn ceiling hold into the timeout", () => { + const decision = decideAdmission( + { admitted: false, position: 0, poolBusy: 3, ceilingHeld: true }, + ENQUEUED_AT, + ENQUEUED_AT + MAX_QUEUE_AGE, + MAX_QUEUE_AGE, + ); + expect(decision).toMatchObject({ _kind: "timeout", ceilingHeld: true }); + }); + it("still waits one tick before the ceiling", () => { const decision = decideAdmission( observed(), diff --git a/packages/core/src/run-admission.ts b/packages/core/src/run-admission.ts index e35abfd5..5211d5b3 100644 --- a/packages/core/src/run-admission.ts +++ b/packages/core/src/run-admission.ts @@ -44,6 +44,12 @@ export type AdmissionObservation = { readonly position: number; /** Live admitted rows in our pool (slots currently in use). */ readonly poolBusy: number; + /** + * The burn ceiling, not the pool cap, held the claim back: our pool had a + * free slot, but the admitted hourly rates plus ours exceed the ceiling + * (ADR-0006 § Burn ceiling). Absent = false. + */ + readonly ceilingHeld?: boolean; }; /** @@ -69,6 +75,8 @@ export type AdmissionDecision = readonly queuedForMs: number; readonly position: number; readonly poolBusy: number; + /** The final attempt saw the burn ceiling, not the pool cap, holding it. */ + readonly ceilingHeld: boolean; }; /** @@ -96,6 +104,7 @@ export const decideAdmission = ( queuedForMs, position: observed.position, poolBusy: observed.poolBusy, + ceilingHeld: observed.ceilingHeld === true, }; } return { diff --git a/packages/core/src/services/child-runs.ts b/packages/core/src/services/child-runs.ts index 3179db15..f8f7dd99 100644 --- a/packages/core/src/services/child-runs.ts +++ b/packages/core/src/services/child-runs.ts @@ -23,7 +23,7 @@ // § Long-running test handling + § Platform limits. import { Context, Effect } from "effect"; -import type { ChildSpawnFailed } from "../errors"; +import type { ChildSpawnFailed, RunSkipped } from "../errors"; /** * The result of a spawn. `executionId` and `instanceId` are equal — the @@ -115,6 +115,14 @@ export const isTerminalChildStatus = (status: ChildRunStatus): boolean => TERMIN /** The service contract a runtime Layer implements. */ export interface ChildRunsService { readonly spawn: (opts: SpawnChildRunOpts) => Effect.Effect; + /** + * Read the deploy's month-to-date spend before a fan-out spawns children of + * `run` (ADR-0006 § Monthly budget). At or over the monthly budget it fails + * `RunSkipped` with the budget reason, and the parent spawns none; an + * unreadable spend fails `ChildSpawnFailed` rather than reading as open. + * Call it inside the spawn step, so a replay reads the checkpointed answer. + */ + readonly spendGate: (run: string) => Effect.Effect; /** * Release the parent's global admission slot after it has finished its own * sandbox work. A parent waiting on children must not occupy a slot that a diff --git a/packages/core/src/spend.test.ts b/packages/core/src/spend.test.ts new file mode 100644 index 00000000..238f14ee --- /dev/null +++ b/packages/core/src/spend.test.ts @@ -0,0 +1,149 @@ +// Spend-guard math — the limits a deploy resolves, the month window, the +// budget decision and the burn-ceiling fit (specs/adr/0006-spend-guard.md). + +import { describe, expect, it } from "vitest"; +import { containerMicroUsdPerHour } from "./cost"; +import { + decideBudget, + fitsBurnCeiling, + monthToDateMicroUsd, + parseUsdLimit, + resolveSpendLimits, + spendCeilingReason, + spendMonth, + type SpendLimits, +} from "./spend"; + +const LIMITS: SpendLimits = { + budgetMicroUsd: 90_000_000, + burnMaxMicroUsdPerHour: 1_500_000, + overheadFactor: 1.1, + warnRatio: 0.8, +}; + +describe("resolveSpendLimits", () => { + it("caps a deploy that sets nothing at 90 USD a month and 1.50 USD an hour", () => { + expect(resolveSpendLimits({})).toEqual(LIMITS); + }); + + it("reads the vars, and a CONFIG_KV override wins over them", () => { + const vars = { SPEND_BUDGET_MONTHLY_USD: "200", SPEND_BURN_MAX_USD_PER_HOUR: "3" }; + expect(resolveSpendLimits(vars)).toMatchObject({ + budgetMicroUsd: 200_000_000, + burnMaxMicroUsdPerHour: 3_000_000, + }); + expect( + resolveSpendLimits(vars, { budgetMonthlyUsd: "250", burnMaxUsdPerHour: null }), + ).toMatchObject({ budgetMicroUsd: 250_000_000, burnMaxMicroUsdPerHour: 3_000_000 }); + }); + + it("disables a limit on `unlimited`, from either source", () => { + expect(resolveSpendLimits({ SPEND_BUDGET_MONTHLY_USD: "unlimited" }).budgetMicroUsd).toBeNull(); + expect( + resolveSpendLimits({}, { burnMaxUsdPerHour: " Unlimited " }).burnMaxMicroUsdPerHour, + ).toBeNull(); + }); + + it("degrades a typo to the default, never to no limit", () => { + expect(parseUsdLimit("ninety", 90)).toBe(90_000_000); + expect(parseUsdLimit("-5", 90)).toBe(90_000_000); + expect(parseUsdLimit("", 90)).toBe(90_000_000); + expect(resolveSpendLimits({ SPEND_OVERHEAD_FACTOR: "0.5", SPEND_WARN_RATIO: "2" })).toMatchObject({ + overheadFactor: 1.1, + warnRatio: 0.8, + }); + }); + + it("treats a blank override as absent", () => { + expect( + resolveSpendLimits({ SPEND_BUDGET_MONTHLY_USD: "40" }, { budgetMonthlyUsd: " " }).budgetMicroUsd, + ).toBe(40_000_000); + }); +}); + +describe("spendMonth", () => { + it("is the UTC calendar month and resets at the next one", () => { + const month = spendMonth(Date.UTC(2026, 9, 22, 13, 5)); + expect(month).toEqual({ + key: "2026-10", + startMs: Date.UTC(2026, 9, 1), + resetsAtMs: Date.UTC(2026, 10, 1), + }); + expect(spendMonth(Date.UTC(2026, 11, 31, 23, 59)).resetsAtMs).toBe(Date.UTC(2027, 0, 1)); + }); +}); + +describe("monthToDateMicroUsd", () => { + it("adds finished and in-flight spend, times the overhead factor", () => { + expect( + monthToDateMicroUsd( + { finishedMicroUsd: 10_000_000, inFlightMicroUsd: 2_000_000, admittedMicroUsdPerHour: 0, admittedCount: 0 }, + 1.1, + ), + ).toBe(13_200_000); + }); +}); + +describe("decideBudget", () => { + it("stays open under the budget and warns past the warn ratio", () => { + expect(decideBudget(10_000_000, LIMITS)).toEqual({ _kind: "open", warn: false }); + expect(decideBudget(72_000_000, LIMITS)).toEqual({ _kind: "open", warn: true }); + }); + + it("closes at the budget with the skip reason", () => { + expect(decideBudget(90_000_000, LIMITS)).toEqual({ + _kind: "closed", + reason: "spend budget reached: $90.00 of $90.00 this month", + }); + expect(decideBudget(91_234_567, LIMITS)).toMatchObject({ + reason: "spend budget reached: $91.23 of $90.00 this month", + }); + }); + + it("never closes an unlimited budget", () => { + expect(decideBudget(10 ** 12, { ...LIMITS, budgetMicroUsd: null })).toEqual({ + _kind: "open", + warn: false, + }); + }); +}); + +describe("fitsBurnCeiling", () => { + const fits = (admitted: readonly number[], own: number, ceiling: number | null = 1_500_000) => + fitsBurnCeiling({ + admittedMicroUsdPerHour: admitted.reduce((a, b) => a + b, 0), + admittedCount: admitted.length, + ownMicroUsdPerHour: own, + burnMaxMicroUsdPerHour: ceiling, + }); + + it("admits three standard-4 containers, or eleven standard-2, under the default ceiling", () => { + const s4 = containerMicroUsdPerHour("standard-4"); + const s2 = containerMicroUsdPerHour("standard-2"); + expect(fits([s4, s4], s4)).toBe(true); + expect(fits([s4, s4, s4], s4)).toBe(false); + expect(fits(Array(10).fill(s2), s2)).toBe(true); + expect(fits(Array(11).fill(s2), s2)).toBe(false); + }); + + it("admits a claimant whose own rate exceeds the ceiling into an empty set only", () => { + expect(fits([], 2_000_000)).toBe(true); + expect(fits([100_000], 2_000_000)).toBe(false); + }); + + it("always fits with no ceiling", () => { + expect(fits([10_000_000], 10_000_000, null)).toBe(true); + }); +}); + +describe("spendCeilingReason", () => { + it("names the burn and the ceiling", () => { + expect( + spendCeilingReason({ + admittedMicroUsdPerHour: 1_203_120, + ownMicroUsdPerHour: 401_040, + burnMaxMicroUsdPerHour: 1_500_000, + }), + ).toBe("spend ceiling: admitted runs burn $1.20/h of the $1.50/h ceiling, this run needs $0.40/h"); + }); +}); diff --git a/packages/core/src/spend.ts b/packages/core/src/spend.ts new file mode 100644 index 00000000..32a4e873 --- /dev/null +++ b/packages/core/src/spend.ts @@ -0,0 +1,198 @@ +// @fractalboxdev/flare-dispatch-core — spend guard: the pure budget and burn-ceiling math. +// +// Cloudflare has no spend cap for Containers, so admission is where a deploy +// caps its own spend: every run passes the admission semaphore before it +// starts a container. Two limits bind there: +// +// * a MONTHLY BUDGET — month-to-date spend at or over it closes admission, +// and the run ends `RunSkipped` (a neutral check, a `skipped` row); +// * a BURN CEILING — the summed hourly rate of every admitted execution, +// across all pools, stays at or under it. The claim SQL enforces it +// (runtime-cf run-admission-d1.ts); `fitsBurnCeiling` is its pure mirror. +// +// Money is integer micro-USD throughout, like the cost engine (cost.ts). This +// module holds no I/O and no clock: the runtime reads D1 and CONFIG_KV inside +// a durable step and hands the readings here. +// +// Spec: specs/adr/0006-spend-guard.md. + +import { MICRO_USD_PER_USD } from "./cost"; + +/** The monthly budget a deploy that sets nothing gets, in USD. */ +export const SPEND_BUDGET_MONTHLY_USD_DEFAULT = 90; +/** The burn ceiling a deploy that sets nothing gets, in USD per hour. */ +export const SPEND_BURN_MAX_USD_PER_HOUR_DEFAULT = 1.5; +/** Multiplier for Workflow steps and Durable Objects on top of container + model spend. */ +export const SPEND_OVERHEAD_FACTOR_DEFAULT = 1.1; +/** Fraction of the budget whose crossing sends the month's one warning. */ +export const SPEND_WARN_RATIO_DEFAULT = 0.8; + +/** The value that disables a limit, in a var or a `CONFIG_KV` override. */ +export const SPEND_UNLIMITED = "unlimited"; + +/** The spend-guard settings an admission decides with. `null` = unlimited. */ +export type SpendLimits = { + readonly budgetMicroUsd: number | null; + readonly burnMaxMicroUsdPerHour: number | null; + readonly overheadFactor: number; + readonly warnRatio: number; +}; + +/** The raw spend settings: wrangler vars, each overridable from `CONFIG_KV`. */ +export type SpendVars = { + readonly SPEND_BUDGET_MONTHLY_USD?: string | undefined; + readonly SPEND_BURN_MAX_USD_PER_HOUR?: string | undefined; + readonly SPEND_OVERHEAD_FACTOR?: string | undefined; + readonly SPEND_WARN_RATIO?: string | undefined; +}; + +/** The `CONFIG_KV` overrides an operator writes to change a limit without a deploy. */ +export type SpendOverrides = { + readonly budgetMonthlyUsd?: string | null | undefined; + readonly burnMaxUsdPerHour?: string | null | undefined; +}; + +const toMicro = (usd: number): number => Math.round(usd * MICRO_USD_PER_USD); + +/** A present, non-blank override wins; otherwise the var. */ +const pick = (override: string | null | undefined, value: string | undefined): string | undefined => + override !== null && override !== undefined && override.trim() !== "" ? override : value; + +/** + * Parse a USD limit into micro-USD. `unlimited` → `null`; a finite number ≥ 0 + * → its micro-USD value; absent, blank or garbage → `fallbackUsd`. A typo + * degrades to the default limit, never to no limit. + */ +export const parseUsdLimit = (raw: string | null | undefined, fallbackUsd: number): number | null => { + const text = raw?.trim(); + if (text === undefined || text === "") return toMicro(fallbackUsd); + if (text.toLowerCase() === SPEND_UNLIMITED) return null; + const usd = Number(text); + return Number.isFinite(usd) && usd >= 0 ? toMicro(usd) : toMicro(fallbackUsd); +}; + +/** Parse a bounded ratio; outside `[min, max]` or unparsable → `fallback`. */ +const parseRatio = (raw: string | undefined, fallback: number, min: number, max: number): number => { + const text = raw?.trim(); + if (text === undefined || text === "") return fallback; + const value = Number(text); + return Number.isFinite(value) && value >= min && value <= max ? value : fallback; +}; + +/** + * Resolve the limits: a present `CONFIG_KV` override wins over the var, and + * the var over the code default. The defaults live here rather than in the + * committed wrangler.jsonc because an operator overlay replaces that file. + */ +export const resolveSpendLimits = (vars: SpendVars, overrides: SpendOverrides = {}): SpendLimits => ({ + budgetMicroUsd: parseUsdLimit( + pick(overrides.budgetMonthlyUsd, vars.SPEND_BUDGET_MONTHLY_USD), + SPEND_BUDGET_MONTHLY_USD_DEFAULT, + ), + burnMaxMicroUsdPerHour: parseUsdLimit( + pick(overrides.burnMaxUsdPerHour, vars.SPEND_BURN_MAX_USD_PER_HOUR), + SPEND_BURN_MAX_USD_PER_HOUR_DEFAULT, + ), + overheadFactor: parseRatio(vars.SPEND_OVERHEAD_FACTOR, SPEND_OVERHEAD_FACTOR_DEFAULT, 1, 10), + warnRatio: parseRatio(vars.SPEND_WARN_RATIO, SPEND_WARN_RATIO_DEFAULT, Number.MIN_VALUE, 1), +}); + +/** The current UTC calendar month: its bounds and its `YYYY-MM` key. */ +export type SpendMonth = { + readonly key: string; + readonly startMs: number; + /** When the month resets — the first instant of the next UTC month. */ + readonly resetsAtMs: number; +}; + +/** The UTC calendar month containing `nowMs`. */ +export const spendMonth = (nowMs: number): SpendMonth => { + const now = new Date(nowMs); + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + return { + key: `${year}-${String(month + 1).padStart(2, "0")}`, + startMs: Date.UTC(year, month, 1), + resetsAtMs: Date.UTC(year, month + 1, 1), + }; +}; + +/** + * What one spend read observed in D1 (runtime-cf spend-d1.ts), before the + * overhead factor: + * + * * `finishedMicroUsd` — `cost_micro_usd` of executions admitted this month; + * * `inFlightMicroUsd` — each admitted, unfinished execution's hourly rate × + * the time since its admission; + * * `admittedMicroUsdPerHour` / `admittedCount` — the live admitted set the + * burn ceiling sums, across every pool. + */ +export type SpendReading = { + readonly finishedMicroUsd: number; + readonly inFlightMicroUsd: number; + readonly admittedMicroUsdPerHour: number; + readonly admittedCount: number; +}; + +/** Month-to-date spend: finished + in-flight, times the overhead factor. */ +export const monthToDateMicroUsd = (reading: SpendReading, overheadFactor: number): number => + Math.round((reading.finishedMicroUsd + reading.inFlightMicroUsd) * overheadFactor); + +/** `$12.34` — two decimals, the precision a budget line reads at. */ +export const formatUsd = (microUsd: number): string => `$${(microUsd / MICRO_USD_PER_USD).toFixed(2)}`; + +/** + * The monthly-budget decision for one admission or spawn. + * + * - `open` — under budget (or unlimited); `warn` is true once spend sits at + * or over `warnRatio` of the budget. + * - `closed` — at or over budget: the run skips with `reason`. + */ +export type BudgetDecision = + | { readonly _kind: "open"; readonly warn: boolean } + | { readonly _kind: "closed"; readonly reason: string }; + +/** Decide the monthly budget from month-to-date spend. Pure. */ +export const decideBudget = (spendMicroUsd: number, limits: SpendLimits): BudgetDecision => { + const budget = limits.budgetMicroUsd; + if (budget === null) return { _kind: "open", warn: false }; + if (spendMicroUsd >= budget) { + return { + _kind: "closed", + reason: `spend budget reached: ${formatUsd(spendMicroUsd)} of ${formatUsd(budget)} this month`, + }; + } + return { _kind: "open", warn: spendMicroUsd >= budget * limits.warnRatio }; +}; + +/** + * Whether a claimant at `ownMicroUsdPerHour` fits under the burn ceiling — the + * pure mirror of the claim SQL's ceiling clause. The admitted rates plus the + * claimant's own must fit; a claimant whose own rate alone exceeds the + * ceiling fits only an empty admitted set, so it never waits forever. + */ +export const fitsBurnCeiling = (opts: { + readonly admittedMicroUsdPerHour: number; + readonly admittedCount: number; + readonly ownMicroUsdPerHour: number; + readonly burnMaxMicroUsdPerHour: number | null; +}): boolean => + opts.burnMaxMicroUsdPerHour === null || + opts.admittedCount === 0 || + opts.admittedMicroUsdPerHour + opts.ownMicroUsdPerHour <= opts.burnMaxMicroUsdPerHour; + +/** The `RunSkipped` reason for a run the burn ceiling held in the queue past its age. */ +export const spendCeilingReason = (opts: { + readonly admittedMicroUsdPerHour: number; + readonly ownMicroUsdPerHour: number; + readonly burnMaxMicroUsdPerHour: number; +}): string => + `spend ceiling: admitted runs burn ${formatUsd(opts.admittedMicroUsdPerHour)}/h of the ` + + `${formatUsd(opts.burnMaxMicroUsdPerHour)}/h ceiling, this run needs ${formatUsd(opts.ownMicroUsdPerHour)}/h`; + +/** The `CONFIG_KV` key that overrides the monthly budget (USD, or `unlimited`). */ +export const SPEND_BUDGET_KV_KEY = "spend.budgetMonthlyUsd"; +/** The `CONFIG_KV` key that overrides the burn ceiling (USD per hour, or `unlimited`). */ +export const SPEND_BURN_KV_KEY = "spend.burnMaxUsdPerHour"; +/** The `CONFIG_KV` marker that a month's warning went out. */ +export const spendWarnedKvKey = (monthKey: string): string => `spend.warned:${monthKey}`; diff --git a/packages/runtime-cf/package.json b/packages/runtime-cf/package.json index 8d6f6579..94d47d85 100644 --- a/packages/runtime-cf/package.json +++ b/packages/runtime-cf/package.json @@ -10,6 +10,7 @@ "./model-gateway": "./src/model-gateway-cf.ts", "./log-names": "./src/log-names.ts", "./admission-cap": "./src/run-admission-d1.ts", + "./spend": "./src/spend-d1.ts", "./run-supersede": "./src/run-supersede-d1.ts" }, "scripts": { diff --git a/packages/runtime-cf/src/child-runs-cf.ts b/packages/runtime-cf/src/child-runs-cf.ts index 759ad07a..b538cad5 100644 --- a/packages/runtime-cf/src/child-runs-cf.ts +++ b/packages/runtime-cf/src/child-runs-cf.ts @@ -35,6 +35,7 @@ import { ChildSpawnFailed, type ChildStatusRecord, isTerminalChildStatus, + RunSkipped, } from "@fractalboxdev/flare-dispatch-core"; import { makeRunAdmissionD1 } from "./run-admission-d1"; @@ -85,6 +86,14 @@ export type ChildRunsLiveConfig = { * Workflow has no request to infer it from). Absent → relative paths. */ readonly origin?: string; + /** + * The deploy's monthly-budget read (ADR-0006): resolves the `RunSkipped` + * reason when month-to-date spend is at or over the budget, `undefined` + * while it is open; rejects when spend is unreadable. The dispatcher wires + * it over D1 + `CONFIG_KV`. Absent = a runtime with no spend guard, the + * same as an `unlimited` budget. + */ + readonly spendGate?: () => Promise; }; /** Map a raw `executions.status` cell to the capability's `ChildRunStatus`. */ @@ -154,7 +163,19 @@ export const deriveChildInstanceId = (opts: { /** Build the live `ChildRuns` Layer bound to a CF `Workflow` binding. */ export const makeChildRunsLive = (cfg: ChildRunsLiveConfig): Layer.Layer => { + const spendGate = cfg.spendGate; const service: ChildRunsService = { + spendGate: (run) => + spendGate === undefined + ? Effect.void + : Effect.tryPromise({ + try: spendGate, + catch: (cause) => new ChildSpawnFailed({ run, instanceId: "spend-gate", cause }), + }).pipe( + Effect.flatMap((closed) => + closed === undefined ? Effect.void : Effect.fail(new RunSkipped({ reason: closed })), + ), + ), handoffAdmission: () => makeRunAdmissionD1(cfg.db).release(cfg.parentExecutionId).pipe(Effect.orDie), spawn: ({ run, input, instanceId }) => { diff --git a/packages/runtime-cf/src/deferred.ts b/packages/runtime-cf/src/deferred.ts index 11cf1a51..b9121cf8 100644 --- a/packages/runtime-cf/src/deferred.ts +++ b/packages/runtime-cf/src/deferred.ts @@ -79,6 +79,8 @@ export const ChildRunsDeferred: Layer.Layer = Layer.succeed( Effect.die(`spawnChildRun: no RUNS_WORKFLOW binding on this runtime (run="${run}")`), handoffAdmission: () => Effect.die("handoffChildAdmission: no RUNS_WORKFLOW binding on this runtime"), + spendGate: (run) => + Effect.die(`fanOut: no RUNS_WORKFLOW binding on this runtime (run="${run}")`), poll: () => Effect.die("waitForChildren: no RUNS_WORKFLOW binding on this runtime"), }))(), ); diff --git a/packages/runtime-cf/src/execution-cost.test.ts b/packages/runtime-cf/src/execution-cost.test.ts new file mode 100644 index 00000000..6f62659e --- /dev/null +++ b/packages/runtime-cf/src/execution-cost.test.ts @@ -0,0 +1,49 @@ +// The sandbox → container class → instance type table the meter and the burn +// ceiling price runs from, checked against the committed wrangler.jsonc: a +// container class resized there fails here until the table follows +// (specs/adr/0006-spend-guard.md § Meter). + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { containerMicroUsdPerHour, type SandboxImage } from "@fractalboxdev/flare-dispatch-core"; +import { hourlyRateForSandboxImage, instanceForSandboxImage, SANDBOX_CONTAINER } from "./execution-cost"; + +/** The committed `wrangler.jsonc`, comments and trailing commas stripped. */ +const wrangler = (() => { + const raw = readFileSync(new URL("../../../wrangler.jsonc", import.meta.url), "utf8"); + return JSON.parse( + raw + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/(^|[^:])\/\/.*$/gm, "$1") + .replace(/,(\s*[}\]])/g, "$1"), + ) as { + containers: { class_name: string; instance_type: string }[]; + durable_objects: { bindings: { name: string; class_name: string }[] }; + }; +})(); + +/** The Container binding each sandbox image routes to (sandbox-routing.ts). */ +const IMAGE_BINDING: Record = { + lean: "RUNS_SANDBOX", + browser: "RUNS_SANDBOX_BROWSER", + agent: "RUNS_SANDBOX_AGENT", + release: "RUNS_SANDBOX_RELEASE", +}; + +describe("SANDBOX_CONTAINER — the one instance table", () => { + it("matches each image's container class and instance_type in wrangler.jsonc", () => { + for (const image of Object.keys(SANDBOX_CONTAINER) as SandboxImage[]) { + const binding = wrangler.durable_objects.bindings.find((b) => b.name === IMAGE_BINDING[image]); + expect(binding?.class_name, `${image} binding`).toBe(SANDBOX_CONTAINER[image].className); + const container = wrangler.containers.find((c) => c.class_name === SANDBOX_CONTAINER[image].className); + expect(container?.instance_type, `${image} container`).toBe(SANDBOX_CONTAINER[image].instance); + } + }); + + it("prices the lean image on its standard-4 class, and an unset image as lean", () => { + expect(instanceForSandboxImage("lean")).toBe("standard-4"); + expect(instanceForSandboxImage(undefined)).toBe("standard-4"); + expect(instanceForSandboxImage("browser")).toBe("standard-2"); + expect(hourlyRateForSandboxImage("agent")).toBe(containerMicroUsdPerHour("standard-3")); + }); +}); diff --git a/packages/runtime-cf/src/execution-cost.ts b/packages/runtime-cf/src/execution-cost.ts index caa09b98..1a53752c 100644 --- a/packages/runtime-cf/src/execution-cost.ts +++ b/packages/runtime-cf/src/execution-cost.ts @@ -5,32 +5,60 @@ // // * METERED model tokens — SUMmed from `execution_model_usage` (written per // call by the live `modelGateway` wrapper, model-gateway-cf.ts); -// * MODELED container compute — `instance vCPU/mem rate × active seconds`, -// where active seconds is the execution wall-time (an upper-bound proxy: -// it includes model round-trips + any in-body waits; refining this to the -// true sandbox boot→destroy window is a follow-up). +// * MODELED container compute — the list rate of the run's container class, +// priced on provisioned vCPU, memory and disk, × container-active seconds, +// which run from admission (`executions.admitted_at`) to completion. Queue +// wait costs nothing; an execution never admitted has no container time. // // via the pure `@fractalboxdev/flare-dispatch-core` cost engine, which decides the honest -// `metered | mixed | modeled | unmetered` basis. Best-effort: a cost-rollup -// failure logs a warning and never flips the run's verdict. +// `metered | mixed | modeled | unmetered` basis. Each row it writes carries +// `cost_meter = 'admitted-active'`; rows priced on wall time before this rule +// carry `'modeled-wall'` (migration 0011). Best-effort: a cost-rollup failure +// logs a warning and never flips the run's verdict. // -// Spec: specs/06-cost.md, infra/migrations/0005_execution_cost.sql. +// Spec: specs/06-cost.md, specs/adr/0006-spend-guard.md § Meter, +// infra/migrations/0005_execution_cost.sql, 0011_spend_guard.sql. import { Effect } from "effect"; import { type InstanceType, type SandboxImage, INSTANCE_SPECS, + containerMicroUsdPerHour, estimateExecutionCost, } from "@fractalboxdev/flare-dispatch-core"; /** - * The container instance type a run's `sandboxImage` lands on. The Durable - * Object classes in wrangler.jsonc bind `lean`/`browser` → `standard-2` and - * `agent` → `standard-3`; an un-set image defaults to `lean`. + * The container class and instance type each sandbox image (= admission pool) + * runs on — the one table the meter, the burn ceiling and the finish path + * read. `execution-cost.test.ts` checks it against the committed + * wrangler.jsonc `containers` entries, so a class resized there fails CI until + * this table follows. + */ +export const SANDBOX_CONTAINER: Record< + SandboxImage, + { readonly className: string; readonly instance: InstanceType } +> = { + lean: { className: "RunSandbox", instance: "standard-4" }, + browser: { className: "RunSandboxBrowser", instance: "standard-2" }, + agent: { className: "RunSandboxAgent", instance: "standard-3" }, + release: { className: "RunSandboxRelease", instance: "standard-4" }, +}; + +/** + * The container instance type a run lands on. Pass the ADMISSION POOL (the + * image after the dispatcher's degrade-to-lean routing), so a run is priced on + * the container it actually consumes; an un-set image is `lean`. */ export const instanceForSandboxImage = (sandboxImage: SandboxImage | undefined): InstanceType => - sandboxImage === "agent" ? "standard-3" : sandboxImage === "release" ? "standard-4" : "standard-2"; + SANDBOX_CONTAINER[sandboxImage ?? "lean"].instance; + +/** A sandbox image's list rate per hour, in micro-USD — what the burn ceiling sums. */ +export const hourlyRateForSandboxImage = (sandboxImage: SandboxImage | undefined): number => + containerMicroUsdPerHour(instanceForSandboxImage(sandboxImage)); + +/** The `executions.cost_meter` label for a row priced by this rule. */ +export const COST_METER_ADMITTED_ACTIVE = "admitted-active"; type UsageRow = { readonly model: string; @@ -40,13 +68,13 @@ type UsageRow = { }; /** - * Compute and persist this execution's cost rollup. Reads its own wall-time and + * Compute and persist this execution's cost rollup. Reads its admitted window and * model-usage rows from D1, models container compute from the instance type, and * writes the `executions` cost columns. Returns `void`; all failure is absorbed. * * @param db D1 binding (`env.RUNS_METADATA`). * @param executionId the row to roll up. - * @param instance the container instance type (`instanceForSandboxImage`). + * @param instance the container instance type (`instanceForSandboxImage(pool)`). */ export const recordExecutionCost = (opts: { readonly db: D1Database; @@ -56,16 +84,17 @@ export const recordExecutionCost = (opts: { Effect.tryPromise(async () => { const { db, executionId, instance } = opts; - // Wall-time → modeled container-active seconds (upper-bound proxy). + // Admission → completion is the container-active window; queue wait and + // a run never admitted cost no container time. const exec = await db - .prepare(`SELECT started_at, completed_at FROM executions WHERE id = ?`) + .prepare(`SELECT admitted_at, completed_at FROM executions WHERE id = ?`) .bind(executionId) - .first<{ started_at: number | null; completed_at: number | null }>(); - const startedAt = exec?.started_at ?? null; + .first<{ admitted_at: number | null; completed_at: number | null }>(); + const admittedAt = exec?.admitted_at ?? null; const completedAt = exec?.completed_at ?? null; const activeSeconds = - startedAt !== null && completedAt !== null && completedAt > startedAt - ? (completedAt - startedAt) / 1000 + admittedAt !== null && completedAt !== null && completedAt > admittedAt + ? (completedAt - admittedAt) / 1000 : 0; // SUM model token usage; pick the dominant model for the rate + the column. @@ -109,13 +138,14 @@ export const recordExecutionCost = (opts: { await db .prepare( `UPDATE executions - SET cost_micro_usd = ?, cost_basis = ?, input_tokens = ?, + SET cost_micro_usd = ?, cost_basis = ?, cost_meter = ?, input_tokens = ?, output_tokens = ?, vcpu_seconds = ?, model = ? WHERE id = ?`, ) .bind( cost.totalMicroUsd, cost.basis, + COST_METER_ADMITTED_ACTIVE, rows.length > 0 ? inTok : null, rows.length > 0 ? outTok : null, vcpuSeconds, diff --git a/packages/runtime-cf/src/execution-cost.workers.test.ts b/packages/runtime-cf/src/execution-cost.workers.test.ts index 234e87c6..115b43de 100644 --- a/packages/runtime-cf/src/execution-cost.workers.test.ts +++ b/packages/runtime-cf/src/execution-cost.workers.test.ts @@ -12,16 +12,26 @@ import { makeTestBindings, type TestBindings } from "./test-support-workers"; const EXEC = "01TEST0000000000000000COST"; -/** Insert a finished executions row spanning `wallMs` of wall-time. */ -const seedExecution = (db: D1Database, id: string, wallMs: number) => - db +/** + * Insert a finished executions row admitted `activeMs` before it completed, + * after `queuedMs` in the admission queue (`admittedAt: null` = never admitted). + */ +const seedExecution = ( + db: D1Database, + id: string, + activeMs: number, + opts: { readonly queuedMs?: number; readonly admitted?: boolean } = {}, +) => { + const admittedAt = 1000 + (opts.queuedMs ?? 0); + return db .prepare( `INSERT INTO executions - (id, run, repo, ref, sha, status, started_at, completed_at, input_json) - VALUES (?, 'pr-review', 'owner/name', 'refs/heads/main', 'abc', 'success', 1000, ?, '{}')`, + (id, run, repo, ref, sha, status, started_at, admitted_at, completed_at, input_json) + VALUES (?, 'pr-review', 'owner/name', 'refs/heads/main', 'abc', 'success', 1000, ?, ?, '{}')`, ) - .bind(id, 1000 + wallMs) + .bind(id, opts.admitted === false ? null : admittedAt, admittedAt + activeMs) .run(); +}; const seedUsage = ( db: D1Database, @@ -50,8 +60,9 @@ describe("recordExecutionCost", () => { }); it("rolls up metered model tokens + modeled container compute as 'mixed'", async () => { - // 480s wall on standard-2 → 16_800 µ$ modeled container; sonnet 20k in / - // 2k out → 20000×3 + 2000×15 = 90_000 µ$ metered model. + // 480s admitted on standard-2 → 17_203 µ$ modeled container (vCPU + + // memory + disk); sonnet 20k in / 2k out → 20000×3 + 2000×15 = 90_000 µ$ + // metered model. await seedExecution(bindings.db, EXEC, 480_000); await seedUsage(bindings.db, EXEC, "anthropic/claude-sonnet-4-6", 20_000, 2_000, 1); @@ -68,7 +79,7 @@ describe("recordExecutionCost", () => { .first>(); expect(row).toMatchObject({ - cost_micro_usd: 106_800, // 16_800 container + 90_000 model + cost_micro_usd: 107_203, // 17_203 container + 90_000 model cost_basis: "mixed", input_tokens: 20_000, output_tokens: 2_000, @@ -92,12 +103,43 @@ describe("recordExecutionCost", () => { .bind(EXEC) .first>(); - expect(row).toMatchObject({ cost_micro_usd: 16_800, cost_basis: "modeled" }); + expect(row).toMatchObject({ cost_micro_usd: 17_203, cost_basis: "modeled" }); expect(row?.input_tokens).toBeNull(); expect(row?.output_tokens).toBeNull(); expect(row?.model).toBeNull(); }); + it("prices admission to completion: queue wait costs nothing, and the row reads admitted-active", async () => { + // 10 minutes queued, then 480 s admitted on standard-4. + await seedExecution(bindings.db, EXEC, 480_000, { queuedMs: 600_000 }); + + await Effect.runPromise( + recordExecutionCost({ db: bindings.db, executionId: EXEC, instance: "standard-4" }), + ); + + const row = await bindings.db + .prepare(`SELECT cost_micro_usd, cost_meter, vcpu_seconds FROM executions WHERE id = ?`) + .bind(EXEC) + .first<{ cost_micro_usd: number; cost_meter: string; vcpu_seconds: number }>(); + // (4 × 20 + 12 × 2.5 + 20 × 0.07) µ$/s × 480 s = 111.4 × 480. + expect(row).toMatchObject({ cost_micro_usd: 53_472, cost_meter: "admitted-active" }); + expect(row?.vcpu_seconds).toBeCloseTo(1920, 3); + }); + + it("charges no container time to an execution never admitted", async () => { + await seedExecution(bindings.db, EXEC, 480_000, { admitted: false }); + + await Effect.runPromise( + recordExecutionCost({ db: bindings.db, executionId: EXEC, instance: "standard-4" }), + ); + + const row = await bindings.db + .prepare(`SELECT cost_micro_usd FROM executions WHERE id = ?`) + .bind(EXEC) + .first<{ cost_micro_usd: number }>(); + expect(row?.cost_micro_usd).toBe(0); + }); + it("upsert-SUM accumulates a same-model fan-out (no double-count on the PK)", async () => { await seedExecution(bindings.db, EXEC, 10_000); // Two writes to the SAME (execution, model) PK — the gateway wrapper's diff --git a/packages/runtime-cf/src/index.ts b/packages/runtime-cf/src/index.ts index a2f91b29..a37a661d 100644 --- a/packages/runtime-cf/src/index.ts +++ b/packages/runtime-cf/src/index.ts @@ -60,6 +60,8 @@ export { } from "./container-lease-d1"; export { makeRunAdmissionD1, + BURN_UNLIMITED, + type BurnCeiling, resolveAdmissionCap, resolvePoolAdmissionCap, ADMISSION_CAP_DEFAULT, @@ -116,7 +118,20 @@ export { type WorkflowBindingLike, } from "./child-runs-cf"; export { makeModelGatewayLive, type AiBinding, type ModelUsageSink } from "./model-gateway-cf"; -export { recordExecutionCost, instanceForSandboxImage } from "./execution-cost"; +export { + recordExecutionCost, + instanceForSandboxImage, + hourlyRateForSandboxImage, + SANDBOX_CONTAINER, + COST_METER_ADMITTED_ACTIVE, +} from "./execution-cost"; +export { + readSpend, + readSpendLimits, + readSpendStatus, + warnSpendOnce, + type SpendStatus, +} from "./spend-d1"; export { BrowserDeferred, ChildRunsDeferred, diff --git a/packages/runtime-cf/src/run-admission-d1.ts b/packages/runtime-cf/src/run-admission-d1.ts index 79f03650..3c900830 100644 --- a/packages/runtime-cf/src/run-admission-d1.ts +++ b/packages/runtime-cf/src/run-admission-d1.ts @@ -26,9 +26,26 @@ // Spec: ADR #109 (global D1-backed run admission semaphore). import { Effect, Schedule } from "effect"; -import type { AdmissionObservation, AdmissionPool } from "@fractalboxdev/flare-dispatch-core"; +import { + type AdmissionObservation, + type AdmissionPool, + fitsBurnCeiling, +} from "@fractalboxdev/flare-dispatch-core"; import { LEASE_TTL_MS } from "./container-lease-d1"; +/** + * The spend-guard burn ceiling one claimant admits under (ADR-0006 § Burn + * ceiling): its own container class's list rate per hour, and the ceiling on + * the summed rate of every admitted row across all pools (`null` = unlimited). + */ +export type BurnCeiling = { + readonly hourlyRateMicroUsd: number; + readonly burnMaxMicroUsdPerHour: number | null; +}; + +/** No ceiling, no rate — the store as it behaves without a spend guard. */ +export const BURN_UNLIMITED: BurnCeiling = { hourlyRateMicroUsd: 0, burnMaxMicroUsdPerHour: null }; + /** * Default admission slots per pool — matches each container class's * `max_instances: 16` in wrangler.jsonc, so admitted runs never exceed the @@ -158,8 +175,10 @@ export interface RunAdmissionStore { /** * One claim attempt: try the atomic FIFO claim; when it does not land, - * refresh our waiter heartbeat and read our queue position + the pool's - * busy count. Returns the observation (+ the clock it was taken at) for + * refresh our waiter heartbeat and read our queue position, the pool's + * busy count, and whether the burn ceiling (not the cap) held us back. A + * landed claim also stamps `executions.admitted_at` — the start of the + * container-active window the meter prices. Returns the observation (+ the clock it was taken at) for * the pure `decideAdmission` — the caller runs the whole attempt inside * one durable `step.do`, so everything here is checkpointed together. */ @@ -185,12 +204,15 @@ export interface RunAdmissionStore { * drive the clock; production passes `Date.now` (always read inside the * caller's `step.do`, so replays see the checkpointed value). `cap` is the * per-pool slot ceiling — the dispatcher resolves it from the - * `ADMISSION_CAP` wrangler var via {@link resolveAdmissionCap}. + * `ADMISSION_CAP` wrangler var via {@link resolveAdmissionCap}. `burn` is + * this claimant's hourly rate and the deploy's burn ceiling; the default + * {@link BURN_UNLIMITED} admits on the cap alone. */ export const makeRunAdmissionD1 = ( db: D1Database, now: () => number = Date.now, cap: number = ADMISSION_CAP_DEFAULT, + burn: BurnCeiling = BURN_UNLIMITED, ): RunAdmissionStore => { const insertQueued = ( executionId: string, @@ -206,15 +228,22 @@ export const makeRunAdmissionD1 = ( // GC'd waiter row restored mid-loop keeps its original FIFO key) → // the parent's row (matrix-children priority inheritance) → now. `INSERT OR IGNORE INTO run_admissions - (execution_id, pool, state, enqueued_at, heartbeat_at) + (execution_id, pool, state, enqueued_at, heartbeat_at, hourly_rate_micro_usd) VALUES (?1, ?2, 'queued', COALESCE(?3, (SELECT enqueued_at FROM run_admissions WHERE execution_id = ?4), ?5), - ?5)`, + ?5, ?6)`, + ) + .bind( + executionId, + pool, + enqueuedAt ?? null, + parentExecutionId ?? null, + ts, + burn.hourlyRateMicroUsd, ) - .bind(executionId, pool, enqueuedAt ?? null, parentExecutionId ?? null, ts) .run(), ).pipe(Effect.asVoid); @@ -247,7 +276,13 @@ export const makeRunAdmissionD1 = ( * - the NOT EXISTS is the no-barge FIFO check among LIVE waiters: nobody * with an earlier `enqueued_at` (execution-id ULID as the tiebreak) may * be skipped, but a waiter silent past `ADMISSION_WAITER_TTL_MS` no - * longer blocks the line. + * longer blocks the line; + * - the burn-ceiling clause admits only while the hourly rates of LIVE + * admitted rows in EVERY pool, plus ours, fit under the ceiling. An + * empty admitted set admits anyone, so a claimant whose own rate + * exceeds the ceiling runs alone rather than waiting forever; a NULL + * ceiling disables the clause. `fitsBurnCeiling` (core spend.ts) is its + * pure mirror. * * `meta.changes === 1` ⇒ our row flipped to 'admitted' — we hold a slot. */ @@ -268,7 +303,12 @@ export const makeRunAdmissionD1 = ( WHERE o.pool = ?3 AND o.state = 'queued' AND o.heartbeat_at > ?6 AND (o.enqueued_at < ?7 - OR (o.enqueued_at = ?7 AND o.execution_id < ?1)))`, + OR (o.enqueued_at = ?7 AND o.execution_id < ?1))) + AND (?8 IS NULL + OR NOT EXISTS (SELECT 1 FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?4) + OR (SELECT COALESCE(SUM(hourly_rate_micro_usd), 0) FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?4) + ?9 <= ?8)`, ) .bind( executionId, @@ -278,10 +318,27 @@ export const makeRunAdmissionD1 = ( cap, ts - ADMISSION_WAITER_TTL_MS, enqueuedAt, + burn.burnMaxMicroUsdPerHour, + burn.hourlyRateMicroUsd, ) .run(), ).pipe(Effect.map((result) => result.meta.changes === 1)); + // Stamp the start of the container-active window on the execution row from + // our admitted row. COALESCE keeps the first stamp across a replay. + const markAdmitted = (executionId: string): Effect.Effect => + d1(() => + db + .prepare( + `UPDATE executions + SET admitted_at = COALESCE(admitted_at, + (SELECT admitted_at FROM run_admissions WHERE execution_id = ?1)) + WHERE id = ?1`, + ) + .bind(executionId) + .run(), + ).pipe(Effect.asVoid); + const attempt = ( executionId: string, pool: AdmissionPool, @@ -302,6 +359,7 @@ export const makeRunAdmissionD1 = ( ); if (own?.state === "admitted") { yield* heartbeat(executionId); + yield* markAdmitted(executionId); return { admitted: true, position: 0, poolBusy: 0, now: ts }; } // Self-heal: a waiter paused past the admitted TTL has had its row @@ -311,11 +369,15 @@ export const makeRunAdmissionD1 = ( yield* insertQueued(executionId, pool, enqueuedAt, undefined, ts); } const claimed = yield* tryClaim(executionId, pool, enqueuedAt, ts); - if (claimed) return { admitted: true, position: 0, poolBusy: 0, now: ts }; + if (claimed) { + yield* markAdmitted(executionId); + return { admitted: true, position: 0, poolBusy: 0, now: ts }; + } // Not admitted: refresh our waiter heartbeat (so we keep blocking // bargers and counting in peers' position reads), then observe where - // we stand — live queued rows ahead of us + live admitted rows. + // we stand — live queued rows ahead of us, live admitted rows in our + // pool, and the live admitted burn across every pool. yield* heartbeat(executionId); const counts = yield* d1(() => db @@ -328,15 +390,31 @@ export const makeRunAdmissionD1 = ( OR (enqueued_at = ?3 AND execution_id < ?4))) AS position, (SELECT COUNT(*) FROM run_admissions WHERE pool = ?1 AND state = 'admitted' - AND heartbeat_at > ?5) AS busy`, + AND heartbeat_at > ?5) AS busy, + (SELECT COUNT(*) FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?5) AS admitted_count, + (SELECT COALESCE(SUM(hourly_rate_micro_usd), 0) FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?5) AS admitted_rate`, ) .bind(pool, ts - ADMISSION_WAITER_TTL_MS, enqueuedAt, executionId, ts - ADMISSION_TTL_MS) - .first<{ position: number; busy: number }>(), + .first<{ position: number; busy: number; admitted_count: number; admitted_rate: number }>(), ); + const poolBusy = counts?.busy ?? 0; + // The ceiling held us when our pool had a free slot but the burn did not + // fit. A full pool is the cap's hold, whatever the burn. + const ceilingHeld = + poolBusy < cap && + !fitsBurnCeiling({ + admittedMicroUsdPerHour: counts?.admitted_rate ?? 0, + admittedCount: counts?.admitted_count ?? 0, + ownMicroUsdPerHour: burn.hourlyRateMicroUsd, + burnMaxMicroUsdPerHour: burn.burnMaxMicroUsdPerHour, + }); return { admitted: false, position: counts?.position ?? 0, - poolBusy: counts?.busy ?? 0, + poolBusy, + ...(ceilingHeld ? { ceilingHeld } : {}), now: ts, }; }); diff --git a/packages/runtime-cf/src/run-admission-d1.workers.test.ts b/packages/runtime-cf/src/run-admission-d1.workers.test.ts index ec571827..6d85cc70 100644 --- a/packages/runtime-cf/src/run-admission-d1.workers.test.ts +++ b/packages/runtime-cf/src/run-admission-d1.workers.test.ts @@ -16,7 +16,7 @@ import { Effect } from "effect"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { decideAdmission, type AdmissionPool } from "@fractalboxdev/flare-dispatch-core"; +import { containerMicroUsdPerHour, decideAdmission, type AdmissionPool } from "@fractalboxdev/flare-dispatch-core"; import { ADMISSION_CAP_DEFAULT, ADMISSION_MAX_QUEUE_AGE_MS, @@ -353,3 +353,120 @@ describe("resolvePoolAdmissionCap — one cap per pool", () => { } }); }); + +// The burn ceiling inside the claim (specs/adr/0006-spend-guard.md § Burn +// ceiling): admitted hourly rates across EVERY pool plus the claimant's own +// must fit, an oversized claimant admits only into an empty set, and a +// non-landing claim reports whether the ceiling or the cap held it. +describe("makeRunAdmissionD1 — the burn ceiling", () => { + let bindings: TestBindings; + beforeEach(async () => { + bindings = await makeTestBindings(); + }); + afterEach(async () => { + await bindings.dispose(); + }); + + const S4 = containerMicroUsdPerHour("standard-4"); // 401_040 µ$/h + const S2 = containerMicroUsdPerHour("standard-2"); // 129_024 µ$/h + const CEILING = 1_500_000; + + const seedAdmitted = (executionId: string, pool: AdmissionPool, rate: number) => + bindings.db + .prepare( + `INSERT INTO run_admissions + (execution_id, pool, state, enqueued_at, admitted_at, heartbeat_at, hourly_rate_micro_usd) + VALUES (?, ?, 'admitted', ?, ?, ?, ?)`, + ) + .bind(executionId, pool, T0 - 1_000, T0 - 1_000, T0, rate) + .run(); + + const claim = async (executionId: string, pool: AdmissionPool, rate: number, ceiling: number | null = CEILING) => { + const store = makeRunAdmissionD1(bindings.db, () => T0, ADMISSION_CAP_DEFAULT, { + hourlyRateMicroUsd: rate, + burnMaxMicroUsdPerHour: ceiling, + }); + const { enqueuedAt } = await Effect.runPromise(store.enqueue(executionId, pool)); + return Effect.runPromise(store.attempt(executionId, pool, enqueuedAt)); + }; + + it("records the claimant's hourly rate on its row", async () => { + await claim(A, "lean", S4); + const row = await bindings.db + .prepare(`SELECT hourly_rate_micro_usd FROM run_admissions WHERE execution_id = ?`) + .bind(A) + .first<{ hourly_rate_micro_usd: number }>(); + expect(row?.hourly_rate_micro_usd).toBe(S4); + }); + + it("admits while the admitted burn plus its own fits, across pools", async () => { + await seedAdmitted("peer-lean", "lean", S4); + await seedAdmitted("peer-release", "release", S4); + expect((await claim(A, "browser", S4)).admitted).toBe(true); + }); + + it("holds a claimant the ceiling cannot fit, with a free slot in its pool", async () => { + // Three standard-4 rows admitted in other pools: 1.203 $/h + 0.401 > 1.50. + await seedAdmitted("peer-1", "release", S4); + await seedAdmitted("peer-2", "release", S4); + await seedAdmitted("peer-3", "browser", S4); + const observed = await claim(A, "lean", S4); + expect(observed).toMatchObject({ admitted: false, poolBusy: 0, ceilingHeld: true }); + // A cheaper claimant still fits the headroom. + expect((await claim(B, "browser", S2)).admitted).toBe(true); + }); + + it("admits a claimant whose own rate exceeds the ceiling only into an empty admitted set", async () => { + expect((await claim(A, "lean", 2_000_000)).admitted).toBe(true); + const observed = await claim(B, "lean", 2_000_000); + expect(observed).toMatchObject({ admitted: false, ceilingHeld: true }); + }); + + it("stops counting a stale admitted row's burn", async () => { + await bindings.db + .prepare( + `INSERT INTO run_admissions + (execution_id, pool, state, enqueued_at, admitted_at, heartbeat_at, hourly_rate_micro_usd) + VALUES ('corpse', 'lean', 'admitted', ?, ?, ?, ?)`, + ) + .bind(T0 - 1_000, T0 - 1_000, T0 - ADMISSION_TTL_MS - 1, CEILING) + .run(); + expect((await claim(A, "lean", S4)).admitted).toBe(true); + }); + + it("admits on the cap alone with no ceiling", async () => { + await seedAdmitted("peer-1", "release", 10_000_000); + expect((await claim(A, "lean", S4, null)).admitted).toBe(true); + }); + + it("reports the cap, not the ceiling, when the pool is full", async () => { + for (let i = 0; i < ADMISSION_CAP_DEFAULT; i++) { + await seedAdmitted(`peer-${String(i).padStart(2, "0")}`, "lean", S4); + } + const observed = await claim(A, "lean", S4); + expect(observed.admitted).toBe(false); + expect(observed.ceilingHeld).toBeUndefined(); + }); + + it("stamps executions.admitted_at from the claim, once", async () => { + await bindings.db + .prepare( + `INSERT INTO executions (id, run, repo, ref, sha, status, started_at, input_json) + VALUES (?, 'check', 'owner/name', 'refs/heads/main', 'abc', 'running', ?, '{}')`, + ) + .bind(A, T0 - 5_000) + .run(); + let clock = T0; + const store = makeRunAdmissionD1(bindings.db, () => clock); + const { enqueuedAt } = await Effect.runPromise(store.enqueue(A, "lean")); + await Effect.runPromise(store.attempt(A, "lean", enqueuedAt)); + // A replayed attempt later keeps the first stamp. + clock = T0 + 60_000; + await Effect.runPromise(store.attempt(A, "lean", enqueuedAt)); + const row = await bindings.db + .prepare(`SELECT admitted_at FROM executions WHERE id = ?`) + .bind(A) + .first<{ admitted_at: number }>(); + expect(row?.admitted_at).toBe(T0); + }); +}); diff --git a/packages/runtime-cf/src/runtime.ts b/packages/runtime-cf/src/runtime.ts index 53e18673..766711d5 100644 --- a/packages/runtime-cf/src/runtime.ts +++ b/packages/runtime-cf/src/runtime.ts @@ -97,6 +97,11 @@ export type CFRuntimeLiveOptions = { * required Dispatcher binding, so the production call site always passes it. */ readonly runsWorkflow?: WorkflowBindingLike; + /** + * The monthly-budget read a fan-out runs before it spawns children + * (ADR-0006) — see `ChildRunsLiveConfig.spendGate`. Absent = no spend guard. + */ + readonly spendGate?: () => Promise; /** This execution's ULID — namespaces D1 rows, R2 keys, the sandbox id. */ readonly executionId: string; /** repo/ref/sha/input the `executions` row requires. */ @@ -413,6 +418,7 @@ export const makeCFRuntimeLive = ( workflow: opts.runsWorkflow, db: opts.db, parentExecutionId: opts.executionId, + ...(opts.spendGate !== undefined ? { spendGate: opts.spendGate } : {}), // Children inherit the parent's public origin so their artifact // links are absolute too (a child posts its own check-run). ...(opts.publicOrigin !== undefined ? { origin: opts.publicOrigin } : {}), diff --git a/packages/runtime-cf/src/spend-d1.ts b/packages/runtime-cf/src/spend-d1.ts new file mode 100644 index 00000000..c551d7c3 --- /dev/null +++ b/packages/runtime-cf/src/spend-d1.ts @@ -0,0 +1,179 @@ +// @fractalboxdev/flare-dispatch-runtime-cf — spend guard I/O: the month-to-date read and the limits. +// +// The I/O shell around the pure spend math (`@fractalboxdev/flare-dispatch-core` +// spend.ts): one indexed D1 query for month-to-date spend and the admitted +// burn, the `CONFIG_KV` overrides for the limits, and the once-per-month +// warning marker. The dispatcher calls these inside durable `step.do` bodies, +// so a replay reads the checkpointed answer rather than a fresh one. +// +// Spec: specs/adr/0006-spend-guard.md § Spend read, § Monthly budget, +// § Warning and visibility. + +import { Effect, Schedule } from "effect"; +import { + type BudgetDecision, + decideBudget, + formatUsd, + monthToDateMicroUsd, + resolveSpendLimits, + SPEND_BUDGET_KV_KEY, + SPEND_BURN_KV_KEY, + type SpendLimits, + spendMonth, + type SpendMonth, + type SpendReading, + type SpendVars, + spendWarnedKvKey, +} from "@fractalboxdev/flare-dispatch-core"; +import { ADMISSION_TTL_MS } from "./run-admission-d1"; + +const MS_PER_HOUR = 3_600_000; + +/** + * Read month-to-date spend and the live admitted burn — one statement: + * + * * `finished` — `cost_micro_usd` of executions admitted since the month + * started (the `executions_admitted_at` range scan); + * * `in_flight_rate_ms` — Σ hourly rate × ms since admission over LIVE + * admitted `run_admissions` rows admitted this month. A row is deleted at + * release, before the finish path writes `cost_micro_usd`, so no execution + * counts in both terms; + * * `admitted_rate` / `admitted_count` — the live admitted set the burn + * ceiling sums, across every pool. + * + * A row written before migration 0011 has a NULL rate and counts as zero. + */ +export const readSpend = (db: D1Database, nowMs: number): Effect.Effect => + Effect.tryPromise({ + try: () => + db + .prepare( + `SELECT + (SELECT COALESCE(SUM(cost_micro_usd), 0) FROM executions + WHERE admitted_at >= ?1 AND cost_micro_usd IS NOT NULL) AS finished, + (SELECT COALESCE(SUM(COALESCE(hourly_rate_micro_usd, 0) * (?2 - admitted_at)), 0) + FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?3 + AND admitted_at >= ?1 AND admitted_at <= ?2) AS in_flight_rate_ms, + (SELECT COALESCE(SUM(hourly_rate_micro_usd), 0) FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?3) AS admitted_rate, + (SELECT COUNT(*) FROM run_admissions + WHERE state = 'admitted' AND heartbeat_at > ?3) AS admitted_count`, + ) + .bind(spendMonth(nowMs).startMs, nowMs, nowMs - ADMISSION_TTL_MS) + .first<{ + finished: number; + in_flight_rate_ms: number; + admitted_rate: number; + admitted_count: number; + }>(), + catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))), + }).pipe( + Effect.retry(Schedule.recurs(2).pipe(Schedule.addDelay(() => "200 millis"))), + Effect.map((row) => ({ + finishedMicroUsd: row?.finished ?? 0, + inFlightMicroUsd: Math.round((row?.in_flight_rate_ms ?? 0) / MS_PER_HOUR), + admittedMicroUsdPerHour: row?.admitted_rate ?? 0, + admittedCount: row?.admitted_count ?? 0, + })), + ); + +/** + * Resolve the limits: `CONFIG_KV` `spend.budgetMonthlyUsd` / + * `spend.burnMaxUsdPerHour` override the vars, and the vars override the code + * defaults. A KV read that fails falls back to the vars and logs: the limit + * then binds at the configured value, never at none. + */ +export const readSpendLimits = ( + kv: KVNamespace | undefined, + vars: SpendVars, +): Effect.Effect => + (kv === undefined + ? Effect.succeed({}) + : Effect.tryPromise(async () => { + const [budgetMonthlyUsd, burnMaxUsdPerHour] = await Promise.all([ + kv.get(SPEND_BUDGET_KV_KEY), + kv.get(SPEND_BURN_KV_KEY), + ]); + return { budgetMonthlyUsd, burnMaxUsdPerHour }; + }).pipe( + Effect.catchAll((cause) => + Effect.logWarning(`spend: CONFIG_KV limit overrides unreadable, using vars — ${cause}`).pipe( + Effect.as({}), + ), + ), + ) + ).pipe(Effect.map((overrides) => resolveSpendLimits(vars, overrides))); + +/** One spend read with its verdict against the budget. */ +export type SpendStatus = { + readonly month: SpendMonth; + readonly limits: SpendLimits; + readonly reading: SpendReading; + /** Finished + in-flight, times the overhead factor. */ + readonly monthToDateMicroUsd: number; + readonly decision: BudgetDecision; +}; + +/** Read spend and decide the monthly budget against `limits`. */ +export const readSpendStatus = ( + db: D1Database, + limits: SpendLimits, + nowMs: number, +): Effect.Effect => + Effect.map(readSpend(db, nowMs), (reading) => { + const spend = monthToDateMicroUsd(reading, limits.overheadFactor); + return { + month: spendMonth(nowMs), + limits, + reading, + monthToDateMicroUsd: spend, + decision: decideBudget(spend, limits), + }; + }); + +/** + * Send the month's one spend warning when `status` crossed the warn ratio and + * the month carries no `spend.warned:` marker yet. Writes the marker + * before notifying, so a failed notification is not retried by every later + * admission. Returns whether this call sent it. Best-effort: a KV or notify + * failure logs and returns `false`; it never touches the run. + * + * Without `CONFIG_KV` there is nowhere to mark the month, so it logs + * `spend.warn` and sends nothing. + */ +export const warnSpendOnce = (opts: { + readonly kv: KVNamespace | undefined; + readonly status: SpendStatus; + readonly notify: (text: string, monthKey: string) => Promise; +}): Effect.Effect => { + const { status } = opts; + if (status.decision._kind !== "open" || !status.decision.warn) return Effect.succeed(false); + const budget = status.limits.budgetMicroUsd ?? 0; + const text = + `FlareDispatch spend reached ${formatUsd(status.monthToDateMicroUsd)} of the ` + + `${formatUsd(budget)} monthly budget for ${status.month.key}. ` + + `Checks skip once it reaches the budget; raise it with CONFIG_KV ${SPEND_BUDGET_KV_KEY}.`; + const kv = opts.kv; + if (kv === undefined) { + return Effect.logWarning(`spend.warn ${text} (no CONFIG_KV — not notified)`).pipe(Effect.as(false)); + } + const marker = spendWarnedKvKey(status.month.key); + return Effect.gen(function* () { + const warned = yield* Effect.tryPromise(() => kv.get(marker)); + if (warned !== null) return false; + yield* Effect.tryPromise(() => + kv.put(marker, String(status.monthToDateMicroUsd), { + // Outlive the month, then expire — a stale marker never silences a later year. + expiration: Math.floor(status.month.resetsAtMs / 1000) + 7 * 24 * 3600, + }), + ); + yield* Effect.logWarning(`spend.warn ${text}`); + yield* Effect.tryPromise(() => opts.notify(text, status.month.key)); + return true; + }).pipe( + Effect.catchAll((cause) => + Effect.logWarning(`spend.warn not delivered — ${cause}`).pipe(Effect.as(false)), + ), + ); +}; diff --git a/packages/runtime-cf/src/spend-d1.workers.test.ts b/packages/runtime-cf/src/spend-d1.workers.test.ts new file mode 100644 index 00000000..775461ac --- /dev/null +++ b/packages/runtime-cf/src/spend-d1.workers.test.ts @@ -0,0 +1,118 @@ +// Integration tests for the spend guard's I/O against real D1 and KV: the +// month-to-date read, the CONFIG_KV limit overrides, and the once-per-month +// warning marker (specs/adr/0006-spend-guard.md). The pure math is covered by +// core spend.test.ts. + +import { Effect } from "effect"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { resolveSpendLimits, type SpendLimits } from "@fractalboxdev/flare-dispatch-core"; +import { readSpend, readSpendLimits, readSpendStatus, warnSpendOnce } from "./spend-d1"; +import { ADMISSION_TTL_MS } from "./run-admission-d1"; +import { makeTestBindings, type TestBindings } from "./test-support-workers"; + +const NOW = Date.UTC(2026, 9, 22, 12, 0); +const MONTH_START = Date.UTC(2026, 9, 1); +const HOUR = 3_600_000; + +describe("spend-d1", () => { + let bindings: TestBindings; + beforeEach(async () => { + bindings = await makeTestBindings(); + }); + afterEach(async () => { + await bindings.dispose(); + }); + + const seedFinished = (id: string, admittedAt: number | null, cost: number) => + bindings.db + .prepare( + `INSERT INTO executions + (id, run, repo, ref, sha, status, started_at, admitted_at, completed_at, input_json, cost_micro_usd) + VALUES (?, 'check', 'owner/name', 'refs/heads/main', 'abc', 'success', ?, ?, ?, '{}', ?)`, + ) + .bind(id, admittedAt ?? NOW, admittedAt, NOW, cost) + .run(); + + const seedAdmitted = (id: string, admittedAt: number, rate: number, heartbeatAt = NOW) => + bindings.db + .prepare( + `INSERT INTO run_admissions + (execution_id, pool, state, enqueued_at, admitted_at, heartbeat_at, hourly_rate_micro_usd) + VALUES (?, 'lean', 'admitted', ?, ?, ?, ?)`, + ) + .bind(id, admittedAt, admittedAt, heartbeatAt, rate) + .run(); + + it("sums finished executions admitted this month and in-flight burn since admission", async () => { + await seedFinished("f1", MONTH_START + HOUR, 3_000_000); + await seedFinished("f2", NOW - HOUR, 1_000_000); + // Admitted last month: not this month's spend. + await seedFinished("old", MONTH_START - HOUR, 50_000_000); + // Priced before the meter (no admission stamp): not counted. + await seedFinished("legacy", null, 7_000_000); + // In flight for two hours at $0.40/h; a stale row counts for nothing. + await seedAdmitted("live", NOW - 2 * HOUR, 400_000); + await seedAdmitted("corpse", NOW - 2 * HOUR, 400_000, NOW - ADMISSION_TTL_MS - 1); + + const reading = await Effect.runPromise(readSpend(bindings.db, NOW)); + expect(reading).toEqual({ + finishedMicroUsd: 4_000_000, + inFlightMicroUsd: 800_000, + admittedMicroUsdPerHour: 400_000, + admittedCount: 1, + }); + }); + + it("closes the budget once spend times the overhead reaches it", async () => { + await seedFinished("f1", NOW - HOUR, 82_000_000); + const limits = resolveSpendLimits({}); + const status = await Effect.runPromise(readSpendStatus(bindings.db, limits, NOW)); + // 82 × 1.10 = 90.20 ≥ 90. + expect(status.monthToDateMicroUsd).toBe(90_200_000); + expect(status.decision).toEqual({ + _kind: "closed", + reason: "spend budget reached: $90.20 of $90.00 this month", + }); + expect(status.month.key).toBe("2026-10"); + }); + + it("reads the CONFIG_KV overrides over the vars", async () => { + await bindings.kv.put("spend.budgetMonthlyUsd", "unlimited"); + await bindings.kv.put("spend.burnMaxUsdPerHour", "4"); + const limits = await Effect.runPromise( + readSpendLimits(bindings.kv, { SPEND_BUDGET_MONTHLY_USD: "50", SPEND_BURN_MAX_USD_PER_HOUR: "2" }), + ); + expect(limits).toMatchObject({ budgetMicroUsd: null, burnMaxMicroUsdPerHour: 4_000_000 }); + expect(await Effect.runPromise(readSpendLimits(undefined, { SPEND_BUDGET_MONTHLY_USD: "50" }))).toMatchObject( + { budgetMicroUsd: 50_000_000 }, + ); + }); + + it("warns once per month past the warn ratio, marking the month in CONFIG_KV", async () => { + await seedFinished("f1", NOW - HOUR, 70_000_000); + const limits: SpendLimits = resolveSpendLimits({}); + const status = await Effect.runPromise(readSpendStatus(bindings.db, limits, NOW)); + const sent: string[] = []; + const notify = async (text: string, month: string) => { + sent.push(`${month}: ${text}`); + }; + + expect(await Effect.runPromise(warnSpendOnce({ kv: bindings.kv, status, notify }))).toBe(true); + expect(await Effect.runPromise(warnSpendOnce({ kv: bindings.kv, status, notify }))).toBe(false); + expect(sent).toHaveLength(1); + expect(sent[0]).toContain("2026-10: FlareDispatch spend reached $77.00 of the $90.00 monthly budget"); + expect(await bindings.kv.get("spend.warned:2026-10")).toBe("77000000"); + }); + + it("sends no warning under the warn ratio", async () => { + await seedFinished("f1", NOW - HOUR, 10_000_000); + const status = await Effect.runPromise(readSpendStatus(bindings.db, resolveSpendLimits({}), NOW)); + let calls = 0; + const warned = await Effect.runPromise( + warnSpendOnce({ kv: bindings.kv, status, notify: async () => void calls++ }), + ); + expect(warned).toBe(false); + expect(calls).toBe(0); + expect(await bindings.kv.get("spend.warned:2026-10")).toBeNull(); + }); +}); diff --git a/runs/contextful-gate.test.ts b/runs/contextful-gate.test.ts index a2ff3801..5310f4d6 100644 --- a/runs/contextful-gate.test.ts +++ b/runs/contextful-gate.test.ts @@ -912,6 +912,23 @@ describe("contextful-gate", () => { }).pipe(Effect.provide(layer)); }); + it.effect("spawns no part once the monthly spend budget is reached, and skips", () => { + const reason = "spend budget reached: $90.00 of $90.00 this month"; + const { layer, handles } = makeCFRuntimeTest({ + config: { "contextful-gate.window:fractalboxdev/contextful": "1" }, + sandboxProgram: three, + childRuns: { spendClosed: reason }, + }); + return Effect.gen(function* () { + const failure = yield* Effect.flip(contextfulGate.run(input)); + expect(failure).toMatchObject({ _tag: "RunSkipped", reason: `${reason} — 3 of 3 parts not run` }); + expect(handles.childRuns.spawned).toHaveLength(0); + expect(cancelledChecks(handles)).toEqual(["pins", "workspace.compile", "formal"].map((stage) => ({ + name: `flare-dispatch/check:${stage}`, conclusion: "cancelled", summary: `not run — ${reason}`, + }))); + }).pipe(Effect.provide(layer)); + }); + it.effect("keeps spawning after an ordinary red part", () => { const { layer, handles } = makeCFRuntimeTest({ config: { "contextful-gate.window:fractalboxdev/contextful": "1" }, diff --git a/runs/contextful-gate.ts b/runs/contextful-gate.ts index 3284dc57..ea2321e5 100644 --- a/runs/contextful-gate.ts +++ b/runs/contextful-gate.ts @@ -84,8 +84,16 @@ export const parseWindow = (raw: string | undefined): number => { export const partCheckName = (stage: string): string => `flare-dispatch/check:${stage}`; export const NOT_AFFECTED_SUMMARY = "not affected by this change"; -/** Why a gate stops spawning before every part has run. */ -export type StopReason = "head superseded" | "pull request closed" | "admission backlog"; +/** + * Why a gate stops spawning before every part has run. A `spend budget + * reached: …` reason is the deploy's monthly budget closing before a window + * spawns (specs/adr/0006-spend-guard.md). + */ +export type StopReason = + | "head superseded" + | "pull request closed" + | "admission backlog" + | `spend budget reached: ${string}`; /** Whether the gated pull request is still open at the gated commit. */ export type Liveness = { readonly live: true } | { readonly live: false; readonly reason: StopReason }; @@ -491,7 +499,7 @@ export const contextfulGate = defineRun({ Effect.gen(function* () { const liveness = yield* readLiveness(input); if (!liveness.live) return { stopped: liveness.reason }; - const handles = yield* fanOut({ + const fanned = yield* fanOut({ run: "check", items: batch, concurrency: batch.length, @@ -509,8 +517,15 @@ export const contextfulGate = defineRun({ : { restore: [{ key: prebuilt.key, base: prebuilt.base, paths: prebuilt.paths }] }), secrets: [], }), - }); - return { handles, spawnedAt: yield* io.now }; + }).pipe( + // The monthly budget closed: spawn nothing more, post the + // unrun parts, and end the gate skipped with that reason. + Effect.catchTag("RunSkipped", (skip) => + Effect.succeed({ stopped: skip.reason as StopReason }), + ), + ); + if ("stopped" in fanned) return fanned; + return { handles: fanned, spawnedAt: yield* io.now }; }), ); if ("stopped" in spawn) { From 7cc95c2c8196348c6a9017d39adbab49fe4042b7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 22 Oct 2026 16:54:42 +0800 Subject: [PATCH 2/4] fix(spend): a release whose budget closes mid-window lets running cells settle, then skips contextful-release caught no RunSkipped from fanOut, so a budget closing at a later spawn ended the parent while running cells still wrote to the draft release, and the typed failure thrown inside the step was retried by the platform before it surfaced. The spawn steps now checkpoint the skip as an answer: no further cell starts, the running ones settle, and the release ends RunSkipped naming the unstarted cells. A closed budget at the formula spawn skips the same way. The child-runs fake's spendClosed also takes a function, so a test closes the budget mid-run. The cost.ts header names the SANDBOX_CONTAINER table. --- packages/core/src/cost.ts | 2 +- packages/core/src/fakes/child-runs-fake.ts | 15 +++++--- runs/contextful-release.test.ts | 42 +++++++++++++++++++++- runs/contextful-release.ts | 38 +++++++++++++++----- 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/packages/core/src/cost.ts b/packages/core/src/cost.ts index 79964767..580119d5 100644 --- a/packages/core/src/cost.ts +++ b/packages/core/src/cost.ts @@ -42,7 +42,7 @@ export const MICRO_USD_PER_USD = 1_000_000; // Provisioned vCPU, memory and disk per Cloudflare Containers instance type // (specs/05-byoc.md § Wrangler config, "Instance types (2026-05)"). A run's // instance type is fixed by the container class its `sandboxImage` selects in -// wrangler.jsonc — the one table is `SANDBOX_INSTANCE` in runtime-cf +// wrangler.jsonc — the one table is `SANDBOX_CONTAINER` in runtime-cf // execution-cost.ts, checked against that file by its test. export type InstanceType = diff --git a/packages/core/src/fakes/child-runs-fake.ts b/packages/core/src/fakes/child-runs-fake.ts index 7bde7070..6deb0ba3 100644 --- a/packages/core/src/fakes/child-runs-fake.ts +++ b/packages/core/src/fakes/child-runs-fake.ts @@ -49,8 +49,12 @@ export const makeChildRunsFake = (opts?: { existing?: Iterable; /** Reject selected spawns before recording them, for partial fan-out tests. */ failSpawn?: (opts: SpawnChildRunOpts) => boolean; - /** When set, `spendGate` fails `RunSkipped` with this reason (budget reached). */ - spendClosed?: string; + /** + * When set, `spendGate` fails `RunSkipped` with this reason (budget reached). + * A function is read on every gate call, so a test closes the budget mid-run; + * it returns `undefined` while the budget is open. + */ + spendClosed?: string | (() => string | undefined); /** * Static per-id poll statuses. An id absent from the map polls as `missing`. * For a loop test that flips `running → success`, supply `pollFn` instead. @@ -80,9 +84,10 @@ export const makeChildRunsFake = (opts?: { const service: ChildRunsService = { spendGate: () => - opts?.spendClosed !== undefined - ? Effect.fail(new RunSkipped({ reason: opts.spendClosed })) - : Effect.void, + Effect.suspend(() => { + const closed = typeof opts?.spendClosed === "function" ? opts.spendClosed() : opts?.spendClosed; + return closed !== undefined ? Effect.fail(new RunSkipped({ reason: closed })) : Effect.void; + }), handoffAdmission: () => Effect.sync(() => { state.admissionHandoffs += 1; diff --git a/runs/contextful-release.test.ts b/runs/contextful-release.test.ts index db6865a3..a3e614cd 100644 --- a/runs/contextful-release.test.ts +++ b/runs/contextful-release.test.ts @@ -310,12 +310,13 @@ describe("contextful-release", () => { const releaseFixture = ( status: (cell: Cell, poll: number, now: number, attempt: number) => Verdict, existing: readonly string[] = [], + spendClosed?: () => string | undefined, ) => { let spawned: readonly { readonly input: unknown; readonly instanceId: string }[] = []; let now = 0; const runtime = makeCFRuntimeTest({ github: { files: { [`${repo}:Cargo.toml`]: '[workspace.package]\nversion = "0.5.0"\n' } }, - childRuns: { existing, pollFn: (ids, poll) => ids.map((id) => { + childRuns: { existing, ...(spendClosed !== undefined ? { spendClosed } : {}), pollFn: (ids, poll) => ids.map((id) => { if (id.startsWith("contextful-release-formula")) { return { executionId: id, status: "success", summaryJson: JSON.stringify({ assets: FORMULA_ASSETS, notes: "## Consumer compatibility\n" }) }; @@ -425,6 +426,45 @@ describe("contextful-release", () => { }).pipe(Effect.provide(layer)); }); + const skipOf = (exit: Exit.Exit) => { + if (!Exit.isFailure(exit)) return undefined; + const failure = Option.getOrUndefined(Cause.failureOption(exit.cause)) as + { _tag?: string; reason?: string } | undefined; + expect(failure?._tag).toBe("RunSkipped"); + return failure?.reason; + }; + const budgetReason = "spend budget reached: $90.00 of $90.00 this month"; + + it.effect("a budget closed before the first spawn starts no cell and skips the release", () => { + const { layer, handles, provideClock } = releaseFixture(() => "success", [], () => budgetReason); + return Effect.gen(function* () { + const exit = yield* Effect.exit(provideClock(contextfulRelease.run({ repo, tag: "v0.5.0", sha, dryRun: true }))); + expect(skipOf(exit)).toBe(`${budgetReason} — 10 of 10 release cells not started`); + expect(handles.childRuns.spawned).toHaveLength(0); + expect(handles.childRuns.admissionHandoffs).toBe(0); + }).pipe(Effect.provide(layer)); + }); + + it.effect("a budget that closes mid-release starts no further cell, lets the running ones settle, then skips", () => { + // The first window spawns; the budget is closed for every later spawn. + let gateCalls = 0; + const { layer, handles, provideClock } = releaseFixture( + (cell, poll) => (cell.profile === "contextful-full" && cell.target === "x86_64-unknown-linux-musl") || poll >= 3 + ? "success" : "running", + [], + () => (gateCalls++ === 0 ? undefined : budgetReason), + ); + return Effect.gen(function* () { + const exit = yield* Effect.exit(provideClock(contextfulRelease.run({ repo, tag: "v0.5.0", sha, dryRun: true }))); + expect(skipOf(exit)).toBe(`${budgetReason} — 2 of 10 release cells not started`); + expect(cellsOf(handles)).toHaveLength(8); + expect(handles.childRuns.spawned.some((child) => child.run === "contextful-release-formula")).toBe(false); + // Every running cell was awaited to its terminal poll before the release ended. + const awaits = stepNames(handles).filter((name) => name.startsWith("await-cells-")); + expect(awaits.length).toBeGreaterThan(1); + }).pipe(Effect.provide(layer)); + }); + it.effect("a cell the platform lost respawns as a fresh child, at most twice, and the release completes", () => { const lost = (cell: Cell, tag: string, attempts: number) => (c: Cell, attempt: number) => c.profile === cell.profile && c.target === cell.target && attempt < attempts ? { failure: tag } : undefined; diff --git a/runs/contextful-release.ts b/runs/contextful-release.ts index ae2be4d4..e22fa093 100644 --- a/runs/contextful-release.ts +++ b/runs/contextful-release.ts @@ -1,7 +1,7 @@ import { Effect, Schema } from "effect"; import { AcceptanceFailed, artifact, checkpointSleep, defineRun, GitHubApiError, github, handoffChildAdmission, io, - isTerminalChildStatus, sandbox, step, StepFailed, type ChildStatusRecord, + isTerminalChildStatus, RunSkipped, sandbox, step, StepFailed, type ChildStatusRecord, } from "@fractalboxdev/flare-dispatch-core"; import { DURABLE_EXEC_RETRY_ON, durableExec, ensureWorkspace, fanOut, waitForAnyChildDurably, waitForChildren, workspace, @@ -436,15 +436,21 @@ export const contextfulRelease = defineRun({ const timedOut = new Set(); const cellFailed = (child: ChildStatusRecord) => child.status !== "success" || !child.summaryJson; let spawned = 0; + // The monthly budget closing at a spawn (specs/adr/0006-spend-guard.md) + // stops the window like a failed cell: no further cell starts, and the + // running ones settle before the release ends skipped. + let budgetClosed: string | undefined; for (let round = 0; queue.length > 0 || running.length > 0; round++) { - const failed = timedOut.size > 0 || [...settledByCell.values()].some(cellFailed); + const failed = budgetClosed !== undefined || timedOut.size > 0 || [...settledByCell.values()].some(cellFailed); const free = RELEASE_WINDOW - running.length; // After a failure no further cell starts; the running ones settle or // outlive their budget, so the verdict names every failed cell. They // write to the same draft release, so the parent never ends before them. if (!failed && queue.length > 0 && free > 0) { - const batch = queue.splice(0, free); - const { handles, spawnedAt } = yield* step(`spawn-cells-${spawned}`, () => Effect.gen(function* () { + const batch = queue.slice(0, free); + // The skip is caught inside the step, so it checkpoints as an answer + // rather than a failure the platform retries. + const spawn = yield* step(`spawn-cells-${spawned}`, () => Effect.gen(function* () { const handles = yield* fanOut({ run: contextfulReleaseCell.name, items: batch, @@ -453,7 +459,14 @@ export const contextfulRelease = defineRun({ releaseId: release.id, dryRun, ...(attempt > 0 ? { attempt } : {}) }), }); return { handles, spawnedAt: yield* io.now }; - })); + }).pipe(Effect.catchTag("RunSkipped", (skip) => Effect.succeed({ budgetClosed: skip.reason })))); + if ("budgetClosed" in spawn) { + budgetClosed = spawn.budgetClosed; + if (running.length === 0) break; + continue; + } + const { handles, spawnedAt } = spawn; + queue.splice(0, batch.length); if (spawned === 0) yield* step("handoff-admission", () => handoffChildAdmission()); batch.forEach(({ cell, attempt }, index) => running.push({ index: CELLS.indexOf(cell), @@ -499,6 +512,11 @@ export const contextfulRelease = defineRun({ const child = settledByCell.get(index); return child !== undefined && cellFailed(child); }); + if (budgetClosed !== undefined && failedCells.length === 0 && timedOut.size === 0) { + return yield* Effect.fail(new RunSkipped({ + reason: `${budgetClosed} — ${queue.length} of ${CELLS.length} release cells not started`, + })); + } if (failedCells.length > 0 || timedOut.size > 0) { // A typed failure, not a defect: the check-run renders `summaryMd`, so // the verdict names every failed, timed-out and unstarted cell there. @@ -507,7 +525,7 @@ export const contextfulRelease = defineRun({ const verdict = [ ["Failed", failedCells], ["Timed out", CELLS.filter((_, index) => timedOut.has(index))], - ["Not started", queue.map(({ cell }) => cell)], + [budgetClosed === undefined ? "Not started" : `Not started (${budgetClosed})`, queue.map(({ cell }) => cell)], ] as const; return yield* Effect.fail(new AcceptanceFailed({ exitCode: 1, summaryMd: `**Release cells**\n\n${verdict .filter(([, cells]) => cells.length > 0) @@ -528,11 +546,15 @@ export const contextfulRelease = defineRun({ return yield* Effect.die(new Error("release tag moved during build")); } } - const formulaHandle = yield* step("spawn-formula", () => fanOut({ run: contextfulReleaseFormula.name, + const formulaSpawn = yield* step("spawn-formula", () => fanOut({ run: contextfulReleaseFormula.name, items: [JSON.stringify(manifests)], toInput: (manifestJson) => ({ repo: input.repo, tag: input.tag, sha: tag.commitSha, releaseId: release.id, dryRun, manifests: manifestJson }), - })); + }).pipe(Effect.catchTag("RunSkipped", (skip) => Effect.succeed({ budgetClosed: skip.reason })))); + if ("budgetClosed" in formulaSpawn) { + return yield* Effect.fail(new RunSkipped({ reason: `${formulaSpawn.budgetClosed} — release formula not started` })); + } + const formulaHandle = formulaSpawn; const formulaChildren = yield* step("await-formula", () => waitForChildren({ ids: formulaHandle.map((handle) => handle.executionId), timeout: "8 hours", })); From 2ac442bc12fd1deee94a8afb69126309a1f4d114 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 22 Oct 2026 17:05:05 +0800 Subject: [PATCH 3/4] feat(admission): position-scaled polls, interactive class, browser cap and start jitter (ADR-0008, ADR-0009) - A waiter sleeps min(60 s, 20 s x ceil(position / cap)) between claims, from the position its claim step observed; ADMISSION_WAITER_TTL_MS is three of the longest sleeps. - limits.admissionClass (default batch); oxlint, pr-review, offload-test and worker-deploy are interactive. The claim and the queue position rank interactive ahead of batch, FIFO within a class; migration 0011 adds run_admissions.admission_class. - ADMISSION_CAP_BROWSER bounds the browser pool; routing tests assert every pool's cap fits its container's max_instances. - An admitted run waits 0-2 s, derived from its execution id, before its container starts. --- apps/dispatcher/src/env.ts | 8 + apps/dispatcher/src/sandbox-routing.test.ts | 24 +++ apps/dispatcher/src/workflow.ts | 52 ++++- infra/migrations/0011_spend_guard.sql | 9 +- packages/core/src/define-run.ts | 7 + packages/core/src/index.ts | 7 + packages/core/src/run-admission.test.ts | 60 ++++++ packages/core/src/run-admission.ts | 64 ++++++ packages/runtime-cf/src/index.ts | 1 + packages/runtime-cf/src/run-admission-d1.ts | 102 +++++++--- .../src/run-admission-d1.workers.test.ts | 191 +++++++++++++++++- runs/admission-class.test.ts | 29 +++ runs/offload-test.ts | 2 + runs/oxlint.ts | 2 + runs/pr-review.ts | 3 +- runs/worker-deploy.ts | 2 + wrangler.jsonc | 6 +- 17 files changed, 518 insertions(+), 51 deletions(-) create mode 100644 runs/admission-class.test.ts diff --git a/apps/dispatcher/src/env.ts b/apps/dispatcher/src/env.ts index ab54d364..c7423567 100644 --- a/apps/dispatcher/src/env.ts +++ b/apps/dispatcher/src/env.ts @@ -464,6 +464,14 @@ export interface Env { */ readonly ADMISSION_CAP_AGENT?: string; + /** + * The browser pool's admission slots — a var, carried as a string. The + * browser pool admits at most the smaller of this and `ADMISSION_CAP`; keep + * it ≤ `RunSandboxBrowser`'s `max_instances`. Absent / unparsable → 16 + * (`BROWSER_ADMISSION_CAP_DEFAULT`). + */ + readonly ADMISSION_CAP_BROWSER?: string; + /** * The monthly spend budget in USD (specs/adr/0006-spend-guard.md) — a var. * At or over it, admission skips every run with a neutral check. Unset or diff --git a/apps/dispatcher/src/sandbox-routing.test.ts b/apps/dispatcher/src/sandbox-routing.test.ts index 4b49a16e..6bff05fc 100644 --- a/apps/dispatcher/src/sandbox-routing.test.ts +++ b/apps/dispatcher/src/sandbox-routing.test.ts @@ -64,6 +64,30 @@ describe("selectAdmission — the pool a run is metered against and its slot cap } }); + it("admits no pool past its container's max_instances on the code defaults", () => { + for (const image of ["lean", "browser", "agent", "release"] as const) { + const { pool, cap } = selectAdmission(image, bound); + const container = wrangler.containers.find((entry) => entry.class_name === POOL_CLASS[pool])!; + expect(cap, `${pool} pool`).toBeLessThanOrEqual(container.max_instances); + } + }); + + it("meters every sandbox container class as one pool", () => { + const sandboxClasses = wrangler.containers + .map((entry) => entry.class_name) + .filter((name) => name.startsWith("RunSandbox")) + .sort(); + expect(sandboxClasses).toEqual(Object.values(POOL_CLASS).sort()); + }); + + it("caps the browser pool by ADMISSION_CAP_BROWSER", () => { + expect(wrangler.vars.ADMISSION_CAP_BROWSER).toBeDefined(); + expect(selectAdmission("browser", { ADMISSION_CAP: "16", ADMISSION_CAP_BROWSER: "4", ...bound })).toEqual({ + pool: "browser", + cap: 4, + }); + }); + it("caps the release and agent pools at their eight containers", () => { expect(selectAdmission("release", { ...wrangler.vars, ...bound })).toEqual({ pool: "release", cap: 8 }); expect(selectAdmission("agent", { ...wrangler.vars, ...bound })).toEqual({ pool: "agent", cap: 8 }); diff --git a/apps/dispatcher/src/workflow.ts b/apps/dispatcher/src/workflow.ts index 59af0331..32059378 100644 --- a/apps/dispatcher/src/workflow.ts +++ b/apps/dispatcher/src/workflow.ts @@ -61,6 +61,9 @@ import { getSandbox } from "@cloudflare/sandbox"; import { Duration, Effect, Exit, identity, Runtime, Schedule, Schema } from "effect"; import { admissionAcquireAttempts, + ADMISSION_CLASS_DEFAULT, + admissionJitterMs, + admissionPollSleepMs, AdmissionTimedOut, Checks, decideAdmission, @@ -442,9 +445,10 @@ export class RunWorkflow extends WorkflowEntrypoint { // selection above (browser/agent/release image without a bound container // degrades to lean), so a run is always metered against the pool whose // container it will actually consume — and that pool's slot cap: the plain - // `ADMISSION_CAP` wrangler var (unset/unparsable → 16), and for the agent - // and release pools the smaller of that and `ADMISSION_CAP_AGENT` / - // `ADMISSION_CAP_RELEASE` (unset → 8). Keep every cap ≤ its container's + // `ADMISSION_CAP` wrangler var (unset/unparsable → 16), and for the agent, + // browser and release pools the smaller of that and `ADMISSION_CAP_AGENT` / + // `ADMISSION_CAP_BROWSER` / `ADMISSION_CAP_RELEASE` (unset → 8, 16, 8). + // Keep every cap ≤ its container's // `max_instances` (sandbox-routing.test.ts reads wrangler.jsonc to check). const { pool: admissionPool, cap: admissionCap } = selectAdmission(run.sandboxImage, this.env); @@ -923,10 +927,16 @@ export class RunWorkflow extends WorkflowEntrypoint { ); let spend = yield* spendCheck("spend-check-enqueue", limits); const burnMaxMicroUsdPerHour = limits.burnMaxMicroUsdPerHour; - const claimer = makeRunAdmissionD1(db, Date.now, admissionCap, { - hourlyRateMicroUsd, - burnMaxMicroUsdPerHour, - }); + // The run's admission class (ADR-0009): interactive waiters are + // admitted ahead of batch ones in the same pool. The burn ceiling + // applies to both and never reorders the line. + const claimer = makeRunAdmissionD1( + db, + Date.now, + admissionCap, + { hourlyRateMicroUsd, burnMaxMicroUsdPerHour }, + run.limits.admissionClass ?? ADMISSION_CLASS_DEFAULT, + ); // A queue timeout while the burn ceiling, not the pool cap, held the // run back is a spend skip; under the cap alone it stays a failure. const queueTimeout = ( @@ -954,7 +964,11 @@ export class RunWorkflow extends WorkflowEntrypoint { ); // Bounded claim/sleep loop uses a count, so the bound is replay-stable - // and trivial against the Workflows step cap (#83 precedent). + // and trivial against the Workflows step cap (#83 precedent). Each + // attempt is three steps (spend check, claim, sleep). The count is + // derived from the SHORTEST sleep, so it covers the queue age however + // the sleeps scale; a waiter sleeping longer reaches the age limit, + // read inside its claim step, in fewer attempts. const maxAttempts = admissionAcquireAttempts( admissionMaxQueueAgeMs, ADMISSION_POLL_EVERY_MS, @@ -1029,6 +1043,11 @@ export class RunWorkflow extends WorkflowEntrypoint { }), ), ); + // Spread a burst of admissions over up to two seconds before the + // container starts (ADR-0009). Derived from the execution id and + // slept durably, so a replay neither re-draws nor re-sleeps it. + const jitterMs = admissionJitterMs(payload.executionId); + if (jitterMs > 0) yield* stepSleep("admission-jitter", jitterMs); return; } if (decision._kind === "timeout") { @@ -1070,10 +1089,21 @@ export class RunWorkflow extends WorkflowEntrypoint { ); } - // Durable, free hibernation between polls. Don't sleep after the - // final attempt — fall straight through to timeout. + // Durable, free hibernation between polls, scaled with the position + // the claim step observed (ADR-0008): 20 s at the head, up to 60 s + // several slots back, never past the age limit by more than one + // 20 s poll. Both inputs are checkpointed, so a replay sleeps the + // same length. Don't sleep after the final attempt — fall straight + // through to timeout. if (i < maxAttempts - 1) { - yield* stepSleep(`admission-wait-${i}`, ADMISSION_POLL_EVERY_MS); + yield* stepSleep( + `admission-wait-${i}`, + admissionPollSleepMs( + decision.position, + admissionCap, + enqueuedAt + admissionMaxQueueAgeMs - observed.now, + ), + ); } else { return yield* queueTimeout( observed, diff --git a/infra/migrations/0011_spend_guard.sql b/infra/migrations/0011_spend_guard.sql index e74462b4..d8671064 100644 --- a/infra/migrations/0011_spend_guard.sql +++ b/infra/migrations/0011_spend_guard.sql @@ -1,7 +1,8 @@ -- Migration number: 0011 spend guard -- -- The meter, the monthly budget and the burn ceiling of --- specs/adr/0006-spend-guard.md: +-- specs/adr/0006-spend-guard.md, and the interactive admission class of +-- specs/adr/0009-fast-trustworthy-verdicts.md: -- -- * `executions.admitted_at` — when the run's admission claim landed. A -- container is active from admission to completion, so the finish path @@ -14,6 +15,11 @@ -- container class per hour. The claim admits only while the live admitted -- rates plus the claimant's own fit under the burn ceiling, across every -- pool (runtime-cf run-admission-d1.ts). +-- * `run_admissions.admission_class` — the waiter's line +-- (specs/adr/0009-fast-trustworthy-verdicts.md): the claim admits the +-- oldest live `interactive` waiter in a pool before any `batch` waiter, +-- and FIFO order and the queue position apply within a class. A NULL row +-- (pre-0011) ranks as `batch`. -- -- The month-to-date spend read (runtime-cf spend-d1.ts) sums `cost_micro_usd` -- over executions admitted this month; `executions_admitted_at` serves that @@ -25,3 +31,4 @@ UPDATE executions SET cost_meter = 'modeled-wall' WHERE cost_micro_usd IS NOT NU CREATE INDEX IF NOT EXISTS executions_admitted_at ON executions(admitted_at); ALTER TABLE run_admissions ADD COLUMN hourly_rate_micro_usd INTEGER; -- NULL rows (pre-0011) count as 0 +ALTER TABLE run_admissions ADD COLUMN admission_class TEXT; -- interactive | batch; NULL = batch diff --git a/packages/core/src/define-run.ts b/packages/core/src/define-run.ts index d422d4a9..3a0cd427 100644 --- a/packages/core/src/define-run.ts +++ b/packages/core/src/define-run.ts @@ -9,12 +9,19 @@ import type { Effect, Schema } from "effect"; import type { RunContext } from "./context"; import type { RunError } from "./errors"; +import type { AdmissionClass } from "./run-admission"; import type { WritebackSpec } from "./writeback"; export type RunLimits = { readonly maxDurationSec: number; /** Maximum admission wait; defaults to the dispatcher's standard queue age. */ readonly admissionMaxQueueAgeSec?: number | ((input: I) => number); + /** + * The admission class this run waits in; `interactive` runs are admitted + * ahead of `batch` ones in the same pool. Defaults to `batch` + * (specs/adr/0009-fast-trustworthy-verdicts.md). + */ + readonly admissionClass?: AdmissionClass; readonly maxConcurrency?: number; readonly requiresBrowser?: boolean; }; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 11eb6731..44ef60e6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -82,6 +82,13 @@ export { export { decideAdmission, admissionAcquireAttempts, + admissionPollSleepMs, + admissionJitterMs, + ADMISSION_CLASS_DEFAULT, + ADMISSION_POLL_MIN_MS, + ADMISSION_POLL_MAX_MS, + ADMISSION_JITTER_MAX_MS, + type AdmissionClass, type AdmissionPool, type AdmissionObservation, type AdmissionDecision, diff --git a/packages/core/src/run-admission.test.ts b/packages/core/src/run-admission.test.ts index 1016daaf..50049501 100644 --- a/packages/core/src/run-admission.test.ts +++ b/packages/core/src/run-admission.test.ts @@ -8,6 +8,11 @@ import { describe, expect, it } from "vitest"; import { admissionAcquireAttempts, + admissionJitterMs, + admissionPollSleepMs, + ADMISSION_JITTER_MAX_MS, + ADMISSION_POLL_MAX_MS, + ADMISSION_POLL_MIN_MS, type AdmissionObservation, decideAdmission, } from "./run-admission"; @@ -115,3 +120,58 @@ describe("admissionAcquireAttempts", () => { expect(admissionAcquireAttempts(1_000, 0)).toBeGreaterThanOrEqual(1); }); }); + +describe("admissionPollSleepMs — min(60 s, 20 s × ceil(position / cap))", () => { + it("keeps the head of the queue on the 20-second poll", () => { + expect(admissionPollSleepMs(0, 16)).toBe(20_000); + expect(admissionPollSleepMs(16, 16)).toBe(20_000); + }); + + it("scales with the slots between the waiter and the head", () => { + expect(admissionPollSleepMs(17, 16)).toBe(40_000); + expect(admissionPollSleepMs(32, 16)).toBe(40_000); + expect(admissionPollSleepMs(33, 16)).toBe(60_000); + expect(admissionPollSleepMs(9, 8)).toBe(40_000); + }); + + it("never sleeps longer than 60 seconds", () => { + expect(admissionPollSleepMs(500, 16)).toBe(ADMISSION_POLL_MAX_MS); + expect(admissionPollSleepMs(500, 1)).toBe(ADMISSION_POLL_MAX_MS); + }); + + it("shortens a long sleep to the time left, never below the 20-second poll", () => { + expect(admissionPollSleepMs(500, 16, 45_000)).toBe(45_000); + expect(admissionPollSleepMs(500, 16, 5_000)).toBe(ADMISSION_POLL_MIN_MS); + expect(admissionPollSleepMs(0, 16, -1_000)).toBe(ADMISSION_POLL_MIN_MS); + }); + + it("treats a zero cap as one slot", () => { + expect(admissionPollSleepMs(2, 0)).toBe(40_000); + }); + + it("covers the queue age within the attempts derived from the shortest sleep", () => { + // Every sleep is ≥ the shortest, so attempts × shortest ≥ the queue age. + const attempts = admissionAcquireAttempts(MAX_QUEUE_AGE, ADMISSION_POLL_MIN_MS); + expect(attempts * ADMISSION_POLL_MIN_MS).toBeGreaterThanOrEqual(MAX_QUEUE_AGE); + // A waiter far back reaches the age limit in a third of the attempts. + expect(Math.ceil(MAX_QUEUE_AGE / admissionPollSleepMs(500, 16))).toBe(attempts / 3); + }); +}); + +describe("admissionJitterMs", () => { + it("is the same for the same execution id", () => { + expect(admissionJitterMs("01ARZ3NDEKTSV4RRFFQ69G5FAA")).toBe(admissionJitterMs("01ARZ3NDEKTSV4RRFFQ69G5FAA")); + }); + + it("stays within 0..2 s and spreads a burst", () => { + const draws = Array.from({ length: 200 }, (_, i) => admissionJitterMs(`01ARZ3NDEKTSV4RRFFQ69G${String(i).padStart(4, "0")}`)); + for (const ms of draws) { + expect(ms).toBeGreaterThanOrEqual(0); + expect(ms).toBeLessThanOrEqual(ADMISSION_JITTER_MAX_MS); + expect(Number.isInteger(ms)).toBe(true); + } + expect(new Set(draws).size).toBeGreaterThan(150); + expect(Math.min(...draws)).toBeLessThan(500); + expect(Math.max(...draws)).toBeGreaterThan(1_500); + }); +}); diff --git a/packages/core/src/run-admission.ts b/packages/core/src/run-admission.ts index 5211d5b3..b643264c 100644 --- a/packages/core/src/run-admission.ts +++ b/packages/core/src/run-admission.ts @@ -31,6 +31,17 @@ /** The container pool a run is admitted into — one semaphore per pool. */ export type AdmissionPool = "lean" | "browser" | "agent" | "release"; +/** + * The admission class a run waits in (ADR-0009 § Interactive admission class). + * A claim admits the oldest live `interactive` waiter in a pool before any + * `batch` waiter; FIFO order and the queue position apply within a class, + * interactive ahead of batch. A run that declares no class is `batch`. + */ +export type AdmissionClass = "interactive" | "batch"; + +/** The class of a run that declares none. */ +export const ADMISSION_CLASS_DEFAULT: AdmissionClass = "batch"; + /** * What one claim attempt observed, read atomically by the runtime layer: * whether OUR atomic claim landed, and — when it did not — where we stand. @@ -121,6 +132,59 @@ export const decideAdmission = ( * read, so the loop stays replay-deterministic across a Workflow checkpoint * resume — exactly the bound `leaseAcquireAttempts` and `waitForChildren` * use for their poll loops. + * + * With position-scaled sleeps ({@link admissionPollSleepMs}) pass the SHORTEST + * sleep: every sleep is at least that long, so the count covers the whole + * queue age, and the age read inside each claim step (`decideAdmission`) ends + * a slower-polling loop at the deadline in fewer attempts. The count is the + * ceiling on steps, the age is the ceiling on time. */ export const admissionAcquireAttempts = (maxQueueAgeMs: number, pollEveryMs: number): number => Math.max(1, Math.ceil(maxQueueAgeMs / Math.max(1, pollEveryMs))); + +/** The head-of-queue poll interval: the shortest sleep between claims. */ +export const ADMISSION_POLL_MIN_MS = 20_000; + +/** The longest sleep between claims, however far back a waiter stands. */ +export const ADMISSION_POLL_MAX_MS = 60_000; + +/** + * The sleep before a waiter's next claim (ADR-0008 § Position-scaled admission + * polls): `min(60 s, 20 s × ceil(position / cap))`, with the head (position 0) + * on the 20-second poll. A waiter moves at most one place per released slot, + * so polling far from the head admits nothing sooner. + * + * `position` is the one observed inside the claim step, so a replay sleeps the + * same length. `remainingMs` (time left before the dispatch-age ceiling, from + * the same step's clock) shortens a long sleep so the final claim lands at the + * deadline instead of up to a minute past it; the sleep never drops below the + * 20-second poll. + */ +export const admissionPollSleepMs = ( + position: number, + cap: number, + remainingMs: number = Number.POSITIVE_INFINITY, +): number => { + const slotsBack = Math.max(1, Math.ceil(Math.max(0, position) / Math.max(1, cap))); + const scaled = Math.min(ADMISSION_POLL_MAX_MS, ADMISSION_POLL_MIN_MS * slotsBack); + return Math.max(ADMISSION_POLL_MIN_MS, Math.min(scaled, remainingMs)); +}; + +/** The longest jitter an admitted run waits before its container starts. */ +export const ADMISSION_JITTER_MAX_MS = 2_000; + +/** + * The pause between an admitting claim and the container start (ADR-0009 § + * Pools sized under the spend guard), `0..ADMISSION_JITTER_MAX_MS` ms. A burst + * of admissions spreads over two seconds instead of hitting the container + * start-rate limit at once. Derived from the execution id (FNV-1a), so a + * replay waits the same length. + */ +export const admissionJitterMs = (executionId: string): number => { + let hash = 0x811c9dc5; + for (let i = 0; i < executionId.length; i++) { + hash ^= executionId.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash % (ADMISSION_JITTER_MAX_MS + 1); +}; diff --git a/packages/runtime-cf/src/index.ts b/packages/runtime-cf/src/index.ts index a37a661d..9dcb3351 100644 --- a/packages/runtime-cf/src/index.ts +++ b/packages/runtime-cf/src/index.ts @@ -67,6 +67,7 @@ export { ADMISSION_CAP_DEFAULT, RELEASE_ADMISSION_CAP_DEFAULT, AGENT_ADMISSION_CAP_DEFAULT, + BROWSER_ADMISSION_CAP_DEFAULT, ADMISSION_POLL_EVERY_MS, ADMISSION_MAX_QUEUE_AGE_MS, ADMISSION_TTL_MS, diff --git a/packages/runtime-cf/src/run-admission-d1.ts b/packages/runtime-cf/src/run-admission-d1.ts index 3c900830..ccf8da03 100644 --- a/packages/runtime-cf/src/run-admission-d1.ts +++ b/packages/runtime-cf/src/run-admission-d1.ts @@ -27,6 +27,10 @@ import { Effect, Schedule } from "effect"; import { + ADMISSION_CLASS_DEFAULT, + ADMISSION_POLL_MAX_MS, + ADMISSION_POLL_MIN_MS, + type AdmissionClass, type AdmissionObservation, type AdmissionPool, fitsBurnCeiling, @@ -83,26 +87,35 @@ export const RELEASE_ADMISSION_CAP_DEFAULT = 8; */ export const AGENT_ADMISSION_CAP_DEFAULT = 8; +/** + * The browser pool's slot cap when `ADMISSION_CAP_BROWSER` is unset — the + * `RunSandboxBrowser` container's `max_instances: 16` in wrangler.jsonc. + */ +export const BROWSER_ADMISSION_CAP_DEFAULT = 16; + /** The admission vars a pool's cap reads. */ export type AdmissionCapVars = { readonly ADMISSION_CAP?: string; readonly ADMISSION_CAP_AGENT?: string; + readonly ADMISSION_CAP_BROWSER?: string; readonly ADMISSION_CAP_RELEASE?: string; }; /** - * The slot cap for one pool. `ADMISSION_CAP` bounds every pool; the agent and - * release pools, whose containers are fewer, are further bounded by - * `ADMISSION_CAP_AGENT` and `ADMISSION_CAP_RELEASE` (defaults - * {@link AGENT_ADMISSION_CAP_DEFAULT}, {@link RELEASE_ADMISSION_CAP_DEFAULT}), - * so their cap is the smaller of the two. Lean and browser keep - * `ADMISSION_CAP` alone. Garbage degrades to the default, as in - * {@link resolveAdmissionCap}. + * The slot cap for one pool — an upper bound; the spend guard's burn ceiling + * sets the operating concurrency (ADR-0009 § Pools sized under the spend + * guard). `ADMISSION_CAP` bounds every pool; the agent, browser and release + * pools are further bounded by `ADMISSION_CAP_AGENT`, `ADMISSION_CAP_BROWSER` + * and `ADMISSION_CAP_RELEASE` (defaults {@link AGENT_ADMISSION_CAP_DEFAULT}, + * {@link BROWSER_ADMISSION_CAP_DEFAULT}, {@link RELEASE_ADMISSION_CAP_DEFAULT}), + * so their cap is the smaller of the two. Lean keeps `ADMISSION_CAP` alone. + * Garbage degrades to the default, as in {@link resolveAdmissionCap}. */ export const resolvePoolAdmissionCap = (pool: AdmissionPool, vars: AdmissionCapVars): number => { const shared = resolveAdmissionCap(vars.ADMISSION_CAP); const own = pool === "release" ? { raw: vars.ADMISSION_CAP_RELEASE, fallback: RELEASE_ADMISSION_CAP_DEFAULT } : pool === "agent" ? { raw: vars.ADMISSION_CAP_AGENT, fallback: AGENT_ADMISSION_CAP_DEFAULT } + : pool === "browser" ? { raw: vars.ADMISSION_CAP_BROWSER, fallback: BROWSER_ADMISSION_CAP_DEFAULT } : undefined; if (own === undefined) return shared; const parsed = own.raw === undefined ? NaN : Number.parseInt(own.raw, 10); @@ -110,12 +123,16 @@ export const resolvePoolAdmissionCap = (pool: AdmissionPool, vars: AdmissionCapV }; /** - * Delay between claim attempts while the pool is full. Each wait is a - * durable `step.sleep`, so a queued run costs nothing while it waits; 20s - * keeps the loop ≤270 claim steps + 270 sleeps over the largest run override - * — trivial against the 10k Workflow step cap (#83 precedent). + * The shortest delay between claim attempts — the head of the queue's poll. + * A waiter further back sleeps up to {@link ADMISSION_POLL_MAX_MS} + * (`admissionPollSleepMs`, core run-admission.ts). Each wait is a durable + * `step.sleep`, so a queued run costs nothing while it waits. The attempt + * count is derived from this shortest sleep, so one attempt costs three steps + * (spend check, claim, sleep): the largest run override (6 h) bounds the loop + * at 1,080 attempts, 3,240 steps, under the 10k Workflow step cap (#83 + * precedent); a waiter several slots back polls up to three times less often. */ -export const ADMISSION_POLL_EVERY_MS = 20_000; +export const ADMISSION_POLL_EVERY_MS = ADMISSION_POLL_MIN_MS; /** * Dispatch-age ceiling — a run queued longer than this fails @@ -136,11 +153,13 @@ export const ADMISSION_TTL_MS = LEASE_TTL_MS; /** * Heartbeat staleness ceiling for QUEUED rows. A waiter refreshes its - * heartbeat on every claim attempt (every `ADMISSION_POLL_EVERY_MS`), so a - * row silent for ~5 polls is a dead waiter — excluded from the FIFO no-barge - * check and the position count, so a corpse never blocks the line. + * heartbeat on every claim attempt, at most {@link ADMISSION_POLL_MAX_MS} + * apart, so a row silent for three of the longest sleeps is a dead waiter — + * excluded from the FIFO no-barge check and the position count, so a corpse + * never blocks the line. A slow-polling live waiter never reads as dead + * (ADR-0008 § Position-scaled admission polls). */ -export const ADMISSION_WAITER_TTL_MS = 100_000; +export const ADMISSION_WAITER_TTL_MS = 3 * ADMISSION_POLL_MAX_MS; /** * One D1 statement, retried twice on a transient error then surfaced — the @@ -199,6 +218,15 @@ export interface RunAdmissionStore { readonly release: (executionId: string) => Effect.Effect; } +/** + * The rank of a class in the FIFO line: interactive (0) ahead of batch (1). A + * row with no class (pre-0011) ranks as batch. The SQL computes the same rank + * from `run_admissions.admission_class` with {@link CLASS_RANK_SQL}. + */ +const classRank = (admissionClass: AdmissionClass): number => (admissionClass === "interactive" ? 0 : 1); +const CLASS_RANK_SQL = (alias: string): string => + `(CASE WHEN ${alias}admission_class = 'interactive' THEN 0 ELSE 1 END)`; + /** * Build a D1-backed run admission store. `now` is injectable so tests can * drive the clock; production passes `Date.now` (always read inside the @@ -206,14 +234,17 @@ export interface RunAdmissionStore { * per-pool slot ceiling — the dispatcher resolves it from the * `ADMISSION_CAP` wrangler var via {@link resolveAdmissionCap}. `burn` is * this claimant's hourly rate and the deploy's burn ceiling; the default - * {@link BURN_UNLIMITED} admits on the cap alone. + * {@link BURN_UNLIMITED} admits on the cap alone. `admissionClass` is the + * claimant's line: interactive waiters are admitted ahead of batch ones. */ export const makeRunAdmissionD1 = ( db: D1Database, now: () => number = Date.now, cap: number = ADMISSION_CAP_DEFAULT, burn: BurnCeiling = BURN_UNLIMITED, + admissionClass: AdmissionClass = ADMISSION_CLASS_DEFAULT, ): RunAdmissionStore => { + const rank = classRank(admissionClass); const insertQueued = ( executionId: string, pool: AdmissionPool, @@ -228,13 +259,14 @@ export const makeRunAdmissionD1 = ( // GC'd waiter row restored mid-loop keeps its original FIFO key) → // the parent's row (matrix-children priority inheritance) → now. `INSERT OR IGNORE INTO run_admissions - (execution_id, pool, state, enqueued_at, heartbeat_at, hourly_rate_micro_usd) + (execution_id, pool, state, enqueued_at, heartbeat_at, hourly_rate_micro_usd, + admission_class) VALUES (?1, ?2, 'queued', COALESCE(?3, (SELECT enqueued_at FROM run_admissions WHERE execution_id = ?4), ?5), - ?5, ?6)`, + ?5, ?6, ?7)`, ) .bind( executionId, @@ -243,6 +275,7 @@ export const makeRunAdmissionD1 = ( parentExecutionId ?? null, ts, burn.hourlyRateMicroUsd, + admissionClass, ) .run(), ).pipe(Effect.asVoid); @@ -274,15 +307,18 @@ export const makeRunAdmissionD1 = ( * are below the cap (stale heartbeats stop counting — a crashed run's * slot frees itself after the TTL); * - the NOT EXISTS is the no-barge FIFO check among LIVE waiters: nobody - * with an earlier `enqueued_at` (execution-id ULID as the tiebreak) may - * be skipped, but a waiter silent past `ADMISSION_WAITER_TTL_MS` no - * longer blocks the line; + * ahead of us may be skipped. "Ahead" is a live waiter of a higher + * class (interactive ahead of batch), or of our class with an earlier + * `enqueued_at` (execution-id ULID as the tiebreak). A waiter silent + * past `ADMISSION_WAITER_TTL_MS` no longer blocks the line; * - the burn-ceiling clause admits only while the hourly rates of LIVE * admitted rows in EVERY pool, plus ours, fit under the ceiling. An * empty admitted set admits anyone, so a claimant whose own rate * exceeds the ceiling runs alone rather than waiting forever; a NULL * ceiling disables the clause. `fitsBurnCeiling` (core spend.ts) is its - * pure mirror. + * pure mirror. The ceiling never reorders the line: a batch claimant + * whose lower rate would fit still waits behind a live interactive + * waiter the ceiling holds. * * `meta.changes === 1` ⇒ our row flipped to 'admitted' — we hold a slot. */ @@ -302,8 +338,10 @@ export const makeRunAdmissionD1 = ( AND NOT EXISTS (SELECT 1 FROM run_admissions o WHERE o.pool = ?3 AND o.state = 'queued' AND o.heartbeat_at > ?6 - AND (o.enqueued_at < ?7 - OR (o.enqueued_at = ?7 AND o.execution_id < ?1))) + AND (${CLASS_RANK_SQL("o.")} < ?10 + OR (${CLASS_RANK_SQL("o.")} = ?10 + AND (o.enqueued_at < ?7 + OR (o.enqueued_at = ?7 AND o.execution_id < ?1))))) AND (?8 IS NULL OR NOT EXISTS (SELECT 1 FROM run_admissions WHERE state = 'admitted' AND heartbeat_at > ?4) @@ -320,6 +358,7 @@ export const makeRunAdmissionD1 = ( enqueuedAt, burn.burnMaxMicroUsdPerHour, burn.hourlyRateMicroUsd, + rank, ) .run(), ).pipe(Effect.map((result) => result.meta.changes === 1)); @@ -376,8 +415,9 @@ export const makeRunAdmissionD1 = ( // Not admitted: refresh our waiter heartbeat (so we keep blocking // bargers and counting in peers' position reads), then observe where - // we stand — live queued rows ahead of us, live admitted rows in our - // pool, and the live admitted burn across every pool. + // we stand — live queued rows ahead of us (higher class, or our class + // and earlier), live admitted rows in our pool, and the live admitted + // burn across every pool. yield* heartbeat(executionId); const counts = yield* d1(() => db @@ -386,8 +426,10 @@ export const makeRunAdmissionD1 = ( (SELECT COUNT(*) FROM run_admissions WHERE pool = ?1 AND state = 'queued' AND heartbeat_at > ?2 - AND (enqueued_at < ?3 - OR (enqueued_at = ?3 AND execution_id < ?4))) AS position, + AND (${CLASS_RANK_SQL("")} < ?6 + OR (${CLASS_RANK_SQL("")} = ?6 + AND (enqueued_at < ?3 + OR (enqueued_at = ?3 AND execution_id < ?4))))) AS position, (SELECT COUNT(*) FROM run_admissions WHERE pool = ?1 AND state = 'admitted' AND heartbeat_at > ?5) AS busy, @@ -396,7 +438,7 @@ export const makeRunAdmissionD1 = ( (SELECT COALESCE(SUM(hourly_rate_micro_usd), 0) FROM run_admissions WHERE state = 'admitted' AND heartbeat_at > ?5) AS admitted_rate`, ) - .bind(pool, ts - ADMISSION_WAITER_TTL_MS, enqueuedAt, executionId, ts - ADMISSION_TTL_MS) + .bind(pool, ts - ADMISSION_WAITER_TTL_MS, enqueuedAt, executionId, ts - ADMISSION_TTL_MS, rank) .first<{ position: number; busy: number; admitted_count: number; admitted_rate: number }>(), ); const poolBusy = counts?.busy ?? 0; diff --git a/packages/runtime-cf/src/run-admission-d1.workers.test.ts b/packages/runtime-cf/src/run-admission-d1.workers.test.ts index 6d85cc70..b4901c32 100644 --- a/packages/runtime-cf/src/run-admission-d1.workers.test.ts +++ b/packages/runtime-cf/src/run-admission-d1.workers.test.ts @@ -16,7 +16,13 @@ import { Effect } from "effect"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { containerMicroUsdPerHour, decideAdmission, type AdmissionPool } from "@fractalboxdev/flare-dispatch-core"; +import { + ADMISSION_POLL_MAX_MS, + containerMicroUsdPerHour, + decideAdmission, + type AdmissionClass, + type AdmissionPool, +} from "@fractalboxdev/flare-dispatch-core"; import { ADMISSION_CAP_DEFAULT, ADMISSION_MAX_QUEUE_AGE_MS, @@ -27,6 +33,7 @@ import { resolvePoolAdmissionCap, RELEASE_ADMISSION_CAP_DEFAULT, AGENT_ADMISSION_CAP_DEFAULT, + BROWSER_ADMISSION_CAP_DEFAULT, } from "./run-admission-d1"; import { makeTestBindings, type TestBindings } from "./test-support-workers"; @@ -346,11 +353,28 @@ describe("resolvePoolAdmissionCap — one cap per pool", () => { expect(resolvePoolAdmissionCap("agent", { ADMISSION_CAP: "4", ADMISSION_CAP_AGENT: "12" })).toBe(4); }); - it("leaves the lean and browser pools on the shared cap", () => { - for (const pool of ["lean", "browser"] as const) { - expect(resolvePoolAdmissionCap(pool, { ADMISSION_CAP: "16", ADMISSION_CAP_AGENT: "8", ADMISSION_CAP_RELEASE: "8" })).toBe(16); - expect(resolvePoolAdmissionCap(pool, {})).toBe(ADMISSION_CAP_DEFAULT); - } + it("caps the browser pool by ADMISSION_CAP_BROWSER under the shared cap", () => { + expect(BROWSER_ADMISSION_CAP_DEFAULT).toBe(16); + expect(resolvePoolAdmissionCap("browser", { ADMISSION_CAP: "16" })).toBe(16); + expect(resolvePoolAdmissionCap("browser", { ADMISSION_CAP: "16", ADMISSION_CAP_BROWSER: "6" })).toBe(6); + expect(resolvePoolAdmissionCap("browser", { ADMISSION_CAP: "4", ADMISSION_CAP_BROWSER: "12" })).toBe(4); + expect(resolvePoolAdmissionCap("browser", { ADMISSION_CAP: "16", ADMISSION_CAP_BROWSER: "six" })).toBe(16); + }); + + it("leaves the lean pool on the shared cap", () => { + expect( + resolvePoolAdmissionCap("lean", { + ADMISSION_CAP: "16", + ADMISSION_CAP_AGENT: "8", + ADMISSION_CAP_BROWSER: "6", + ADMISSION_CAP_RELEASE: "8", + }), + ).toBe(16); + expect(resolvePoolAdmissionCap("lean", {})).toBe(ADMISSION_CAP_DEFAULT); + }); + + it("keeps a waiter live across three of the longest poll sleeps", () => { + expect(ADMISSION_WAITER_TTL_MS).toBeGreaterThanOrEqual(3 * ADMISSION_POLL_MAX_MS); }); }); @@ -470,3 +494,158 @@ describe("makeRunAdmissionD1 — the burn ceiling", () => { expect(row?.admitted_at).toBe(T0); }); }); + +// The interactive admission class (specs/adr/0009-fast-trustworthy-verdicts.md +// § Interactive admission class): a claim admits the oldest live interactive +// waiter before any batch waiter; FIFO and the queue position apply within a +// class, interactive ahead of batch; the burn ceiling never reorders the line. +describe("makeRunAdmissionD1 — the interactive admission class", () => { + let bindings: TestBindings; + beforeEach(async () => { + bindings = await makeTestBindings(); + }); + afterEach(async () => { + await bindings.dispose(); + }); + + const BATCH_OLD = "01ARZ3NDEKTSV4RRFFQ69G5F01"; + const BATCH_NEW = "01ARZ3NDEKTSV4RRFFQ69G5F02"; + const INTER_1 = "01ARZ3NDEKTSV4RRFFQ69G5F03"; + const INTER_2 = "01ARZ3NDEKTSV4RRFFQ69G5F04"; + + const storeFor = ( + admissionClass: AdmissionClass, + clock: () => number, + cap = 1, + burn = { hourlyRateMicroUsd: 0, burnMaxMicroUsdPerHour: null as number | null }, + ) => makeRunAdmissionD1(bindings.db, clock, cap, burn, admissionClass); + + /** Enqueue `id` at `at` in `admissionClass` and return its store + FIFO key. */ + const enqueueAt = async ( + id: string, + admissionClass: AdmissionClass, + at: number, + cap = 1, + burn?: { hourlyRateMicroUsd: number; burnMaxMicroUsdPerHour: number | null }, + ) => { + const store = storeFor(admissionClass, () => at, cap, burn); + const { enqueuedAt } = await Effect.runPromise(store.enqueue(id, "lean")); + return { store, enqueuedAt }; + }; + + const attemptAt = ( + id: string, + admissionClass: AdmissionClass, + enqueuedAt: number, + at: number, + cap = 1, + burn?: { hourlyRateMicroUsd: number; burnMaxMicroUsdPerHour: number | null }, + ) => Effect.runPromise(storeFor(admissionClass, () => at, cap, burn).attempt(id, "lean", enqueuedAt)); + + const holdSlot = () => + bindings.db + .prepare( + `INSERT INTO run_admissions (execution_id, pool, state, enqueued_at, admitted_at, heartbeat_at) + VALUES ('holder', 'lean', 'admitted', ?, ?, ?)`, + ) + .bind(T0 - 5_000, T0 - 5_000, T0) + .run(); + const freeSlot = () => bindings.db.prepare(`DELETE FROM run_admissions WHERE execution_id = 'holder'`).run(); + + it("records the class on the row; a store without one enqueues batch", async () => { + await enqueueAt(INTER_1, "interactive", T0); + await Effect.runPromise(makeRunAdmissionD1(bindings.db, () => T0).enqueue(BATCH_OLD, "lean")); + const classes = await bindings.db + .prepare(`SELECT execution_id, admission_class FROM run_admissions ORDER BY execution_id`) + .all<{ execution_id: string; admission_class: string }>(); + expect(classes.results).toEqual([ + { execution_id: BATCH_OLD, admission_class: "batch" }, + { execution_id: INTER_1, admission_class: "interactive" }, + ]); + }); + + it("admits a later interactive waiter before an earlier batch waiter", async () => { + await holdSlot(); + const batch = await enqueueAt(BATCH_OLD, "batch", T0 - 60_000); + const inter = await enqueueAt(INTER_1, "interactive", T0); + // Full pool: the interactive waiter is first in line, the batch one second. + expect(await attemptAt(INTER_1, "interactive", inter.enqueuedAt, T0 + 1)).toMatchObject({ + admitted: false, + position: 0, + }); + expect(await attemptAt(BATCH_OLD, "batch", batch.enqueuedAt, T0 + 2)).toMatchObject({ + admitted: false, + position: 1, + }); + await freeSlot(); + // The batch waiter cannot barge past the live interactive one. + expect((await attemptAt(BATCH_OLD, "batch", batch.enqueuedAt, T0 + 3)).admitted).toBe(false); + expect((await attemptAt(INTER_1, "interactive", inter.enqueuedAt, T0 + 4)).admitted).toBe(true); + }); + + it("keeps FIFO within the interactive class", async () => { + await holdSlot(); + const first = await enqueueAt(INTER_1, "interactive", T0 - 10_000); + const second = await enqueueAt(INTER_2, "interactive", T0); + expect((await attemptAt(INTER_2, "interactive", second.enqueuedAt, T0 + 1)).position).toBe(1); + await freeSlot(); + expect((await attemptAt(INTER_2, "interactive", second.enqueuedAt, T0 + 2)).admitted).toBe(false); + expect((await attemptAt(INTER_1, "interactive", first.enqueuedAt, T0 + 3)).admitted).toBe(true); + }); + + it("keeps FIFO within the batch class, behind every interactive waiter", async () => { + await holdSlot(); + await enqueueAt(INTER_1, "interactive", T0); + await enqueueAt(BATCH_OLD, "batch", T0 - 20_000); + const newer = await enqueueAt(BATCH_NEW, "batch", T0 - 10_000); + expect((await attemptAt(BATCH_NEW, "batch", newer.enqueuedAt, T0 + 1)).position).toBe(2); + }); + + it("ranks a pre-0011 row with no class as batch", async () => { + await holdSlot(); + await bindings.db + .prepare( + `INSERT INTO run_admissions (execution_id, pool, state, enqueued_at, heartbeat_at) + VALUES (?, 'lean', 'queued', ?, ?)`, + ) + .bind(BATCH_OLD, T0 - 60_000, T0) + .run(); + const inter = await enqueueAt(INTER_1, "interactive", T0); + expect((await attemptAt(INTER_1, "interactive", inter.enqueuedAt, T0 + 1)).position).toBe(0); + const batch = await enqueueAt(BATCH_NEW, "batch", T0); + expect((await attemptAt(BATCH_NEW, "batch", batch.enqueuedAt, T0 + 2)).position).toBe(2); + }); + + it("stops a dead interactive waiter from blocking the batch line", async () => { + await enqueueAt(INTER_1, "interactive", T0 - ADMISSION_WAITER_TTL_MS - 10); + const batch = await enqueueAt(BATCH_OLD, "batch", T0); + expect((await attemptAt(BATCH_OLD, "batch", batch.enqueuedAt, T0 + 1)).admitted).toBe(true); + }); + + it("holds a batch claimant the ceiling would fit behind an interactive waiter it holds", async () => { + // One admitted row at 1.0 $/h; ceiling 1.5 $/h. The interactive waiter at + // 0.8 $/h does not fit; the batch waiter at 0.2 $/h would, but stays in line. + const ceiling = 1_500_000; + await bindings.db + .prepare( + `INSERT INTO run_admissions + (execution_id, pool, state, enqueued_at, admitted_at, heartbeat_at, hourly_rate_micro_usd) + VALUES ('holder', 'release', 'admitted', ?, ?, ?, 1000000)`, + ) + .bind(T0 - 5_000, T0 - 5_000, T0) + .run(); + const interBurn = { hourlyRateMicroUsd: 800_000, burnMaxMicroUsdPerHour: ceiling }; + const batchBurn = { hourlyRateMicroUsd: 200_000, burnMaxMicroUsdPerHour: ceiling }; + const inter = await enqueueAt(INTER_1, "interactive", T0, 16, interBurn); + const batch = await enqueueAt(BATCH_OLD, "batch", T0 - 60_000, 16, batchBurn); + expect(await attemptAt(INTER_1, "interactive", inter.enqueuedAt, T0 + 1, 16, interBurn)).toMatchObject({ + admitted: false, + ceilingHeld: true, + }); + expect((await attemptAt(BATCH_OLD, "batch", batch.enqueuedAt, T0 + 2, 16, batchBurn)).admitted).toBe(false); + // Burn frees: the interactive waiter admits first, then the batch one fits. + await freeSlot(); + expect((await attemptAt(INTER_1, "interactive", inter.enqueuedAt, T0 + 3, 16, interBurn)).admitted).toBe(true); + expect((await attemptAt(BATCH_OLD, "batch", batch.enqueuedAt, T0 + 4, 16, batchBurn)).admitted).toBe(true); + }); +}); diff --git a/runs/admission-class.test.ts b/runs/admission-class.test.ts new file mode 100644 index 00000000..0d80c55b --- /dev/null +++ b/runs/admission-class.test.ts @@ -0,0 +1,29 @@ +// Catalog-level guard for `limits.admissionClass` +// (specs/adr/0009-fast-trustworthy-verdicts.md § Interactive admission class). +// +// An interactive run is admitted ahead of every batch run in its pool, so the +// set is a deliberate list: the PR verdicts and the deploy someone waits on. +// The assertion derives from the whole catalog, so a new run that declares +// `interactive` fails here until the list is updated on purpose. + +import { describe, expect, it } from "vitest"; +import type { Run } from "@fractalboxdev/flare-dispatch-core"; +import * as catalog from "./index"; + +const runs = Object.values(catalog) as ReadonlyArray>; + +describe("admissionClass catalog", () => { + it("admits exactly the waited-on runs in the interactive class", () => { + const interactive = runs + .filter((r) => r.limits.admissionClass === "interactive") + .map((r) => r.name) + .sort(); + expect(interactive).toEqual(["offload-test", "oxlint", "pr-review", "worker-deploy"]); + }); + + it("leaves every other run in the default batch class", () => { + for (const r of runs) { + expect(["interactive", "batch", undefined]).toContain(r.limits.admissionClass); + } + }); +}); diff --git a/runs/offload-test.ts b/runs/offload-test.ts index b5077783..476dd78f 100644 --- a/runs/offload-test.ts +++ b/runs/offload-test.ts @@ -511,6 +511,8 @@ export const offloadTest = defineRun({ // dispatcher. One documented 1800 s exec plus checkout, install and // upload fits; aligned retries of that exec do not (issue #42). maxDurationSec: 3600, + // A PR verdict someone waits on: admitted ahead of batch work. + admissionClass: "interactive", }, run: (input) => diff --git a/runs/oxlint.ts b/runs/oxlint.ts index 59dcbbfc..e816d793 100644 --- a/runs/oxlint.ts +++ b/runs/oxlint.ts @@ -204,6 +204,8 @@ export const oxlint = defineRun({ // Wall-time ceiling — lint is cheap, but a cold `npx` fetch + a huge tree // gets headroom. maxDurationSec: 600, + // A PR verdict someone waits on: admitted ahead of batch work. + admissionClass: "interactive", }, run: (input) => diff --git a/runs/pr-review.ts b/runs/pr-review.ts index 68e4a4b8..22f66e6d 100644 --- a/runs/pr-review.ts +++ b/runs/pr-review.ts @@ -304,7 +304,8 @@ export const prReview = defineRun({ outputs: ReviewOutput, - limits: { maxDurationSec: 1500, maxConcurrency: FULL_AGENTS.length }, + // A PR verdict someone waits on: admitted ahead of batch work. + limits: { maxDurationSec: 1500, maxConcurrency: FULL_AGENTS.length, admissionClass: "interactive" }, // At most one review per PR per window, across BOTH dispatch paths // (webhook + Action). Default 60 min; override via CONFIG_KV diff --git a/runs/worker-deploy.ts b/runs/worker-deploy.ts index 17bb88a4..ba0bfd48 100644 --- a/runs/worker-deploy.ts +++ b/runs/worker-deploy.ts @@ -334,6 +334,8 @@ export const workerDeploy = defineRun({ limits: { maxDurationSec: 1800, + // A deploy someone waits on: admitted ahead of batch work. + admissionClass: "interactive", }, run: (input) => diff --git a/wrangler.jsonc b/wrangler.jsonc index 8bb58cd2..58da948b 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -89,9 +89,11 @@ // also set the AI_GATEWAY_AUTH_TOKEN secret. Operator overlay. "AI_GATEWAY_ID": "your-ai-gateway", "ADMISSION_CAP": "16", - // The agent and release pools admit at most RunSandboxAgent's and - // RunSandboxRelease's max_instances. + // The agent, browser and release pools admit at most RunSandboxAgent's, + // RunSandboxBrowser's and RunSandboxRelease's max_instances. Each cap is an + // upper bound; SPEND_BURN_MAX_USD_PER_HOUR sets the operating concurrency. "ADMISSION_CAP_AGENT": "8", + "ADMISSION_CAP_BROWSER": "16", "ADMISSION_CAP_RELEASE": "8", // Opt the human-facing viewer surfaces (incl. the GitHub App install flow) // out of the Cloudflare Access gate for the initial dogfood. Configure From 0c97279c8ccf55e90b5ba9bb1270a639913b02b7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 22 Oct 2026 17:48:04 +0800 Subject: [PATCH 4/4] feat(spend): default the monthly budget to 50 USD and size pools as upper bounds with stop headroom (ADR-0006, ADR-0009) --- apps/dispatcher/src/env.ts | 2 +- apps/dispatcher/src/routes/spend.test.ts | 12 ++++++------ apps/dispatcher/src/sandbox-routing.test.ts | 8 ++++---- packages/core/src/spend.test.ts | 4 ++-- packages/core/src/spend.ts | 2 +- packages/runtime-cf/src/run-admission-d1.ts | 2 +- packages/runtime-cf/src/spend-d1.workers.test.ts | 14 +++++++------- wrangler.jsonc | 10 +++++----- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/apps/dispatcher/src/env.ts b/apps/dispatcher/src/env.ts index c7423567..7baddfe2 100644 --- a/apps/dispatcher/src/env.ts +++ b/apps/dispatcher/src/env.ts @@ -475,7 +475,7 @@ export interface Env { /** * The monthly spend budget in USD (specs/adr/0006-spend-guard.md) — a var. * At or over it, admission skips every run with a neutral check. Unset or - * unparsable → 90; `unlimited` disables it. CONFIG_KV + * unparsable → 50; `unlimited` disables it. CONFIG_KV * `spend.budgetMonthlyUsd` overrides it without a deploy. The default lives * in code (core spend.ts), because an operator overlay replaces the * committed wrangler.jsonc. diff --git a/apps/dispatcher/src/routes/spend.test.ts b/apps/dispatcher/src/routes/spend.test.ts index 067ad006..433bbcc1 100644 --- a/apps/dispatcher/src/routes/spend.test.ts +++ b/apps/dispatcher/src/routes/spend.test.ts @@ -38,10 +38,10 @@ const status = (monthToDateMicroUsd: number, admittedMicroUsdPerHour = 401_040): describe("a spend skip concludes neutral", () => { it("records a budget skip as skipped with its reason, and a neutral check", () => { - const decision = decideBudget(90_500_000, resolveSpendLimits({})); + const decision = decideBudget(50_500_000, resolveSpendLimits({})); if (decision._kind !== "closed") throw new Error("expected a closed budget"); const exit = Exit.fail(new RunSkipped({ reason: decision.reason })); - expect(runSkippedReason(exit)).toBe("spend budget reached: $90.50 of $90.00 this month"); + expect(runSkippedReason(exit)).toBe("spend budget reached: $50.50 of $50.00 this month"); expect(checkConclusion("skipped", true)).toBe("neutral"); }); @@ -60,7 +60,7 @@ describe("spendView", () => { expect(spendView(status(12_340_000))).toEqual({ month: "2026-10", monthToDateUsd: 12.34, - budgetUsd: 90, + budgetUsd: 50, burnMaxUsdPerHour: 1.5, admittedUsdPerHour: 0.40104, admittedCount: 1, @@ -96,7 +96,7 @@ describe("GET /api/spend", () => { expect(res.status).toBe(200); expect(await res.json()).toMatchObject({ monthToDateUsd: 0, - budgetUsd: 90, + budgetUsd: 50, burnMaxUsdPerHour: 1.5, budgetReached: false, }); @@ -119,11 +119,11 @@ describe("GET /api/spend", () => { describe("spendLine", () => { it("is the dashboard's one-line summary", () => { expect(spendLine(status(12_340_000))).toBe( - "Spend 2026-10: $12.34 of $90.00 · burn $0.40/h of $1.50/h · resets 2026-11-01", + "Spend 2026-10: $12.34 of $50.00 · burn $0.40/h of $1.50/h · resets 2026-11-01", ); }); it("says when the budget closed", () => { - expect(spendLine(status(95_000_000))).toContain("$95.00 of $90.00 — budget reached, runs skip"); + expect(spendLine(status(55_000_000))).toContain("$55.00 of $50.00 — budget reached, runs skip"); }); }); diff --git a/apps/dispatcher/src/sandbox-routing.test.ts b/apps/dispatcher/src/sandbox-routing.test.ts index 6bff05fc..1ecf0fda 100644 --- a/apps/dispatcher/src/sandbox-routing.test.ts +++ b/apps/dispatcher/src/sandbox-routing.test.ts @@ -55,20 +55,20 @@ const POOL_CLASS = { lean: "RunSandbox", browser: "RunSandboxBrowser", agent: "R const bound = { RUNS_SANDBOX_BROWSER: browser, RUNS_SANDBOX_AGENT: agent, RUNS_SANDBOX_RELEASE: release }; describe("selectAdmission — the pool a run is metered against and its slot cap", () => { - it("admits no pool past its container's max_instances under the committed vars", () => { + it("keeps every pool's cap below its container's max_instances under the committed vars", () => { for (const image of ["lean", "browser", "agent", "release"] as const) { const { pool, cap } = selectAdmission(image, { ...wrangler.vars, ...bound }); expect(pool).toBe(image); const container = wrangler.containers.find((entry) => entry.class_name === POOL_CLASS[pool])!; - expect(cap, `${pool} pool`).toBeLessThanOrEqual(container.max_instances); + expect(cap, `${pool} pool`).toBeLessThan(container.max_instances); } }); - it("admits no pool past its container's max_instances on the code defaults", () => { + it("keeps every pool's cap below its container's max_instances on the code defaults", () => { for (const image of ["lean", "browser", "agent", "release"] as const) { const { pool, cap } = selectAdmission(image, bound); const container = wrangler.containers.find((entry) => entry.class_name === POOL_CLASS[pool])!; - expect(cap, `${pool} pool`).toBeLessThanOrEqual(container.max_instances); + expect(cap, `${pool} pool`).toBeLessThan(container.max_instances); } }); diff --git a/packages/core/src/spend.test.ts b/packages/core/src/spend.test.ts index 238f14ee..02c35881 100644 --- a/packages/core/src/spend.test.ts +++ b/packages/core/src/spend.test.ts @@ -22,8 +22,8 @@ const LIMITS: SpendLimits = { }; describe("resolveSpendLimits", () => { - it("caps a deploy that sets nothing at 90 USD a month and 1.50 USD an hour", () => { - expect(resolveSpendLimits({})).toEqual(LIMITS); + it("caps a deploy that sets nothing at 50 USD a month and 1.50 USD an hour", () => { + expect(resolveSpendLimits({})).toEqual({ ...LIMITS, budgetMicroUsd: 50_000_000 }); }); it("reads the vars, and a CONFIG_KV override wins over them", () => { diff --git a/packages/core/src/spend.ts b/packages/core/src/spend.ts index 32a4e873..68d44ef8 100644 --- a/packages/core/src/spend.ts +++ b/packages/core/src/spend.ts @@ -19,7 +19,7 @@ import { MICRO_USD_PER_USD } from "./cost"; /** The monthly budget a deploy that sets nothing gets, in USD. */ -export const SPEND_BUDGET_MONTHLY_USD_DEFAULT = 90; +export const SPEND_BUDGET_MONTHLY_USD_DEFAULT = 50; /** The burn ceiling a deploy that sets nothing gets, in USD per hour. */ export const SPEND_BURN_MAX_USD_PER_HOUR_DEFAULT = 1.5; /** Multiplier for Workflow steps and Durable Objects on top of container + model spend. */ diff --git a/packages/runtime-cf/src/run-admission-d1.ts b/packages/runtime-cf/src/run-admission-d1.ts index ccf8da03..981e922d 100644 --- a/packages/runtime-cf/src/run-admission-d1.ts +++ b/packages/runtime-cf/src/run-admission-d1.ts @@ -57,7 +57,7 @@ export const BURN_UNLIMITED: BurnCeiling = { hourlyRateMicroUsd: 0, burnMaxMicro * `ADMISSION_CAP` wrangler var (see {@link resolveAdmissionCap}); this is * the fallback when the var is unset. */ -export const ADMISSION_CAP_DEFAULT = 16; +export const ADMISSION_CAP_DEFAULT = 48; /** * Parse the `ADMISSION_CAP` wrangler var (vars are always strings) into the diff --git a/packages/runtime-cf/src/spend-d1.workers.test.ts b/packages/runtime-cf/src/spend-d1.workers.test.ts index 775461ac..3a92b7e3 100644 --- a/packages/runtime-cf/src/spend-d1.workers.test.ts +++ b/packages/runtime-cf/src/spend-d1.workers.test.ts @@ -64,14 +64,14 @@ describe("spend-d1", () => { }); it("closes the budget once spend times the overhead reaches it", async () => { - await seedFinished("f1", NOW - HOUR, 82_000_000); + await seedFinished("f1", NOW - HOUR, 46_000_000); const limits = resolveSpendLimits({}); const status = await Effect.runPromise(readSpendStatus(bindings.db, limits, NOW)); - // 82 × 1.10 = 90.20 ≥ 90. - expect(status.monthToDateMicroUsd).toBe(90_200_000); + // 46 × 1.10 = 50.60 ≥ 50. + expect(status.monthToDateMicroUsd).toBe(50_600_000); expect(status.decision).toEqual({ _kind: "closed", - reason: "spend budget reached: $90.20 of $90.00 this month", + reason: "spend budget reached: $50.60 of $50.00 this month", }); expect(status.month.key).toBe("2026-10"); }); @@ -89,7 +89,7 @@ describe("spend-d1", () => { }); it("warns once per month past the warn ratio, marking the month in CONFIG_KV", async () => { - await seedFinished("f1", NOW - HOUR, 70_000_000); + await seedFinished("f1", NOW - HOUR, 40_000_000); const limits: SpendLimits = resolveSpendLimits({}); const status = await Effect.runPromise(readSpendStatus(bindings.db, limits, NOW)); const sent: string[] = []; @@ -100,8 +100,8 @@ describe("spend-d1", () => { expect(await Effect.runPromise(warnSpendOnce({ kv: bindings.kv, status, notify }))).toBe(true); expect(await Effect.runPromise(warnSpendOnce({ kv: bindings.kv, status, notify }))).toBe(false); expect(sent).toHaveLength(1); - expect(sent[0]).toContain("2026-10: FlareDispatch spend reached $77.00 of the $90.00 monthly budget"); - expect(await bindings.kv.get("spend.warned:2026-10")).toBe("77000000"); + expect(sent[0]).toContain("2026-10: FlareDispatch spend reached $44.00 of the $50.00 monthly budget"); + expect(await bindings.kv.get("spend.warned:2026-10")).toBe("44000000"); }); it("sends no warning under the warn ratio", async () => { diff --git a/wrangler.jsonc b/wrangler.jsonc index 58da948b..8911bccb 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -88,7 +88,7 @@ // mutually exclusive). If the gateway has Authenticated Gateway turned on, // also set the AI_GATEWAY_AUTH_TOKEN secret. Operator overlay. "AI_GATEWAY_ID": "your-ai-gateway", - "ADMISSION_CAP": "16", + "ADMISSION_CAP": "48", // The agent, browser and release pools admit at most RunSandboxAgent's, // RunSandboxBrowser's and RunSandboxRelease's max_instances. Each cap is an // upper bound; SPEND_BURN_MAX_USD_PER_HOUR sets the operating concurrency. @@ -193,26 +193,26 @@ "image_build_context": ".", "image_vars": { "WITH_BUILD_TOOLS": "true", "WITH_RUST": "true" }, "instance_type": "standard-4", - "max_instances": 16, + "max_instances": 56, }, { "class_name": "RunSandboxBrowser", "image": "./infra/Dockerfile.sandbox", "instance_type": "standard-2", - "max_instances": 16, + "max_instances": 24, }, { "class_name": "RunSandboxAgent", "image": "./infra/Dockerfile.sandbox", "instance_type": "standard-3", - "max_instances": 8, + "max_instances": 16, }, { "class_name": "RunSandboxRelease", "image": "./infra/Dockerfile.sandbox-release", "image_build_context": ".", "instance_type": "standard-4", - "max_instances": 8, + "max_instances": 16, // Every deploy pushes a new image and rolls every container. A rollout // replaces an active instance once this grace has passed, and a // replaced release container loses its build (`SandboxProcessLost`):