feat(cache): warm the compile cache's base lane from every default-branch head

apps/dispatcher/src/grant-catalog.ts

@@ -144,6 +144,7 @@ export const RUN_GRANTS: Readonly<Record<string, RunGrant>> = {
   "offload-test": { profiles: POLYGLOT_BUILD, rollout: "legacy" },
   check: { profiles: POLYGLOT_BUILD, rollout: "legacy" },
   "contextful-gate": { profiles: POLYGLOT_BUILD, rollout: "legacy" },
+  "contextful-cache-warm": { profiles: POLYGLOT_BUILD, rollout: "legacy" },
   oxlint: { profiles: JS_BUILD, rollout: "legacy" },
   "vitest-shard": { profiles: JS_BUILD, rollout: "legacy" },
   // The parent of a fan-out does no work of its own beyond the checkout; the

apps/dispatcher/src/index.test.ts

@@ -108,6 +108,7 @@ describe("GET /health", () => {
         "cdp-acceptance",
         "check",
         "ci-triage-pr",
+        "contextful-cache-warm",
         "contextful-gate",
         "contextful-measures",
         "contextful-protocol",

apps/dispatcher/src/registry.ts

@@ -14,6 +14,7 @@ import {
   cdpAcceptance,
   check,
   ciTriagePr,
+  contextfulCacheWarm,
   contextfulGate,
   contextfulMeasures,
   contextfulProtocol,
@@ -75,6 +76,7 @@ const RUN_REGISTRY: Record<string, Run<unknown, unknown>> = {
   [oxlint.name]: oxlint as Run<unknown, unknown>,
   [check.name]: check as Run<unknown, unknown>,
   [contextfulGate.name]: contextfulGate as Run<unknown, unknown>,
+  [contextfulCacheWarm.name]: contextfulCacheWarm as Run<unknown, unknown>,
   [vitestShard.name]: vitestShard as Run<unknown, unknown>,
   [emailOtpLogin.name]: emailOtpLogin as Run<unknown, unknown>,
   [finopsAudit.name]: finopsAudit as Run<unknown, unknown>,

runs/contextful-cache-warm.test.ts

@@ -0,0 +1,145 @@
+import { it } from "@effect/vitest";
+import { Cause, Effect, Exit } from "effect";
+import { describe, expect } from "vitest";
+import { makeCFRuntimeTest } from "@fractalboxdev/flare-dispatch-core/testing";
+import { compileCacheCommand } from "@fractalboxdev/flare-dispatch-core/primitives";
+import { contextfulCacheWarm, WARM_PART } from "./contextful-cache-warm";
+import { lightLabelsKey, stageCommand } from "./contextful-gate";
+
+const REPO = "fractalboxdev/contextful";
+const SHA = "a".repeat(40);
+const BEFORE = "b".repeat(40);
+const DIGEST = "c".repeat(64);
+const ENDPOINT = "https://dispatch.example/v1/compile-cache";
+const TOKEN = "cc1.1700000000.ZXhlYw.mac";
+const PUSHED_AT = "2026-10-23T06:00:00Z";
+const input = { repo: REPO, sha: SHA, baseSha: BEFORE, revisedAt: Date.parse(PUSHED_AT) } as const;
+const prebuilt = {
+  key: `contextful-ci-${DIGEST}`,
+  base: "/tmp/flare-dispatch-prebuilt/contextful-ci",
+  paths: ["bin"],
+  path: "/tmp/flare-dispatch-prebuilt/contextful-ci/bin/contextful-ci",
+} as const;
+const keyed = {
+  "cargo tree": {
+    stdout: "contextful-ci v0.16.0 (/workspace/contextful/tools/ci)\n",
+    exitCode: 0,
+  },
+  "sha256sum /tmp/flare-dispatch-prebuilt-key.txt": { stdout: `${DIGEST}  /tmp/flare-dispatch-prebuilt-key.txt\n`, exitCode: 0 },
+};
+const cacheConfig = { "compile-cache.endpoint": ENDPOINT, "compile-cache.token": TOKEN };
+const head = { branchHeads: { [`${REPO}:main`]: SHA } };
+const push = {
+  ref: "refs/heads/main",
+  before: BEFORE,
+  after: SHA,
+  head_commit: { timestamp: PUSHED_AT },
+  repository: { full_name: REPO, default_branch: "main" },
+};
+
+describe("contextful-cache-warm", () => {
+  it("triggers once per default-branch head with no opt-in key", () => {
+    const trigger = contextfulCacheWarm.triggers?.[0];
+    expect(trigger?.event).toBe("push");
+    expect(trigger?.gate?.({ payload: push } as never)).toBe(true);
+    expect(trigger?.idempotencyKey({ payload: push } as never)).toBe(`contextful-cache-warm:${SHA}`);
+    expect(trigger?.inputs({ payload: push } as never)).toEqual(input);
+    for (const other of [
+      { ...push, ref: "refs/heads/feature" },
+      { ...push, ref: "refs/tags/v1.0.0" },
+      { ...push, after: "0".repeat(40) },
+      { ...push, repository: { full_name: "other/repo", default_branch: "main" } },
+      { ...push, ref: "refs/heads/trunk", repository: { full_name: REPO, default_branch: "trunk" } },
+    ]) {
+      expect(trigger?.gate?.({ payload: other } as never)).toBe(false);
+    }
+    expect(contextfulCacheWarm.optIn).toBeUndefined();
+  });
+
+  it("writes the default branch's lane, and a newer head supersedes an older warm", () => {
+    expect(contextfulCacheWarm.compileCache?.(input)).toBe("base");
+    expect(contextfulCacheWarm.supersede?.(input)).toEqual({
+      group: `contextful-cache-warm:${REPO}:main`,
+      order: Date.parse(PUSHED_AT),
+    });
+    const { revisedAt: _, ...unranked } = input;
+    expect(contextfulCacheWarm.supersede?.(unranked)).toEqual({ group: `contextful-cache-warm:${REPO}:main`, order: 0 });
+  });
+
+  it.effect("compiles the gate's compile part exactly as a gate part runs it, in one container", () => {
+    const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: keyed, config: cacheConfig, github: head });
+    return Effect.gen(function* () {
+      expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 0 });
+      expect(WARM_PART).toBe("workspace.compile");
+      const command = compileCacheCommand(stageCommand(WARM_PART, BEFORE, prebuilt), REPO);
+      const warm = handles.sandbox.execs.filter((exec) => exec.command === command);
+      expect(warm).toHaveLength(1);
+      expect(warm[0]?.env).toEqual({ SCCACHE_WEBDAV_ENDPOINT: ENDPOINT, SCCACHE_WEBDAV_TOKEN: TOKEN });
+      expect(handles.sandbox.execs.filter((exec) => exec.command.includes("cargo build"))).toHaveLength(1);
+      expect(handles.sandbox.clones).toEqual([{ repo: REPO, sha: SHA }]);
+      expect(handles.childRuns.spawned).toHaveLength(0);
+    }).pipe(Effect.provide(layer));
+  });
+
+  it.effect("reports a failed compile without failing: the gate gives the verdict", () => {
+    const { layer } = makeCFRuntimeTest({
+      sandboxProgram: { ...keyed, [`--stage ${WARM_PART}`]: { exitCode: 101 } },
+      config: cacheConfig,
+      github: head,
+    });
+    return Effect.gen(function* () {
+      expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 101 });
+    }).pipe(Effect.provide(layer));
+  });
+
+  const skipsBeforeCheckout = (
+    name: string,
+    opts: Parameters<typeof makeCFRuntimeTest>[0],
+    reason: string,
+  ) =>
+    it.effect(name, () => {
+      const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: keyed, ...opts });
+      return Effect.gen(function* () {
+        const exit = yield* Effect.exit(contextfulCacheWarm.run(input));
+        expect(Exit.isFailure(exit)).toBe(true);
+        if (Exit.isFailure(exit)) {
+          expect(Cause.pretty(exit.cause)).toContain("RunSkipped");
+          expect(Cause.pretty(exit.cause)).toContain(reason);
+        }
+        expect(handles.sandbox.clones).toEqual([]);
+        expect(handles.sandbox.execs).toEqual([]);
+      }).pipe(Effect.provide(layer));
+    });
+
+  skipsBeforeCheckout(
+    "skips a head the default branch has moved past",
+    { config: cacheConfig, github: { branchHeads: { [`${REPO}:main`]: "d".repeat(40) } } },
+    "no longer the head of main",
+  );
+  skipsBeforeCheckout(
+    "skips when the default branch's head cannot be read: only a default-branch head writes base",
+    { config: cacheConfig },
+    "could not be read",
+  );
+  skipsBeforeCheckout(
+    "skips when the deploy injects no compile-cache access",
+    { github: head },
+    "compile cache not injected",
+  );
+  skipsBeforeCheckout(
+    "skips when the two-tier gate's heavy tier already compiles the part on this push",
+    { config: { ...cacheConfig, [lightLabelsKey(REPO)]: "pins,formal" }, github: head },
+    "heavy tier",
+  );
+
+  it.effect("warms when the two-tier gate lists the part as light", () => {
+    const { layer } = makeCFRuntimeTest({
+      sandboxProgram: keyed,
+      config: { ...cacheConfig, [lightLabelsKey(REPO)]: `pins,${WARM_PART}` },
+      github: head,
+    });
+    return Effect.gen(function* () {
+      expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 0 });
+    }).pipe(Effect.provide(layer));
+  });
+});

runs/contextful-cache-warm.ts

@@ -0,0 +1,166 @@
+// `contextful-cache-warm` — fills the compile cache's `base` lane from each
+// default-branch head, so a pull request's first gate reads compiled crates
+// instead of compiling cold.
+//
+// One container per head: checkout, the gate's prebuilt `contextful-ci`, then
+// the gate's compile part under sccache. The part runs through the same
+// command, checkout path, install and cache wrapper as a gate part, because
+// sccache keys an entry on the compile's arguments, working directory and
+// `CARGO_*` environment: any difference turns every pull-request read into a
+// miss. No child runs and no part check-runs.
+//
+// Spec: specs/adr/0007-cost-efficient-execution.md § Base-lane warm.
+
+import { Effect, Schema } from "effect";
+import { config, defineRun, github, io, RunSkipped, step } from "@fractalboxdev/flare-dispatch-core";
+import {
+  compileCacheCommand,
+  compileCacheEnv,
+  COMPILE_CACHE_ENDPOINT_KEY,
+  COMPILE_CACHE_TOKEN_KEY,
+  durableExec,
+  ensureWorkspace,
+  prebuildCargoBinary,
+  restorePrebuilt,
+  workspace,
+  type PrebuiltBinary,
+  type Workspace,
+} from "@fractalboxdev/flare-dispatch-core/primitives";
+import { CI_PACKAGE, lightLabelsKey, parseLightLabels, STAGE_TIMEOUT_SEC, stageCommand } from "./contextful-gate";
+
+const REPO = "fractalboxdev/contextful";
+const BRANCH = "main";
+const SHA40 = /^[a-f0-9]{40}$/i;
+const ZERO_SHA = "0".repeat(40);
+
+/**
+ * The gate part this run compiles: `cargo test --workspace --no-run` in the
+ * workspace stage's target directory, the build every `workspace.*` part
+ * links against.
+ */
+export const WARM_PART = "workspace.compile";
+
+const Input = Schema.Struct({
+  repo: Schema.Literal(REPO),
+  sha: Schema.String.pipe(Schema.pattern(SHA40)),
+  /** The push's `before`: the `--base` a gate part passes. */
+  baseSha: Schema.String.pipe(Schema.pattern(SHA40)),
+  /** The pushed head commit's timestamp in ms: the rank within the supersession group. */
+  revisedAt: Schema.optional(Schema.Number.pipe(Schema.int(), Schema.nonNegative())),
+});
+
+type InputType = Schema.Schema.Type<typeof Input>;
+
+const Output = Schema.Struct({
+  part: Schema.String,
+  exitCode: Schema.Number,
+});
+
+export const contextfulCacheWarm = defineRun({
+  name: "contextful-cache-warm",
+  version: "1.0.0",
+  inputs: Input,
+  outputs: Output,
+  triggers: [
+    {
+      event: "push",
+      // A redelivered push, or a second push of the same head, collapses onto
+      // the execution that already warmed it.
+      idempotencyKey: ({ payload }) => `contextful-cache-warm:${String(payload.after ?? "")}`,
+      gate: ({ payload }) =>
+        payload.repository?.full_name === REPO &&
+        payload.repository?.default_branch === BRANCH &&
+        payload.ref === `refs/heads/${BRANCH}` &&
+        typeof payload.after === "string" && SHA40.test(payload.after) && payload.after !== ZERO_SHA &&
+        typeof payload.before === "string" && SHA40.test(payload.before) && payload.before !== ZERO_SHA,
+      inputs: ({ payload }): InputType => {
+        const revisedAt = Date.parse(String(payload.head_commit?.timestamp ?? ""));
+        return {
+          repo: REPO,
+          sha: String(payload.after),
+          baseSha: String(payload.before),
+          ...(Number.isInteger(revisedAt) && revisedAt >= 0 ? { revisedAt } : {}),
+        };
+      },
+    },
+  ],
+  // Only default-branch heads write `base`; the body refuses any other sha
+  // before the token reaches a container.
+  compileCache: () => "base",
+  // A newer head cancels the older warm, queued or running: its entries
+  // already written stay, and the newer head's compile reads them.
+  supersede: (input) => ({ group: `contextful-cache-warm:${input.repo}:${BRANCH}`, order: input.revisedAt ?? 0 }),
+  limits: { maxDurationSec: 5400, admissionMaxQueueAgeSec: 90 * 60 },
+  run: (input) =>
+    Effect.gen(function* () {
+      // A configured two-tier gate compiles its heavy parts on this push into
+      // `base` already; warm only when the compile part is light.
+      const lightLabels = yield* step("resolve-tiers", () =>
+        config.get(lightLabelsKey(input.repo)).pipe(Effect.map((raw) => parseLightLabels(raw) ?? null)),
+      );
+      if (lightLabels !== null && !lightLabels.includes(WARM_PART)) {
+        return yield* Effect.fail(
+          new RunSkipped({ reason: `the two-tier gate's heavy tier compiles ${WARM_PART} into base on this push` }),
+        );
+      }
+      const current = yield* step("read-head", () =>
+        github.branchHead({ repo: input.repo, branch: BRANCH }).pipe(
+          Effect.map((sha): string | null => sha),
+          Effect.orElseSucceed((): string | null => null),
+        ),
+      );
+      if (current === null) {
+        return yield* Effect.fail(
+          new RunSkipped({ reason: `the head of ${BRANCH} could not be read; only a default-branch head writes base` }),
+        );
+      }
+      if (current.toLowerCase() !== input.sha.toLowerCase()) {
+        return yield* Effect.fail(new RunSkipped({ reason: `${input.sha} is no longer the head of ${BRANCH}` }));
+      }
+      // Read INLINE, as `check` does: the token is a credential and stays out
+      // of checkpointed state.
+      const endpoint = yield* config.get(COMPILE_CACHE_ENDPOINT_KEY);
+      const token = yield* config.get(COMPILE_CACHE_TOKEN_KEY);
+      if (endpoint === undefined || endpoint === "" || token === undefined || token === "") {
+        return yield* Effect.fail(new RunSkipped({ reason: "compile cache not injected on this deploy" }));
+      }
+
+      const { container, dir } = yield* step("checkout", () =>
+        workspace({ repo: input.repo, sha: input.sha, install: true }),
+      );
+      const prebuilt = yield* step(
+        "prebuild-ci-binary",
+        () =>
+          ensureWorkspace({ current: { container, dir }, repo: input.repo, sha: input.sha, install: true }).pipe(
+            Effect.flatMap((ws) => prebuildCargoBinary({ workspace: ws, pkg: CI_PACKAGE, bin: CI_PACKAGE })),
+          ),
+        { timeoutSec: 1920, retries: 1, retryOn: ["CheckoutFailed"] },
+      ).pipe(Effect.orElseSucceed((): PrebuiltBinary | null => null));
+      // A re-cloned container lost the binary; restore it as a gate part does.
+      const restore = (ws: Workspace) =>
+        prebuilt === null
+          ? Effect.succeed(ws)
+          : restorePrebuilt({ container: ws.container, dir: ws.dir, entry: prebuilt }).pipe(Effect.as(ws));
+      const { result } = yield* durableExec({
+        name: "warm",
+        workspace: { container, dir },
+        ensure: (ws) =>
+          ensureWorkspace({ current: ws, repo: input.repo, sha: input.sha, install: true }).pipe(
+            Effect.flatMap((next) => (next === ws ? Effect.succeed(next) : restore(next))),
+          ),
+        exec: (ws) => ({
+          cwd: ws.dir,
+          command: compileCacheCommand(stageCommand(WARM_PART, input.baseSha, prebuilt), input.repo),
+          env: compileCacheEnv({ endpoint, token }),
+          redactValues: [token],
+          timeoutSec: STAGE_TIMEOUT_SEC,
+        }),
+      });
+      // The gate gives the verdict on this head; a failed compile still
+      // leaves every entry it wrote.
+      if (result.exitCode !== 0) {
+        yield* io.log("warn", `contextful-cache-warm: ${WARM_PART} exited ${result.exitCode}; base holds what it compiled`);
+      }
+      return { part: WARM_PART, exitCode: result.exitCode };
+    }),
+});

runs/contextful-gate.ts

@@ -30,7 +30,7 @@ import { check } from "./check";
 const REPO = "fractalboxdev/contextful";
 const SHA40 = /^[a-f0-9]{40}$/i;
 const ZERO_SHA = "0".repeat(40);
-const CI_PACKAGE = "contextful-ci";
+export const CI_PACKAGE = "contextful-ci";
 const CARGO_RUN_CI = `cargo run --locked -q -p ${CI_PACKAGE} --`;
 const DISCOVER = `${CARGO_RUN_CI} stages --parts`;
 const STAGE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,31}$/;
@@ -44,7 +44,7 @@ const checkoutCommand = (sha: string): string =>
  * takes 550–760 s, and its ordinary suite another ~1100 s, so a single part needs
  * up to ~1750 s; 2700 s keeps that at about 65% of the deadline.
  */
-const STAGE_TIMEOUT_SEC = 2700;
+export const STAGE_TIMEOUT_SEC = 2700;
 /** Admission queue age a part may wait before it starts. */
 const STAGE_QUEUE_SEC = 90 * 60;
 /** A spawned part's whole budget: its queue age plus the check run's ceiling. */

runs/index.ts

@@ -31,6 +31,7 @@ export { contextfulRelease, contextfulReleaseCell, contextfulReleaseFormula } fr
 export { oxlint } from "./oxlint";
 export { check } from "./check";
 export { contextfulGate } from "./contextful-gate";
+export { contextfulCacheWarm } from "./contextful-cache-warm";
 export { emailOtpLogin } from "./email-otp-login";
 export { finopsAudit } from "./finops-audit";
 export { contextfulMeasures } from "./contextful-measures";