diff --git a/apps/dispatcher/src/run-end.test.ts b/apps/dispatcher/src/run-end.test.ts index 2c3d882..247bd86 100644 --- a/apps/dispatcher/src/run-end.test.ts +++ b/apps/dispatcher/src/run-end.test.ts @@ -9,7 +9,7 @@ const EXPIRES = 1_800_003_600; const ports = () => { const kill = vi.fn(async () => {}); - const end = vi.fn(async (_: { executionId: string; expiresAt: number }) => {}); + const end = vi.fn<(input: { executionId: string; expiresAt: number }) => Promise>(async () => {}); return { kill, end, diff --git a/runs/contextful-cache-warm.test.ts b/runs/contextful-cache-warm.test.ts index 667ce8a..1cbb9b6 100644 --- a/runs/contextful-cache-warm.test.ts +++ b/runs/contextful-cache-warm.test.ts @@ -3,7 +3,7 @@ import { Cause, Effect, Exit } from "effect"; import { describe, expect } from "vitest"; import { makeCFRuntimeTest } from "@fractalboxdev/flare-dispatch-core/testing"; import { compileCacheCommand } from "@fractalboxdev/flare-dispatch-core/primitives"; -import { contextfulCacheWarm, WARM_PART } from "./contextful-cache-warm"; +import { contextfulCacheWarm, WARM_CHILDREN, WARM_PART } from "./contextful-cache-warm"; import { lightLabelsKey, stageCommand } from "./contextful-gate"; const REPO = "fractalboxdev/contextful"; @@ -29,6 +29,17 @@ const keyed = { }; const cacheConfig = { "compile-cache.endpoint": ENDPOINT, "compile-cache.token": TOKEN }; const head = { branchHeads: { [`${REPO}:main`]: SHA } }; +/** The compile groups a tree with `stages --parts --compile-groups` lists, per tier. */ +const PR_GROUPS = ["workspace.compile", "workspace.cli-1", "workspace.cli-formal", "workspace.foundation", "evaluate.cli"]; +const NIGHTLY_GROUPS = ["features.binary-all"]; +const FOLDS = "suite.cli workspace.cli-formal evaluate.cli\nsuite.foundation workspace.foundation evaluate.foundation\n"; +const listing = { + "--compile-groups --tier pr": { stdout: `${PR_GROUPS.join("\n")}\n`, exitCode: 0 }, + "--compile-groups --tier nightly": { stdout: `${NIGHTLY_GROUPS.join("\n")}\n`, exitCode: 0 }, + "stages --folds": { stdout: FOLDS, exitCode: 0 }, +}; +/** A part's compile-only command, as a gate part runs it under the cache wrapper. */ +const warmed = (part: string) => compileCacheCommand(stageCommand(part, BEFORE, prebuilt, { compileOnly: true }), REPO); const push = { ref: "refs/heads/main", before: BEFORE, @@ -66,10 +77,14 @@ describe("contextful-cache-warm", () => { expect(contextfulCacheWarm.supersede?.(unranked)).toEqual({ group: `contextful-cache-warm:${REPO}:main`, order: 0 }); }); - it.effect("compiles the gate's compile part exactly as a gate part runs it, in one container", () => { - const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: keyed, config: cacheConfig, github: head }); + it.effect("falls back to the gate's compile part, exactly as a gate part runs it, when the tree lists no compile groups", () => { + const { layer, handles } = makeCFRuntimeTest({ + sandboxProgram: { ...keyed, "--compile-groups": { exitCode: 2, stderr: "error: unexpected argument '--compile-groups' found\n\nUsage: contextful-ci stages" } }, + config: cacheConfig, + github: head, + }); return Effect.gen(function* () { - expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 0 }); + expect(yield* contextfulCacheWarm.run(input)).toEqual({ parts: [{ part: WARM_PART, exitCode: 0 }], unstarted: [] }); expect(WARM_PART).toBe("workspace.compile"); const command = compileCacheCommand(stageCommand(WARM_PART, BEFORE, prebuilt), REPO); const warm = handles.sandbox.execs.filter((exec) => exec.command === command); @@ -88,7 +103,82 @@ describe("contextful-cache-warm", () => { github: head, }); return Effect.gen(function* () { - expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 101 }); + expect(yield* contextfulCacheWarm.run(input)).toEqual({ parts: [{ part: WARM_PART, exitCode: 101 }], unstarted: [] }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("spreads the compile groups over at most WARM_CHILDREN children of its own run and joins them", () => { + const { layer, handles } = makeCFRuntimeTest({ + sandboxProgram: { ...keyed, ...listing }, + config: cacheConfig, + github: head, + childRuns: { statuses: Object.fromEntries([0, 1, 2, 3].map((i) => [`contextful-cache-warm:${i}`, i === 2 ? "failure" : "success"])) }, + }); + return Effect.gen(function* () { + const groups = [...PR_GROUPS, ...NIGHTLY_GROUPS]; + const out = yield* contextfulCacheWarm.run(input); + expect(WARM_CHILDREN).toBe(4); + const spawned = handles.childRuns.spawned; + expect(spawned).toHaveLength(WARM_CHILDREN); + expect(spawned.every((child) => child.run === "contextful-cache-warm")).toBe(true); + const chunks = spawned.map((child) => (child.input as { groups: string[] }).groups); + expect(chunks.flat().sort()).toEqual([...groups].sort()); + expect(chunks[0]).toEqual([groups[0], groups[4]]); + expect(spawned.map((child) => child.input)).toEqual(chunks.map((parts) => ({ ...input, groups: parts }))); + expect(out).toEqual({ parts: [], unstarted: [], children: chunks.map((parts, i) => ({ parts, status: i === 2 ? "failure" : "success" })) }); + // The parent compiles nothing itself, and gives its container and slot to the children. + expect(handles.sandbox.execs.filter((exec) => exec.command.includes("--compile-only"))).toHaveLength(0); + expect(handles.sandbox.execs.filter((exec) => exec.command.includes("gate --predecessors"))).toHaveLength(0); + expect(handles.sandbox.destroyed).toHaveLength(1); + expect(handles.childRuns.admissionHandoffs).toBe(1); + // A child joins no supersession group: the parent's cancels the whole tree. + expect(contextfulCacheWarm.supersede?.({ ...input, groups: chunks[0]! })).toBeUndefined(); + expect(contextfulCacheWarm.compileCache?.({ ...input, groups: chunks[0]! })).toBe("base"); + expect(contextfulCacheWarm.joinsChildren).toBe(true); + }).pipe(Effect.provide(layer)); + }); + + it.effect("a child compiles each of its groups in order, compile-only, past a failing one", () => { + const { layer, handles } = makeCFRuntimeTest({ + sandboxProgram: { ...keyed, ...listing, "--stage workspace.cli-1 ": { exitCode: 101 } }, + config: cacheConfig, + github: head, + }); + return Effect.gen(function* () { + const groups = ["workspace.compile", "workspace.cli-1", "features.binary-all"]; + const out = yield* contextfulCacheWarm.run({ ...input, groups }); + expect(out).toEqual({ + parts: [ + { part: "workspace.compile", exitCode: 0 }, + { part: "workspace.cli-1", exitCode: 101 }, + { part: "features.binary-all", exitCode: 0 }, + ], + unstarted: [], + }); + const commands = handles.sandbox.execs.map((exec) => exec.command).filter((command) => command.includes("--compile-only")); + expect(commands).toEqual(groups.map(warmed)); + for (const exec of handles.sandbox.execs.filter((e) => e.command.includes("--compile-only"))) { + expect(exec.env).toEqual({ SCCACHE_WEBDAV_ENDPOINT: ENDPOINT, SCCACHE_WEBDAV_TOKEN: TOKEN }); + } + expect(handles.sandbox.execs.some((exec) => exec.command.includes("--compile-groups"))).toBe(false); + expect(handles.childRuns.spawned).toHaveLength(0); + expect(handles.sandbox.clones).toEqual([{ repo: REPO, sha: SHA }]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("leaves out the groups the two-tier gate's heavy tier compiles into base on this push", () => { + const { layer, handles } = makeCFRuntimeTest({ + sandboxProgram: { ...keyed, ...listing }, + config: { ...cacheConfig, [lightLabelsKey(REPO)]: "pins,suite.cli" }, + github: head, + childRuns: { statuses: Object.fromEntries([0, 1, 2].map((i) => [`contextful-cache-warm:${i}`, "success"])) }, + }); + return Effect.gen(function* () { + yield* contextfulCacheWarm.run(input); + const chunks = handles.childRuns.spawned.map((child) => (child.input as { groups: string[] }).groups); + // `suite.cli` is light: its members warm. The heavy pull-request parts do not; + // the nightly groups, which no heavy tier runs, do. + expect(chunks.flat().sort()).toEqual(["evaluate.cli", "features.binary-all", "workspace.cli-formal"]); }).pipe(Effect.provide(layer)); }); @@ -96,11 +186,12 @@ describe("contextful-cache-warm", () => { name: string, opts: Parameters[0], reason: string, + extra: { groups?: string[] } = {}, ) => it.effect(name, () => { const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: keyed, ...opts }); return Effect.gen(function* () { - const exit = yield* Effect.exit(contextfulCacheWarm.run(input)); + const exit = yield* Effect.exit(contextfulCacheWarm.run({ ...input, ...extra })); expect(Exit.isFailure(exit)).toBe(true); if (Exit.isFailure(exit)) { expect(Cause.pretty(exit.cause)).toContain("RunSkipped"); @@ -127,11 +218,25 @@ describe("contextful-cache-warm", () => { "compile cache not injected", ); skipsBeforeCheckout( - "skips when the two-tier gate's heavy tier already compiles the part on this push", - { config: { ...cacheConfig, [lightLabelsKey(REPO)]: "pins,formal" }, github: head }, - "heavy tier", + "a child skips a head the default branch has moved past", + { config: cacheConfig, github: { branchHeads: { [`${REPO}:main`]: "d".repeat(40) } } }, + "no longer the head of main", + { groups: ["workspace.compile"] }, ); + it.effect("skips when the two-tier gate's heavy tier already compiles every group on this push", () => { + const { layer, handles } = makeCFRuntimeTest({ + sandboxProgram: keyed, + config: { ...cacheConfig, [lightLabelsKey(REPO)]: "pins,formal" }, + github: head, + }); + return Effect.gen(function* () { + const exit = yield* Effect.exit(contextfulCacheWarm.run(input)); + expect(Exit.isFailure(exit) && Cause.pretty(exit.cause).includes("heavy tier")).toBe(true); + expect(handles.sandbox.execs.filter((exec) => exec.command.includes("gate --predecessors"))).toHaveLength(0); + }).pipe(Effect.provide(layer)); + }); + it.effect("warms when the two-tier gate lists the part as light", () => { const { layer } = makeCFRuntimeTest({ sandboxProgram: keyed, @@ -139,7 +244,7 @@ describe("contextful-cache-warm", () => { github: head, }); return Effect.gen(function* () { - expect(yield* contextfulCacheWarm.run(input)).toEqual({ part: WARM_PART, exitCode: 0 }); + expect(yield* contextfulCacheWarm.run(input)).toEqual({ parts: [{ part: WARM_PART, exitCode: 0 }], unstarted: [] }); }).pipe(Effect.provide(layer)); }); }); diff --git a/runs/contextful-cache-warm.ts b/runs/contextful-cache-warm.ts index bdda93d..994571d 100644 --- a/runs/contextful-cache-warm.ts +++ b/runs/contextful-cache-warm.ts @@ -2,17 +2,34 @@ // default-branch head, so a pull request's first gate reads compiled crates // instead of compiling cold. // -// One container per head: checkout, the gate's prebuilt `contextful-ci`, then -// the gate's compile part under sccache. The part runs through the same -// command, checkout path, install and cache wrapper as a gate part, because -// sccache keys an entry on the compile's arguments, working directory and -// `CARGO_*` environment: any difference turns every pull-request read into a -// miss. No child runs and no part check-runs. +// The head lists its compile groups (`contextful-ci stages --parts +// --compile-groups`): one gate part per distinct set of `cargo test` +// invocations. Each group compiles through `gate --stage +// --compile-only`, which runs the part's own cargo invocations with `--no-run`, +// under the same command, checkout path, install and cache wrapper as a gate +// part, because sccache keys an entry on the compile's arguments, working +// directory and `CARGO_*` environment: any difference turns every +// pull-request read into a miss. The groups spread over at most +// `WARM_CHILDREN` children of this run, each compiling its share in one +// container; a tree whose `contextful-ci` lists no groups compiles the gate's +// `workspace.compile` part in the parent's container. No part check-runs. // // Spec: specs/adr/0007-cost-efficient-execution.md § Base-lane warm. -import { Effect, Schema } from "effect"; -import { config, defineRun, github, io, RunSkipped, step } from "@fractalboxdev/flare-dispatch-core"; +import { Effect, Exit, Schema } from "effect"; +import { + ChildRuns, + config, + defineRun, + github, + handoffChildAdmission, + io, + RunSkipped, + sandbox, + spawnChildRun, + step, + type ChildStatusRecord, +} from "@fractalboxdev/flare-dispatch-core"; import { compileCacheCommand, compileCacheEnv, @@ -22,11 +39,22 @@ import { ensureWorkspace, prebuildCargoBinary, restorePrebuilt, + waitForChildrenDurably, workspace, type PrebuiltBinary, type Workspace, } from "@fractalboxdev/flare-dispatch-core/primitives"; -import { CI_PACKAGE, lightLabelsKey, parseLightLabels, STAGE_TIMEOUT_SEC, stageCommand } from "./contextful-gate"; +import { + CI_PACKAGE, + ciCommand, + lightLabelsKey, + parseLightLabels, + parseStages, + STAGE_TIMEOUT_SEC, + stageCommand, +} from "./contextful-gate"; + +const NAME = "contextful-cache-warm"; const REPO = "fractalboxdev/contextful"; const BRANCH = "main"; @@ -34,12 +62,49 @@ const SHA40 = /^[a-f0-9]{40}$/i; const ZERO_SHA = "0".repeat(40); /** - * The gate part this run compiles: `cargo test --workspace --no-run` in the - * workspace stage's target directory, the build every `workspace.*` part - * links against. + * The gate part a tree without compile groups warms: `cargo test --workspace + * --no-run` in the workspace stage's target directory, the build every + * `workspace.*` part links against. */ export const WARM_PART = "workspace.compile"; +/** Children the compile groups spread over, round-robin in run order. */ +export const WARM_CHILDREN = 4; +/** Queue age a run of this definition, parent or child, may wait before it starts. */ +const WARM_QUEUE_SEC = 90 * 60; +/** + * A child starts no group past this many seconds after it starts: with the last + * group's own `STAGE_TIMEOUT_SEC`, a child ends within ~2.75 h. + */ +const GROUP_START_CUTOFF_SEC = 2 * 3600; +/** The parent's join: a child's queue age, its start cutoff and its last group's deadline. */ +const CHILD_WAIT_SEC = WARM_QUEUE_SEC + GROUP_START_CUTOFF_SEC + STAGE_TIMEOUT_SEC + 600; + +/** `stages --folds` lines, `