ci(cli): add a full benchmark for integration-heavy commands

.github/scripts/cli-bench/full-plan.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, test } from "bun:test";
+import { planFullRun, readManifest, type PlanDeps, type PlanInput } from "./full-plan.ts";
+
+const REPO = "supabase/cli";
+const manifest = readManifest(
+  JSON.stringify({
+    workloads: [
+      { id: "start", runtime: "docker", estimatedSeconds: 400 },
+      { id: "db-reset", runtime: "docker", estimatedSeconds: 400 },
+      { id: "stack-start-native", runtime: "native", estimatedSeconds: 100 },
+    ],
+  }),
+);
+
+const input = (overrides: Partial<PlanInput> = {}): PlanInput => ({
+  event: "workflow_dispatch",
+  workloads: "all",
+  runtime: "both",
+  base: "merge-base",
+  refName: "feature/x",
+  repository: REPO,
+  manifest,
+  ...overrides,
+});
+
+const deps = (overrides: Partial<PlanDeps> = {}): PlanDeps => ({
+  latestRelease: async () => "v3.0.0",
+  openPullRequests: async () => [{ number: 12, headRepoFullName: REPO }],
+  ...overrides,
+});
+
+describe("planFullRun", () => {
+  test("measures everything against the latest stable release on a schedule, with no pull request", async () => {
+    const plan = await planFullRun(
+      input({ event: "schedule", workloads: "", runtime: "", base: "", refName: "develop" }),
+      deps(),
+    );
+
+    expect(plan).toMatchObject({
+      workloads: "all",
+      runtime: "both",
+      baseKind: "release",
+      baseTag: "v3.0.0",
+      prNumber: "",
+      runLightweight: true,
+      runIntegration: true,
+    });
+  });
+
+  test("attaches a merge-base run to the branch's single open pull request", async () => {
+    const plan = await planFullRun(input(), deps());
+
+    expect(plan).toMatchObject({ baseKind: "merge-base", baseTag: "", prNumber: "12" });
+  });
+
+  test.each([
+    ["no open pull request", [] as Array<{ number: number; headRepoFullName: string }>],
+    [
+      "two open pull requests",
+      [
+        { number: 1, headRepoFullName: REPO },
+        { number: 2, headRepoFullName: REPO },
+      ],
+    ],
+    ["a pull request from a fork", [{ number: 3, headRepoFullName: "someone/cli" }]],
+  ])("attaches no pull request when the branch has %s", async (_name, pulls) => {
+    const plan = await planFullRun(input(), deps({ openPullRequests: async () => pulls }));
+
+    expect(plan.prNumber).toBe("");
+  });
+
+  test("leaves the pull request comment alone for a run against a release tag", async () => {
+    const plan = await planFullRun(input({ base: "v3.0.0" }), deps());
+
+    expect(plan).toMatchObject({ baseKind: "release", baseTag: "v3.0.0", prNumber: "" });
+  });
+
+  test.each(["main", "v3", "v3.0.0; rm -rf /", "$(id)", "../v3.0.0"])(
+    "rejects the base %p",
+    async (base) => {
+      await expect(planFullRun(input({ base }), deps())).rejects.toThrow("Unsupported base");
+    },
+  );
+
+  test.each([
+    ["lightweight", { runLightweight: true, runIntegration: false, shards: 0 }],
+    ["integration-heavy", { runLightweight: false, runIntegration: true, shards: 3 }],
+    ["all", { runLightweight: true, runIntegration: true, shards: 3 }],
+  ])("plans shards for workloads=%s", async (workloads, expected) => {
+    const plan = await planFullRun(input({ workloads }), deps());
+
+    expect(plan.runLightweight).toBe(expected.runLightweight);
+    expect(plan.runIntegration).toBe(expected.runIntegration);
+    expect(plan.matrix.include).toHaveLength(expected.shards);
+  });
+
+  test("keeps only the requested runtime's shards", async () => {
+    const plan = await planFullRun(input({ runtime: "native" }), deps());
+
+    expect(plan.matrix.include).toEqual([
+      expect.objectContaining({ runtime: "native", workloads: "stack-start-native" }),
+    ]);
+  });
+
+  test("rejects an unknown runtime or workloads choice", async () => {
+    await expect(planFullRun(input({ runtime: "podman" }), deps())).rejects.toThrow("runtime");
+    await expect(planFullRun(input({ workloads: "none" }), deps())).rejects.toThrow("workloads");
+  });
+});

.github/scripts/cli-bench/full-plan.ts

@@ -0,0 +1,194 @@
+/**
+ * Planner for the full benchmark workflow: resolves what a run measures and against which base,
+ * attaches the run to its pull request, and splits the workloads into shards.
+ *
+ * A scheduled run measures everything against the latest stable release. A dispatched run uses its
+ * inputs; its base is the merge base with `develop` unless a release tag is given. Only a
+ * merge-base run on a branch with exactly one open pull request into `develop` from this
+ * repository updates that pull request's comment. Needs read access only.
+ * Run in CI as: `bun .github/scripts/cli-bench/full-plan.ts`.
+ */
+import { appendFileSync, readFileSync } from "node:fs";
+
+import { packShards, type PackedWorkload } from "../../../apps/cli/scripts/bench-pack.ts";
+import { githubFetch, requireEnv } from "./github.ts";
+
+export const WORKLOAD_CHOICES = ["all", "lightweight", "integration-heavy"] as const;
+export const RUNTIME_CHOICES = ["both", "docker", "native"] as const;
+export type WorkloadChoice = (typeof WORKLOAD_CHOICES)[number];
+export type RuntimeChoice = (typeof RUNTIME_CHOICES)[number];
+
+const RELEASE_TAG = /^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u;
+const MANIFEST = "apps/cli/bench/workloads.full.json";
+
+export interface PlanInput {
+  event: string;
+  workloads: string;
+  runtime: string;
+  base: string;
+  refName: string;
+  repository: string;
+  manifest: ReadonlyArray<PackedWorkload>;
+}
+
+export interface PullRequestRef {
+  number: number;
+  headRepoFullName: string;
+}
+
+export interface PlanDeps {
+  /** Tag of the latest stable release. */
+  latestRelease: () => Promise<string>;
+  /** Open pull requests into `develop` whose head is `refName`. */
+  openPullRequests: (refName: string) => Promise<ReadonlyArray<PullRequestRef>>;
+}
+
+export interface FullPlan {
+  workloads: WorkloadChoice;
+  runtime: RuntimeChoice;
+  baseKind: "merge-base" | "release";
+  /** The release tag compared against; empty for a merge-base run. */
+  baseTag: string;
+  /** Pull request whose comment this run updates; empty when none. */
+  prNumber: string;
+  runLightweight: boolean;
+  runIntegration: boolean;
+  matrix: { include: ReadonlyArray<Record<string, unknown>> };
+}
+
+const isChoice = <T extends string>(choices: ReadonlyArray<T>, value: string): value is T =>
+  (choices as ReadonlyArray<string>).includes(value);
+
+/** What a run measures, against what, and which shards run it. Throws on invalid inputs. */
+export async function planFullRun(input: PlanInput, deps: PlanDeps): Promise<FullPlan> {
+  const scheduled = input.event === "schedule";
+  const workloads = scheduled ? "all" : input.workloads;
+  const runtime = scheduled ? "both" : input.runtime;
+  if (!isChoice(WORKLOAD_CHOICES, workloads)) {
+    throw new Error(`Unsupported workloads "${workloads}".`);
+  }
+  if (!isChoice(RUNTIME_CHOICES, runtime)) throw new Error(`Unsupported runtime "${runtime}".`);
+
+  let baseKind: FullPlan["baseKind"] = "merge-base";
+  let baseTag = "";
+  if (scheduled) {
+    baseKind = "release";
+    baseTag = await deps.latestRelease();
+  } else if (input.base !== "merge-base") {
+    baseKind = "release";
+    baseTag = input.base;
+  }
+  if (baseKind === "release" && !RELEASE_TAG.test(baseTag)) {
+    throw new Error(
+      `Unsupported base "${baseTag}"; use merge-base or a release tag such as v3.0.0.`,
+    );
+  }
+
+  let prNumber = "";
+  if (!scheduled && baseKind === "merge-base") {
+    const candidates = (await deps.openPullRequests(input.refName)).filter(
+      (pr) => pr.headRepoFullName === input.repository,
+    );
+    if (candidates.length === 1) prNumber = String(candidates[0]?.number);
+  }
+
+  const runIntegration = workloads === "all" || workloads === "integration-heavy";
+  const wanted = (shardRuntime: string) => runtime === "both" || runtime === shardRuntime;
+  const include = runIntegration
+    ? packShards(input.manifest)
+        .filter((shard) => wanted(shard.runtime))
+        .map((shard) => ({ ...shard, workloads: shard.workloads.join(",") }))
+    : [];
+  return {
+    workloads,
+    runtime,
+    baseKind,
+    baseTag,
+    prNumber,
+    runLightweight: workloads === "all" || workloads === "lightweight",
+    runIntegration,
+    matrix: { include },
+  };
+}
+
+/** Reads the `id`, `runtime` and `estimatedSeconds` fields of a workload manifest. */
+export function readManifest(text: string): ReadonlyArray<PackedWorkload> {
+  const parsed: unknown = JSON.parse(text);
+  const list = (parsed as { workloads?: unknown }).workloads;
+  if (!Array.isArray(list)) throw new Error("The workload manifest has no workloads list.");
+  return list.map((entry: Record<string, unknown>) => ({
+    id: String(entry.id),
+    ...(entry.runtime === "docker" || entry.runtime === "native" ? { runtime: entry.runtime } : {}),
+    ...(typeof entry.estimatedSeconds === "number"
+      ? { estimatedSeconds: entry.estimatedSeconds }
+      : {}),
+  }));
+}
+
+async function main(): Promise<void> {
+  const token = requireEnv("GITHUB_TOKEN");
+  const repository = requireEnv("GITHUB_REPOSITORY");
+  const [owner = ""] = repository.split("/");
+  const base = `https://api.github.com/repos/${repository}`;
+
+  const plan = await planFullRun(
+    {
+      event: requireEnv("EVENT_NAME"),
+      workloads: process.env.INPUT_WORKLOADS ?? "all",
+      runtime: process.env.INPUT_RUNTIME ?? "both",
+      base: process.env.INPUT_BASE ?? "merge-base",
+      refName: requireEnv("REF_NAME"),
+      repository,
+      manifest: readManifest(readFileSync(MANIFEST, "utf8")),
+    },
+    {
+      latestRelease: async () => {
+        const latest = (await (await githubFetch(`${base}/releases/latest`, token)).json()) as {
+          tag_name: string;
+        };
+        return latest.tag_name;
+      },
+      openPullRequests: async (refName) => {
+        const url =
+          `${base}/pulls?state=open&base=develop&per_page=100` +
+          `&head=${encodeURIComponent(`${owner}:${refName}`)}`;
+        const pulls = (await (await githubFetch(url, token)).json()) as Array<{
+          number: number;
+          head: { repo: { full_name: string } | null };
+        }>;
+        return pulls.map((pr) => ({
+          number: pr.number,
+          headRepoFullName: pr.head.repo?.full_name ?? "",
+        }));
+      },
+    },
+  );
+
+  const outputs: Record<string, string> = {
+    base_kind: plan.baseKind,
+    base_tag: plan.baseTag,
+    pr_number: plan.prNumber,
+    run_lightweight: String(plan.runLightweight),
+    run_integration: String(plan.runIntegration),
+    has_shards: String(plan.matrix.include.length > 0),
+    matrix: JSON.stringify(plan.matrix),
+  };
+  console.log(
+    `CLI full benchmark plan: workloads=${plan.workloads} runtime=${plan.runtime} ` +
+      `base=${plan.baseKind === "release" ? plan.baseTag : "merge-base"} pr=${plan.prNumber || "none"} ` +
+      `shards=${plan.matrix.include.length}`,
+  );
+  appendFileSync(
+    requireEnv("GITHUB_OUTPUT"),
+    Object.entries(outputs)
+      .map(([key, value]) => `${key}=${value}\n`)
+      .join(""),
+  );
+}
+
+if (import.meta.main) {
+  main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}

.github/scripts/cli-bench/history.test.ts

@@ -19,11 +19,25 @@ const entry = (n: number, overrides: Partial<RunEntry> = {}): RunEntry => ({
 
 describe("headlineOf", () => {
   test("reads the title of the rendered report", () => {
+    expect(
+      headlineOf(
+        "<!-- cli-bench:pr -->\n\n## CLI benchmark (Lightweight Commands): 2 things to check\n\nbody",
+      ),
+    ).toBe("2 things to check");
+  });
+
+  test("reads the title of a lightweight report under the previous heading", () => {
     expect(headlineOf("<!-- cli-bench:pr -->\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe(
       "2 things to check",
     );
   });
 
+  test("reads the title of the integration-heavy report", () => {
+    expect(headlineOf("## CLI benchmark (Integration Heavy Commands): 1 thing to check")).toBe(
+      "1 thing to check",
+    );
+  });
+
   test("falls back when the report has no title", () => {
     expect(headlineOf("nothing here")).toBe("unknown");
   });
@@ -92,6 +106,13 @@ describe("renderHistory and decodeHistory", () => {
     expect(rendered).not.toContain("0000003` ·");
   });
 
+  test("lists every run, the newest included, while a run is in progress", () => {
+    const rendered = renderHistory([entry(3), entry(2)], (t) => t, true);
+
+    expect(rendered).toContain("`0000003` ·");
+    expect(rendered).toContain("`0000002` ·");
+  });
+
   test("shows no list for a first run", () => {
     expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs");
   });

.github/scripts/cli-bench/history.ts

@@ -1,5 +1,5 @@
 /**
- * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier
+ * Run history kept inside one benchmark comment. The newest run is the comment itself; earlier
  * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on.
  * The marker is read back only from the workflow bot's own comment, and anything that does not
  * decode cleanly counts as no history.
@@ -39,9 +39,9 @@ const isEntry = (value: unknown): value is RunEntry => {
   );
 };
 
-/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */
+/** The title the renderer puts after the `## CLI benchmark` heading, flattened to one bounded line. */
 export function headlineOf(markdown: string): string {
-  const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown";
+  const title = /^## CLI benchmark(?: \([^)\n]*\))?: (.+)$/mu.exec(markdown)?.[1] ?? "unknown";
   return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS);
 }
 
@@ -87,14 +87,16 @@ const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`;
 
 /**
  * The hidden marker for `entries` (current run first), preceded by a collapsed list of the
- * earlier ones when there are any. `clean` makes a headline safe to render.
+ * earlier ones when there are any. While a run is in progress the comment has no current result,
+ * so every entry is listed. `clean` makes a headline safe to render.
  */
 export function renderHistory(
   entries: ReadonlyArray<RunEntry>,
   clean: (text: string) => string,
+  running = false,
 ): string {
   const marker = `<!-- cli-bench:history ${Buffer.from(JSON.stringify(entries)).toString("base64")} -->`;
-  const earlier = entries.slice(1);
+  const earlier = running ? entries : entries.slice(1);
   if (earlier.length === 0) return marker;
   const lines = earlier.map(
     (entry) =>