test(cli): temporary stack benchmark dry run (do not merge)

.github/scripts/cli-bench/github.ts

@@ -67,3 +67,22 @@ export async function listIssueComments(
     if (batch.length < 100) return comments;
   }
 }
+
+/** One comment by id, or undefined when it no longer exists. */
+export async function getIssueComment(
+  token: string,
+  base: string,
+  id: number,
+): Promise<IssueComment | undefined> {
+  try {
+    const entry = (await (await githubFetch(`${base}/issues/comments/${id}`, token)).json()) as {
+      id: number;
+      body: string | null;
+      user: { login: string } | null;
+    };
+    return { id: entry.id, body: entry.body ?? "", authorLogin: entry.user?.login ?? "" };
+  } catch (error) {
+    if (error instanceof GithubRequestError && error.status === 404) return undefined;
+    throw error;
+  }
+}

.github/scripts/cli-bench/history.test.ts

@@ -5,23 +5,33 @@ import {
   makeEntry,
   MAX_PREVIOUS_RUNS,
   mergeHistory,
+  previousEntries,
   renderHistory,
   type RunEntry,
 } from "./history.ts";
 
 const entry = (n: number, overrides: Partial<RunEntry> = {}): RunEntry => ({
+  section: "lightweight",
   sha: n.toString(16).padStart(7, "0"),
   at: "2026-10-28T09:05Z",
   headline: "no changes found",
   url: `https://github.com/o/r/actions/runs/${n}`,
   ...overrides,
 });
+const heavy = (n: number, overrides: Partial<RunEntry> = {}) =>
+  entry(n, { section: "integration-heavy", ...overrides });
 
 describe("headlineOf", () => {
-  test("reads the title of the rendered report", () => {
-    expect(headlineOf("<!-- cli-bench:pr -->\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe(
-      "2 things to check",
-    );
+  test.each([
+    ["a comment section", "### Lightweight Commands: 2 things to check\n\nbody"],
+    ["an integration-heavy section", "### Integration Heavy Commands: 2 things to check\n\nbody"],
+    ["a job summary", "## CLI benchmark: 2 things to check\n\nbody"],
+    [
+      "an integration-heavy job summary",
+      "## CLI benchmark (Integration Heavy Commands): 2 things to check",
+    ],
+  ])("reads the title of %s", (_name, markdown) => {
+    expect(headlineOf(markdown)).toBe("2 things to check");
   });
 
   test("falls back when the report has no title", () => {
@@ -32,17 +42,23 @@ describe("headlineOf", () => {
 describe("makeEntry", () => {
   test("shortens the commit and rounds the time down to the minute in UTC", () => {
     const made = makeEntry({
+      section: "integration-heavy",
       sha: "0123456789abcdef0123456789abcdef01234567",
       now: new Date("2026-10-28T09:05:59Z"),
       headline: "no changes found",
       url: "https://github.com/o/r/actions/runs/9/attempts/2",
     });
 
-    expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" });
+    expect(made).toMatchObject({
+      section: "integration-heavy",
+      sha: "0123456",
+      at: "2026-10-28T09:05Z",
+    });
   });
 
   test("refuses a run link that is not a workflow run", () => {
     const made = makeEntry({
+      section: "lightweight",
       sha: "abcdef1",
       now: new Date(),
       headline: "h",
@@ -61,39 +77,78 @@ describe("mergeHistory", () => {
     ]);
   });
 
-  test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => {
-    const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1));
+  test("keeps each section's newest run plus at most 20 older ones across both sections", () => {
+    const previous = [
+      ...Array.from({ length: 15 }, (_, i) => entry(100 - i)),
+      heavy(50),
+      ...Array.from({ length: 15 }, (_, i) => heavy(40 - i)),
+    ];
+
+    const merged = mergeHistory(previous, entry(200));
+
+    const older = merged.length - 2;
+    expect(older).toBe(MAX_PREVIOUS_RUNS);
+    expect(merged[0]).toEqual(entry(200));
+    expect(merged.filter((e) => e.section === "integration-heavy")[0]).toEqual(heavy(50));
+  });
+
+  test("keeps a section's newest run even when the other section has filled the list", () => {
+    const previous = [...Array.from({ length: 25 }, (_, i) => entry(100 - i)), heavy(5)];
+
+    const merged = mergeHistory(previous, entry(200));
+
+    expect(merged.at(-1)).toEqual(heavy(5));
+  });
+});
+
+describe("previousEntries", () => {
+  test("leaves out each section's newest entry, which is its current result", () => {
+    const entries = [entry(4), heavy(3), entry(2), heavy(1)];
+
+    expect(previousEntries(entries)).toEqual([entry(2), heavy(1)]);
+  });
 
-    const merged = mergeHistory(previous, entry(100));
+  test("lists every entry of a section that is running, since it has no current result", () => {
+    const entries = [entry(4), heavy(3), entry(2), heavy(1)];
 
-    expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1);
-    expect(merged[0]).toEqual(entry(100));
-    expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS));
+    expect(previousEntries(entries, new Set(["integration-heavy"]))).toEqual([
+      heavy(3),
+      entry(2),
+      heavy(1),
+    ]);
   });
 });
 
 describe("renderHistory and decodeHistory", () => {
+  test("reads an entry written before sections existed as a lightweight run", () => {
+    const legacy = { ...entry(1), section: undefined };
+    const body = `<!-- cli-bench:history ${Buffer.from(JSON.stringify([legacy])).toString("base64")} -->`;
+
+    expect(decodeHistory(body)).toEqual([entry(1)]);
+  });
+
   test("round-trips the entries through the hidden marker", () => {
-    const entries = [entry(3), entry(2), entry(1)];
+    const entries = [entry(3), heavy(2), entry(1)];
 
     expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries);
   });
 
-  test("lists earlier runs newest first and leaves the current run out of the list", () => {
+  test("names the section of each replaced run and leaves current results out of the list", () => {
     const rendered = renderHistory(
-      [entry(3), entry(2, { headline: "1 thing to check" })],
+      [entry(4), heavy(3), heavy(2, { headline: "1 thing to check" })],
       (t) => t,
     );
 
     expect(rendered).toContain("<summary>Previous runs</summary>");
     expect(rendered).toContain(
-      "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)",
+      "- Integration Heavy Commands · `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)",
     );
     expect(rendered).not.toContain("0000003` ·");
+    expect(rendered).not.toContain("0000004` ·");
   });
 
-  test("shows no list for a first run", () => {
-    expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs");
+  test("shows no list while every section has a single run", () => {
+    expect(renderHistory([entry(1), heavy(2)], (t) => t)).not.toContain("Previous runs");
   });
 
   test.each([
@@ -106,7 +161,7 @@ describe("renderHistory and decodeHistory", () => {
     ],
     [
       "too many entries",
-      `<!-- cli-bench:history ${Buffer.from(JSON.stringify(Array.from({ length: 22 }, (_, i) => entry(i + 1)))).toString("base64")} -->`,
+      `<!-- cli-bench:history ${Buffer.from(JSON.stringify(Array.from({ length: 23 }, (_, i) => entry(i + 1)))).toString("base64")} -->`,
     ],
   ])("treats %s as empty history", (_name, body) => {
     expect(decodeHistory(body)).toEqual([]);

.github/scripts/cli-bench/history.ts

@@ -1,11 +1,19 @@
 /**
- * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier
- * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on.
- * The marker is read back only from the workflow bot's own comment, and anything that does not
- * decode cleanly counts as no history.
+ * Run history kept inside the benchmark comment. Each section's newest result is the section
+ * itself; the results it replaced sit in a collapsed list, backed by a hidden marker so the next
+ * update can carry them on. The marker is read back only from the workflow bot's own comment, and
+ * anything that does not decode cleanly counts as no history.
  */
 
-/** Earlier runs kept below the current one. */
+/** Parts of the comment, each written by its own run. */
+export type Section = "lightweight" | "integration-heavy";
+export const SECTIONS: ReadonlyArray<Section> = ["lightweight", "integration-heavy"];
+export const SECTION_NAMES: Record<Section, string> = {
+  lightweight: "Lightweight Commands",
+  "integration-heavy": "Integration Heavy Commands",
+};
+
+/** Replaced results kept below the current ones. */
 export const MAX_PREVIOUS_RUNS = 20;
 const HISTORY_MARKER = /<!-- cli-bench:history ([A-Za-z0-9+/=]+) -->/u;
 const MAX_ENCODED_CHARS = 30_000;
@@ -15,6 +23,7 @@ const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u;
 const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u;
 
 export interface RunEntry {
+  section: Section;
   /** Short commit SHA. */
   sha: string;
   /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */
@@ -23,10 +32,14 @@ export interface RunEntry {
   url: string;
 }
 
+const isSection = (value: unknown): value is Section =>
+  typeof value === "string" && (SECTIONS as ReadonlyArray<string>).includes(value);
+
 const isEntry = (value: unknown): value is RunEntry => {
   if (typeof value !== "object" || value === null) return false;
-  const { sha, at, headline, url } = value as Record<string, unknown>;
+  const { section, sha, at, headline, url } = value as Record<string, unknown>;
   return (
+    isSection(section) &&
     typeof sha === "string" &&
     SHA.test(sha) &&
     typeof at === "string" &&
@@ -39,20 +52,25 @@ const isEntry = (value: unknown): value is RunEntry => {
   );
 };
 
-/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */
+/** The title a section heading carries after its name, flattened to one bounded line. */
 export function headlineOf(markdown: string): string {
-  const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown";
+  const title =
+    /^#{2,3} (?:CLI benchmark(?: \([^)\n]*\))?|Lightweight Commands|Integration Heavy Commands): (.+)$/mu.exec(
+      markdown,
+    )?.[1] ?? "unknown";
   return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS);
 }
 
 /** An entry for a run, or undefined when the commit or run link is not in the expected shape. */
 export function makeEntry(input: {
+  section: Section;
   sha: string;
   now: Date;
   headline: string;
   url: string;
 }): RunEntry | undefined {
   const entry = {
+    section: input.section,
     sha: input.sha.slice(0, 7),
     at: `${input.now.toISOString().slice(0, 16)}Z`,
     headline: input.headline,
@@ -67,38 +85,77 @@ export function decodeHistory(body: string): Array<RunEntry> {
   if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return [];
   try {
     const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8"));
-    return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry)
-      ? value
-      : [];
+    if (!Array.isArray(value) || value.length > MAX_PREVIOUS_RUNS + SECTIONS.length) return [];
+    // Entries written before comments had sections belong to the lightweight run.
+    const entries = value.map((entry: unknown) =>
+      typeof entry === "object" && entry !== null && !("section" in entry)
+        ? { ...entry, section: "lightweight" }
+        : entry,
+    );
+    return entries.every(isEntry) ? entries : [];
   } catch {
     return [];
   }
 }
 
-/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */
+/**
+ * The entries a comment shows as previous runs: everything but each section's newest entry,
+ * which is the section's current result. A running section has no current result, so all of its
+ * entries are previous.
+ */
+export function previousEntries(
+  entries: ReadonlyArray<RunEntry>,
+  running: ReadonlySet<Section> = new Set(),
+): Array<RunEntry> {
+  const seen = new Set<Section>();
+  return entries.filter((entry) => {
+    if (running.has(entry.section)) return true;
+    if (!seen.has(entry.section)) {
+      seen.add(entry.section);
+      return false;
+    }
+    return true;
+  });
+}
+
+/**
+ * The new result first, then the earlier entries, keeping every section's newest entry and at
+ * most {@link MAX_PREVIOUS_RUNS} older ones across both sections.
+ */
 export function mergeHistory(
   previous: ReadonlyArray<RunEntry>,
   current: RunEntry,
 ): Array<RunEntry> {
-  return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1);
+  const all = [current, ...previous];
+  const seen = new Set<Section>();
+  let older = 0;
+  return all.filter((entry) => {
+    if (!seen.has(entry.section)) {
+      seen.add(entry.section);
+      return true;
+    }
+    older += 1;
+    return older <= MAX_PREVIOUS_RUNS;
+  });
 }
 
 const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`;
 
 /**
- * The hidden marker for `entries` (current run first), preceded by a collapsed list of the
- * earlier ones when there are any. `clean` makes a headline safe to render.
+ * The hidden marker for `entries` (newest first), preceded by a collapsed list of the previous
+ * ones when there are any. `clean` makes a headline safe to render.
  */
 export function renderHistory(
   entries: ReadonlyArray<RunEntry>,
   clean: (text: string) => string,
+  running: ReadonlySet<Section> = new Set(),
 ): string {
   const marker = `<!-- cli-bench:history ${Buffer.from(JSON.stringify(entries)).toString("base64")} -->`;
-  const earlier = entries.slice(1);
-  if (earlier.length === 0) return marker;
-  const lines = earlier.map(
+  const previous = previousEntries(entries, running);
+  if (previous.length === 0) return marker;
+  const lines = previous.map(
     (entry) =>
-      `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`,
+      `- ${SECTION_NAMES[entry.section]} · \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`,
   );
   return ["<details><summary>Previous runs</summary>", "", ...lines, "", "</details>", marker].join(
     "\n",