From ad897e4fbfbb2de6add748555f845dda2e375c3c Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 15:33:51 +0530 Subject: [PATCH 01/11] ci(cli): add benchmark comment sections and the /bench full command The benchmark comment holds a Lightweight Commands and an Integration Heavy Commands section. Each writer replaces only its own section, verifies it landed, and retries when another writer overwrote it. Previous runs record the result each section replaced. /bench full dispatches the stack benchmark workflow on the pull request's branch and refreshes the lightweight section only when it is behind the head commit. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/cli-bench/github.ts | 19 + .github/scripts/cli-bench/history.test.ts | 88 +++- .github/scripts/cli-bench/history.ts | 88 +++- .../scripts/cli-bench/post-comment.test.ts | 345 +++++++++++++--- .github/scripts/cli-bench/post-comment.ts | 386 ++++++++++++++---- .github/scripts/cli-bench/rerun.test.ts | 199 ++++++++- .github/scripts/cli-bench/rerun.ts | 221 ++++++++-- .github/scripts/cli-bench/resolve.test.ts | 28 +- .github/scripts/cli-bench/resolve.ts | 9 +- .github/workflows/cli-bench.yml | 19 +- 10 files changed, 1183 insertions(+), 219 deletions(-) diff --git a/.github/scripts/cli-bench/github.ts b/.github/scripts/cli-bench/github.ts index 4859009a68..ddd69aaa3a 100644 --- a/.github/scripts/cli-bench/github.ts +++ b/.github/scripts/cli-bench/github.ts @@ -67,3 +67,22 @@ export async function listIssueComments( if (batch.length < 100) return comments; } } + +/** One comment by id, or undefined when it no longer exists. */ +export async function getIssueComment( + token: string, + base: string, + id: number, +): Promise { + try { + const entry = (await (await githubFetch(`${base}/issues/comments/${id}`, token)).json()) as { + id: number; + body: string | null; + user: { login: string } | null; + }; + return { id: entry.id, body: entry.body ?? "", authorLogin: entry.user?.login ?? "" }; + } catch (error) { + if (error instanceof GithubRequestError && error.status === 404) return undefined; + throw error; + } +} diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts index 0d096bce41..f302079806 100644 --- a/.github/scripts/cli-bench/history.test.ts +++ b/.github/scripts/cli-bench/history.test.ts @@ -5,23 +5,33 @@ import { makeEntry, MAX_PREVIOUS_RUNS, mergeHistory, + previousEntries, renderHistory, type RunEntry, } from "./history.ts"; const entry = (n: number, overrides: Partial = {}): RunEntry => ({ + section: "lightweight", sha: n.toString(16).padStart(7, "0"), at: "2026-10-28T09:05Z", headline: "no changes found", url: `https://github.com/o/r/actions/runs/${n}`, ...overrides, }); +const heavy = (n: number, overrides: Partial = {}) => + entry(n, { section: "integration-heavy", ...overrides }); describe("headlineOf", () => { - test("reads the title of the rendered report", () => { - expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( - "2 things to check", - ); + test.each([ + ["a comment section", "### Lightweight Commands: 2 things to check\n\nbody"], + ["an integration-heavy section", "### Integration Heavy Commands: 2 things to check\n\nbody"], + ["a job summary", "## CLI benchmark: 2 things to check\n\nbody"], + [ + "an integration-heavy job summary", + "## CLI benchmark (Integration Heavy Commands): 2 things to check", + ], + ])("reads the title of %s", (_name, markdown) => { + expect(headlineOf(markdown)).toBe("2 things to check"); }); test("falls back when the report has no title", () => { @@ -32,17 +42,23 @@ describe("headlineOf", () => { describe("makeEntry", () => { test("shortens the commit and rounds the time down to the minute in UTC", () => { const made = makeEntry({ + section: "integration-heavy", sha: "0123456789abcdef0123456789abcdef01234567", now: new Date("2026-10-28T09:05:59Z"), headline: "no changes found", url: "https://github.com/o/r/actions/runs/9/attempts/2", }); - expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" }); + expect(made).toMatchObject({ + section: "integration-heavy", + sha: "0123456", + at: "2026-10-28T09:05Z", + }); }); test("refuses a run link that is not a workflow run", () => { const made = makeEntry({ + section: "lightweight", sha: "abcdef1", now: new Date(), headline: "h", @@ -61,52 +77,88 @@ describe("mergeHistory", () => { ]); }); - test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => { - const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1)); + test("keeps each section's newest run plus at most 20 older ones across both sections", () => { + const previous = [ + ...Array.from({ length: 15 }, (_, i) => entry(100 - i)), + heavy(50), + ...Array.from({ length: 15 }, (_, i) => heavy(40 - i)), + ]; - const merged = mergeHistory(previous, entry(100)); + const merged = mergeHistory(previous, entry(200)); - expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1); - expect(merged[0]).toEqual(entry(100)); - expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS)); + const older = merged.length - 2; + expect(older).toBe(MAX_PREVIOUS_RUNS); + expect(merged[0]).toEqual(entry(200)); + expect(merged.filter((e) => e.section === "integration-heavy")[0]).toEqual(heavy(50)); + }); + + test("keeps a section's newest run even when the other section has filled the list", () => { + const previous = [...Array.from({ length: 25 }, (_, i) => entry(100 - i)), heavy(5)]; + + const merged = mergeHistory(previous, entry(200)); + + expect(merged.at(-1)).toEqual(heavy(5)); + }); +}); + +describe("previousEntries", () => { + test("leaves out each section's newest entry, which is its current result", () => { + const entries = [entry(4), heavy(3), entry(2), heavy(1)]; + + expect(previousEntries(entries)).toEqual([entry(2), heavy(1)]); + }); + + test("lists every entry of a section that is running, since it has no current result", () => { + const entries = [entry(4), heavy(3), entry(2), heavy(1)]; + + expect(previousEntries(entries, new Set(["integration-heavy"]))).toEqual([ + heavy(3), + entry(2), + heavy(1), + ]); }); }); describe("renderHistory and decodeHistory", () => { test("round-trips the entries through the hidden marker", () => { - const entries = [entry(3), entry(2), entry(1)]; + const entries = [entry(3), heavy(2), entry(1)]; expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries); }); - test("lists earlier runs newest first and leaves the current run out of the list", () => { + test("names the section of each replaced run and leaves current results out of the list", () => { const rendered = renderHistory( - [entry(3), entry(2, { headline: "1 thing to check" })], + [entry(4), heavy(3), heavy(2, { headline: "1 thing to check" })], (t) => t, ); expect(rendered).toContain("Previous runs"); expect(rendered).toContain( - "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", + "- Integration Heavy Commands · `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", ); expect(rendered).not.toContain("0000003` ·"); + expect(rendered).not.toContain("0000004` ·"); }); - test("shows no list for a first run", () => { - expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs"); + test("shows no list while every section has a single run", () => { + expect(renderHistory([entry(1), heavy(2)], (t) => t)).not.toContain("Previous runs"); }); test.each([ ["no marker", "plain comment"], ["not base64 json", ""], ["wrong shape", ``], + [ + "an entry without a section", + ``, + ], [ "a forged run link", ``, ], [ "too many entries", - ``, + ``, ], ])("treats %s as empty history", (_name, body) => { expect(decodeHistory(body)).toEqual([]); diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts index 9602383119..82b695117e 100644 --- a/.github/scripts/cli-bench/history.ts +++ b/.github/scripts/cli-bench/history.ts @@ -1,11 +1,19 @@ /** - * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier - * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on. - * The marker is read back only from the workflow bot's own comment, and anything that does not - * decode cleanly counts as no history. + * Run history kept inside the benchmark comment. Each section's newest result is the section + * itself; the results it replaced sit in a collapsed list, backed by a hidden marker so the next + * update can carry them on. The marker is read back only from the workflow bot's own comment, and + * anything that does not decode cleanly counts as no history. */ -/** Earlier runs kept below the current one. */ +/** Parts of the comment, each written by its own run. */ +export type Section = "lightweight" | "integration-heavy"; +export const SECTIONS: ReadonlyArray
= ["lightweight", "integration-heavy"]; +export const SECTION_NAMES: Record = { + lightweight: "Lightweight Commands", + "integration-heavy": "Integration Heavy Commands", +}; + +/** Replaced results kept below the current ones. */ export const MAX_PREVIOUS_RUNS = 20; const HISTORY_MARKER = //u; const MAX_ENCODED_CHARS = 30_000; @@ -15,6 +23,7 @@ const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u; const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; export interface RunEntry { + section: Section; /** Short commit SHA. */ sha: string; /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */ @@ -23,10 +32,14 @@ export interface RunEntry { url: string; } +const isSection = (value: unknown): value is Section => + typeof value === "string" && (SECTIONS as ReadonlyArray).includes(value); + const isEntry = (value: unknown): value is RunEntry => { if (typeof value !== "object" || value === null) return false; - const { sha, at, headline, url } = value as Record; + const { section, sha, at, headline, url } = value as Record; return ( + isSection(section) && typeof sha === "string" && SHA.test(sha) && typeof at === "string" && @@ -39,20 +52,25 @@ const isEntry = (value: unknown): value is RunEntry => { ); }; -/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */ +/** The title a section heading carries after its name, flattened to one bounded line. */ export function headlineOf(markdown: string): string { - const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; + const title = + /^#{2,3} (?:CLI benchmark(?: \([^)\n]*\))?|Lightweight Commands|Integration Heavy Commands): (.+)$/mu.exec( + markdown, + )?.[1] ?? "unknown"; return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS); } /** An entry for a run, or undefined when the commit or run link is not in the expected shape. */ export function makeEntry(input: { + section: Section; sha: string; now: Date; headline: string; url: string; }): RunEntry | undefined { const entry = { + section: input.section, sha: input.sha.slice(0, 7), at: `${input.now.toISOString().slice(0, 16)}Z`, headline: input.headline, @@ -67,7 +85,9 @@ export function decodeHistory(body: string): Array { if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return []; try { const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); - return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry) + return Array.isArray(value) && + value.length <= MAX_PREVIOUS_RUNS + SECTIONS.length && + value.every(isEntry) ? value : []; } catch { @@ -75,30 +95,64 @@ export function decodeHistory(body: string): Array { } } -/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */ +/** + * The entries a comment shows as previous runs: everything but each section's newest entry, + * which is the section's current result. A running section has no current result, so all of its + * entries are previous. + */ +export function previousEntries( + entries: ReadonlyArray, + running: ReadonlySet
= new Set(), +): Array { + const seen = new Set
(); + return entries.filter((entry) => { + if (running.has(entry.section)) return true; + if (!seen.has(entry.section)) { + seen.add(entry.section); + return false; + } + return true; + }); +} + +/** + * The new result first, then the earlier entries, keeping every section's newest entry and at + * most {@link MAX_PREVIOUS_RUNS} older ones across both sections. + */ export function mergeHistory( previous: ReadonlyArray, current: RunEntry, ): Array { - return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1); + const all = [current, ...previous]; + const seen = new Set
(); + let older = 0; + return all.filter((entry) => { + if (!seen.has(entry.section)) { + seen.add(entry.section); + return true; + } + older += 1; + return older <= MAX_PREVIOUS_RUNS; + }); } const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`; /** - * The hidden marker for `entries` (current run first), preceded by a collapsed list of the - * earlier ones when there are any. `clean` makes a headline safe to render. + * The hidden marker for `entries` (newest first), preceded by a collapsed list of the previous + * ones when there are any. `clean` makes a headline safe to render. */ export function renderHistory( entries: ReadonlyArray, clean: (text: string) => string, + running: ReadonlySet
= new Set(), ): string { const marker = ``; - const earlier = entries.slice(1); - if (earlier.length === 0) return marker; - const lines = earlier.map( + const previous = previousEntries(entries, running); + if (previous.length === 0) return marker; + const lines = previous.map( (entry) => - `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, + `- ${SECTION_NAMES[entry.section]} · \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, ); return ["
Previous runs", "", ...lines, "", "
", marker].join( "\n", diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index fb890a8952..2e5142013b 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -1,96 +1,179 @@ import { describe, expect, test } from "bun:test"; import { BENCH_MARKER, - FAILURE_TITLE, + composeBody, + failureMarkdown, + findBenchComment, MAX_BODY_CHARS, + MAX_WRITE_ATTEMPTS, + parseSections, planComment, - prepareBody, + RERUN_NOTE, sanitize, + writeSection, + type CommentIo, + type ExistingComment, } from "./post-comment.ts"; import { decodeHistory, type RunEntry } from "./history.ts"; const BOT = "github-actions[bot]"; +const RUN = "https://github.com/o/r/actions/runs/"; +const NOW = new Date("2026-10-28T09:05:00Z"); +const SHA = "0123456789abcdef"; + +const withSections = (id: number, body: string): ExistingComment => ({ + id, + body, + authorLogin: BOT, +}); + +/** An in-memory comment thread with hooks to interleave other writers' updates. */ +function fakeThread(initial: Array = []) { + const comments = [...initial]; + let nextId = 100; + const hooks: { afterUpdate?: () => void; afterCreate?: () => void } = {}; + const io: CommentIo = { + list: async () => comments.map((comment) => ({ ...comment })), + get: async (id) => comments.find((comment) => comment.id === id), + create: async (body) => { + const created = withSections(nextId++, body); + comments.push(created); + hooks.afterCreate?.(); + return created.id; + }, + update: async (id, body) => { + const target = comments.find((comment) => comment.id === id); + if (target !== undefined) target.body = body; + hooks.afterUpdate?.(); + }, + remove: async (id) => { + comments.splice( + comments.findIndex((comment) => comment.id === id), + 1, + ); + }, + }; + return { comments, io, hooks }; +} + +const report = (title: string, section: "Lightweight" | "Integration Heavy") => + `### ${section} Commands: ${title}\n\ndetails`; describe("planComment", () => { test("creates the comment when none carries the marker", () => { - expect(planComment("create-if-missing", [{ id: 1, body: "hi", authorLogin: BOT }])).toEqual({ - kind: "create", - }); + expect( + planComment("create-if-missing", "lightweight", [{ id: 1, body: "hi", authorLogin: BOT }]), + ).toEqual({ kind: "create" }); }); - test("leaves an existing marker comment alone on a later push", () => { - const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + test("leaves a section that already reports a result alone on a later push", () => { + const existing = withSections( + 5, + composeBody({ lightweight: { sha: "abc1234", content: report("old", "Lightweight") } }), + ); - expect(planComment("create-if-missing", [existing])).toEqual({ kind: "skip" }); + expect(planComment("create-if-missing", "lightweight", [existing])).toEqual({ kind: "skip" }); }); - test("replaces a first-run failure notice on a later push", () => { - const failed = { - id: 5, - body: `${BENCH_MARKER}\n${FAILURE_TITLE}\nbuild failed`, - authorLogin: BOT, - }; + test("fills a section the comment does not have yet, even on a later push", () => { + const existing = withSections( + 5, + composeBody({ + "integration-heavy": { sha: "abc1234", content: report("old", "Integration Heavy") }, + }), + ); - expect(planComment("create-if-missing", [failed])).toEqual({ + expect(planComment("create-if-missing", "lightweight", [existing])).toEqual({ kind: "update", id: 5, - previousBody: failed.body, + previousBody: existing.body, }); }); - test("updates the marker comment in place for a re-run", () => { - const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + test("replaces a first-run failure notice on a later push", () => { + const failed = withSections( + 5, + composeBody({ lightweight: { sha: "abc1234", content: failureMarkdown("lightweight") } }), + ); - expect(planComment("upsert", [existing])).toEqual({ - kind: "update", - id: 5, - previousBody: existing.body, - }); + expect(planComment("create-if-missing", "lightweight", [failed]).kind).toBe("update"); + }); + + test.each(["upsert", "running"] as const)("updates a reported section for %s", (mode) => { + const existing = withSections( + 5, + composeBody({ lightweight: { sha: "abc1234", content: report("old", "Lightweight") } }), + ); + + expect(planComment(mode, "lightweight", [existing]).kind).toBe("update"); }); test("ignores a marker pasted by someone else", () => { const forged = { id: 6, body: BENCH_MARKER, authorLogin: "mallory" }; - expect(planComment("create-if-missing", [forged])).toEqual({ kind: "create" }); + expect(planComment("create-if-missing", "lightweight", [forged])).toEqual({ kind: "create" }); + }); + + test("keeps the older comment when two writers each created one", () => { + const older = withSections(3, BENCH_MARKER); + const newer = withSections(9, BENCH_MARKER); + + expect(findBenchComment([newer, older])?.id).toBe(3); }); }); -describe("prepareBody", () => { - test("puts the marker first exactly once", () => { - const body = prepareBody(`text ${BENCH_MARKER} more`); +describe("composeBody", () => { + test("puts the marker first exactly once and ends with the rerun note for both commands", () => { + const body = composeBody({ + lightweight: { sha: "abc1234", content: `text ${BENCH_MARKER} more` }, + }); expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(body.split(BENCH_MARKER)).toHaveLength(2); + expect(body.split(BENCH_MARKER).length - 1).toBeGreaterThanOrEqual(1); + expect(body).toContain(RERUN_NOTE); + expect(RERUN_NOTE).toBe( + "Comment `/bench` (only includes lightweight benchmarks) or `/bench full` to run it again on the latest commit.", + ); + expect(body).not.toContain("Comment `/bench` to run it again"); + }); + + test("keeps each section's commit and content readable from the body", () => { + const body = composeBody({ + lightweight: { sha: "abc1234", content: report("a", "Lightweight") }, + "integration-heavy": { sha: "def5678", content: report("b", "Integration Heavy") }, + }); + + expect(parseSections(body)).toEqual({ + lightweight: { sha: "abc1234", content: report("a", "Lightweight") }, + "integration-heavy": { sha: "def5678", content: report("b", "Integration Heavy") }, + }); }); - test("cuts an oversized report below GitHub's limit and says so", () => { - const body = prepareBody("x".repeat(200_000)); + test("cuts oversized sections below GitHub's limit and says so", () => { + const body = composeBody({ + lightweight: { sha: "abc1234", content: "x".repeat(200_000) }, + "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, + }); expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); expect(body).toContain("truncated"); + expect(parseSections(body).lightweight).toBeDefined(); + expect(parseSections(body)["integration-heavy"]).toBeDefined(); }); -}); -describe("prepareBody history", () => { const run = (n: number): RunEntry => ({ + section: "lightweight", sha: n.toString(16).padStart(7, "0"), at: "2026-10-28T09:05Z", headline: "no changes found", - url: `https://github.com/o/r/actions/runs/${n}`, + url: `${RUN}${n}`, }); - test("lists earlier runs after the run link and carries them in the marker", () => { - const body = prepareBody("report", "https://github.com/o/r/actions/runs/3", [run(3), run(2)]); - - expect(body.indexOf("[Workflow run]")).toBeLessThan(body.indexOf("Previous runs")); - expect(decodeHistory(body)).toEqual([run(3), run(2)]); - }); - - test("drops the oldest runs before cutting the report when the body is too long", () => { + test("drops the oldest runs before cutting a section when the body is too long", () => { const history = Array.from({ length: 21 }, (_, i) => run(21 - i)); - const report = "x".repeat(MAX_BODY_CHARS - 4_000); + const content = "x".repeat(MAX_BODY_CHARS - 4_000); - const body = prepareBody(report, "https://github.com/o/r/actions/runs/21", history); + const body = composeBody({ lightweight: { sha: "abc1234", content } }, history); expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); expect(body).not.toContain("truncated"); @@ -100,13 +183,16 @@ describe("prepareBody history", () => { expect(kept[0]).toEqual(history[0]); }); - test("renders an earlier run's headline through the sanitizer", () => { + test("renders a previous run's headline through the sanitizer", () => { const hostile = { ...run(1), headline: "@team " }; - const body = prepareBody("report", undefined, [run(2), hostile]); + const body = composeBody( + { lightweight: { sha: "abc1234", content: report("new", "Lightweight") } }, + [run(3), run(2), hostile], + ); expect(body).not.toContain(" { const clean = sanitize(hostile); - expect(clean).not.toMatch(/@(?!\u200b)/u); + expect(clean).not.toMatch(/@(?!​)/u); expect(clean).not.toContain("@"); expect(clean).not.toMatch(/(? { expect(clean).toContain("
"); }); - test("pins the failure title and marker literals the renderer emits", () => { - expect(FAILURE_TITLE).toBe("## CLI benchmark: could not run"); - expect(BENCH_MARKER).toBe(""); + test("keeps a report from forging a section marker", () => { + const forged = sanitize(""); + + expect(forged).not.toContain(""; +/** The one note under every comment, naming both commands. */ +export const RERUN_NOTE = + "Comment `/bench` (only includes lightweight benchmarks) or `/bench full` to run it again on the latest commit."; /** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ const WORKFLOW_BOT_LOGIN = "github-actions[bot]"; /** GitHub rejects comment bodies longer than 65536 characters. */ export const MAX_BODY_CHARS = 65_000; const TRUNCATION_NOTICE = "\n\n
\n\n_The report was truncated; see the workflow run for the rest._\n"; +/** Writes attempted before giving up on a section that keeps being overwritten. */ +export const MAX_WRITE_ATTEMPTS = 3; -export type Mode = "create-if-missing" | "upsert"; +export type Mode = "create-if-missing" | "upsert" | "running"; export type ExistingComment = IssueComment; @@ -35,10 +58,9 @@ export type CommentPlan = | { kind: "update"; id: number; previousBody: string }; const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); -const ZERO_WIDTH_SPACE = "\u200b"; -/** Title the renderer gives a run that produced no comparison. */ -export const FAILURE_TITLE = "## CLI benchmark: could not run"; +const ZERO_WIDTH_SPACE = "​"; export const MAX_FILE_BYTES = 1_000_000; +const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; /** Keeps mentions, links, images and raw HTML in untrusted markdown from rendering. */ export function sanitize(markdown: string): string { @@ -54,94 +76,310 @@ export function sanitize(markdown: string): string { .replaceAll("]", "\\]"); } -/** - * The comment body: the marker first, the sanitized report, the trusted run link, then the run - * history. `history` lists the current run first; the oldest runs are dropped until the body fits. - */ -export function prepareBody( - markdown: string, - runUrl?: string, - history: ReadonlyArray = [], +export interface SectionState { + /** Short SHA of the commit the section measured. */ + sha: string; + /** The section's Markdown, from its heading to its run link. */ + content: string; +} +export type Sections = Partial>; + +const startMarker = (section: Section, sha: string) => + ``; +const endMarker = (section: Section) => ``; + +/** The sections recorded in a comment body. */ +export function parseSections(body: string): Sections { + const sections: Sections = {}; + for (const section of SECTIONS) { + const match = new RegExp( + `\\n([\\s\\S]*?)\\n${endMarker(section)}`, + "u", + ).exec(body); + if (match?.[1] !== undefined && match[2] !== undefined) { + sections[section] = { sha: match[1], content: match[2] }; + } + } + return sections; +} + +/** Whether a section's heading says the run produced no comparison. */ +export const sectionFailed = (content: string) => /^### [^:\n]+: could not run$/mu.test(content); +const sectionRunning = (content: string) => /^### [^:\n]+: running$/mu.test(content); + +const short = (sha: string) => sha.slice(0, 7); + +/** The placeholder shown while a run is in progress. */ +export function runningContent( + section: Section, + sha: string, + now: Date, + runUrl: string | undefined, ): string { - const link = runUrl === undefined || runUrl === "" ? "" : `\n\n[Workflow run](${runUrl})\n`; + const link = runUrl !== undefined && RUN_URL.test(runUrl) ? ` [Workflow run](${runUrl})` : ""; + return ( + `### ${SECTION_NAMES[section]}: running\n\n` + + `Started ${now.toISOString().slice(0, 16).replace("T", " ")} UTC for commit \`${short(sha)}\`.${link}` + ); +} + +/** The section for a run that produced no report. */ +export function failureMarkdown(section: Section): string { + return `### ${SECTION_NAMES[section]}: could not run\n\nThe benchmark did not finish. See the workflow run.`; +} + +/** The sanitized report, then the trusted run link. */ +export function sectionContent(markdown: string, runUrl: string | undefined): string { const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); - const body = `${BENCH_MARKER}\n${content}\n`; - for (let kept = history.length; kept >= 1; kept--) { - const tail = `${link}\n${renderHistory(history.slice(0, kept), sanitize)}\n`; - if (body.length + tail.length <= MAX_BODY_CHARS) return body + tail; + const link = runUrl !== undefined && RUN_URL.test(runUrl) ? `\n\n[Workflow run](${runUrl})` : ""; + return `${content}${link}`; +} + +function assemble(sections: Sections, entries: ReadonlyArray): string { + const running = new Set( + SECTIONS.filter((section) => { + const state = sections[section]; + return state !== undefined && sectionRunning(state.content); + }), + ); + const blocks = SECTIONS.flatMap((section) => { + const state = sections[section]; + return state === undefined + ? [] + : [`${startMarker(section, state.sha)}\n${state.content}\n${endMarker(section)}`]; + }); + return ( + [ + BENCH_MARKER, + "## CLI benchmark", + ...blocks, + RERUN_NOTE, + renderHistory(entries, sanitize, running), + ].join("\n\n") + "\n" + ); +} + +/** + * The comment body: the marker, each section, the re-run note, then the run history. `entries` + * lists the newest run first; the oldest are dropped until the body fits, and then each section is + * cut to its share of the limit. + */ +export function composeBody(sections: Sections, entries: ReadonlyArray = []): string { + for (let kept = entries.length; kept >= 0; kept--) { + const body = assemble(sections, entries.slice(0, kept)); + if (body.length <= MAX_BODY_CHARS) return body; + } + const present = SECTIONS.filter((section) => sections[section] !== undefined); + const overhead = assemble( + Object.fromEntries(present.map((section) => [section, { sha: "0000000", content: "" }])), + [], + ).length; + const share = Math.floor((MAX_BODY_CHARS - overhead) / Math.max(present.length, 1)); + const cut: Sections = {}; + for (const section of present) { + const state = sections[section]; + if (state === undefined) continue; + cut[section] = { + sha: state.sha, + content: + state.content.length <= share + ? state.content + : state.content.slice(0, share - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE, + }; } - const tail = `${link}\n${history.length === 0 ? "" : `${renderHistory(history.slice(0, 1), sanitize)}\n`}`; - const limit = MAX_BODY_CHARS - tail.length; - const fitted = - body.length <= limit - ? body - : body.slice(0, limit - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE; - return fitted + tail; + return assemble(cut, []); } -/** The workflow bot's benchmark comment, if any. Only its marker counts. */ +/** The workflow bot's benchmark comment, if any; the oldest wins when concurrent writers made two. */ export function findBenchComment( comments: ReadonlyArray, ): ExistingComment | undefined { - return comments.find( - (comment) => comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), - ); + return [...comments] + .filter( + (comment) => + comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), + ) + .sort((a, b) => a.id - b.id)[0]; } -/** What to do with the pull request's benchmark comment. */ -export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { +/** What to do with the pull request's benchmark comment for `section`. */ +export function planComment( + mode: Mode, + section: Section, + comments: ReadonlyArray, +): CommentPlan { const existing = findBenchComment(comments); if (existing === undefined) return { kind: "create" }; + const state = parseSections(existing.body)[section]; // A first run that could not compare anything must not block the next push's real result. - const replaceable = mode === "upsert" || existing.body.includes(FAILURE_TITLE); + const replaceable = + mode !== "create-if-missing" || state === undefined || sectionFailed(state.content); return replaceable ? { kind: "update", id: existing.id, previousBody: existing.body } : { kind: "skip" }; } +export interface CommentIo { + list: () => Promise>; + get: (id: number) => Promise; + create: (body: string) => Promise; + update: (id: number, body: string) => Promise; + remove: (id: number) => Promise; +} + +export interface WriteInput { + mode: Mode; + section: Section; + /** The commit the run measured. */ + sha: string; + /** The rendered section; absent for `running`. */ + markdown: string | undefined; + runUrl: string | undefined; + now: Date; +} + +export type WriteResult = + | { kind: "skipped" } + | { kind: "written"; id: number; created: boolean; attempts: number } + | { kind: "unverified"; attempts: number }; + +/** + * Writes `section` into the comment and verifies it by reading the comment back, retrying up to + * {@link MAX_WRITE_ATTEMPTS} times when a concurrent writer overwrote it. A comment created while + * another writer created one too is removed in favour of the older one. + */ +export async function writeSection(io: CommentIo, input: WriteInput): Promise { + const { mode, section } = input; + const content = + mode === "running" || input.markdown === undefined + ? runningContent(section, input.sha, input.now, input.runUrl) + : sectionContent(input.markdown, input.runUrl); + const entry = + mode === "running" || input.markdown === undefined + ? undefined + : makeEntry({ + section, + sha: input.sha, + now: input.now, + headline: headlineOf(input.markdown), + url: input.runUrl ?? "", + }); + + for (let attempt = 1; attempt <= MAX_WRITE_ATTEMPTS; attempt++) { + const plan = planComment(mode, section, await io.list()); + if (plan.kind === "skip") return { kind: "skipped" }; + const earlier = plan.kind === "update" ? plan.previousBody : ""; + const sections = parseSections(earlier); + sections[section] = { sha: short(input.sha), content }; + const history = decodeHistory(earlier); + const body = composeBody( + sections, + entry === undefined ? history : mergeHistory(history, entry), + ); + + let id: number; + if (plan.kind === "update") { + id = plan.id; + await io.update(id, body); + } else { + id = await io.create(body); + } + + const written = await io.get(id); + // Compare with the section as composed, which may have been truncated to fit. + const intended = parseSections(body)[section]; + const actual = written === undefined ? undefined : parseSections(written.body)[section]; + const landed = + intended !== undefined && actual?.sha === intended.sha && actual.content === intended.content; + if (plan.kind === "create") { + const canonical = findBenchComment(await io.list()); + if (canonical !== undefined && canonical.id !== id) { + await io.remove(id); + continue; + } + } + if (landed) return { kind: "written", id, created: plan.kind === "create", attempts: attempt }; + } + return { kind: "unverified", attempts: MAX_WRITE_ATTEMPTS }; +} + async function main(): Promise { const token = requireEnv("GITHUB_TOKEN"); const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; const pr = Number(requireEnv("PR_NUMBER")); const mode = requireEnv("MODE"); - if (mode !== "create-if-missing" && mode !== "upsert") { + if (mode !== "create-if-missing" && mode !== "upsert" && mode !== "running") { throw new Error(`Unsupported MODE "${mode}".`); } - const file = requireEnv("BODY_FILE"); - const stat = lstatSync(file); - if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { - throw new Error("The benchmark comment file is not a regular file under the size limit."); + const section = process.env.SECTION ?? "lightweight"; + if (section !== "lightweight" && section !== "integration-heavy") { + throw new Error(`Unsupported SECTION "${section}".`); + } + + let markdown: string | undefined; + if (mode !== "running") { + const file = requireEnv("BODY_FILE"); + let stat: ReturnType | undefined; + try { + stat = lstatSync(file); + } catch { + stat = undefined; + } + if (stat === undefined) { + if (process.env.MISSING_BODY !== "failure") { + console.log("The benchmark job produced no comment."); + return; + } + markdown = failureMarkdown(section); + } else { + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { + throw new Error("The benchmark comment file is not a regular file under the size limit."); + } + markdown = readFileSync(file, "utf8"); + } } - const markdown = readFileSync(file, "utf8"); - const runUrl = process.env.RUN_URL; - const entry = makeEntry({ + + const io: CommentIo = { + list: () => listIssueComments(token, base, pr), + get: (id) => getIssueComment(token, base, id), + create: async (body) => { + const created = (await ( + await githubFetch(`${base}/issues/${pr}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }) + ).json()) as { id: number }; + return created.id; + }, + update: async (id, body) => { + await githubFetch(`${base}/issues/comments/${id}`, token, { + method: "PATCH", + body: JSON.stringify({ body }), + }); + }, + remove: async (id) => { + await githubFetch(`${base}/issues/comments/${id}`, token, { method: "DELETE" }); + }, + }; + + const result = await writeSection(io, { + mode, + section, sha: requireEnv("HEAD_SHA"), + markdown, + runUrl: process.env.RUN_URL, now: new Date(), - headline: headlineOf(markdown), - url: runUrl ?? "", }); - - const plan = planComment(mode, await listIssueComments(token, base, pr)); - const earlier = plan.kind === "update" ? decodeHistory(plan.previousBody) : []; - const body = prepareBody( - markdown, - runUrl, - entry === undefined ? [] : mergeHistory(earlier, entry), - ); - if (plan.kind === "skip") { - console.log(`PR #${pr} already has a benchmark comment; leaving it untouched.`); - } else if (plan.kind === "update") { - await githubFetch(`${base}/issues/comments/${plan.id}`, token, { - method: "PATCH", - body: JSON.stringify({ body }), - }); - console.log(`Updated benchmark comment ${plan.id} on PR #${pr}.`); + if (result.kind === "skipped") { + console.log(`PR #${pr} already has a ${section} result; leaving it untouched.`); + } else if (result.kind === "written") { + console.log( + `${result.created ? "Created" : "Updated"} benchmark comment ${result.id} on PR #${pr} (${section}).`, + ); } else { - await githubFetch(`${base}/issues/${pr}/comments`, token, { - method: "POST", - body: JSON.stringify({ body }), - }); - console.log(`Created benchmark comment on PR #${pr}.`); + throw new Error( + `The ${section} section did not land after ${result.attempts} attempts; another writer kept replacing it.`, + ); } } diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts index fa37cad648..036d3ba5de 100644 --- a/.github/scripts/cli-bench/rerun.test.ts +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -3,8 +3,11 @@ import { GithubRequestError } from "./github.ts"; import { type BenchPr, type CommandIo, + coversIntegrationHeavy, handleBenchCommand, RERUN_WINDOW_MS, + type StackRun, + type StackWorkloads, type WorkflowRun, } from "./rerun.ts"; @@ -15,6 +18,7 @@ const pr = (overrides: Partial = {}): BenchPr => ({ state: "open", draft: false, headSha: HEAD, + headRef: "feature/x", headRepoFullName: REPO, baseRepoFullName: REPO, baseRef: "develop", @@ -33,12 +37,17 @@ function setup( pr?: BenchPr; runs?: ReadonlyArray; rerunError?: Error; + stackRuns?: ReadonlyArray; + dispatchError?: Error; + /** Short SHA the Lightweight Commands section measured. */ + lightweightSha?: string; } = {}, ) { const calls = { reactions: [] as Array, replies: [] as Array, reruns: [] as Array, + dispatches: [] as Array<{ ref: string; workloads: StackWorkloads }>, }; const io: CommandIo = { fetchPr: async () => options.pr ?? pr(), @@ -48,6 +57,12 @@ function setup( if (options.rerunError !== undefined) throw options.rerunError; calls.reruns.push(id); }, + listStackRuns: async () => options.stackRuns ?? [], + dispatchStack: async (ref, workloads) => { + if (options.dispatchError !== undefined) throw options.dispatchError; + calls.dispatches.push({ ref, workloads }); + }, + lightweightSha: async () => options.lightweightSha, reactToComment: async (id) => { calls.reactions.push(id); }, @@ -78,7 +93,7 @@ describe("handleBenchCommand", () => { expect(calls.replies).toEqual([]); }); - test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( + test.each(["/bench fully", "/benchmark", "please /bench", "/Bench", "/bench full now"])( "ignores %p without a reply", async (body) => { const { io, calls } = setup(); @@ -86,7 +101,7 @@ describe("handleBenchCommand", () => { const outcome = await handleBenchCommand(command(body), io); expect(outcome.kind).toBe("ignored"); - expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [], dispatches: [] }); }, ); @@ -116,7 +131,7 @@ describe("handleBenchCommand", () => { const outcome = await handleBenchCommand(command("/bench"), io); expect(outcome.kind).toBe("ignored"); - expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [], dispatches: [] }); }); test("tells the pull request there is no run when none exists for the head commit", async () => { @@ -182,3 +197,181 @@ describe("handleBenchCommand", () => { expect(calls.reruns).toEqual([]); }); }); + +const stackRun = (overrides: Partial = {}): StackRun => ({ + id: 31, + status: "in_progress", + displayTitle: "CLI benchmark stack (integration-heavy): feature/x", + url: "https://github.com/supabase/cli/actions/runs/31", + ...overrides, +}); + +describe("handleBenchCommand with /bench full", () => { + test("dispatches the integration-heavy workloads on the branch and leaves a current lightweight section alone", async () => { + const { io, calls } = setup({ lightweightSha: "aaaaaaa" }); + + const outcome = await handleBenchCommand(command("/bench full\n"), io); + + expect(outcome).toEqual({ + kind: "dispatched", + workloads: "integration-heavy", + rerunId: undefined, + }); + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "integration-heavy" }]); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([99]); + expect(calls.replies).toEqual([]); + }); + + test("also re-runs the lightweight run when its section is behind the head commit", async () => { + const { io, calls } = setup({ lightweightSha: "bbbbbbb", runs: [run({ id: 4 })] }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ kind: "dispatched", workloads: "integration-heavy", rerunId: 4 }); + expect(calls.reruns).toEqual([4]); + }); + + test("leaves the lightweight part to a pull request run that is already in flight", async () => { + const { io, calls } = setup({ + lightweightSha: "bbbbbbb", + runs: [run({ status: "queued" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "integration-heavy" }]); + expect(calls.reruns).toEqual([]); + expect(calls.replies).toEqual([]); + }); + + test.each([ + ["no pull request run exists for the head commit", [] as Array], + [ + "its pull request run is older than 30 days", + [run({ createdAt: new Date(NOW.getTime() - RERUN_WINDOW_MS - 1000).toISOString() })], + ], + ])("measures the lightweight workloads in the stack run when %s", async (_name, runs) => { + const { io, calls } = setup({ lightweightSha: undefined, runs }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ kind: "dispatched", workloads: "all", rerunId: undefined }); + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "all" }]); + expect(calls.reruns).toEqual([]); + }); + + test("replies with the link of an integration-heavy run already in progress and starts nothing", async () => { + const { io, calls } = setup({ stackRuns: [stackRun()] }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("https://github.com/supabase/cli/actions/runs/31"); + expect(calls.dispatches).toEqual([]); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("starts a run when the one in progress measures only the lightweight workloads", async () => { + const { io, calls } = setup({ + lightweightSha: "aaaaaaa", + stackRuns: [stackRun({ displayTitle: "CLI benchmark stack (lightweight): feature/x" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toHaveLength(1); + }); + + test("starts a run when earlier ones have finished", async () => { + const { io, calls } = setup({ + lightweightSha: "aaaaaaa", + stackRuns: [stackRun({ status: "completed" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toHaveLength(1); + }); + + test.each([ + ["a missing workflow file", new GithubRequestError(404, "Not Found", "u")], + [ + "a branch whose workflow cannot be dispatched", + new GithubRequestError(422, "Workflow does not have 'workflow_dispatch' trigger", "u"), + ], + ])("tells the author to merge develop on %s", async (_name, dispatchError) => { + const { io, calls } = setup({ dispatchError }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("Merge `develop` into the branch"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("surfaces GitHub's refusal of the dispatch with its status and message", async () => { + const { io, calls } = setup({ + dispatchError: new GithubRequestError(403, "Resource not accessible by integration", "u"), + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.replies[0]).toContain("403"); + expect(calls.replies[0]).toContain("Resource not accessible by integration"); + expect(calls.reactions).toEqual([]); + }); + + test.each([ + ["a draft", pr({ draft: true }), "draft"], + ["a fork", pr({ headRepoFullName: "someone/cli" }), "from this repository"], + ["a pull request into another branch", pr({ baseRef: "release" }), "into develop"], + ])("replies that %s is not benchmarked and starts nothing", async (_name, head, text) => { + const { io, calls } = setup({ pr: head }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.replies[0]).toContain(text); + expect(calls.dispatches).toEqual([]); + }); + + test("ignores a commenter without write permission without a reply", async () => { + const { io, calls } = setup({ permissions: { maintainer: "read" } }); + + const outcome = await handleBenchCommand(command("/bench full", "MEMBER"), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls.replies).toEqual([]); + expect(calls.dispatches).toEqual([]); + }); + + test("keeps a started stack run when re-running the lightweight run is refused", async () => { + const { io, calls } = setup({ + lightweightSha: "bbbbbbb", + rerunError: new GithubRequestError(403, "nope", "u"), + }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ + kind: "dispatched", + workloads: "integration-heavy", + rerunId: undefined, + }); + expect(calls.dispatches).toHaveLength(1); + expect(calls.reactions).toEqual([99]); + }); +}); + +describe("coversIntegrationHeavy", () => { + test.each([ + ["CLI benchmark stack (all): develop", true], + ["CLI benchmark stack (integration-heavy): feature/x", true], + ["CLI benchmark stack (lightweight): feature/x", false], + ["CLI benchmark stack (scheduled)", false], + ])("reads %p as %p", (title, expected) => { + expect(coversIntegrationHeavy(title)).toBe(expected); + }); +}); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index b0020185b8..ed31bb6f46 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -1,22 +1,29 @@ /** - * `/bench` handler: re-runs the benchmark workflow run of a pull request's current head commit. + * `/bench` and `/bench full` handler. * * Runs default branch code and never checks out or runs pull request code. The commenter must be * the repository owner or hold write access, and the pull request must be an open, ready one from - * this repository into `develop`. The command re-runs the existing `pull_request` run, which - * benchmarks and updates the comment; the pull request gets a reply when no run can be re-run. + * this repository into `develop`. `/bench` re-runs the existing `pull_request` run, which + * benchmarks and updates the Lightweight Commands section. `/bench full` always re-measures the + * Integration Heavy Commands section by dispatching the stack benchmark workflow on the pull + * request's branch, and re-runs the lightweight part only when that section is behind the head + * commit; the pull request gets a reply when nothing can run. * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. */ import { appendFileSync } from "node:fs"; import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; -import { githubFetch, GithubRequestError, requireEnv } from "./github.ts"; +import { githubFetch, GithubRequestError, listIssueComments, requireEnv } from "./github.ts"; +import { findBenchComment, parseSections } from "./post-comment.ts"; export const BENCH_COMMAND = "/bench"; +export const BENCH_FULL_COMMAND = "/bench full"; /** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; /** The workflow file whose runs `/bench` re-runs. */ export const WORKFLOW_FILE = "cli-bench.yml"; +/** The workflow `/bench full` dispatches. */ +export const STACK_WORKFLOW_FILE = "cli-bench-stack.yml"; /** GitHub refuses to re-run a workflow run older than this. */ export const RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; @@ -24,6 +31,8 @@ export interface BenchPr { state: "open" | "closed"; draft: boolean; headSha: string; + /** Name of the head branch. */ + headRef: string; /** `owner/name` of the head repository, empty when it was deleted. */ headRepoFullName: string; baseRepoFullName: string; @@ -36,6 +45,17 @@ export interface WorkflowRun { createdAt: string; } +export interface StackRun { + id: number; + status: string; + /** The run name, which carries the `workloads` input. */ + displayTitle: string; + url: string; +} + +/** What a dispatched stack run measures. */ +export type StackWorkloads = "integration-heavy" | "all"; + export interface CommandInput { prNumber: number; comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; @@ -48,6 +68,11 @@ export interface CommandIo { /** Runs of the benchmark workflow triggered by `pull_request` for this commit, any order. */ listRuns: (headSha: string) => Promise>; rerun: (runId: number) => Promise; + /** Dispatched runs of the stack workflow on a branch, any order. */ + listStackRuns: (headRef: string) => Promise>; + dispatchStack: (headRef: string, workloads: StackWorkloads) => Promise; + /** Short SHA of the commit the comment's Lightweight Commands section measured, if it has one. */ + lightweightSha: (prNumber: number) => Promise; reactToComment: (commentId: number) => Promise; replyToPr: (prNumber: number, body: string) => Promise; } @@ -55,7 +80,8 @@ export interface CommandIo { export type CommandOutcome = | { kind: "ignored"; reason: string } | { kind: "replied"; reason: string } - | { kind: "rerun"; runId: number }; + | { kind: "rerun"; runId: number } + | { kind: "dispatched"; workloads: StackWorkloads; rerunId: number | undefined }; const short = (sha: string) => sha.slice(0, 7); @@ -77,18 +103,46 @@ export const replies = { refused: (sha: string, status: number, detail: string) => `The benchmark could not be re-run for commit \`${short(sha)}\`: GitHub refused the request ` + `(${status} ${quoted(detail)}).`, + notEligible: () => + `\`${BENCH_FULL_COMMAND}\` runs only on pull requests from this repository into ` + + `${BENCH_BASE_BRANCHES.join(" or ")}.`, + stackRunning: (url: string) => + `An integration-heavy benchmark is already queued or running for this pull request: ${url}. ` + + "The comment updates when it finishes.", + noStackWorkflow: () => + "This branch does not have the stack benchmark workflow yet. Merge `develop` into the branch " + + `and comment \`${BENCH_FULL_COMMAND}\` again.`, + stackRefused: (status: number, detail: string) => + `The integration-heavy benchmark could not be started: GitHub refused the request ` + + `(${status} ${quoted(detail)}).`, }; -/** Handles one `/bench` comment; a rejected command is ignored without a reply. */ +const STACK_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; + +/** Whether a stack run's name says it measures the integration-heavy workloads. */ +export function coversIntegrationHeavy(displayTitle: string): boolean { + const workloads = STACK_RUN_WORKLOADS.exec(displayTitle)?.[1]; + return workloads === "integration-heavy" || workloads === "all"; +} + +const isCommand = (body: string) => body.trim() === BENCH_COMMAND; +const isFullCommand = (body: string) => body.trim() === BENCH_FULL_COMMAND; + +/** Handles one `/bench` or `/bench full` comment; a rejected command is ignored without a reply. */ export async function handleBenchCommand( input: CommandInput, io: CommandIo, ): Promise { const { comment, prNumber } = input; - // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. - if (comment.body.trim() !== BENCH_COMMAND) { - return { kind: "ignored", reason: `Comment is not exactly ${BENCH_COMMAND}.` }; + const full = isFullCommand(comment.body); + // The workflow's `if:` only pre-filters on `startsWith`, so any other text must be rejected here. + if (!full && !isCommand(comment.body)) { + return { + kind: "ignored", + reason: `Comment is not exactly ${BENCH_COMMAND} or ${BENCH_FULL_COMMAND}.`, + }; } + const command = full ? BENCH_FULL_COMMAND : BENCH_COMMAND; // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. const permission = await io.fetchPermission(comment.authorLogin); const authorized = @@ -98,55 +152,103 @@ export async function handleBenchCommand( return { kind: "ignored", reason: - `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + + `Commenter @${comment.authorLogin} is not authorized to run ${command} ` + `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, }; } const pr = await io.fetchPr(prNumber); if (pr.state === "closed") return { kind: "ignored", reason: `PR #${prNumber} is closed.` }; - if (pr.headRepoFullName !== pr.baseRepoFullName) { - return { - kind: "ignored", - reason: `${BENCH_COMMAND} runs only on pull requests from this repository.`, - }; - } - if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { - return { - kind: "ignored", - reason: `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, - }; - } const reply = async (text: string): Promise => { await io.replyToPr(prNumber, text); return { kind: "replied", reason: text }; }; + const eligible = + pr.headRepoFullName === pr.baseRepoFullName && BENCH_BASE_BRANCHES.includes(pr.baseRef); + if (!eligible) { + return full + ? reply(replies.notEligible()) + : { + kind: "ignored", + reason: + pr.headRepoFullName !== pr.baseRepoFullName + ? `${BENCH_COMMAND} runs only on pull requests from this repository.` + : `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, + }; + } if (pr.draft) return reply(replies.draft()); const latest = [...(await io.listRuns(pr.headSha))].sort((a, b) => b.createdAt.localeCompare(a.createdAt), )[0]; - if (latest === undefined) return reply(replies.noRun(pr.headSha)); - if (latest.status !== "completed") return reply(replies.running(pr.headSha)); - if (input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS) { - return reply(replies.expired(pr.headSha)); + const expired = + latest !== undefined && input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS; + + if (!full) { + if (latest === undefined) return reply(replies.noRun(pr.headSha)); + if (latest.status !== "completed") return reply(replies.running(pr.headSha)); + if (expired) return reply(replies.expired(pr.headSha)); + try { + await io.rerun(latest.id); + } catch (error) { + if (error instanceof GithubRequestError) { + return reply(replies.refused(pr.headSha, error.status, error.detail)); + } + throw error; + } + await react(io, comment.id); + return { kind: "rerun", runId: latest.id }; } + const active = (await io.listStackRuns(pr.headRef)).find( + (stackRun) => stackRun.status !== "completed" && coversIntegrationHeavy(stackRun.displayTitle), + ); + if (active !== undefined) return reply(replies.stackRunning(active.url)); + + // The lightweight part refreshes only when its section is behind the head commit and no + // lightweight run is already in flight; with no run to re-run, the stack run measures it too. + const sectionSha = await io.lightweightSha(prNumber); + const lightweightCurrent = sectionSha !== undefined && pr.headSha.startsWith(sectionSha); + const lightweightInFlight = latest !== undefined && latest.status !== "completed"; + const needsLightweight = !lightweightCurrent && !lightweightInFlight; + const rerunnable = latest !== undefined && !expired ? latest : undefined; + const workloads: StackWorkloads = + needsLightweight && rerunnable === undefined ? "all" : "integration-heavy"; + try { - await io.rerun(latest.id); + await io.dispatchStack(pr.headRef, workloads); } catch (error) { - if (error instanceof GithubRequestError) { - return reply(replies.refused(pr.headSha, error.status, error.detail)); + if (!(error instanceof GithubRequestError)) throw error; + const missing = + error.status === 404 || (error.status === 422 && /workflow_dispatch/u.test(error.detail)); + return reply( + missing ? replies.noStackWorkflow() : replies.stackRefused(error.status, error.detail), + ); + } + + let rerunId: number | undefined; + if (needsLightweight && rerunnable !== undefined) { + try { + await io.rerun(rerunnable.id); + rerunId = rerunnable.id; + } catch (error) { + // The stack run is already under way; a refused lightweight re-run must not hide it. + console.warn( + `Could not re-run the lightweight benchmark run ${rerunnable.id}: ${String(error)}`, + ); } - throw error; } - // Cosmetic: a failed reaction must never hide an accepted re-run. + await react(io, comment.id); + return { kind: "dispatched", workloads, rerunId }; +} + +/** Cosmetic: a failed reaction must never hide an accepted command. */ +async function react(io: CommandIo, commentId: number): Promise { try { - await io.reactToComment(comment.id); + await io.reactToComment(commentId); } catch (error) { - console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); + console.warn(`Could not react to comment ${commentId}: ${String(error)}`); } - return { kind: "rerun", runId: latest.id }; } async function main(): Promise { @@ -160,13 +262,14 @@ async function main(): Promise { const value = (await (await githubFetch(`${base}/pulls/${n}`, token)).json()) as { state: "open" | "closed"; draft: boolean; - head: { sha: string; repo: { full_name: string } | null }; + head: { sha: string; ref: string; repo: { full_name: string } | null }; base: { ref: string; repo: { full_name: string } }; }; return { state: value.state, draft: value.draft, headSha: value.head.sha, + headRef: value.head.ref, headRepoFullName: value.head.repo?.full_name ?? "", baseRepoFullName: value.base.repo.full_name, baseRef: value.base.ref, @@ -187,6 +290,48 @@ async function main(): Promise { rerun: async (runId) => { await githubFetch(`${base}/actions/runs/${runId}/rerun`, token, { method: "POST" }); }, + listStackRuns: async (headRef) => { + const url = + `${base}/actions/workflows/${STACK_WORKFLOW_FILE}/runs` + + `?event=workflow_dispatch&branch=${encodeURIComponent(headRef)}&per_page=50`; + const payload = await githubFetch(url, token).then( + (response) => + response.json() as Promise<{ + workflow_runs: Array<{ + id: number; + status: string; + display_title: string; + html_url: string; + }>; + }>, + (error: unknown) => { + // A branch without the workflow file has no runs of it. + if (error instanceof GithubRequestError && error.status === 404) { + return { workflow_runs: [] }; + } + throw error; + }, + ); + return payload.workflow_runs.map((run) => ({ + id: run.id, + status: run.status, + displayTitle: run.display_title, + url: run.html_url, + })); + }, + dispatchStack: async (headRef, workloads) => { + await githubFetch(`${base}/actions/workflows/${STACK_WORKFLOW_FILE}/dispatches`, token, { + method: "POST", + body: JSON.stringify({ + ref: headRef, + inputs: { workloads, runtime: "both", base: "merge-base" }, + }), + }); + }, + lightweightSha: async (n) => { + const existing = findBenchComment(await listIssueComments(token, base, n)); + return existing === undefined ? undefined : parseSections(existing.body).lightweight?.sha; + }, reactToComment: async (commentId) => { await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { method: "POST", @@ -217,7 +362,11 @@ async function main(): Promise { const message = outcome.kind === "rerun" ? `Re-running benchmark run ${outcome.runId} for PR #${prNumber}.` - : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; + : outcome.kind === "dispatched" + ? `Dispatched the stack benchmark (${outcome.workloads}) for PR #${prNumber}${ + outcome.rerunId === undefined ? "" : ` and re-running benchmark run ${outcome.rerunId}` + }.` + : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; console.log(message); const summaryFile = process.env.GITHUB_STEP_SUMMARY; if (summaryFile) appendFileSync(summaryFile, `${message}\n`); diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 7ac55287b2..8fd2178c6b 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -1,13 +1,14 @@ import { describe, expect, test } from "bun:test"; -import { BENCH_MARKER, FAILURE_TITLE } from "./post-comment.ts"; +import { composeBody, failureMarkdown } from "./post-comment.ts"; import { resolveBench, SKIP_REASON } from "./resolve.ts"; const BOT = "github-actions[bot]"; -const reported = { +const lightweight = (content: string) => ({ id: 5, - body: `${BENCH_MARKER}\n## CLI benchmark: no changes found`, + body: composeBody({ lightweight: { sha: "abc1234", content } }), authorLogin: BOT, -}; +}); +const reported = lightweight("### Lightweight Commands: no changes found"); describe("resolveBench", () => { test("benchmarks the first push of a pull request that has no comment yet", () => { @@ -22,11 +23,28 @@ describe("resolveBench", () => { }); test("benchmarks again when the comment reports a failed run", () => { - const failed = { ...reported, body: `${BENCH_MARKER}\n${FAILURE_TITLE}` }; + const failed = lightweight(failureMarkdown("lightweight")); expect(resolveBench({ runAttempt: 1, isFork: false, comments: [failed] }).shouldRun).toBe(true); }); + test("benchmarks when the comment holds only an integration-heavy result", () => { + const stackOnly = { + id: 5, + body: composeBody({ + "integration-heavy": { + sha: "abc1234", + content: "### Integration Heavy Commands: no changes found", + }, + }), + authorLogin: BOT, + }; + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [stackOnly] }).shouldRun).toBe( + true, + ); + }); + test("benchmarks every re-run even when the comment reports a result", () => { expect(resolveBench({ runAttempt: 2, isFork: false, comments: [reported] }).shouldRun).toBe( true, diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index 03882d2450..b59c278ba7 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -1,15 +1,15 @@ /** * CLI benchmark resolver for `pull_request` runs: decides whether the benchmark job runs. * - * The first run on a pull request benchmarks and comments. Later pushes skip while the comment - * exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first + * The first run on a pull request benchmarks and comments. Later pushes skip while the comment's + * Lightweight Commands section exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first * run, or a fork pull request, which never has a comment, benchmarks. Needs read access only. * Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. */ import { appendFileSync } from "node:fs"; import { listIssueComments, requireEnv, type IssueComment } from "./github.ts"; -import { FAILURE_TITLE, findBenchComment } from "./post-comment.ts"; +import { findBenchComment, parseSections, sectionFailed } from "./post-comment.ts"; export interface ResolveInput { runAttempt: number; @@ -30,7 +30,8 @@ export const SKIP_REASON = export function resolveBench(input: ResolveInput): ResolveResult { if (input.isFork || input.runAttempt > 1) return { shouldRun: true }; const existing = findBenchComment(input.comments); - if (existing === undefined || existing.body.includes(FAILURE_TITLE)) return { shouldRun: true }; + const section = existing === undefined ? undefined : parseSections(existing.body).lightweight; + if (section === undefined || sectionFailed(section.content)) return { shouldRun: true }; return { shouldRun: false, skipReason: SKIP_REASON }; } diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 73f76a6ac5..cc4449680d 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -8,10 +8,12 @@ run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.i # The first push to a pull request from this repository benchmarks and posts one comment tagged # ``; later pushes skip while that comment reports a result. A maintainer # commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does -# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment, keeping -# earlier runs in a collapsed list. Pull requests from forks benchmark every push and write a job -# summary only. A pull request that changes no benchmarked path, or more than the 300 files -# GitHub's path filter inspects, has no run to re-run, and `/bench` says so. +# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment's +# Lightweight Commands section, keeping earlier runs in a collapsed list. A maintainer commenting +# exactly `/bench full` also dispatches cli-bench-stack.yml on the pull request's branch to +# re-measure the Integration Heavy Commands section. Pull requests from forks benchmark every push +# and write a job summary only. A pull request that changes no benchmarked path, or more than the +# 300 files GitHub's path filter inspects, has no run to re-run, and `/bench` says so. # See apps/cli/docs/tracing-monitoring.md. on: pull_request: @@ -30,7 +32,8 @@ on: - "turbo.json" - ".github/workflows/cli-bench.yml" - ".github/scripts/cli-bench/**" - # `/bench` runs default branch code only: it never checks out or runs pull request code. + # `/bench` and `/bench full` run default branch code only: they never check out or run pull + # request code. # jactionlint ignore=dangerous-triggers issue_comment: types: @@ -76,8 +79,9 @@ jobs: RUN_ATTEMPT: ${{ github.run_attempt }} run: bun .github/scripts/cli-bench/resolve.ts - # `/bench`: re-runs this workflow's run for the pull request's current head commit. The only job - # with `actions: write`; it runs default branch code and no pull request code. + # `/bench`: re-runs this workflow's run for the pull request's current head commit. `/bench full` + # also dispatches the stack benchmark workflow on the pull request's branch. The only job with + # `actions: write`; it runs default branch code and no pull request code. rerun: name: Re-run benchmark # Cheap, non-authoritative pre-filter; rerun.ts does the exact command match and the @@ -290,6 +294,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} MODE: ${{ github.run_attempt > 1 && 'upsert' || 'create-if-missing' }} + SECTION: lightweight HEAD_SHA: ${{ github.event.pull_request.head.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} From 88db0a9d908ce42f5d1673a6488eb06afff4d9b0 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 16:59:41 +0530 Subject: [PATCH 02/11] feat(cli): benchmark stack workloads with setup, teardown, and container counts The benchmark harness runs workloads that need a local stack: untimed setup and teardown shared per run or per block of launches, a stack switch, a runtime, fixtures, a shared native artifact cache, per-workload run counts, and tracing limits. Reports add container operation counts, a runtime column, a section name, shard packing, and a report merge. Every launch disables product telemetry. The stack manifest and fixtures are generated and committed. Co-Authored-By: Claude Opus 5.5 --- .../migrations/20260101000001_bench_01.sql | 7 + .../migrations/20260101000002_bench_02.sql | 6 + .../migrations/20260101000003_bench_03.sql | 6 + .../migrations/20260101000004_bench_04.sql | 6 + .../migrations/20260101000005_bench_05.sql | 6 + .../migrations/20260101000006_bench_06.sql | 6 + .../migrations/20260101000007_bench_07.sql | 6 + .../migrations/20260101000008_bench_08.sql | 6 + .../migrations/20260101000009_bench_09.sql | 6 + .../migrations/20260101000010_bench_10.sql | 6 + .../migrations/20260101000011_bench_11.sql | 6 + .../migrations/20260101000012_bench_12.sql | 6 + .../migrations/20260101000013_bench_13.sql | 6 + .../migrations/20260101000014_bench_14.sql | 6 + .../migrations/20260101000015_bench_15.sql | 6 + .../migrations/20260101000016_bench_16.sql | 6 + .../migrations/20260101000017_bench_17.sql | 6 + .../migrations/20260101000018_bench_18.sql | 6 + .../migrations/20260101000019_bench_19.sql | 6 + .../migrations/20260101000020_bench_20.sql | 6 + apps/cli/bench/fixtures/migrations/seed.sql | 2 + .../20260101000001_bench_checks.sql | 4 + .../pgtap/tests/database/bench.test.sql | 8 + .../20260101000001_bench_schema.sql | 2779 +++++++++++++++++ .../20260101000001_bench_schema.sql | 694 ++++ .../20260101000001_bench_schema.sql | 138 + apps/cli/bench/workloads.stack.json | 385 +++ .../cli/scripts/bench-cli.integration.test.ts | 306 +- apps/cli/scripts/bench-cli.ts | 484 ++- apps/cli/scripts/bench-cli.unit.test.ts | 68 + apps/cli/scripts/bench-fixtures.ts | 130 + apps/cli/scripts/bench-merge.ts | 74 + apps/cli/scripts/bench-pack.ts | 68 + apps/cli/scripts/bench-pack.unit.test.ts | 65 + apps/cli/scripts/bench-report.ts | 169 +- apps/cli/scripts/bench-stack.unit.test.ts | 226 ++ apps/cli/scripts/bench-summary.ts | 152 +- apps/cli/scripts/bench-summary.unit.test.ts | 133 +- apps/cli/scripts/bench-workloads.unit.test.ts | 60 + 39 files changed, 5905 insertions(+), 161 deletions(-) create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql create mode 100644 apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql create mode 100644 apps/cli/bench/fixtures/migrations/seed.sql create mode 100644 apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql create mode 100644 apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql create mode 100644 apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql create mode 100644 apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql create mode 100644 apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql create mode 100644 apps/cli/bench/workloads.stack.json create mode 100644 apps/cli/scripts/bench-fixtures.ts create mode 100644 apps/cli/scripts/bench-merge.ts create mode 100644 apps/cli/scripts/bench-pack.ts create mode 100644 apps/cli/scripts/bench-pack.unit.test.ts create mode 100644 apps/cli/scripts/bench-stack.unit.test.ts diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql new file mode 100644 index 0000000000..91673d12aa --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql @@ -0,0 +1,7 @@ +create table public.bench_items ( + id bigint generated always as identity primary key, + label text not null, + amount numeric not null, + created_at timestamptz not null default now() +); +create index bench_items_label_idx on public.bench_items (label); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql new file mode 100644 index 0000000000..a50e4ce1d3 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql @@ -0,0 +1,6 @@ +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_002_item_idx on public.bench_table_002 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql new file mode 100644 index 0000000000..5b68d67841 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql @@ -0,0 +1,6 @@ +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_003_item_idx on public.bench_table_003 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql new file mode 100644 index 0000000000..feff4efd96 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql @@ -0,0 +1,6 @@ +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_004_item_idx on public.bench_table_004 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql new file mode 100644 index 0000000000..c1431ae655 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql @@ -0,0 +1,6 @@ +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_005_item_idx on public.bench_table_005 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql new file mode 100644 index 0000000000..8a954b1805 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql @@ -0,0 +1,6 @@ +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_006_item_idx on public.bench_table_006 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql new file mode 100644 index 0000000000..4cee582f77 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql @@ -0,0 +1,6 @@ +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_007_item_idx on public.bench_table_007 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql new file mode 100644 index 0000000000..f6198057b0 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql @@ -0,0 +1,6 @@ +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_008_item_idx on public.bench_table_008 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql new file mode 100644 index 0000000000..a9ec7f6083 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql @@ -0,0 +1,6 @@ +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_009_item_idx on public.bench_table_009 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql new file mode 100644 index 0000000000..1e9819111b --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql @@ -0,0 +1,6 @@ +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_010_item_idx on public.bench_table_010 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql new file mode 100644 index 0000000000..19837f3ddf --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql @@ -0,0 +1,6 @@ +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_011_item_idx on public.bench_table_011 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql new file mode 100644 index 0000000000..5877718517 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql @@ -0,0 +1,6 @@ +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_012_item_idx on public.bench_table_012 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql new file mode 100644 index 0000000000..4db2608229 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql @@ -0,0 +1,6 @@ +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_013_item_idx on public.bench_table_013 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql new file mode 100644 index 0000000000..7b32a4dcc7 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql @@ -0,0 +1,6 @@ +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_014_item_idx on public.bench_table_014 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql new file mode 100644 index 0000000000..68f9dd68d8 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql @@ -0,0 +1,6 @@ +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_015_item_idx on public.bench_table_015 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql new file mode 100644 index 0000000000..2c56106853 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql @@ -0,0 +1,6 @@ +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_016_item_idx on public.bench_table_016 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql new file mode 100644 index 0000000000..9064dd0f34 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql @@ -0,0 +1,6 @@ +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_017_item_idx on public.bench_table_017 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql new file mode 100644 index 0000000000..285e5f0b42 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql @@ -0,0 +1,6 @@ +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_018_item_idx on public.bench_table_018 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql new file mode 100644 index 0000000000..f899cc45ac --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql @@ -0,0 +1,6 @@ +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_019_item_idx on public.bench_table_019 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql new file mode 100644 index 0000000000..5cc1eef527 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql @@ -0,0 +1,6 @@ +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_020_item_idx on public.bench_table_020 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/seed.sql b/apps/cli/bench/fixtures/migrations/seed.sql new file mode 100644 index 0000000000..a557250dd3 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/seed.sql @@ -0,0 +1,2 @@ +insert into public.bench_items (label, amount) +select 'item-' || n, (n % 97) * 1.5 from generate_series(1, 1000) as n; diff --git a/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql b/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql new file mode 100644 index 0000000000..93dd159907 --- /dev/null +++ b/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql @@ -0,0 +1,4 @@ +create table public.bench_checks ( + id bigint generated always as identity primary key, + label text not null +); diff --git a/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql b/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql new file mode 100644 index 0000000000..4b1b9687f0 --- /dev/null +++ b/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql @@ -0,0 +1,8 @@ +begin; +select plan(4); +select has_table('public', 'bench_checks', 'bench_checks exists'); +select has_column('public', 'bench_checks', 'label', 'label column exists'); +select col_not_null('public', 'bench_checks', 'label', 'label is required'); +select col_is_pk('public', 'bench_checks', 'id', 'id is the primary key'); +select * from finish(); +rollback; diff --git a/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..1623748429 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,2779 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; + +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_011_owner_idx on public.bench_table_011 (owner_id); +create index bench_table_011_status_idx on public.bench_table_011 (status, created_at); +alter table public.bench_table_011 enable row level security; +create policy bench_table_011_read on public.bench_table_011 for select using (true); + +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_012_owner_idx on public.bench_table_012 (owner_id); +create index bench_table_012_status_idx on public.bench_table_012 (status, created_at); +alter table public.bench_table_012 enable row level security; +create policy bench_table_012_read on public.bench_table_012 for select using (true); + +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_013_owner_idx on public.bench_table_013 (owner_id); +create index bench_table_013_status_idx on public.bench_table_013 (status, created_at); +alter table public.bench_table_013 enable row level security; +create policy bench_table_013_read on public.bench_table_013 for select using (true); + +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_014_owner_idx on public.bench_table_014 (owner_id); +create index bench_table_014_status_idx on public.bench_table_014 (status, created_at); +alter table public.bench_table_014 enable row level security; +create policy bench_table_014_read on public.bench_table_014 for select using (true); + +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_015_owner_idx on public.bench_table_015 (owner_id); +create index bench_table_015_status_idx on public.bench_table_015 (status, created_at); +alter table public.bench_table_015 enable row level security; +create policy bench_table_015_read on public.bench_table_015 for select using (true); + +create view public.bench_table_015_summary as + select status, count(*) as total from public.bench_table_015 group by status; + +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_016_owner_idx on public.bench_table_016 (owner_id); +create index bench_table_016_status_idx on public.bench_table_016 (status, created_at); +alter table public.bench_table_016 enable row level security; +create policy bench_table_016_read on public.bench_table_016 for select using (true); + +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_017_owner_idx on public.bench_table_017 (owner_id); +create index bench_table_017_status_idx on public.bench_table_017 (status, created_at); +alter table public.bench_table_017 enable row level security; +create policy bench_table_017_read on public.bench_table_017 for select using (true); + +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_018_owner_idx on public.bench_table_018 (owner_id); +create index bench_table_018_status_idx on public.bench_table_018 (status, created_at); +alter table public.bench_table_018 enable row level security; +create policy bench_table_018_read on public.bench_table_018 for select using (true); + +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_019_owner_idx on public.bench_table_019 (owner_id); +create index bench_table_019_status_idx on public.bench_table_019 (status, created_at); +alter table public.bench_table_019 enable row level security; +create policy bench_table_019_read on public.bench_table_019 for select using (true); + +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_020_owner_idx on public.bench_table_020 (owner_id); +create index bench_table_020_status_idx on public.bench_table_020 (status, created_at); +alter table public.bench_table_020 enable row level security; +create policy bench_table_020_read on public.bench_table_020 for select using (true); + +create view public.bench_table_020_summary as + select status, count(*) as total from public.bench_table_020 group by status; + +create function public.bench_table_020_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_020 $$; + +create table public.bench_table_021 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_021_owner_idx on public.bench_table_021 (owner_id); +create index bench_table_021_status_idx on public.bench_table_021 (status, created_at); +alter table public.bench_table_021 enable row level security; +create policy bench_table_021_read on public.bench_table_021 for select using (true); + +create table public.bench_table_022 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_022_owner_idx on public.bench_table_022 (owner_id); +create index bench_table_022_status_idx on public.bench_table_022 (status, created_at); +alter table public.bench_table_022 enable row level security; +create policy bench_table_022_read on public.bench_table_022 for select using (true); + +create table public.bench_table_023 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_023_owner_idx on public.bench_table_023 (owner_id); +create index bench_table_023_status_idx on public.bench_table_023 (status, created_at); +alter table public.bench_table_023 enable row level security; +create policy bench_table_023_read on public.bench_table_023 for select using (true); + +create table public.bench_table_024 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_024_owner_idx on public.bench_table_024 (owner_id); +create index bench_table_024_status_idx on public.bench_table_024 (status, created_at); +alter table public.bench_table_024 enable row level security; +create policy bench_table_024_read on public.bench_table_024 for select using (true); + +create table public.bench_table_025 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_025_owner_idx on public.bench_table_025 (owner_id); +create index bench_table_025_status_idx on public.bench_table_025 (status, created_at); +alter table public.bench_table_025 enable row level security; +create policy bench_table_025_read on public.bench_table_025 for select using (true); + +create view public.bench_table_025_summary as + select status, count(*) as total from public.bench_table_025 group by status; + +create table public.bench_table_026 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_026_owner_idx on public.bench_table_026 (owner_id); +create index bench_table_026_status_idx on public.bench_table_026 (status, created_at); +alter table public.bench_table_026 enable row level security; +create policy bench_table_026_read on public.bench_table_026 for select using (true); + +create table public.bench_table_027 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_027_owner_idx on public.bench_table_027 (owner_id); +create index bench_table_027_status_idx on public.bench_table_027 (status, created_at); +alter table public.bench_table_027 enable row level security; +create policy bench_table_027_read on public.bench_table_027 for select using (true); + +create table public.bench_table_028 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_028_owner_idx on public.bench_table_028 (owner_id); +create index bench_table_028_status_idx on public.bench_table_028 (status, created_at); +alter table public.bench_table_028 enable row level security; +create policy bench_table_028_read on public.bench_table_028 for select using (true); + +create table public.bench_table_029 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_029_owner_idx on public.bench_table_029 (owner_id); +create index bench_table_029_status_idx on public.bench_table_029 (status, created_at); +alter table public.bench_table_029 enable row level security; +create policy bench_table_029_read on public.bench_table_029 for select using (true); + +create table public.bench_table_030 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_030_owner_idx on public.bench_table_030 (owner_id); +create index bench_table_030_status_idx on public.bench_table_030 (status, created_at); +alter table public.bench_table_030 enable row level security; +create policy bench_table_030_read on public.bench_table_030 for select using (true); + +create view public.bench_table_030_summary as + select status, count(*) as total from public.bench_table_030 group by status; + +create function public.bench_table_030_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_030 $$; + +create table public.bench_table_031 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_031_owner_idx on public.bench_table_031 (owner_id); +create index bench_table_031_status_idx on public.bench_table_031 (status, created_at); +alter table public.bench_table_031 enable row level security; +create policy bench_table_031_read on public.bench_table_031 for select using (true); + +create table public.bench_table_032 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_032_owner_idx on public.bench_table_032 (owner_id); +create index bench_table_032_status_idx on public.bench_table_032 (status, created_at); +alter table public.bench_table_032 enable row level security; +create policy bench_table_032_read on public.bench_table_032 for select using (true); + +create table public.bench_table_033 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_033_owner_idx on public.bench_table_033 (owner_id); +create index bench_table_033_status_idx on public.bench_table_033 (status, created_at); +alter table public.bench_table_033 enable row level security; +create policy bench_table_033_read on public.bench_table_033 for select using (true); + +create table public.bench_table_034 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_034_owner_idx on public.bench_table_034 (owner_id); +create index bench_table_034_status_idx on public.bench_table_034 (status, created_at); +alter table public.bench_table_034 enable row level security; +create policy bench_table_034_read on public.bench_table_034 for select using (true); + +create table public.bench_table_035 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_035_owner_idx on public.bench_table_035 (owner_id); +create index bench_table_035_status_idx on public.bench_table_035 (status, created_at); +alter table public.bench_table_035 enable row level security; +create policy bench_table_035_read on public.bench_table_035 for select using (true); + +create view public.bench_table_035_summary as + select status, count(*) as total from public.bench_table_035 group by status; + +create table public.bench_table_036 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_036_owner_idx on public.bench_table_036 (owner_id); +create index bench_table_036_status_idx on public.bench_table_036 (status, created_at); +alter table public.bench_table_036 enable row level security; +create policy bench_table_036_read on public.bench_table_036 for select using (true); + +create table public.bench_table_037 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_037_owner_idx on public.bench_table_037 (owner_id); +create index bench_table_037_status_idx on public.bench_table_037 (status, created_at); +alter table public.bench_table_037 enable row level security; +create policy bench_table_037_read on public.bench_table_037 for select using (true); + +create table public.bench_table_038 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_038_owner_idx on public.bench_table_038 (owner_id); +create index bench_table_038_status_idx on public.bench_table_038 (status, created_at); +alter table public.bench_table_038 enable row level security; +create policy bench_table_038_read on public.bench_table_038 for select using (true); + +create table public.bench_table_039 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_039_owner_idx on public.bench_table_039 (owner_id); +create index bench_table_039_status_idx on public.bench_table_039 (status, created_at); +alter table public.bench_table_039 enable row level security; +create policy bench_table_039_read on public.bench_table_039 for select using (true); + +create table public.bench_table_040 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_040_owner_idx on public.bench_table_040 (owner_id); +create index bench_table_040_status_idx on public.bench_table_040 (status, created_at); +alter table public.bench_table_040 enable row level security; +create policy bench_table_040_read on public.bench_table_040 for select using (true); + +create view public.bench_table_040_summary as + select status, count(*) as total from public.bench_table_040 group by status; + +create function public.bench_table_040_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_040 $$; + +create table public.bench_table_041 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_041_owner_idx on public.bench_table_041 (owner_id); +create index bench_table_041_status_idx on public.bench_table_041 (status, created_at); +alter table public.bench_table_041 enable row level security; +create policy bench_table_041_read on public.bench_table_041 for select using (true); + +create table public.bench_table_042 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_042_owner_idx on public.bench_table_042 (owner_id); +create index bench_table_042_status_idx on public.bench_table_042 (status, created_at); +alter table public.bench_table_042 enable row level security; +create policy bench_table_042_read on public.bench_table_042 for select using (true); + +create table public.bench_table_043 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_043_owner_idx on public.bench_table_043 (owner_id); +create index bench_table_043_status_idx on public.bench_table_043 (status, created_at); +alter table public.bench_table_043 enable row level security; +create policy bench_table_043_read on public.bench_table_043 for select using (true); + +create table public.bench_table_044 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_044_owner_idx on public.bench_table_044 (owner_id); +create index bench_table_044_status_idx on public.bench_table_044 (status, created_at); +alter table public.bench_table_044 enable row level security; +create policy bench_table_044_read on public.bench_table_044 for select using (true); + +create table public.bench_table_045 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_045_owner_idx on public.bench_table_045 (owner_id); +create index bench_table_045_status_idx on public.bench_table_045 (status, created_at); +alter table public.bench_table_045 enable row level security; +create policy bench_table_045_read on public.bench_table_045 for select using (true); + +create view public.bench_table_045_summary as + select status, count(*) as total from public.bench_table_045 group by status; + +create table public.bench_table_046 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_046_owner_idx on public.bench_table_046 (owner_id); +create index bench_table_046_status_idx on public.bench_table_046 (status, created_at); +alter table public.bench_table_046 enable row level security; +create policy bench_table_046_read on public.bench_table_046 for select using (true); + +create table public.bench_table_047 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_047_owner_idx on public.bench_table_047 (owner_id); +create index bench_table_047_status_idx on public.bench_table_047 (status, created_at); +alter table public.bench_table_047 enable row level security; +create policy bench_table_047_read on public.bench_table_047 for select using (true); + +create table public.bench_table_048 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_048_owner_idx on public.bench_table_048 (owner_id); +create index bench_table_048_status_idx on public.bench_table_048 (status, created_at); +alter table public.bench_table_048 enable row level security; +create policy bench_table_048_read on public.bench_table_048 for select using (true); + +create table public.bench_table_049 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_049_owner_idx on public.bench_table_049 (owner_id); +create index bench_table_049_status_idx on public.bench_table_049 (status, created_at); +alter table public.bench_table_049 enable row level security; +create policy bench_table_049_read on public.bench_table_049 for select using (true); + +create table public.bench_table_050 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_050_owner_idx on public.bench_table_050 (owner_id); +create index bench_table_050_status_idx on public.bench_table_050 (status, created_at); +alter table public.bench_table_050 enable row level security; +create policy bench_table_050_read on public.bench_table_050 for select using (true); + +create view public.bench_table_050_summary as + select status, count(*) as total from public.bench_table_050 group by status; + +create function public.bench_table_050_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_050 $$; + +create table public.bench_table_051 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_051_owner_idx on public.bench_table_051 (owner_id); +create index bench_table_051_status_idx on public.bench_table_051 (status, created_at); +alter table public.bench_table_051 enable row level security; +create policy bench_table_051_read on public.bench_table_051 for select using (true); + +create table public.bench_table_052 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_052_owner_idx on public.bench_table_052 (owner_id); +create index bench_table_052_status_idx on public.bench_table_052 (status, created_at); +alter table public.bench_table_052 enable row level security; +create policy bench_table_052_read on public.bench_table_052 for select using (true); + +create table public.bench_table_053 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_053_owner_idx on public.bench_table_053 (owner_id); +create index bench_table_053_status_idx on public.bench_table_053 (status, created_at); +alter table public.bench_table_053 enable row level security; +create policy bench_table_053_read on public.bench_table_053 for select using (true); + +create table public.bench_table_054 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_054_owner_idx on public.bench_table_054 (owner_id); +create index bench_table_054_status_idx on public.bench_table_054 (status, created_at); +alter table public.bench_table_054 enable row level security; +create policy bench_table_054_read on public.bench_table_054 for select using (true); + +create table public.bench_table_055 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_055_owner_idx on public.bench_table_055 (owner_id); +create index bench_table_055_status_idx on public.bench_table_055 (status, created_at); +alter table public.bench_table_055 enable row level security; +create policy bench_table_055_read on public.bench_table_055 for select using (true); + +create view public.bench_table_055_summary as + select status, count(*) as total from public.bench_table_055 group by status; + +create table public.bench_table_056 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_056_owner_idx on public.bench_table_056 (owner_id); +create index bench_table_056_status_idx on public.bench_table_056 (status, created_at); +alter table public.bench_table_056 enable row level security; +create policy bench_table_056_read on public.bench_table_056 for select using (true); + +create table public.bench_table_057 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_057_owner_idx on public.bench_table_057 (owner_id); +create index bench_table_057_status_idx on public.bench_table_057 (status, created_at); +alter table public.bench_table_057 enable row level security; +create policy bench_table_057_read on public.bench_table_057 for select using (true); + +create table public.bench_table_058 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_058_owner_idx on public.bench_table_058 (owner_id); +create index bench_table_058_status_idx on public.bench_table_058 (status, created_at); +alter table public.bench_table_058 enable row level security; +create policy bench_table_058_read on public.bench_table_058 for select using (true); + +create table public.bench_table_059 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_059_owner_idx on public.bench_table_059 (owner_id); +create index bench_table_059_status_idx on public.bench_table_059 (status, created_at); +alter table public.bench_table_059 enable row level security; +create policy bench_table_059_read on public.bench_table_059 for select using (true); + +create table public.bench_table_060 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_060_owner_idx on public.bench_table_060 (owner_id); +create index bench_table_060_status_idx on public.bench_table_060 (status, created_at); +alter table public.bench_table_060 enable row level security; +create policy bench_table_060_read on public.bench_table_060 for select using (true); + +create view public.bench_table_060_summary as + select status, count(*) as total from public.bench_table_060 group by status; + +create function public.bench_table_060_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_060 $$; + +create table public.bench_table_061 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_061_owner_idx on public.bench_table_061 (owner_id); +create index bench_table_061_status_idx on public.bench_table_061 (status, created_at); +alter table public.bench_table_061 enable row level security; +create policy bench_table_061_read on public.bench_table_061 for select using (true); + +create table public.bench_table_062 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_062_owner_idx on public.bench_table_062 (owner_id); +create index bench_table_062_status_idx on public.bench_table_062 (status, created_at); +alter table public.bench_table_062 enable row level security; +create policy bench_table_062_read on public.bench_table_062 for select using (true); + +create table public.bench_table_063 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_063_owner_idx on public.bench_table_063 (owner_id); +create index bench_table_063_status_idx on public.bench_table_063 (status, created_at); +alter table public.bench_table_063 enable row level security; +create policy bench_table_063_read on public.bench_table_063 for select using (true); + +create table public.bench_table_064 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_064_owner_idx on public.bench_table_064 (owner_id); +create index bench_table_064_status_idx on public.bench_table_064 (status, created_at); +alter table public.bench_table_064 enable row level security; +create policy bench_table_064_read on public.bench_table_064 for select using (true); + +create table public.bench_table_065 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_065_owner_idx on public.bench_table_065 (owner_id); +create index bench_table_065_status_idx on public.bench_table_065 (status, created_at); +alter table public.bench_table_065 enable row level security; +create policy bench_table_065_read on public.bench_table_065 for select using (true); + +create view public.bench_table_065_summary as + select status, count(*) as total from public.bench_table_065 group by status; + +create table public.bench_table_066 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_066_owner_idx on public.bench_table_066 (owner_id); +create index bench_table_066_status_idx on public.bench_table_066 (status, created_at); +alter table public.bench_table_066 enable row level security; +create policy bench_table_066_read on public.bench_table_066 for select using (true); + +create table public.bench_table_067 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_067_owner_idx on public.bench_table_067 (owner_id); +create index bench_table_067_status_idx on public.bench_table_067 (status, created_at); +alter table public.bench_table_067 enable row level security; +create policy bench_table_067_read on public.bench_table_067 for select using (true); + +create table public.bench_table_068 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_068_owner_idx on public.bench_table_068 (owner_id); +create index bench_table_068_status_idx on public.bench_table_068 (status, created_at); +alter table public.bench_table_068 enable row level security; +create policy bench_table_068_read on public.bench_table_068 for select using (true); + +create table public.bench_table_069 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_069_owner_idx on public.bench_table_069 (owner_id); +create index bench_table_069_status_idx on public.bench_table_069 (status, created_at); +alter table public.bench_table_069 enable row level security; +create policy bench_table_069_read on public.bench_table_069 for select using (true); + +create table public.bench_table_070 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_070_owner_idx on public.bench_table_070 (owner_id); +create index bench_table_070_status_idx on public.bench_table_070 (status, created_at); +alter table public.bench_table_070 enable row level security; +create policy bench_table_070_read on public.bench_table_070 for select using (true); + +create view public.bench_table_070_summary as + select status, count(*) as total from public.bench_table_070 group by status; + +create function public.bench_table_070_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_070 $$; + +create table public.bench_table_071 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_071_owner_idx on public.bench_table_071 (owner_id); +create index bench_table_071_status_idx on public.bench_table_071 (status, created_at); +alter table public.bench_table_071 enable row level security; +create policy bench_table_071_read on public.bench_table_071 for select using (true); + +create table public.bench_table_072 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_072_owner_idx on public.bench_table_072 (owner_id); +create index bench_table_072_status_idx on public.bench_table_072 (status, created_at); +alter table public.bench_table_072 enable row level security; +create policy bench_table_072_read on public.bench_table_072 for select using (true); + +create table public.bench_table_073 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_073_owner_idx on public.bench_table_073 (owner_id); +create index bench_table_073_status_idx on public.bench_table_073 (status, created_at); +alter table public.bench_table_073 enable row level security; +create policy bench_table_073_read on public.bench_table_073 for select using (true); + +create table public.bench_table_074 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_074_owner_idx on public.bench_table_074 (owner_id); +create index bench_table_074_status_idx on public.bench_table_074 (status, created_at); +alter table public.bench_table_074 enable row level security; +create policy bench_table_074_read on public.bench_table_074 for select using (true); + +create table public.bench_table_075 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_075_owner_idx on public.bench_table_075 (owner_id); +create index bench_table_075_status_idx on public.bench_table_075 (status, created_at); +alter table public.bench_table_075 enable row level security; +create policy bench_table_075_read on public.bench_table_075 for select using (true); + +create view public.bench_table_075_summary as + select status, count(*) as total from public.bench_table_075 group by status; + +create table public.bench_table_076 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_076_owner_idx on public.bench_table_076 (owner_id); +create index bench_table_076_status_idx on public.bench_table_076 (status, created_at); +alter table public.bench_table_076 enable row level security; +create policy bench_table_076_read on public.bench_table_076 for select using (true); + +create table public.bench_table_077 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_077_owner_idx on public.bench_table_077 (owner_id); +create index bench_table_077_status_idx on public.bench_table_077 (status, created_at); +alter table public.bench_table_077 enable row level security; +create policy bench_table_077_read on public.bench_table_077 for select using (true); + +create table public.bench_table_078 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_078_owner_idx on public.bench_table_078 (owner_id); +create index bench_table_078_status_idx on public.bench_table_078 (status, created_at); +alter table public.bench_table_078 enable row level security; +create policy bench_table_078_read on public.bench_table_078 for select using (true); + +create table public.bench_table_079 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_079_owner_idx on public.bench_table_079 (owner_id); +create index bench_table_079_status_idx on public.bench_table_079 (status, created_at); +alter table public.bench_table_079 enable row level security; +create policy bench_table_079_read on public.bench_table_079 for select using (true); + +create table public.bench_table_080 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_080_owner_idx on public.bench_table_080 (owner_id); +create index bench_table_080_status_idx on public.bench_table_080 (status, created_at); +alter table public.bench_table_080 enable row level security; +create policy bench_table_080_read on public.bench_table_080 for select using (true); + +create view public.bench_table_080_summary as + select status, count(*) as total from public.bench_table_080 group by status; + +create function public.bench_table_080_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_080 $$; + +create table public.bench_table_081 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_081_owner_idx on public.bench_table_081 (owner_id); +create index bench_table_081_status_idx on public.bench_table_081 (status, created_at); +alter table public.bench_table_081 enable row level security; +create policy bench_table_081_read on public.bench_table_081 for select using (true); + +create table public.bench_table_082 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_082_owner_idx on public.bench_table_082 (owner_id); +create index bench_table_082_status_idx on public.bench_table_082 (status, created_at); +alter table public.bench_table_082 enable row level security; +create policy bench_table_082_read on public.bench_table_082 for select using (true); + +create table public.bench_table_083 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_083_owner_idx on public.bench_table_083 (owner_id); +create index bench_table_083_status_idx on public.bench_table_083 (status, created_at); +alter table public.bench_table_083 enable row level security; +create policy bench_table_083_read on public.bench_table_083 for select using (true); + +create table public.bench_table_084 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_084_owner_idx on public.bench_table_084 (owner_id); +create index bench_table_084_status_idx on public.bench_table_084 (status, created_at); +alter table public.bench_table_084 enable row level security; +create policy bench_table_084_read on public.bench_table_084 for select using (true); + +create table public.bench_table_085 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_085_owner_idx on public.bench_table_085 (owner_id); +create index bench_table_085_status_idx on public.bench_table_085 (status, created_at); +alter table public.bench_table_085 enable row level security; +create policy bench_table_085_read on public.bench_table_085 for select using (true); + +create view public.bench_table_085_summary as + select status, count(*) as total from public.bench_table_085 group by status; + +create table public.bench_table_086 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_086_owner_idx on public.bench_table_086 (owner_id); +create index bench_table_086_status_idx on public.bench_table_086 (status, created_at); +alter table public.bench_table_086 enable row level security; +create policy bench_table_086_read on public.bench_table_086 for select using (true); + +create table public.bench_table_087 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_087_owner_idx on public.bench_table_087 (owner_id); +create index bench_table_087_status_idx on public.bench_table_087 (status, created_at); +alter table public.bench_table_087 enable row level security; +create policy bench_table_087_read on public.bench_table_087 for select using (true); + +create table public.bench_table_088 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_088_owner_idx on public.bench_table_088 (owner_id); +create index bench_table_088_status_idx on public.bench_table_088 (status, created_at); +alter table public.bench_table_088 enable row level security; +create policy bench_table_088_read on public.bench_table_088 for select using (true); + +create table public.bench_table_089 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_089_owner_idx on public.bench_table_089 (owner_id); +create index bench_table_089_status_idx on public.bench_table_089 (status, created_at); +alter table public.bench_table_089 enable row level security; +create policy bench_table_089_read on public.bench_table_089 for select using (true); + +create table public.bench_table_090 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_090_owner_idx on public.bench_table_090 (owner_id); +create index bench_table_090_status_idx on public.bench_table_090 (status, created_at); +alter table public.bench_table_090 enable row level security; +create policy bench_table_090_read on public.bench_table_090 for select using (true); + +create view public.bench_table_090_summary as + select status, count(*) as total from public.bench_table_090 group by status; + +create function public.bench_table_090_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_090 $$; + +create table public.bench_table_091 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_091_owner_idx on public.bench_table_091 (owner_id); +create index bench_table_091_status_idx on public.bench_table_091 (status, created_at); +alter table public.bench_table_091 enable row level security; +create policy bench_table_091_read on public.bench_table_091 for select using (true); + +create table public.bench_table_092 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_092_owner_idx on public.bench_table_092 (owner_id); +create index bench_table_092_status_idx on public.bench_table_092 (status, created_at); +alter table public.bench_table_092 enable row level security; +create policy bench_table_092_read on public.bench_table_092 for select using (true); + +create table public.bench_table_093 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_093_owner_idx on public.bench_table_093 (owner_id); +create index bench_table_093_status_idx on public.bench_table_093 (status, created_at); +alter table public.bench_table_093 enable row level security; +create policy bench_table_093_read on public.bench_table_093 for select using (true); + +create table public.bench_table_094 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_094_owner_idx on public.bench_table_094 (owner_id); +create index bench_table_094_status_idx on public.bench_table_094 (status, created_at); +alter table public.bench_table_094 enable row level security; +create policy bench_table_094_read on public.bench_table_094 for select using (true); + +create table public.bench_table_095 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_095_owner_idx on public.bench_table_095 (owner_id); +create index bench_table_095_status_idx on public.bench_table_095 (status, created_at); +alter table public.bench_table_095 enable row level security; +create policy bench_table_095_read on public.bench_table_095 for select using (true); + +create view public.bench_table_095_summary as + select status, count(*) as total from public.bench_table_095 group by status; + +create table public.bench_table_096 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_096_owner_idx on public.bench_table_096 (owner_id); +create index bench_table_096_status_idx on public.bench_table_096 (status, created_at); +alter table public.bench_table_096 enable row level security; +create policy bench_table_096_read on public.bench_table_096 for select using (true); + +create table public.bench_table_097 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_097_owner_idx on public.bench_table_097 (owner_id); +create index bench_table_097_status_idx on public.bench_table_097 (status, created_at); +alter table public.bench_table_097 enable row level security; +create policy bench_table_097_read on public.bench_table_097 for select using (true); + +create table public.bench_table_098 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_098_owner_idx on public.bench_table_098 (owner_id); +create index bench_table_098_status_idx on public.bench_table_098 (status, created_at); +alter table public.bench_table_098 enable row level security; +create policy bench_table_098_read on public.bench_table_098 for select using (true); + +create table public.bench_table_099 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_099_owner_idx on public.bench_table_099 (owner_id); +create index bench_table_099_status_idx on public.bench_table_099 (status, created_at); +alter table public.bench_table_099 enable row level security; +create policy bench_table_099_read on public.bench_table_099 for select using (true); + +create table public.bench_table_100 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_100_owner_idx on public.bench_table_100 (owner_id); +create index bench_table_100_status_idx on public.bench_table_100 (status, created_at); +alter table public.bench_table_100 enable row level security; +create policy bench_table_100_read on public.bench_table_100 for select using (true); + +create view public.bench_table_100_summary as + select status, count(*) as total from public.bench_table_100 group by status; + +create function public.bench_table_100_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_100 $$; + +create table public.bench_table_101 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_101_owner_idx on public.bench_table_101 (owner_id); +create index bench_table_101_status_idx on public.bench_table_101 (status, created_at); +alter table public.bench_table_101 enable row level security; +create policy bench_table_101_read on public.bench_table_101 for select using (true); + +create table public.bench_table_102 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_102_owner_idx on public.bench_table_102 (owner_id); +create index bench_table_102_status_idx on public.bench_table_102 (status, created_at); +alter table public.bench_table_102 enable row level security; +create policy bench_table_102_read on public.bench_table_102 for select using (true); + +create table public.bench_table_103 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_103_owner_idx on public.bench_table_103 (owner_id); +create index bench_table_103_status_idx on public.bench_table_103 (status, created_at); +alter table public.bench_table_103 enable row level security; +create policy bench_table_103_read on public.bench_table_103 for select using (true); + +create table public.bench_table_104 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_104_owner_idx on public.bench_table_104 (owner_id); +create index bench_table_104_status_idx on public.bench_table_104 (status, created_at); +alter table public.bench_table_104 enable row level security; +create policy bench_table_104_read on public.bench_table_104 for select using (true); + +create table public.bench_table_105 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_105_owner_idx on public.bench_table_105 (owner_id); +create index bench_table_105_status_idx on public.bench_table_105 (status, created_at); +alter table public.bench_table_105 enable row level security; +create policy bench_table_105_read on public.bench_table_105 for select using (true); + +create view public.bench_table_105_summary as + select status, count(*) as total from public.bench_table_105 group by status; + +create table public.bench_table_106 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_106_owner_idx on public.bench_table_106 (owner_id); +create index bench_table_106_status_idx on public.bench_table_106 (status, created_at); +alter table public.bench_table_106 enable row level security; +create policy bench_table_106_read on public.bench_table_106 for select using (true); + +create table public.bench_table_107 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_107_owner_idx on public.bench_table_107 (owner_id); +create index bench_table_107_status_idx on public.bench_table_107 (status, created_at); +alter table public.bench_table_107 enable row level security; +create policy bench_table_107_read on public.bench_table_107 for select using (true); + +create table public.bench_table_108 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_108_owner_idx on public.bench_table_108 (owner_id); +create index bench_table_108_status_idx on public.bench_table_108 (status, created_at); +alter table public.bench_table_108 enable row level security; +create policy bench_table_108_read on public.bench_table_108 for select using (true); + +create table public.bench_table_109 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_109_owner_idx on public.bench_table_109 (owner_id); +create index bench_table_109_status_idx on public.bench_table_109 (status, created_at); +alter table public.bench_table_109 enable row level security; +create policy bench_table_109_read on public.bench_table_109 for select using (true); + +create table public.bench_table_110 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_110_owner_idx on public.bench_table_110 (owner_id); +create index bench_table_110_status_idx on public.bench_table_110 (status, created_at); +alter table public.bench_table_110 enable row level security; +create policy bench_table_110_read on public.bench_table_110 for select using (true); + +create view public.bench_table_110_summary as + select status, count(*) as total from public.bench_table_110 group by status; + +create function public.bench_table_110_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_110 $$; + +create table public.bench_table_111 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_111_owner_idx on public.bench_table_111 (owner_id); +create index bench_table_111_status_idx on public.bench_table_111 (status, created_at); +alter table public.bench_table_111 enable row level security; +create policy bench_table_111_read on public.bench_table_111 for select using (true); + +create table public.bench_table_112 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_112_owner_idx on public.bench_table_112 (owner_id); +create index bench_table_112_status_idx on public.bench_table_112 (status, created_at); +alter table public.bench_table_112 enable row level security; +create policy bench_table_112_read on public.bench_table_112 for select using (true); + +create table public.bench_table_113 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_113_owner_idx on public.bench_table_113 (owner_id); +create index bench_table_113_status_idx on public.bench_table_113 (status, created_at); +alter table public.bench_table_113 enable row level security; +create policy bench_table_113_read on public.bench_table_113 for select using (true); + +create table public.bench_table_114 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_114_owner_idx on public.bench_table_114 (owner_id); +create index bench_table_114_status_idx on public.bench_table_114 (status, created_at); +alter table public.bench_table_114 enable row level security; +create policy bench_table_114_read on public.bench_table_114 for select using (true); + +create table public.bench_table_115 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_115_owner_idx on public.bench_table_115 (owner_id); +create index bench_table_115_status_idx on public.bench_table_115 (status, created_at); +alter table public.bench_table_115 enable row level security; +create policy bench_table_115_read on public.bench_table_115 for select using (true); + +create view public.bench_table_115_summary as + select status, count(*) as total from public.bench_table_115 group by status; + +create table public.bench_table_116 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_116_owner_idx on public.bench_table_116 (owner_id); +create index bench_table_116_status_idx on public.bench_table_116 (status, created_at); +alter table public.bench_table_116 enable row level security; +create policy bench_table_116_read on public.bench_table_116 for select using (true); + +create table public.bench_table_117 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_117_owner_idx on public.bench_table_117 (owner_id); +create index bench_table_117_status_idx on public.bench_table_117 (status, created_at); +alter table public.bench_table_117 enable row level security; +create policy bench_table_117_read on public.bench_table_117 for select using (true); + +create table public.bench_table_118 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_118_owner_idx on public.bench_table_118 (owner_id); +create index bench_table_118_status_idx on public.bench_table_118 (status, created_at); +alter table public.bench_table_118 enable row level security; +create policy bench_table_118_read on public.bench_table_118 for select using (true); + +create table public.bench_table_119 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_119_owner_idx on public.bench_table_119 (owner_id); +create index bench_table_119_status_idx on public.bench_table_119 (status, created_at); +alter table public.bench_table_119 enable row level security; +create policy bench_table_119_read on public.bench_table_119 for select using (true); + +create table public.bench_table_120 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_120_owner_idx on public.bench_table_120 (owner_id); +create index bench_table_120_status_idx on public.bench_table_120 (status, created_at); +alter table public.bench_table_120 enable row level security; +create policy bench_table_120_read on public.bench_table_120 for select using (true); + +create view public.bench_table_120_summary as + select status, count(*) as total from public.bench_table_120 group by status; + +create function public.bench_table_120_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_120 $$; + +create table public.bench_table_121 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_121_owner_idx on public.bench_table_121 (owner_id); +create index bench_table_121_status_idx on public.bench_table_121 (status, created_at); +alter table public.bench_table_121 enable row level security; +create policy bench_table_121_read on public.bench_table_121 for select using (true); + +create table public.bench_table_122 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_122_owner_idx on public.bench_table_122 (owner_id); +create index bench_table_122_status_idx on public.bench_table_122 (status, created_at); +alter table public.bench_table_122 enable row level security; +create policy bench_table_122_read on public.bench_table_122 for select using (true); + +create table public.bench_table_123 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_123_owner_idx on public.bench_table_123 (owner_id); +create index bench_table_123_status_idx on public.bench_table_123 (status, created_at); +alter table public.bench_table_123 enable row level security; +create policy bench_table_123_read on public.bench_table_123 for select using (true); + +create table public.bench_table_124 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_124_owner_idx on public.bench_table_124 (owner_id); +create index bench_table_124_status_idx on public.bench_table_124 (status, created_at); +alter table public.bench_table_124 enable row level security; +create policy bench_table_124_read on public.bench_table_124 for select using (true); + +create table public.bench_table_125 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_125_owner_idx on public.bench_table_125 (owner_id); +create index bench_table_125_status_idx on public.bench_table_125 (status, created_at); +alter table public.bench_table_125 enable row level security; +create policy bench_table_125_read on public.bench_table_125 for select using (true); + +create view public.bench_table_125_summary as + select status, count(*) as total from public.bench_table_125 group by status; + +create table public.bench_table_126 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_126_owner_idx on public.bench_table_126 (owner_id); +create index bench_table_126_status_idx on public.bench_table_126 (status, created_at); +alter table public.bench_table_126 enable row level security; +create policy bench_table_126_read on public.bench_table_126 for select using (true); + +create table public.bench_table_127 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_127_owner_idx on public.bench_table_127 (owner_id); +create index bench_table_127_status_idx on public.bench_table_127 (status, created_at); +alter table public.bench_table_127 enable row level security; +create policy bench_table_127_read on public.bench_table_127 for select using (true); + +create table public.bench_table_128 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_128_owner_idx on public.bench_table_128 (owner_id); +create index bench_table_128_status_idx on public.bench_table_128 (status, created_at); +alter table public.bench_table_128 enable row level security; +create policy bench_table_128_read on public.bench_table_128 for select using (true); + +create table public.bench_table_129 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_129_owner_idx on public.bench_table_129 (owner_id); +create index bench_table_129_status_idx on public.bench_table_129 (status, created_at); +alter table public.bench_table_129 enable row level security; +create policy bench_table_129_read on public.bench_table_129 for select using (true); + +create table public.bench_table_130 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_130_owner_idx on public.bench_table_130 (owner_id); +create index bench_table_130_status_idx on public.bench_table_130 (status, created_at); +alter table public.bench_table_130 enable row level security; +create policy bench_table_130_read on public.bench_table_130 for select using (true); + +create view public.bench_table_130_summary as + select status, count(*) as total from public.bench_table_130 group by status; + +create function public.bench_table_130_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_130 $$; + +create table public.bench_table_131 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_131_owner_idx on public.bench_table_131 (owner_id); +create index bench_table_131_status_idx on public.bench_table_131 (status, created_at); +alter table public.bench_table_131 enable row level security; +create policy bench_table_131_read on public.bench_table_131 for select using (true); + +create table public.bench_table_132 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_132_owner_idx on public.bench_table_132 (owner_id); +create index bench_table_132_status_idx on public.bench_table_132 (status, created_at); +alter table public.bench_table_132 enable row level security; +create policy bench_table_132_read on public.bench_table_132 for select using (true); + +create table public.bench_table_133 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_133_owner_idx on public.bench_table_133 (owner_id); +create index bench_table_133_status_idx on public.bench_table_133 (status, created_at); +alter table public.bench_table_133 enable row level security; +create policy bench_table_133_read on public.bench_table_133 for select using (true); + +create table public.bench_table_134 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_134_owner_idx on public.bench_table_134 (owner_id); +create index bench_table_134_status_idx on public.bench_table_134 (status, created_at); +alter table public.bench_table_134 enable row level security; +create policy bench_table_134_read on public.bench_table_134 for select using (true); + +create table public.bench_table_135 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_135_owner_idx on public.bench_table_135 (owner_id); +create index bench_table_135_status_idx on public.bench_table_135 (status, created_at); +alter table public.bench_table_135 enable row level security; +create policy bench_table_135_read on public.bench_table_135 for select using (true); + +create view public.bench_table_135_summary as + select status, count(*) as total from public.bench_table_135 group by status; + +create table public.bench_table_136 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_136_owner_idx on public.bench_table_136 (owner_id); +create index bench_table_136_status_idx on public.bench_table_136 (status, created_at); +alter table public.bench_table_136 enable row level security; +create policy bench_table_136_read on public.bench_table_136 for select using (true); + +create table public.bench_table_137 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_137_owner_idx on public.bench_table_137 (owner_id); +create index bench_table_137_status_idx on public.bench_table_137 (status, created_at); +alter table public.bench_table_137 enable row level security; +create policy bench_table_137_read on public.bench_table_137 for select using (true); + +create table public.bench_table_138 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_138_owner_idx on public.bench_table_138 (owner_id); +create index bench_table_138_status_idx on public.bench_table_138 (status, created_at); +alter table public.bench_table_138 enable row level security; +create policy bench_table_138_read on public.bench_table_138 for select using (true); + +create table public.bench_table_139 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_139_owner_idx on public.bench_table_139 (owner_id); +create index bench_table_139_status_idx on public.bench_table_139 (status, created_at); +alter table public.bench_table_139 enable row level security; +create policy bench_table_139_read on public.bench_table_139 for select using (true); + +create table public.bench_table_140 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_140_owner_idx on public.bench_table_140 (owner_id); +create index bench_table_140_status_idx on public.bench_table_140 (status, created_at); +alter table public.bench_table_140 enable row level security; +create policy bench_table_140_read on public.bench_table_140 for select using (true); + +create view public.bench_table_140_summary as + select status, count(*) as total from public.bench_table_140 group by status; + +create function public.bench_table_140_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_140 $$; + +create table public.bench_table_141 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_141_owner_idx on public.bench_table_141 (owner_id); +create index bench_table_141_status_idx on public.bench_table_141 (status, created_at); +alter table public.bench_table_141 enable row level security; +create policy bench_table_141_read on public.bench_table_141 for select using (true); + +create table public.bench_table_142 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_142_owner_idx on public.bench_table_142 (owner_id); +create index bench_table_142_status_idx on public.bench_table_142 (status, created_at); +alter table public.bench_table_142 enable row level security; +create policy bench_table_142_read on public.bench_table_142 for select using (true); + +create table public.bench_table_143 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_143_owner_idx on public.bench_table_143 (owner_id); +create index bench_table_143_status_idx on public.bench_table_143 (status, created_at); +alter table public.bench_table_143 enable row level security; +create policy bench_table_143_read on public.bench_table_143 for select using (true); + +create table public.bench_table_144 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_144_owner_idx on public.bench_table_144 (owner_id); +create index bench_table_144_status_idx on public.bench_table_144 (status, created_at); +alter table public.bench_table_144 enable row level security; +create policy bench_table_144_read on public.bench_table_144 for select using (true); + +create table public.bench_table_145 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_145_owner_idx on public.bench_table_145 (owner_id); +create index bench_table_145_status_idx on public.bench_table_145 (status, created_at); +alter table public.bench_table_145 enable row level security; +create policy bench_table_145_read on public.bench_table_145 for select using (true); + +create view public.bench_table_145_summary as + select status, count(*) as total from public.bench_table_145 group by status; + +create table public.bench_table_146 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_146_owner_idx on public.bench_table_146 (owner_id); +create index bench_table_146_status_idx on public.bench_table_146 (status, created_at); +alter table public.bench_table_146 enable row level security; +create policy bench_table_146_read on public.bench_table_146 for select using (true); + +create table public.bench_table_147 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_147_owner_idx on public.bench_table_147 (owner_id); +create index bench_table_147_status_idx on public.bench_table_147 (status, created_at); +alter table public.bench_table_147 enable row level security; +create policy bench_table_147_read on public.bench_table_147 for select using (true); + +create table public.bench_table_148 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_148_owner_idx on public.bench_table_148 (owner_id); +create index bench_table_148_status_idx on public.bench_table_148 (status, created_at); +alter table public.bench_table_148 enable row level security; +create policy bench_table_148_read on public.bench_table_148 for select using (true); + +create table public.bench_table_149 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_149_owner_idx on public.bench_table_149 (owner_id); +create index bench_table_149_status_idx on public.bench_table_149 (status, created_at); +alter table public.bench_table_149 enable row level security; +create policy bench_table_149_read on public.bench_table_149 for select using (true); + +create table public.bench_table_150 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_150_owner_idx on public.bench_table_150 (owner_id); +create index bench_table_150_status_idx on public.bench_table_150 (status, created_at); +alter table public.bench_table_150 enable row level security; +create policy bench_table_150_read on public.bench_table_150 for select using (true); + +create view public.bench_table_150_summary as + select status, count(*) as total from public.bench_table_150 group by status; + +create function public.bench_table_150_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_150 $$; + +create table public.bench_table_151 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_151_owner_idx on public.bench_table_151 (owner_id); +create index bench_table_151_status_idx on public.bench_table_151 (status, created_at); +alter table public.bench_table_151 enable row level security; +create policy bench_table_151_read on public.bench_table_151 for select using (true); + +create table public.bench_table_152 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_152_owner_idx on public.bench_table_152 (owner_id); +create index bench_table_152_status_idx on public.bench_table_152 (status, created_at); +alter table public.bench_table_152 enable row level security; +create policy bench_table_152_read on public.bench_table_152 for select using (true); + +create table public.bench_table_153 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_153_owner_idx on public.bench_table_153 (owner_id); +create index bench_table_153_status_idx on public.bench_table_153 (status, created_at); +alter table public.bench_table_153 enable row level security; +create policy bench_table_153_read on public.bench_table_153 for select using (true); + +create table public.bench_table_154 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_154_owner_idx on public.bench_table_154 (owner_id); +create index bench_table_154_status_idx on public.bench_table_154 (status, created_at); +alter table public.bench_table_154 enable row level security; +create policy bench_table_154_read on public.bench_table_154 for select using (true); + +create table public.bench_table_155 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_155_owner_idx on public.bench_table_155 (owner_id); +create index bench_table_155_status_idx on public.bench_table_155 (status, created_at); +alter table public.bench_table_155 enable row level security; +create policy bench_table_155_read on public.bench_table_155 for select using (true); + +create view public.bench_table_155_summary as + select status, count(*) as total from public.bench_table_155 group by status; + +create table public.bench_table_156 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_156_owner_idx on public.bench_table_156 (owner_id); +create index bench_table_156_status_idx on public.bench_table_156 (status, created_at); +alter table public.bench_table_156 enable row level security; +create policy bench_table_156_read on public.bench_table_156 for select using (true); + +create table public.bench_table_157 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_157_owner_idx on public.bench_table_157 (owner_id); +create index bench_table_157_status_idx on public.bench_table_157 (status, created_at); +alter table public.bench_table_157 enable row level security; +create policy bench_table_157_read on public.bench_table_157 for select using (true); + +create table public.bench_table_158 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_158_owner_idx on public.bench_table_158 (owner_id); +create index bench_table_158_status_idx on public.bench_table_158 (status, created_at); +alter table public.bench_table_158 enable row level security; +create policy bench_table_158_read on public.bench_table_158 for select using (true); + +create table public.bench_table_159 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_159_owner_idx on public.bench_table_159 (owner_id); +create index bench_table_159_status_idx on public.bench_table_159 (status, created_at); +alter table public.bench_table_159 enable row level security; +create policy bench_table_159_read on public.bench_table_159 for select using (true); + +create table public.bench_table_160 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_160_owner_idx on public.bench_table_160 (owner_id); +create index bench_table_160_status_idx on public.bench_table_160 (status, created_at); +alter table public.bench_table_160 enable row level security; +create policy bench_table_160_read on public.bench_table_160 for select using (true); + +create view public.bench_table_160_summary as + select status, count(*) as total from public.bench_table_160 group by status; + +create function public.bench_table_160_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_160 $$; + +create table public.bench_table_161 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_161_owner_idx on public.bench_table_161 (owner_id); +create index bench_table_161_status_idx on public.bench_table_161 (status, created_at); +alter table public.bench_table_161 enable row level security; +create policy bench_table_161_read on public.bench_table_161 for select using (true); + +create table public.bench_table_162 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_162_owner_idx on public.bench_table_162 (owner_id); +create index bench_table_162_status_idx on public.bench_table_162 (status, created_at); +alter table public.bench_table_162 enable row level security; +create policy bench_table_162_read on public.bench_table_162 for select using (true); + +create table public.bench_table_163 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_163_owner_idx on public.bench_table_163 (owner_id); +create index bench_table_163_status_idx on public.bench_table_163 (status, created_at); +alter table public.bench_table_163 enable row level security; +create policy bench_table_163_read on public.bench_table_163 for select using (true); + +create table public.bench_table_164 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_164_owner_idx on public.bench_table_164 (owner_id); +create index bench_table_164_status_idx on public.bench_table_164 (status, created_at); +alter table public.bench_table_164 enable row level security; +create policy bench_table_164_read on public.bench_table_164 for select using (true); + +create table public.bench_table_165 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_165_owner_idx on public.bench_table_165 (owner_id); +create index bench_table_165_status_idx on public.bench_table_165 (status, created_at); +alter table public.bench_table_165 enable row level security; +create policy bench_table_165_read on public.bench_table_165 for select using (true); + +create view public.bench_table_165_summary as + select status, count(*) as total from public.bench_table_165 group by status; + +create table public.bench_table_166 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_166_owner_idx on public.bench_table_166 (owner_id); +create index bench_table_166_status_idx on public.bench_table_166 (status, created_at); +alter table public.bench_table_166 enable row level security; +create policy bench_table_166_read on public.bench_table_166 for select using (true); + +create table public.bench_table_167 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_167_owner_idx on public.bench_table_167 (owner_id); +create index bench_table_167_status_idx on public.bench_table_167 (status, created_at); +alter table public.bench_table_167 enable row level security; +create policy bench_table_167_read on public.bench_table_167 for select using (true); + +create table public.bench_table_168 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_168_owner_idx on public.bench_table_168 (owner_id); +create index bench_table_168_status_idx on public.bench_table_168 (status, created_at); +alter table public.bench_table_168 enable row level security; +create policy bench_table_168_read on public.bench_table_168 for select using (true); + +create table public.bench_table_169 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_169_owner_idx on public.bench_table_169 (owner_id); +create index bench_table_169_status_idx on public.bench_table_169 (status, created_at); +alter table public.bench_table_169 enable row level security; +create policy bench_table_169_read on public.bench_table_169 for select using (true); + +create table public.bench_table_170 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_170_owner_idx on public.bench_table_170 (owner_id); +create index bench_table_170_status_idx on public.bench_table_170 (status, created_at); +alter table public.bench_table_170 enable row level security; +create policy bench_table_170_read on public.bench_table_170 for select using (true); + +create view public.bench_table_170_summary as + select status, count(*) as total from public.bench_table_170 group by status; + +create function public.bench_table_170_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_170 $$; + +create table public.bench_table_171 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_171_owner_idx on public.bench_table_171 (owner_id); +create index bench_table_171_status_idx on public.bench_table_171 (status, created_at); +alter table public.bench_table_171 enable row level security; +create policy bench_table_171_read on public.bench_table_171 for select using (true); + +create table public.bench_table_172 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_172_owner_idx on public.bench_table_172 (owner_id); +create index bench_table_172_status_idx on public.bench_table_172 (status, created_at); +alter table public.bench_table_172 enable row level security; +create policy bench_table_172_read on public.bench_table_172 for select using (true); + +create table public.bench_table_173 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_173_owner_idx on public.bench_table_173 (owner_id); +create index bench_table_173_status_idx on public.bench_table_173 (status, created_at); +alter table public.bench_table_173 enable row level security; +create policy bench_table_173_read on public.bench_table_173 for select using (true); + +create table public.bench_table_174 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_174_owner_idx on public.bench_table_174 (owner_id); +create index bench_table_174_status_idx on public.bench_table_174 (status, created_at); +alter table public.bench_table_174 enable row level security; +create policy bench_table_174_read on public.bench_table_174 for select using (true); + +create table public.bench_table_175 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_175_owner_idx on public.bench_table_175 (owner_id); +create index bench_table_175_status_idx on public.bench_table_175 (status, created_at); +alter table public.bench_table_175 enable row level security; +create policy bench_table_175_read on public.bench_table_175 for select using (true); + +create view public.bench_table_175_summary as + select status, count(*) as total from public.bench_table_175 group by status; + +create table public.bench_table_176 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_176_owner_idx on public.bench_table_176 (owner_id); +create index bench_table_176_status_idx on public.bench_table_176 (status, created_at); +alter table public.bench_table_176 enable row level security; +create policy bench_table_176_read on public.bench_table_176 for select using (true); + +create table public.bench_table_177 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_177_owner_idx on public.bench_table_177 (owner_id); +create index bench_table_177_status_idx on public.bench_table_177 (status, created_at); +alter table public.bench_table_177 enable row level security; +create policy bench_table_177_read on public.bench_table_177 for select using (true); + +create table public.bench_table_178 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_178_owner_idx on public.bench_table_178 (owner_id); +create index bench_table_178_status_idx on public.bench_table_178 (status, created_at); +alter table public.bench_table_178 enable row level security; +create policy bench_table_178_read on public.bench_table_178 for select using (true); + +create table public.bench_table_179 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_179_owner_idx on public.bench_table_179 (owner_id); +create index bench_table_179_status_idx on public.bench_table_179 (status, created_at); +alter table public.bench_table_179 enable row level security; +create policy bench_table_179_read on public.bench_table_179 for select using (true); + +create table public.bench_table_180 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_180_owner_idx on public.bench_table_180 (owner_id); +create index bench_table_180_status_idx on public.bench_table_180 (status, created_at); +alter table public.bench_table_180 enable row level security; +create policy bench_table_180_read on public.bench_table_180 for select using (true); + +create view public.bench_table_180_summary as + select status, count(*) as total from public.bench_table_180 group by status; + +create function public.bench_table_180_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_180 $$; + +create table public.bench_table_181 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_181_owner_idx on public.bench_table_181 (owner_id); +create index bench_table_181_status_idx on public.bench_table_181 (status, created_at); +alter table public.bench_table_181 enable row level security; +create policy bench_table_181_read on public.bench_table_181 for select using (true); + +create table public.bench_table_182 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_182_owner_idx on public.bench_table_182 (owner_id); +create index bench_table_182_status_idx on public.bench_table_182 (status, created_at); +alter table public.bench_table_182 enable row level security; +create policy bench_table_182_read on public.bench_table_182 for select using (true); + +create table public.bench_table_183 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_183_owner_idx on public.bench_table_183 (owner_id); +create index bench_table_183_status_idx on public.bench_table_183 (status, created_at); +alter table public.bench_table_183 enable row level security; +create policy bench_table_183_read on public.bench_table_183 for select using (true); + +create table public.bench_table_184 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_184_owner_idx on public.bench_table_184 (owner_id); +create index bench_table_184_status_idx on public.bench_table_184 (status, created_at); +alter table public.bench_table_184 enable row level security; +create policy bench_table_184_read on public.bench_table_184 for select using (true); + +create table public.bench_table_185 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_185_owner_idx on public.bench_table_185 (owner_id); +create index bench_table_185_status_idx on public.bench_table_185 (status, created_at); +alter table public.bench_table_185 enable row level security; +create policy bench_table_185_read on public.bench_table_185 for select using (true); + +create view public.bench_table_185_summary as + select status, count(*) as total from public.bench_table_185 group by status; + +create table public.bench_table_186 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_186_owner_idx on public.bench_table_186 (owner_id); +create index bench_table_186_status_idx on public.bench_table_186 (status, created_at); +alter table public.bench_table_186 enable row level security; +create policy bench_table_186_read on public.bench_table_186 for select using (true); + +create table public.bench_table_187 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_187_owner_idx on public.bench_table_187 (owner_id); +create index bench_table_187_status_idx on public.bench_table_187 (status, created_at); +alter table public.bench_table_187 enable row level security; +create policy bench_table_187_read on public.bench_table_187 for select using (true); + +create table public.bench_table_188 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_188_owner_idx on public.bench_table_188 (owner_id); +create index bench_table_188_status_idx on public.bench_table_188 (status, created_at); +alter table public.bench_table_188 enable row level security; +create policy bench_table_188_read on public.bench_table_188 for select using (true); + +create table public.bench_table_189 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_189_owner_idx on public.bench_table_189 (owner_id); +create index bench_table_189_status_idx on public.bench_table_189 (status, created_at); +alter table public.bench_table_189 enable row level security; +create policy bench_table_189_read on public.bench_table_189 for select using (true); + +create table public.bench_table_190 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_190_owner_idx on public.bench_table_190 (owner_id); +create index bench_table_190_status_idx on public.bench_table_190 (status, created_at); +alter table public.bench_table_190 enable row level security; +create policy bench_table_190_read on public.bench_table_190 for select using (true); + +create view public.bench_table_190_summary as + select status, count(*) as total from public.bench_table_190 group by status; + +create function public.bench_table_190_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_190 $$; + +create table public.bench_table_191 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_191_owner_idx on public.bench_table_191 (owner_id); +create index bench_table_191_status_idx on public.bench_table_191 (status, created_at); +alter table public.bench_table_191 enable row level security; +create policy bench_table_191_read on public.bench_table_191 for select using (true); + +create table public.bench_table_192 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_192_owner_idx on public.bench_table_192 (owner_id); +create index bench_table_192_status_idx on public.bench_table_192 (status, created_at); +alter table public.bench_table_192 enable row level security; +create policy bench_table_192_read on public.bench_table_192 for select using (true); + +create table public.bench_table_193 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_193_owner_idx on public.bench_table_193 (owner_id); +create index bench_table_193_status_idx on public.bench_table_193 (status, created_at); +alter table public.bench_table_193 enable row level security; +create policy bench_table_193_read on public.bench_table_193 for select using (true); + +create table public.bench_table_194 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_194_owner_idx on public.bench_table_194 (owner_id); +create index bench_table_194_status_idx on public.bench_table_194 (status, created_at); +alter table public.bench_table_194 enable row level security; +create policy bench_table_194_read on public.bench_table_194 for select using (true); + +create table public.bench_table_195 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_195_owner_idx on public.bench_table_195 (owner_id); +create index bench_table_195_status_idx on public.bench_table_195 (status, created_at); +alter table public.bench_table_195 enable row level security; +create policy bench_table_195_read on public.bench_table_195 for select using (true); + +create view public.bench_table_195_summary as + select status, count(*) as total from public.bench_table_195 group by status; + +create table public.bench_table_196 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_196_owner_idx on public.bench_table_196 (owner_id); +create index bench_table_196_status_idx on public.bench_table_196 (status, created_at); +alter table public.bench_table_196 enable row level security; +create policy bench_table_196_read on public.bench_table_196 for select using (true); + +create table public.bench_table_197 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_197_owner_idx on public.bench_table_197 (owner_id); +create index bench_table_197_status_idx on public.bench_table_197 (status, created_at); +alter table public.bench_table_197 enable row level security; +create policy bench_table_197_read on public.bench_table_197 for select using (true); + +create table public.bench_table_198 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_198_owner_idx on public.bench_table_198 (owner_id); +create index bench_table_198_status_idx on public.bench_table_198 (status, created_at); +alter table public.bench_table_198 enable row level security; +create policy bench_table_198_read on public.bench_table_198 for select using (true); + +create table public.bench_table_199 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_199_owner_idx on public.bench_table_199 (owner_id); +create index bench_table_199_status_idx on public.bench_table_199 (status, created_at); +alter table public.bench_table_199 enable row level security; +create policy bench_table_199_read on public.bench_table_199 for select using (true); + +create table public.bench_table_200 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_200_owner_idx on public.bench_table_200 (owner_id); +create index bench_table_200_status_idx on public.bench_table_200 (status, created_at); +alter table public.bench_table_200 enable row level security; +create policy bench_table_200_read on public.bench_table_200 for select using (true); + +create view public.bench_table_200_summary as + select status, count(*) as total from public.bench_table_200 group by status; + +create function public.bench_table_200_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_200 $$; diff --git a/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..24049cf329 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,694 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; + +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_011_owner_idx on public.bench_table_011 (owner_id); +create index bench_table_011_status_idx on public.bench_table_011 (status, created_at); +alter table public.bench_table_011 enable row level security; +create policy bench_table_011_read on public.bench_table_011 for select using (true); + +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_012_owner_idx on public.bench_table_012 (owner_id); +create index bench_table_012_status_idx on public.bench_table_012 (status, created_at); +alter table public.bench_table_012 enable row level security; +create policy bench_table_012_read on public.bench_table_012 for select using (true); + +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_013_owner_idx on public.bench_table_013 (owner_id); +create index bench_table_013_status_idx on public.bench_table_013 (status, created_at); +alter table public.bench_table_013 enable row level security; +create policy bench_table_013_read on public.bench_table_013 for select using (true); + +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_014_owner_idx on public.bench_table_014 (owner_id); +create index bench_table_014_status_idx on public.bench_table_014 (status, created_at); +alter table public.bench_table_014 enable row level security; +create policy bench_table_014_read on public.bench_table_014 for select using (true); + +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_015_owner_idx on public.bench_table_015 (owner_id); +create index bench_table_015_status_idx on public.bench_table_015 (status, created_at); +alter table public.bench_table_015 enable row level security; +create policy bench_table_015_read on public.bench_table_015 for select using (true); + +create view public.bench_table_015_summary as + select status, count(*) as total from public.bench_table_015 group by status; + +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_016_owner_idx on public.bench_table_016 (owner_id); +create index bench_table_016_status_idx on public.bench_table_016 (status, created_at); +alter table public.bench_table_016 enable row level security; +create policy bench_table_016_read on public.bench_table_016 for select using (true); + +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_017_owner_idx on public.bench_table_017 (owner_id); +create index bench_table_017_status_idx on public.bench_table_017 (status, created_at); +alter table public.bench_table_017 enable row level security; +create policy bench_table_017_read on public.bench_table_017 for select using (true); + +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_018_owner_idx on public.bench_table_018 (owner_id); +create index bench_table_018_status_idx on public.bench_table_018 (status, created_at); +alter table public.bench_table_018 enable row level security; +create policy bench_table_018_read on public.bench_table_018 for select using (true); + +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_019_owner_idx on public.bench_table_019 (owner_id); +create index bench_table_019_status_idx on public.bench_table_019 (status, created_at); +alter table public.bench_table_019 enable row level security; +create policy bench_table_019_read on public.bench_table_019 for select using (true); + +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_020_owner_idx on public.bench_table_020 (owner_id); +create index bench_table_020_status_idx on public.bench_table_020 (status, created_at); +alter table public.bench_table_020 enable row level security; +create policy bench_table_020_read on public.bench_table_020 for select using (true); + +create view public.bench_table_020_summary as + select status, count(*) as total from public.bench_table_020 group by status; + +create function public.bench_table_020_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_020 $$; + +create table public.bench_table_021 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_021_owner_idx on public.bench_table_021 (owner_id); +create index bench_table_021_status_idx on public.bench_table_021 (status, created_at); +alter table public.bench_table_021 enable row level security; +create policy bench_table_021_read on public.bench_table_021 for select using (true); + +create table public.bench_table_022 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_022_owner_idx on public.bench_table_022 (owner_id); +create index bench_table_022_status_idx on public.bench_table_022 (status, created_at); +alter table public.bench_table_022 enable row level security; +create policy bench_table_022_read on public.bench_table_022 for select using (true); + +create table public.bench_table_023 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_023_owner_idx on public.bench_table_023 (owner_id); +create index bench_table_023_status_idx on public.bench_table_023 (status, created_at); +alter table public.bench_table_023 enable row level security; +create policy bench_table_023_read on public.bench_table_023 for select using (true); + +create table public.bench_table_024 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_024_owner_idx on public.bench_table_024 (owner_id); +create index bench_table_024_status_idx on public.bench_table_024 (status, created_at); +alter table public.bench_table_024 enable row level security; +create policy bench_table_024_read on public.bench_table_024 for select using (true); + +create table public.bench_table_025 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_025_owner_idx on public.bench_table_025 (owner_id); +create index bench_table_025_status_idx on public.bench_table_025 (status, created_at); +alter table public.bench_table_025 enable row level security; +create policy bench_table_025_read on public.bench_table_025 for select using (true); + +create view public.bench_table_025_summary as + select status, count(*) as total from public.bench_table_025 group by status; + +create table public.bench_table_026 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_026_owner_idx on public.bench_table_026 (owner_id); +create index bench_table_026_status_idx on public.bench_table_026 (status, created_at); +alter table public.bench_table_026 enable row level security; +create policy bench_table_026_read on public.bench_table_026 for select using (true); + +create table public.bench_table_027 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_027_owner_idx on public.bench_table_027 (owner_id); +create index bench_table_027_status_idx on public.bench_table_027 (status, created_at); +alter table public.bench_table_027 enable row level security; +create policy bench_table_027_read on public.bench_table_027 for select using (true); + +create table public.bench_table_028 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_028_owner_idx on public.bench_table_028 (owner_id); +create index bench_table_028_status_idx on public.bench_table_028 (status, created_at); +alter table public.bench_table_028 enable row level security; +create policy bench_table_028_read on public.bench_table_028 for select using (true); + +create table public.bench_table_029 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_029_owner_idx on public.bench_table_029 (owner_id); +create index bench_table_029_status_idx on public.bench_table_029 (status, created_at); +alter table public.bench_table_029 enable row level security; +create policy bench_table_029_read on public.bench_table_029 for select using (true); + +create table public.bench_table_030 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_030_owner_idx on public.bench_table_030 (owner_id); +create index bench_table_030_status_idx on public.bench_table_030 (status, created_at); +alter table public.bench_table_030 enable row level security; +create policy bench_table_030_read on public.bench_table_030 for select using (true); + +create view public.bench_table_030_summary as + select status, count(*) as total from public.bench_table_030 group by status; + +create function public.bench_table_030_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_030 $$; + +create table public.bench_table_031 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_031_owner_idx on public.bench_table_031 (owner_id); +create index bench_table_031_status_idx on public.bench_table_031 (status, created_at); +alter table public.bench_table_031 enable row level security; +create policy bench_table_031_read on public.bench_table_031 for select using (true); + +create table public.bench_table_032 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_032_owner_idx on public.bench_table_032 (owner_id); +create index bench_table_032_status_idx on public.bench_table_032 (status, created_at); +alter table public.bench_table_032 enable row level security; +create policy bench_table_032_read on public.bench_table_032 for select using (true); + +create table public.bench_table_033 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_033_owner_idx on public.bench_table_033 (owner_id); +create index bench_table_033_status_idx on public.bench_table_033 (status, created_at); +alter table public.bench_table_033 enable row level security; +create policy bench_table_033_read on public.bench_table_033 for select using (true); + +create table public.bench_table_034 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_034_owner_idx on public.bench_table_034 (owner_id); +create index bench_table_034_status_idx on public.bench_table_034 (status, created_at); +alter table public.bench_table_034 enable row level security; +create policy bench_table_034_read on public.bench_table_034 for select using (true); + +create table public.bench_table_035 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_035_owner_idx on public.bench_table_035 (owner_id); +create index bench_table_035_status_idx on public.bench_table_035 (status, created_at); +alter table public.bench_table_035 enable row level security; +create policy bench_table_035_read on public.bench_table_035 for select using (true); + +create view public.bench_table_035_summary as + select status, count(*) as total from public.bench_table_035 group by status; + +create table public.bench_table_036 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_036_owner_idx on public.bench_table_036 (owner_id); +create index bench_table_036_status_idx on public.bench_table_036 (status, created_at); +alter table public.bench_table_036 enable row level security; +create policy bench_table_036_read on public.bench_table_036 for select using (true); + +create table public.bench_table_037 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_037_owner_idx on public.bench_table_037 (owner_id); +create index bench_table_037_status_idx on public.bench_table_037 (status, created_at); +alter table public.bench_table_037 enable row level security; +create policy bench_table_037_read on public.bench_table_037 for select using (true); + +create table public.bench_table_038 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_038_owner_idx on public.bench_table_038 (owner_id); +create index bench_table_038_status_idx on public.bench_table_038 (status, created_at); +alter table public.bench_table_038 enable row level security; +create policy bench_table_038_read on public.bench_table_038 for select using (true); + +create table public.bench_table_039 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_039_owner_idx on public.bench_table_039 (owner_id); +create index bench_table_039_status_idx on public.bench_table_039 (status, created_at); +alter table public.bench_table_039 enable row level security; +create policy bench_table_039_read on public.bench_table_039 for select using (true); + +create table public.bench_table_040 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_040_owner_idx on public.bench_table_040 (owner_id); +create index bench_table_040_status_idx on public.bench_table_040 (status, created_at); +alter table public.bench_table_040 enable row level security; +create policy bench_table_040_read on public.bench_table_040 for select using (true); + +create view public.bench_table_040_summary as + select status, count(*) as total from public.bench_table_040 group by status; + +create function public.bench_table_040_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_040 $$; + +create table public.bench_table_041 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_041_owner_idx on public.bench_table_041 (owner_id); +create index bench_table_041_status_idx on public.bench_table_041 (status, created_at); +alter table public.bench_table_041 enable row level security; +create policy bench_table_041_read on public.bench_table_041 for select using (true); + +create table public.bench_table_042 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_042_owner_idx on public.bench_table_042 (owner_id); +create index bench_table_042_status_idx on public.bench_table_042 (status, created_at); +alter table public.bench_table_042 enable row level security; +create policy bench_table_042_read on public.bench_table_042 for select using (true); + +create table public.bench_table_043 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_043_owner_idx on public.bench_table_043 (owner_id); +create index bench_table_043_status_idx on public.bench_table_043 (status, created_at); +alter table public.bench_table_043 enable row level security; +create policy bench_table_043_read on public.bench_table_043 for select using (true); + +create table public.bench_table_044 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_044_owner_idx on public.bench_table_044 (owner_id); +create index bench_table_044_status_idx on public.bench_table_044 (status, created_at); +alter table public.bench_table_044 enable row level security; +create policy bench_table_044_read on public.bench_table_044 for select using (true); + +create table public.bench_table_045 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_045_owner_idx on public.bench_table_045 (owner_id); +create index bench_table_045_status_idx on public.bench_table_045 (status, created_at); +alter table public.bench_table_045 enable row level security; +create policy bench_table_045_read on public.bench_table_045 for select using (true); + +create view public.bench_table_045_summary as + select status, count(*) as total from public.bench_table_045 group by status; + +create table public.bench_table_046 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_046_owner_idx on public.bench_table_046 (owner_id); +create index bench_table_046_status_idx on public.bench_table_046 (status, created_at); +alter table public.bench_table_046 enable row level security; +create policy bench_table_046_read on public.bench_table_046 for select using (true); + +create table public.bench_table_047 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_047_owner_idx on public.bench_table_047 (owner_id); +create index bench_table_047_status_idx on public.bench_table_047 (status, created_at); +alter table public.bench_table_047 enable row level security; +create policy bench_table_047_read on public.bench_table_047 for select using (true); + +create table public.bench_table_048 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_048_owner_idx on public.bench_table_048 (owner_id); +create index bench_table_048_status_idx on public.bench_table_048 (status, created_at); +alter table public.bench_table_048 enable row level security; +create policy bench_table_048_read on public.bench_table_048 for select using (true); + +create table public.bench_table_049 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_049_owner_idx on public.bench_table_049 (owner_id); +create index bench_table_049_status_idx on public.bench_table_049 (status, created_at); +alter table public.bench_table_049 enable row level security; +create policy bench_table_049_read on public.bench_table_049 for select using (true); + +create table public.bench_table_050 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_050_owner_idx on public.bench_table_050 (owner_id); +create index bench_table_050_status_idx on public.bench_table_050 (status, created_at); +alter table public.bench_table_050 enable row level security; +create policy bench_table_050_read on public.bench_table_050 for select using (true); + +create view public.bench_table_050_summary as + select status, count(*) as total from public.bench_table_050 group by status; + +create function public.bench_table_050_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_050 $$; diff --git a/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..2ddae3c330 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,138 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; diff --git a/apps/cli/bench/workloads.stack.json b/apps/cli/bench/workloads.stack.json new file mode 100644 index 0000000000..21fc4a233d --- /dev/null +++ b/apps/cli/bench/workloads.stack.json @@ -0,0 +1,385 @@ +{ + "workloads": [ + { + "id": "start", + "args": ["start"], + "cwdSetup": "init", + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 290 + }, + { + "id": "stop", + "args": ["stop", "--no-backup"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "db-start", + "args": ["db", "start"], + "cwdSetup": "init", + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 170 + }, + { + "id": "status", + "args": ["status"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "db-query", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "db-reset", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 530 + }, + { + "id": "test-db", + "args": ["test", "db"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "pgtap", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 60 + }, + { + "id": "gen-types-small", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-small", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "gen-types-medium", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "gen-types-large", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-large", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 100 + }, + { + "id": "db-diff-cold", + "args": ["db", "diff", "--use-pg-delta"], + "cwdSetup": "init", + "setup": [ + ["start"], + ["db", "reset"], + ["db", "query", "--local", "create table public.bench_drift (id int)"] + ], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 620 + }, + { + "id": "db-diff-warm", + "args": ["db", "diff", "--use-pg-delta"], + "cwdSetup": "init", + "setup": [ + ["start"], + ["db", "reset"], + ["db", "query", "--local", "create table public.bench_drift (id int)"], + ["db", "diff", "--use-pg-delta"] + ], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 660 + }, + { + "id": "db-dump", + "args": ["db", "dump", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 100 + }, + { + "id": "db-lint", + "args": ["db", "lint", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "migration-up", + "args": ["migration", "up", "--local"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "db-push-local", + "args": ["db", "push", "--local"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "migration-list", + "args": ["migration", "list", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "migrations", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "stack-start-docker", + "args": ["stack", "start", "--runtime", "docker", "--eager"], + "cwdSetup": "init", + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 190 + }, + { + "id": "stack-status-docker", + "args": ["stack", "status"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-stop-docker", + "args": ["stack", "stop"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 200 + }, + { + "id": "stack-db-query-docker", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-db-reset-docker", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 370 + }, + { + "id": "stack-start-native", + "args": ["stack", "start", "--runtime", "native", "--eager"], + "cwdSetup": "init", + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 290 + }, + { + "id": "stack-status-native", + "args": ["stack", "status"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "native", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-stop-native", + "args": ["stack", "stop"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 190 + }, + { + "id": "stack-db-query-native", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "native", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-db-reset-native", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 350 + } + ] +} diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts index 6caeed1fcb..14bd58dd8e 100644 --- a/apps/cli/scripts/bench-cli.integration.test.ts +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -5,13 +5,37 @@ import { FetchHttpClient } from "effect/http"; import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; import { LAUNCH_TIMEOUT_EXIT_CODE, + prepareArtifacts, runLaunch, runWorkloadBench, type BenchOptions, } from "./bench-cli.ts"; +import { analyzeWorkload, collectFlags, type Workload } from "./bench-report.ts"; const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; +const baseOptions = (binary: string, overrides: Partial = {}): BenchOptions => ({ + base: binary, + branch: binary, + runs: 2, + warmup: 1, + runsExplicit: false, + warmupExplicit: false, + commands: [], + workloadsFile: undefined, + only: undefined, + runtime: undefined, + artifactCache: undefined, + cwdSetupInit: false, + apiUrl: undefined, + trace: false, + json: false, + out: undefined, + cpuProf: false, + updateCheck: false, + ...overrides, +}); + // Stands in for a CLI build: every launch makes one request to the profile's API and, when asked // to trace, records it as an HTTP client span. const FAKE_CLI = `#!/bin/sh @@ -23,6 +47,74 @@ if [ -n "$SUPABASE_TRACE_FILE" ]; then fi `; +// Stands in for a stack-aware CLI build. It logs every call with the launch environment, hangs on +// `slow`, and reports one container call per launch when traced. +const LOGGING_CLI = `#!/bin/sh +LOG="$(dirname "$0")/calls.log" +echo "$(basename "$0") $* stack=\${SUPABASE_EXPERIMENTAL_STACK:-} dnt=\${DO_NOT_TRACK:-} tel=\${SUPABASE_TELEMETRY_DISABLED:-} home=\${SUPABASE_HOME:-} cwd=$PWD" >> "$LOG" +case "$1" in + slow) exec /bin/sleep 30 ;; + check) if [ -f supabase/seed.sql ]; then echo "fixture present" >> "$LOG"; fi ;; + cache) if [ -L "$SUPABASE_HOME/cache/stack" ]; then echo "cache linked to $(readlink "$SUPABASE_HOME/cache/stack")" >> "$LOG"; fi ;; +esac +if [ -n "$SUPABASE_TRACE_FILE" ]; then + printf '{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"t","spanId":"s","name":"ContainerCli.spawn","startTimeUnixNano":"0","endTimeUnixNano":"1000000","status":{"code":1},"attributes":[{"key":"process.subcommand","value":{"stringValue":"container inspect"}}]}]}]}]}\\n' >> "$SUPABASE_TRACE_FILE" +fi +exit 0 +`; + +interface Call { + readonly build: string; + readonly command: string; + readonly stack: string; + readonly dnt: string; + readonly tel: string; + readonly home: string; + readonly cwd: string; +} + +const parseCalls = (log: string): ReadonlyArray => + log.split("\n").flatMap((line) => { + const match = /^(\S+) (.*) stack=(\S*) dnt=(\S*) tel=(\S*) home=(\S*) cwd=(\S*)$/u.exec(line); + return match === null + ? [] + : [ + { + build: match[1]!, + command: match[2]!, + stack: match[3]!, + dnt: match[4]!, + tel: match[5]!, + home: match[6]!, + cwd: match[7]!, + }, + ]; + }); + +/** Two stand-in builds sharing one call log, so the order of launches across builds is visible. */ +const makeBuilds = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-logging-cli-" }); + for (const name of ["base", "branch"]) { + yield* fs.writeFileString(`${dir}/${name}`, LOGGING_CLI); + yield* fs.chmod(`${dir}/${name}`, 0o755); + } + const calls = Effect.map( + fs.readFileString(`${dir}/calls.log`).pipe(Effect.orElseSucceed(() => "")), + (text) => text, + ); + return { dir, base: `${dir}/base`, branch: `${dir}/branch`, calls }; +}); + +const stackWorkload = (overrides: Partial = {}): Workload => ({ + id: "w", + args: ["work"], + cwdSetup: "none", + stack: true, + runtime: "docker", + ...overrides, +}); + describe("runWorkloadBench", () => { it.effect("counts only the measured launch's API calls, not setup or earlier launches", () => Effect.scoped( @@ -36,24 +128,9 @@ describe("runWorkloadBench", () => { const binary = `${dir}/supabase`; yield* fs.writeFileString(binary, FAKE_CLI); yield* fs.chmod(binary, 0o755); - const options: BenchOptions = { - base: binary, - branch: binary, - runs: 2, - warmup: 1, - commands: [], - workloadsFile: undefined, - cwdSetupInit: false, - apiUrl: server.url, - trace: true, - json: false, - out: undefined, - cpuProf: false, - updateCheck: false, - }; const { report } = yield* runWorkloadBench( - options, + baseOptions(binary, { apiUrl: server.url, trace: true }), { id: "w", args: ["projects", "list"], @@ -74,6 +151,203 @@ describe("runWorkloadBench", () => { }), ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), ); + + it.effect( + "runs setup, the launch and teardown in one directory and home per run-scoped unit", + () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 1 }), + stackWorkload({ + setup: [["boot"]], + teardown: [["halt"]], + }), + yield* Ref.make(false), + ); + + const calls = parseCalls(yield* builds.calls); + expect(report.base).toHaveLength(1); + expect(report.branch).toHaveLength(1); + // Warmup and measured iteration, base and branch each: four units of three calls. + expect(calls.map((call) => call.command)).toEqual( + Array.from({ length: 4 }, () => ["boot", "work", "halt"]).flat(), + ); + for (let unit = 0; unit < 4; unit++) { + const [setup, launch, teardown] = calls.slice(unit * 3, unit * 3 + 3); + expect(launch?.home).toBe(setup?.home); + expect(teardown?.home).toBe(setup?.home); + expect(launch?.cwd).toBe(setup?.cwd); + } + expect(new Set(calls.map((call) => call.home)).size).toBe(4); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("starts one stack per block of launches and alternates the builds between blocks", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 6, warmup: 1 }), + stackWorkload({ setupScope: "block", setup: [["boot"]], teardown: [["halt"]] }), + yield* Ref.make(false), + ); + + const calls = parseCalls(yield* builds.calls); + expect(report.base).toHaveLength(6); + expect(report.branch).toHaveLength(6); + const boots = calls.filter((call) => call.command === "boot"); + expect(boots.map((call) => call.build)).toEqual(["base", "branch", "branch", "base"]); + expect(calls.filter((call) => call.command === "halt")).toHaveLength(4); + // One warmup plus five measured launches share the first block's stack. + const firstBlock = calls.filter((call) => call.home === boots[0]?.home); + expect(firstBlock.map((call) => call.command)).toEqual([ + "boot", + ...Array.from({ length: 6 }, () => "work"), + "halt", + ]); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("opens stack mode only for stack workloads and never lets telemetry through", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + const run = (workload: Workload) => + runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 0 }), + workload, + Ref.make(false).pipe(Effect.runSync), + ); + + yield* run(stackWorkload({ id: "stack", args: ["on"], setup: [["boot"]] })); + yield* run({ id: "plain", args: ["off"], cwdSetup: "none" }); + + const calls = parseCalls(yield* builds.calls); + for (const call of calls) { + expect(call.dnt).toBe("1"); + expect(call.tel).toBe("1"); + } + expect( + calls + .filter((call) => ["boot", "on"].includes(call.command)) + .every((call) => call.stack === "1"), + ).toBe(true); + expect( + calls.filter((call) => call.command === "off").every((call) => call.stack === ""), + ).toBe(true); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.live("runs teardown after a launch that outlives its timeout and records the timeout", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 0 }), + stackWorkload({ args: ["slow"], teardown: [["halt"]], timeoutSeconds: 1 }), + yield* Ref.make(false), + ); + + const commands = parseCalls(yield* builds.calls).map((call) => call.command); + expect(commands).toEqual(["slow", "halt", "slow", "halt"]); + expect(report.base[0]?.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + expect(report.branch[0]?.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect( + "copies the workload's fixture into the project and links the shared artifact cache into every home", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const builds = yield* makeBuilds; + const manifestDir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-manifest-" }); + yield* fs.makeDirectory(`${manifestDir}/fixtures/seeded`, { recursive: true }); + yield* fs.writeFileString(`${manifestDir}/fixtures/seeded/seed.sql`, "select 1;"); + const cache = yield* fs.makeTempDirectoryScoped({ prefix: "bench-cache-" }); + + yield* runWorkloadBench( + baseOptions(builds.base, { + branch: builds.branch, + runs: 1, + warmup: 0, + workloadsFile: `${manifestDir}/workloads.json`, + artifactCache: cache, + }), + stackWorkload({ args: ["check"], setup: [["cache"]], fixture: "seeded" }), + yield* Ref.make(false), + ); + + const log = yield* builds.calls; + expect(log.match(/fixture present/gu)).toHaveLength(2); + expect(log.match(new RegExp(`cache linked to ${cache}`, "gu"))).toHaveLength(2); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect( + "traces only the workload's traceRuns launches per build and reports container calls from them", + () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 3, warmup: 0, trace: true }), + { id: "w", args: ["work"], cwdSetup: "none", traceRuns: 1 }, + yield* Ref.make(false), + ); + + for (const samples of [report.base, report.branch]) { + expect(samples.map((sample) => sample.trace !== undefined)).toEqual([ + true, + false, + false, + ]); + } + const analysis = analyzeWorkload(report, true); + expect(analysis.base.containers).toEqual({ "container inspect": 1 }); + expect(collectFlags(analysis).map((flag) => flag.kind)).not.toContain("trace-incomplete"); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); +}); + +describe("prepareArtifacts", () => { + it.effect("installs each stack runtime once per build and skips default-mode workloads", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + yield* prepareArtifacts(baseOptions(builds.base, { branch: builds.branch }), [ + stackWorkload({ id: "a", runtime: "native" }), + stackWorkload({ id: "b", runtime: "native" }), + stackWorkload({ id: "c", runtime: "docker" }), + { id: "plain", args: ["start"], cwdSetup: "none", runtime: "docker" }, + ]); + + const installs = parseCalls(yield* builds.calls) + .filter((call) => call.command !== "init") + .map((call) => `${call.build}: ${call.command}`); + expect(installs).toEqual([ + "base: stack install --runtime docker", + "base: stack install --runtime native", + "branch: stack install --runtime docker", + "branch: stack install --runtime native", + ]); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); }); describe("runLaunch", () => { diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 5699c7aba8..ba34abe7e5 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -45,10 +45,13 @@ import { readRequestLogCount, resetReplay } from "./replay-control.ts"; import { readSpans } from "./trace-report.ts"; const USAGE = - 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--workloads ] [--api-url ] [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + + 'usage: bun scripts/bench-cli.ts --base --branch [--runs N] [--warmup N] [--command ""]... [--workloads ] [--only ] [--runtime docker|native] [--artifact-cache ] [--api-url ] [--cwd-setup init] [--trace] [--json] [--out ] [--cpu-prof] [--update-check]\n' + " --command splits on whitespace; it does not support shell quoting. Use --workloads for quoted arguments.\n" + " --workloads reads a manifest of workloads and cannot be combined with --command or --cwd-setup.\n" + " --api-url points every launch at a replay server through a profile file, and counts Management API calls from the traced launches; it requires --trace.\n" + + " --only keeps the listed workload ids (comma separated); --runtime keeps workloads for that runtime, treating workloads with no runtime as docker.\n" + + " --artifact-cache links one directory in as the stack artifact cache of every launch's home, so native archives download once.\n" + + " --runs and --warmup, when given, override the per-workload values in a manifest.\n" + " --update-check keeps the GitHub release check, which otherwise adds network time to every launch."; export interface BenchOptions { @@ -56,8 +59,14 @@ export interface BenchOptions { readonly branch: string; readonly runs: number; readonly warmup: number; + /** Whether `--runs` was passed, which then overrides each workload's own `runs`. */ + readonly runsExplicit: boolean; + readonly warmupExplicit: boolean; readonly commands: ReadonlyArray>; readonly workloadsFile: string | undefined; + readonly only: ReadonlyArray | undefined; + readonly runtime: "docker" | "native" | undefined; + readonly artifactCache: string | undefined; readonly cwdSetupInit: boolean; readonly apiUrl: string | undefined; readonly trace: boolean; @@ -67,6 +76,11 @@ export interface BenchOptions { readonly updateCheck: boolean; } +const DEFAULT_RUNS = 10; +const DEFAULT_WARMUP = 2; +/** The one variable a `stack: true` workload sets on its launches; it opens stack mode. */ +const STACK_FLAG = "SUPABASE_EXPERIMENTAL_STACK"; + const DEFAULT_COMMANDS: ReadonlyArray> = [["--version"], ["--help"]]; /** Dummy token for launches pointed at a replay server; it never reaches a real API. */ @@ -98,10 +112,13 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { options: { base: { type: "string" }, branch: { type: "string" }, - runs: { type: "string", default: "10" }, - warmup: { type: "string", default: "2" }, + runs: { type: "string" }, + warmup: { type: "string" }, command: { type: "string", multiple: true }, workloads: { type: "string" }, + only: { type: "string" }, + runtime: { type: "string" }, + "artifact-cache": { type: "string" }, "api-url": { type: "string" }, "cwd-setup": { type: "string" }, trace: { type: "boolean", default: false }, @@ -115,14 +132,17 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { if (values.base === undefined || values.branch === undefined) { throw new Error(USAGE); } - const runs = Number(values.runs); - const warmup = Number(values.warmup); + const runs = Number(values.runs ?? DEFAULT_RUNS); + const warmup = Number(values.warmup ?? DEFAULT_WARMUP); if (!Number.isInteger(runs) || runs < 1) { throw new Error("--runs must be a positive integer"); } if (!Number.isInteger(warmup) || warmup < 0) { throw new Error("--warmup must be a non-negative integer"); } + if (values.runtime !== undefined && values.runtime !== "docker" && values.runtime !== "native") { + throw new Error('--runtime must be "docker" or "native"'); + } if (values["cwd-setup"] !== undefined && values["cwd-setup"] !== "init") { throw new Error('--cwd-setup only supports "init"'); } @@ -147,10 +167,19 @@ export function parseBenchArgs(argv: ReadonlyArray): BenchOptions { branch: hostPath.resolve(values.branch), runs, warmup, + runsExplicit: values.runs !== undefined, + warmupExplicit: values.warmup !== undefined, commands: hasCommands ? values.command!.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)) : DEFAULT_COMMANDS, workloadsFile: values.workloads, + only: + values.only === undefined ? undefined : values.only.split(",").filter((id) => id.length > 0), + runtime: values.runtime, + artifactCache: + values["artifact-cache"] === undefined + ? undefined + : hostPath.resolve(values["artifact-cache"]), cwdSetupInit: values["cwd-setup"] === "init", apiUrl: values["api-url"], trace: values.trace ?? false, @@ -221,9 +250,20 @@ export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* const fs = yield* FileSystem.FileSystem; return yield* parseWorkloadManifest(yield* fs.readFileString(options.workloadsFile!)); }); - const problems = validateWorkloads(workloads, { apiUrl: options.apiUrl }); + const problems = [...validateWorkloads(workloads, { apiUrl: options.apiUrl })]; + if (options.only !== undefined) { + for (const id of options.only) { + if (!workloads.some((workload) => workload.id === id)) { + problems.push(`--only names unknown workload "${id}"`); + } + } + } if (problems.length > 0) return yield* new WorkloadValidationError({ problems }); - return workloads; + return workloads.filter( + (workload) => + (options.only === undefined || options.only.includes(workload.id)) && + (options.runtime === undefined || (workload.runtime ?? "docker") === options.runtime), + ); }); interface LaunchResult { @@ -232,7 +272,8 @@ interface LaunchResult { } /** Longest a single launch may run before it is killed and counted as a failed launch. */ -export const LAUNCH_TIMEOUT = Duration.seconds(60); +export const LAUNCH_TIMEOUT_SECONDS = 60; +export const LAUNCH_TIMEOUT = Duration.seconds(LAUNCH_TIMEOUT_SECONDS); /** Exit code recorded for a launch that hit {@link LAUNCH_TIMEOUT}. */ export const LAUNCH_TIMEOUT_EXIT_CODE = 124; @@ -282,29 +323,38 @@ export function profileYaml(apiUrl: string): string { ].join("\n"); } +/** Variables set on every launch: no product telemetry, no keyring, and no update check unless asked. */ const baseLaunchEnv = ( options: BenchOptions, + workload: Workload, supabaseHome: string, profilePath: string, ): Record => ({ SUPABASE_HOME: supabaseHome, DO_NOT_TRACK: "1", + SUPABASE_TELEMETRY_DISABLED: "1", SUPABASE_NO_KEYRING: "1", ...(options.updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(workload.stack === true ? { [STACK_FLAG]: "1" } : {}), ...(options.apiUrl === undefined ? {} - : { - SUPABASE_PROFILE: profilePath, - SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN, - SUPABASE_TELEMETRY_DISABLED: "1", - }), + : { SUPABASE_PROFILE: profilePath, SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN }), }); +/** Longest an untimed setup, teardown, or artifact step may run. */ +export const STEP_TIMEOUT = Duration.minutes(5); + +export interface Unit { + readonly projectDir: string; + readonly env: Record; +} + /** - * A fresh project directory and `SUPABASE_HOME`, with the replay profile written and the - * workload's `cwdSetup` applied. + * A project directory and `SUPABASE_HOME` shared by one unit's setup, launches and teardown, with + * the replay profile written, the workload's `cwdSetup` applied, its fixture copied in, and the + * shared artifact cache linked into the home. */ -const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string) => +const prepareUnit = (options: BenchOptions, workload: Workload, binary: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -314,7 +364,12 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string if (options.apiUrl !== undefined) { yield* fs.writeFileString(profilePath, profileYaml(options.apiUrl)); } - const env = baseLaunchEnv(options, supabaseHome, profilePath); + if (options.artifactCache !== undefined) { + yield* fs.makeDirectory(options.artifactCache, { recursive: true }); + yield* fs.makeDirectory(path.join(supabaseHome, "cache"), { recursive: true }); + yield* fs.symlink(options.artifactCache, path.join(supabaseHome, "cache", "stack")); + } + const env = baseLaunchEnv(options, workload, supabaseHome, profilePath); if (workload.cwdSetup !== "none") { const init = yield* runLaunch({ binary, @@ -328,115 +383,333 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string ); } } - return { projectDir, env }; + if (workload.fixture !== undefined && options.workloadsFile !== undefined) { + // The fixtures sit outside any `supabase/` directory, which the repository ignores. + const target = path.join(projectDir, "supabase"); + yield* fs.makeDirectory(target, { recursive: true }); + yield* fs.copy( + path.join(path.dirname(options.workloadsFile), "fixtures", workload.fixture), + target, + { overwrite: true }, + ); + } + return { projectDir, env } satisfies Unit; }); -export const runWorkloadBench = ( +const runStep = Effect.fn("BenchCli.runStep")(function* ( + kind: "setup" | "teardown", + binary: string, + unit: Unit, + args: ReadonlyArray, +) { + const result = yield* runLaunch({ + binary, + args, + cwd: unit.projectDir, + env: isolatedEnv(process.env, unit.env), + timeout: STEP_TIMEOUT, + }); + yield* Effect.annotateCurrentSpan({ "bench.step": kind, "bench.exit_code": result.exitCode }); + if (result.exitCode !== 0) { + yield* Console.error( + `warning: ${kind} step "${args.join(" ")}" exited ${result.exitCode}; the workload may not reflect a real stack`, + ); + } +}); + +/** Pulls images or downloads archives for each stack runtime, so no measured launch downloads anything. */ +export const prepareArtifacts = Effect.fn("BenchCli.prepareArtifacts")(function* ( options: BenchOptions, - workload: Workload, - profiled: Ref.Ref, -) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const plan = buildRunPlan(options.runs, options.warmup); + workloads: ReadonlyArray, +) { + const runtimes = [ + ...new Set( + workloads.flatMap((workload) => + workload.stack === true && workload.runtime !== undefined ? [workload.runtime] : [], + ), + ), + ].sort(); + const scratch: Workload = { id: "install", args: [], cwdSetup: "init", stack: true }; + for (const binary of [options.base, options.branch]) { + for (const runtime of runtimes) { + yield* Effect.scoped( + Effect.gen(function* () { + const unit = yield* prepareUnit(options, scratch, binary); + yield* runStep("setup", binary, unit, ["stack", "install", "--runtime", runtime]); + }), + ); + } + } +}); - const samples: Record> = { base: [], branch: [] }; - let cpuProfilePath: string | undefined; +type LaunchKind = "warmup" | "timed" | "traced"; - // Resets the replay server after setup, so setup's own requests and a previous launch's - // consumed scenario never reach the launch that follows. - const resetServer = - options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); +interface LaunchOutcome { + readonly kind: LaunchKind; + readonly wallMs: number; + readonly exitCode: number; + readonly trace: RunSample["trace"]; +} - for (const planned of plan) { - const binary = planned.build === "base" ? options.base : options.branch; +/** Planned units of work for one build: a run-scoped unit holds one launch, a block-scoped one many. */ +export interface PlannedUnit { + readonly build: Build; + readonly launches: ReadonlyArray<"warmup" | "timed">; +} - yield* Effect.scoped( - Effect.gen(function* () { - const prepared = yield* prepareLaunch(options, workload, binary); - yield* resetServer; +/** Measured launches per block for `block`-scoped workloads. */ +export const BLOCK_SIZE = 5; + +/** + * Units in execution order. `run` scope reuses {@link buildRunPlan}, one launch per unit. + * `block` scope starts one unit per block of up to `blockSize` measured launches, preceded by + * `warmup` discarded launches, and alternates which build goes first in every other block. + */ +export function buildUnitPlan( + runs: number, + warmup: number, + scope: "run" | "block", + blockSize: number = BLOCK_SIZE, +): ReadonlyArray { + if (scope === "run") { + return buildRunPlan(runs, warmup).map((planned) => ({ + build: planned.build, + launches: [planned.warmup ? "warmup" : "timed"], + })); + } + const units: Array = []; + const blocks = Math.ceil(runs / blockSize); + for (let block = 0; block < blocks; block++) { + const measured = Math.min(blockSize, runs - block * blockSize); + const launches = [ + ...Array.from({ length: warmup }, () => "warmup" as const), + ...Array.from({ length: measured }, () => "timed" as const), + ]; + const order: ReadonlyArray = block % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; + for (const build of order) units.push({ build, launches }); + } + return units; +} + +/** + * Runs one unit: setup, the launches in order, then teardown. Teardown is a finalizer registered + * before setup, so it runs after a failed step or a timed-out launch too. + */ +const runUnit = Effect.fn("BenchCli.runUnit")(function* (params: { + readonly options: BenchOptions; + readonly workload: Workload; + readonly build: Build; + readonly launches: ReadonlyArray; + readonly profiled: Ref.Ref; + readonly onProfile: (path: string) => void; +}) { + const { options, workload, build, launches } = params; + const binary = build === "base" ? options.base : options.branch; + return yield* Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const unit = yield* prepareUnit(options, workload, binary); + yield* Effect.addFinalizer(() => + Effect.forEach(workload.teardown ?? [], (args) => runStep("teardown", binary, unit, args), { + discard: true, + }).pipe(Effect.ignore), + ); + yield* Effect.forEach(workload.setup ?? [], (args) => runStep("setup", binary, unit, args), { + discard: true, + }); + + // Resets the replay server before each launch, so setup's own requests and a previous + // launch's consumed scenario never reach the launch that follows. + const resetServer = + options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); + const timeout = Duration.seconds(workload.timeoutSeconds ?? LAUNCH_TIMEOUT_SECONDS); + const outcomes: Array = []; + + for (const kind of launches) { + yield* resetServer; + if (kind !== "traced") { const result = yield* runLaunch({ binary, args: workload.args, - cwd: prepared.projectDir, - env: isolatedEnv(process.env, prepared.env), + cwd: unit.projectDir, + env: isolatedEnv(process.env, unit.env), + timeout, }); - - if (planned.warmup) return; - - // Tracing and profiling slow the launch they observe, so they run in an extra, untimed - // launch in fresh directories. Both builds get traced launches; only one branch launch - // is profiled. - const traceFile = options.trace - ? yield* fs.makeTempFileScoped({ prefix: "supabase-bench-trace-", suffix: ".jsonl" }) - : undefined; - let bunOptions: string | undefined; - if (options.cpuProf && planned.build === "branch") { - const alreadyProfiled = yield* Ref.getAndSet(profiled, true); - if (!alreadyProfiled) { - const profileDir = tmpdir(); - const profileName = `supabase-bench-branch-${randomUUID()}.cpuprofile`; - cpuProfilePath = path.join(profileDir, profileName); - bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; - } + outcomes.push({ kind, ...result, trace: undefined }); + continue; + } + + // Tracing and profiling slow the launch they observe, so they run in extra, untimed launches. + const traceFile = options.trace + ? yield* fs.makeTempFileScoped({ prefix: "supabase-bench-trace-", suffix: ".jsonl" }) + : undefined; + let bunOptions: string | undefined; + if (options.cpuProf && build === "branch") { + const alreadyProfiled = yield* Ref.getAndSet(params.profiled, true); + if (!alreadyProfiled) { + const profileDir = tmpdir(); + const profileName = `supabase-bench-branch-${randomUUID()}.cpuprofile`; + params.onProfile(path.join(profileDir, profileName)); + bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; } - - let trace: RunSample["trace"]; - let tracedExitCode: number | undefined; - if (traceFile !== undefined || bunOptions !== undefined) { - const observed = yield* prepareLaunch(options, workload, binary); - yield* resetServer; - const traced = yield* runLaunch({ - binary, - args: workload.args, - cwd: observed.projectDir, - env: isolatedEnv(process.env, { - ...observed.env, - ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), - ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), - }), - }); - tracedExitCode = traced.exitCode; - if (traceFile !== undefined) { - const requestLogCount = - options.apiUrl === undefined - ? undefined - : yield* readRequestLogCount(options.apiUrl); - // Trace data is optional; a missing or unreadable file must not discard timings. - const decoded = yield* readSpans(traceFile).pipe(Effect.option); - if (Option.isSome(decoded) && decoded.value.spans.length > 0) { - const { spans, skippedLines } = decoded.value; - const http = - options.apiUrl === undefined - ? undefined - : classifyHttpSpans(spans, options.apiUrl); - trace = { - skippedLines, - spans: summarizeSpanRun(spans), - ...(http === undefined || requestLogCount === undefined - ? {} - : { api: { ...http, requestLogCount } }), - }; - } - } + } + const traced = yield* runLaunch({ + binary, + args: workload.args, + cwd: unit.projectDir, + env: isolatedEnv(process.env, { + ...unit.env, + ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), + ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), + }), + timeout, + }); + let trace: RunSample["trace"]; + if (traceFile !== undefined) { + const requestLogCount = + options.apiUrl === undefined ? undefined : yield* readRequestLogCount(options.apiUrl); + // Trace data is optional; a missing or unreadable file must not discard timings. + const decoded = yield* readSpans(traceFile).pipe(Effect.option); + if (Option.isSome(decoded) && decoded.value.spans.length > 0) { + const { spans, skippedLines } = decoded.value; + const http = + options.apiUrl === undefined ? undefined : classifyHttpSpans(spans, options.apiUrl); + trace = { + skippedLines, + spans: summarizeSpanRun(spans), + ...(http === undefined || requestLogCount === undefined + ? {} + : { api: { ...http, requestLogCount } }), + }; } + } + outcomes.push({ kind, ...traced, trace }); + } + return outcomes; + }), + ); +}); - samples[planned.build].push({ - wallMs: result.wallMs, - exitCode: result.exitCode, - ...(tracedExitCode === undefined ? {} : { tracedExitCode }), - ...(trace === undefined ? {} : { trace }), - }); +/** Traced launches still wanted for `build`, given how many it has had. */ +const wantsTrace = (options: BenchOptions, tracedSoFar: number, traceRuns: number) => + options.trace && tracedSoFar < traceRuns; + +export const runWorkloadBench = ( + options: BenchOptions, + workload: Workload, + profiled: Ref.Ref, +) => + Effect.gen(function* () { + const runs = options.runsExplicit ? options.runs : (workload.runs ?? options.runs); + const warmup = options.warmupExplicit ? options.warmup : (workload.warmup ?? options.warmup); + const traceRuns = options.trace ? Math.min(workload.traceRuns ?? runs, runs) : 0; + const scope = workload.setupScope ?? "run"; + const plan = buildUnitPlan(runs, warmup, scope); + const started = yield* Clock.currentTimeNanos; + + const timed: Record> = { + base: [], + branch: [], + }; + const traced: Record> = { + base: [], + branch: [], + }; + let cpuProfilePath: string | undefined; + const onProfile = (path: string) => { + cpuProfilePath = path; + }; + + // A launch is traced when tracing wants more samples for the build, or a branch launch is + // still to be profiled; the profile needs no trace file. + const wantsExtra = (build: Build, tracedSoFar: number) => + Effect.map( + Ref.get(profiled), + (alreadyProfiled) => + wantsTrace(options, tracedSoFar, traceRuns) || + (options.cpuProf && build === "branch" && !alreadyProfiled), + ); + + const record = (build: Build, outcomes: ReadonlyArray) => { + for (const outcome of outcomes) { + if (outcome.kind === "timed") { + timed[build].push({ wallMs: outcome.wallMs, exitCode: outcome.exitCode }); + } else if (outcome.kind === "traced") { + traced[build].push({ exitCode: outcome.exitCode, trace: outcome.trace }); + } + } + }; + + for (const planned of plan) { + const { build } = planned; + if (scope === "run") { + const outcomes = yield* runUnit({ + options, + workload, + build, + launches: planned.launches, + profiled, + onProfile, + }); + record(build, outcomes); + if (planned.launches[0] === "timed" && (yield* wantsExtra(build, traced[build].length))) { + record( + build, + yield* runUnit({ + options, + workload, + build, + launches: ["traced"], + profiled, + onProfile, + }), + ); + } + continue; + } + const measured = planned.launches.filter((kind) => kind === "timed").length; + const extra: Array<"traced"> = []; + let tracedCount = traced[build].length; + for (let index = 0; index < measured; index++) { + if (!(yield* wantsExtra(build, tracedCount))) break; + extra.push("traced"); + // A profile-only launch is wanted once, not per measured launch. + if (!wantsTrace(options, tracedCount, traceRuns)) break; + tracedCount++; + } + record( + build, + yield* runUnit({ + options, + workload, + build, + launches: [...planned.launches, ...extra], + profiled, + onProfile, }), ); } + const sample = (build: Build): Array => + timed[build].map((launch, index) => { + const extra = traced[build][index]; + return { + ...launch, + ...(extra === undefined ? {} : { tracedExitCode: extra.exitCode }), + ...(extra?.trace === undefined ? {} : { trace: extra.trace }), + }; + }); + const finished = yield* Clock.currentTimeNanos; const report: WorkloadReport = { id: workload.id, args: workload.args, - base: samples.base, - branch: samples.branch, + ...(workload.runtime === undefined ? {} : { runtime: workload.runtime }), + ...(workload.stack === undefined ? {} : { stack: workload.stack }), + ...(options.trace ? { traceRuns } : {}), + elapsedSeconds: Math.round(Number(finished - started) / 1_000_000) / 1000, + base: sample("base"), + branch: sample("branch"), }; return { report, cpuProfilePath }; }); @@ -529,6 +802,7 @@ const main = Effect.gen(function* () { if (options === undefined) return; const workloads = yield* resolveWorkloads(options); + yield* prepareArtifacts(options, workloads); const profiled = yield* Ref.make(false); const reports: Array = []; let cpuProfilePath: string | undefined; diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index 8906f3a380..15f8b80855 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -3,6 +3,7 @@ import { BunServices } from "@effect/platform-bun"; import { Effect } from "effect"; import { buildRunPlan, + buildUnitPlan, isolatedEnv, parseBenchArgs, profileYaml, @@ -108,6 +109,73 @@ describe("buildRunPlan", () => { }); }); +describe("buildUnitPlan", () => { + it("gives a run-scoped workload one launch per unit in the order buildRunPlan chooses", () => { + const plan = buildUnitPlan(2, 1, "run"); + + expect(plan.map((unit) => [unit.build, ...unit.launches])).toEqual([ + ["branch", "warmup"], + ["base", "warmup"], + ["base", "timed"], + ["branch", "timed"], + ["branch", "timed"], + ["base", "timed"], + ]); + }); + + it("splits a block-scoped workload into blocks of measured launches after the warmup launches, swapping the first build every other block", () => { + const plan = buildUnitPlan(7, 1, "block", 5); + + expect(plan.map((unit) => unit.build)).toEqual(["base", "branch", "branch", "base"]); + const measured = (build: "base" | "branch") => + plan + .filter((unit) => unit.build === build) + .map((unit) => unit.launches.filter((kind) => kind === "timed").length); + expect(measured("base")).toEqual([5, 2]); + expect(measured("branch")).toEqual([5, 2]); + expect(plan[0]?.launches).toEqual(["warmup", "timed", "timed", "timed", "timed", "timed"]); + }); +}); + +describe("parseBenchArgs for stack runs", () => { + it("lets explicit --runs and --warmup override the manifest, and defaults otherwise", () => { + expect(parseBenchArgs(required)).toMatchObject({ + runs: 10, + warmup: 2, + runsExplicit: false, + warmupExplicit: false, + }); + expect(parseBenchArgs([...required, "--runs", "3", "--warmup", "0"])).toMatchObject({ + runs: 3, + warmup: 0, + runsExplicit: true, + warmupExplicit: true, + }); + }); + + it("reads --only, --runtime and a resolved --artifact-cache", () => { + const options = parseBenchArgs([ + ...required, + "--only", + "a,b", + "--runtime", + "native", + "--artifact-cache", + "/tmp/cache", + ]); + + expect(options).toMatchObject({ + only: ["a", "b"], + runtime: "native", + artifactCache: "/tmp/cache", + }); + }); + + it("rejects an unknown runtime", () => { + expect(() => parseBenchArgs([...required, "--runtime", "podman"])).toThrow("--runtime"); + }); +}); + describe("profileYaml", () => { it("points the API and dashboard at the replay server and keeps project hosts local", () => { const yaml = profileYaml("http://127.0.0.1:9"); diff --git a/apps/cli/scripts/bench-fixtures.ts b/apps/cli/scripts/bench-fixtures.ts new file mode 100644 index 0000000000..b1797c7393 --- /dev/null +++ b/apps/cli/scripts/bench-fixtures.ts @@ -0,0 +1,130 @@ +/** + * Regenerates the benchmark fixtures under `apps/cli/bench/fixtures/`. The output is deterministic and committed, + * so a series of results keeps measuring the same work; changing it starts a new series. Run as + * `bun apps/cli/scripts/bench-fixtures.ts`. + */ +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Effect, FileSystem, Path } from "effect"; + +/** Tables in each schema fixture. */ +export const SCHEMA_SIZES = { small: 10, medium: 50, large: 200 } as const; +/** Migrations in the `migrations` fixture. */ +export const MIGRATION_COUNT = 20; +/** Rows inserted by the `migrations` fixture's seed. */ +export const SEED_ROWS = 1000; + +const table = (index: number) => `bench_table_${String(index).padStart(3, "0")}`; + +/** One migration creating `count` related tables, indexes, policies, views and functions. */ +export function schemaSql(count: number): string { + const blocks: Array = []; + for (let index = 1; index <= count; index++) { + const name = table(index); + blocks.push( + `create table public.${name} ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index ${name}_owner_idx on public.${name} (owner_id); +create index ${name}_status_idx on public.${name} (status, created_at); +alter table public.${name} enable row level security; +create policy ${name}_read on public.${name} for select using (true);`, + ); + if (index % 5 === 0) { + blocks.push( + `create view public.${name}_summary as + select status, count(*) as total from public.${name} group by status;`, + ); + } + if (index % 10 === 0) { + blocks.push( + `create function public.${name}_count() returns bigint + language sql stable as $$ select count(*) from public.${name} $$;`, + ); + } + } + return `${blocks.join("\n\n")}\n`; +} + +/** Migration `index` of the `migrations` fixture; the first one owns the seeded table. */ +export function migrationSql(index: number): string { + if (index === 1) { + return `create table public.bench_items ( + id bigint generated always as identity primary key, + label text not null, + amount numeric not null, + created_at timestamptz not null default now() +); +create index bench_items_label_idx on public.bench_items (label); +`; + } + const name = table(index); + return `create table public.${name} ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index ${name}_item_idx on public.${name} (item_id); +`; +} + +/** Seed for the `migrations` fixture. */ +export const seedSql = `insert into public.bench_items (label, amount) +select 'item-' || n, (n % 97) * 1.5 from generate_series(1, ${SEED_ROWS}) as n; +`; + +const stamp = (index: number) => `2026010100${String(index).padStart(4, "0")}`; + +/** The pgTAP fixture: one table and a test file over it. */ +export const pgtapMigration = `create table public.bench_checks ( + id bigint generated always as identity primary key, + label text not null +); +`; + +export const pgtapTest = `begin; +select plan(4); +select has_table('public', 'bench_checks', 'bench_checks exists'); +select has_column('public', 'bench_checks', 'label', 'label column exists'); +select col_not_null('public', 'bench_checks', 'label', 'label is required'); +select col_is_pk('public', 'bench_checks', 'id', 'id is the primary key'); +select * from finish(); +rollback; +`; + +const main = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = path.join( + path.dirname(new URL(import.meta.url).pathname), + "..", + "bench", + "fixtures", + ); + const write = Effect.fnUntraced(function* (relative: string, content: string) { + const file = path.join(root, relative); + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(file, content); + }); + + for (const [size, count] of Object.entries(SCHEMA_SIZES)) { + yield* write(`schema-${size}/migrations/${stamp(1)}_bench_schema.sql`, schemaSql(count)); + } + for (let index = 1; index <= MIGRATION_COUNT; index++) { + yield* write( + `migrations/migrations/${stamp(index)}_bench_${String(index).padStart(2, "0")}.sql`, + migrationSql(index), + ); + } + yield* write("migrations/seed.sql", seedSql); + yield* write(`pgtap/migrations/${stamp(1)}_bench_checks.sql`, pgtapMigration); + yield* write("pgtap/tests/database/bench.test.sql", pgtapTest); +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-merge.ts b/apps/cli/scripts/bench-merge.ts new file mode 100644 index 0000000000..2a7b67e74e --- /dev/null +++ b/apps/cli/scripts/bench-merge.ts @@ -0,0 +1,74 @@ +/** + * Merges the per-shard `bench-cli.ts --out` reports of one run into a single report, ordering + * workloads as the manifest does. Run as `bun scripts/bench-merge.ts [--workloads ] + * --out ...`. Binary paths, run counts and settings come from the first + * shard. + */ +import { parseArgs } from "node:util"; +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Effect, FileSystem, Stdio } from "effect"; +import { + decodeBenchReport, + encodeBenchReport, + parseWorkloadManifest, + type BenchReport, +} from "./bench-report.ts"; + +/** Combines shard reports; a workload that appears in several shards keeps its first report. */ +export function mergeReports( + reports: ReadonlyArray, + order: ReadonlyArray, +): BenchReport | undefined { + const [first] = reports; + if (first === undefined) return undefined; + const byId = new Map(); + for (const report of reports) { + for (const workload of report.workloads) { + if (!byId.has(workload.id)) byId.set(workload.id, workload); + } + } + const rank = (id: string) => { + const index = order.indexOf(id); + return index === -1 ? order.length : index; + }; + const cpuProfilePath = reports.find( + (report) => report.cpuProfilePath !== undefined, + )?.cpuProfilePath; + return { + ...first, + workloads: [...byId.values()].sort((a, b) => rank(a.id) - rank(b.id)), + ...(cpuProfilePath === undefined ? {} : { cpuProfilePath }), + }; +} + +const main = Effect.gen(function* () { + const stdio = yield* Stdio.Stdio; + const argv = yield* stdio.args; + const { values, positionals } = parseArgs({ + args: [...argv], + allowPositionals: true, + options: { workloads: { type: "string" }, out: { type: "string" } }, + }); + if (values.out === undefined || positionals.length === 0) { + return yield* Effect.fail( + "usage: bun scripts/bench-merge.ts [--workloads ] --out ...", + ); + } + const fs = yield* FileSystem.FileSystem; + const order = + values.workloads === undefined + ? [] + : (yield* parseWorkloadManifest(yield* fs.readFileString(values.workloads))).map( + (workload) => workload.id, + ); + const reports: Array = []; + for (const file of positionals) { + reports.push(yield* decodeBenchReport(yield* fs.readFileString(file))); + } + const merged = mergeReports(reports, order); + if (merged !== undefined) yield* fs.writeFileString(values.out, yield* encodeBenchReport(merged)); +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-pack.ts b/apps/cli/scripts/bench-pack.ts new file mode 100644 index 0000000000..9d1cb28b62 --- /dev/null +++ b/apps/cli/scripts/bench-pack.ts @@ -0,0 +1,68 @@ +/** + * Packs benchmark workloads into shards for the stack benchmark matrix. Each shard runs on one + * runner, so base and branch of a workload always share a machine and no two workloads share one + * at the same time. Free of runtime dependencies so the workflow's plan job can import it. + */ +import type { Workload } from "./bench-report.ts"; + +export type PackedWorkload = Pick; + +export interface Shard { + readonly runtime: "docker" | "native"; + /** 1-based position within the runtime's shards. */ + readonly shard: number; + /** Shards for this runtime. */ + readonly shards: number; + readonly workloads: ReadonlyArray; + readonly estimatedSeconds: number; +} + +/** Seconds assumed for a workload that has no estimate. */ +export const DEFAULT_ESTIMATE_SECONDS = 60; +/** Seconds of work each shard aims for. */ +export const DEFAULT_TARGET_SECONDS = 600; +export const DEFAULT_MAX_SHARDS = 8; + +const estimateOf = (workload: PackedWorkload) => + workload.estimatedSeconds ?? DEFAULT_ESTIMATE_SECONDS; + +/** + * Splits each runtime's workloads into `ceil(total / target)` shards (at most `maxShards`), + * assigning the longest workloads first to the lightest shard. Shards come back ordered by runtime + * then index, with each shard's workloads in manifest order. + */ +export function packShards( + workloads: ReadonlyArray, + options: { readonly targetSeconds?: number; readonly maxShards?: number } = {}, +): ReadonlyArray { + const target = options.targetSeconds ?? DEFAULT_TARGET_SECONDS; + const maxShards = options.maxShards ?? DEFAULT_MAX_SHARDS; + const shards: Array = []; + for (const runtime of ["docker", "native"] as const) { + const group = workloads.filter((workload) => (workload.runtime ?? "docker") === runtime); + if (group.length === 0) continue; + const total = group.reduce((sum, workload) => sum + estimateOf(workload), 0); + const count = Math.min(maxShards, group.length, Math.max(1, Math.ceil(total / target))); + const bins = Array.from({ length: count }, () => ({ ids: new Set(), seconds: 0 })); + const longestFirst = [...group].sort( + (a, b) => estimateOf(b) - estimateOf(a) || a.id.localeCompare(b.id), + ); + for (const workload of longestFirst) { + const lightest = bins.reduce((best, bin) => (bin.seconds < best.seconds ? bin : best)); + lightest.ids.add(workload.id); + lightest.seconds += estimateOf(workload); + } + bins.forEach((bin, index) => { + shards.push({ + runtime, + shard: index + 1, + shards: count, + workloads: group + .filter((workload) => bin.ids.has(workload.id)) + .map((workload) => workload.id), + estimatedSeconds: bin.seconds, + }); + }); + } + return shards; +} diff --git a/apps/cli/scripts/bench-pack.unit.test.ts b/apps/cli/scripts/bench-pack.unit.test.ts new file mode 100644 index 0000000000..7b9cff2e6d --- /dev/null +++ b/apps/cli/scripts/bench-pack.unit.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "@effect/vitest"; +import { packShards } from "./bench-pack.ts"; + +const workload = (id: string, estimatedSeconds: number, runtime?: "docker" | "native") => ({ + id, + estimatedSeconds, + ...(runtime === undefined ? {} : { runtime }), +}); + +describe("packShards", () => { + it("balances each runtime's workloads across ceil(total / target) shards and never mixes runtimes", () => { + const shards = packShards( + [ + workload("a", 400), + workload("b", 300), + workload("c", 300), + workload("d", 200), + workload("n1", 100, "native"), + ], + { targetSeconds: 600 }, + ); + + const docker = shards.filter((shard) => shard.runtime === "docker"); + expect(docker.map((shard) => shard.estimatedSeconds)).toEqual([600, 600]); + expect(docker.every((shard) => shard.shards === 2)).toBe(true); + expect(shards.filter((shard) => shard.runtime === "native")).toEqual([ + { runtime: "native", shard: 1, shards: 1, workloads: ["n1"], estimatedSeconds: 100 }, + ]); + }); + + it("puts every workload in exactly one shard and keeps manifest order within a shard", () => { + const workloads = ["w1", "w2", "w3", "w4", "w5"].map((id, index) => + workload(id, 150 + index * 10), + ); + + const shards = packShards(workloads, { targetSeconds: 300 }); + + expect(shards.flatMap((shard) => shard.workloads).sort()).toEqual([ + "w1", + "w2", + "w3", + "w4", + "w5", + ]); + for (const shard of shards) { + expect(shard.workloads).toEqual( + workloads.map((w) => w.id).filter((id) => shard.workloads.includes(id)), + ); + } + }); + + it("assumes a default estimate for a workload without one and caps the shard count", () => { + const shards = packShards( + Array.from({ length: 20 }, (_, index) => ({ id: `w${index}` })), + { targetSeconds: 10, maxShards: 3 }, + ); + + expect(shards).toHaveLength(3); + expect(shards.flatMap((shard) => shard.workloads)).toHaveLength(20); + }); + + it("returns no shards for no workloads", () => { + expect(packShards([])).toEqual([]); + }); +}); diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 8e48662b76..918ad7280b 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -142,11 +142,32 @@ export function exitCodesDiffer( /* Workload manifest */ +const CliArgs = Schema.Array(Schema.String); + const Workload = Schema.Struct({ id: Schema.NonEmptyString, - args: Schema.Array(Schema.String), + args: CliArgs, cwdSetup: Schema.Literals(["none", "init"]), scenario: Schema.optionalKey(Schema.String), + /** Untimed commands run with the build under test before the measured launches. */ + setup: Schema.optionalKey(Schema.Array(CliArgs)), + /** Untimed commands that always run after the measured launches, even after a failure or timeout. */ + teardown: Schema.optionalKey(Schema.Array(CliArgs)), + /** `run` repeats setup and teardown for every launch; `block` shares them across a block of launches. */ + setupScope: Schema.optionalKey(Schema.Literals(["run", "block"])), + /** Launches with `SUPABASE_EXPERIMENTAL_STACK=1`. */ + stack: Schema.optionalKey(Schema.Boolean), + /** Container runtime the workload exercises; stack workloads pass it to `stack start` themselves. */ + runtime: Schema.optionalKey(Schema.Literals(["docker", "native"])), + /** Directory under the manifest's `fixtures/` copied into the project's `supabase/` directory after `init`. */ + fixture: Schema.optionalKey(Schema.NonEmptyString), + runs: Schema.optionalKey(Schema.Finite), + warmup: Schema.optionalKey(Schema.Finite), + /** Traced launches per build; defaults to one per measured run. */ + traceRuns: Schema.optionalKey(Schema.Finite), + timeoutSeconds: Schema.optionalKey(Schema.Finite), + /** Expected seconds for one pass of the workload over both builds, used to pack shards. */ + estimatedSeconds: Schema.optionalKey(Schema.Finite), }); export type Workload = typeof Workload.Type; @@ -162,10 +183,39 @@ export function validateWorkloads( const problems: Array = []; const seen = new Set(); for (const workload of workloads) { + const name = `workload "${workload.id}"`; if (seen.has(workload.id)) problems.push(`duplicate workload id "${workload.id}"`); seen.add(workload.id); if (workload.scenario !== undefined && context.apiUrl === undefined) { - problems.push(`workload "${workload.id}" names a scenario and needs --api-url`); + problems.push(`${name} names a scenario and needs --api-url`); + } + for (const field of ["runs", "warmup", "traceRuns", "timeoutSeconds"] as const) { + const value = workload[field]; + const minimum = field === "warmup" || field === "traceRuns" ? 0 : 1; + if (value !== undefined && (!Number.isInteger(value) || value < minimum)) { + problems.push(`${name} has an invalid ${field}`); + } + } + if (workload.estimatedSeconds !== undefined && !(workload.estimatedSeconds > 0)) { + problems.push(`${name} has an invalid estimatedSeconds`); + } + if (workload.stack === true && workload.runtime === undefined) { + problems.push(`${name} is a stack workload and needs a runtime`); + } + if (workload.stack === true && (workload.teardown?.length ?? 0) === 0) { + problems.push(`${name} is a stack workload and needs a teardown that destroys its stack`); + } + if (workload.stack !== true && workload.runtime === "native") { + problems.push(`${name} uses the native runtime, which only stack workloads support`); + } + for (const command of [workload.args, ...(workload.setup ?? [])]) { + if (command[0] !== "stack" || command[1] !== "start" || workload.runtime === undefined) { + continue; + } + const at = command.indexOf("--runtime"); + if (at === -1 || command[at + 1] !== workload.runtime) { + problems.push(`${name} runs stack start without --runtime ${workload.runtime}`); + } } } return problems; @@ -242,6 +292,14 @@ export type RunSample = typeof RunSample.Type; const WorkloadReport = Schema.Struct({ id: Schema.String, args: Schema.Array(Schema.String), + /** Container runtime the workload exercised, when it declares one. */ + runtime: Schema.optionalKey(Schema.Literals(["docker", "native"])), + /** Whether the workload ran against the stack backend. */ + stack: Schema.optionalKey(Schema.Boolean), + /** Traced launches expected per build; absent means one per measured run. */ + traceRuns: Schema.optionalKey(Schema.Finite), + /** Wall seconds across both builds, including setup, teardown and traced launches. */ + elapsedSeconds: Schema.optionalKey(Schema.Finite), base: Schema.Array(RunSample), branch: Schema.Array(RunSample), }); @@ -312,7 +370,9 @@ export interface BuildAnalysis { readonly tracedExitCodes: ReadonlyArray; readonly spans: Record | undefined; readonly api: ApiSummary | undefined; - /** Measured runs without a usable trace when tracing was requested. */ + /** Median container CLI calls per run by verb, summed over `ContainerCli.spawn` and `ContainerCli.run`. */ + readonly containers: Record | undefined; + /** Traced runs expected but without a usable trace when tracing was requested. */ readonly traceMissingRuns: number; readonly skippedTraceLines: number; } @@ -339,8 +399,28 @@ function summarizeApi(samples: ReadonlyArray): ApiSummary | undefined }; } -function summarizeBuild(samples: ReadonlyArray, traced: boolean): BuildAnalysis { +const CONTAINER_SPAN = /^ContainerCli\.(?:spawn|run)(?: \((.+)\))?$/u; + +/** Median per-run container CLI call counts keyed by verb; empty when no container call ran. */ +export function containerCounts(spans: Record): Record { + const counts: Record = {}; + for (const [name, aggregate] of Object.entries(spans)) { + const match = CONTAINER_SPAN.exec(name); + if (match === null) continue; + const verb = match[1] ?? "(unlabelled)"; + counts[verb] = (counts[verb] ?? 0) + aggregate.medianCount; + } + return counts; +} + +function summarizeBuild( + samples: ReadonlyArray, + traced: boolean, + expectedTraces: number, + countContainers: boolean, +): BuildAnalysis { const withTrace = samples.flatMap((sample) => (sample.trace === undefined ? [] : [sample.trace])); + const spans = traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined; return { timing: summarizeTimings(samples.map((sample) => sample.wallMs)), samples: samples.length, @@ -348,9 +428,13 @@ function summarizeBuild(samples: ReadonlyArray, traced: boolean): Bui tracedExitCodes: samples.flatMap((sample) => sample.tracedExitCode === undefined ? [] : [sample.tracedExitCode], ), - spans: traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined, + spans, api: summarizeApi(samples), - traceMissingRuns: traced ? samples.length - withTrace.length : 0, + containers: + spans === undefined || !countContainers || withTrace.length === 0 + ? undefined + : containerCounts(spans), + traceMissingRuns: traced ? Math.max(0, expectedTraces - withTrace.length) : 0, skippedTraceLines: withTrace.reduce((sum, trace) => sum + trace.skippedLines, 0), }; } @@ -361,9 +445,17 @@ export interface EndpointChange { readonly branch: number; } +export interface ContainerChange { + readonly verb: string; + readonly base: number; + readonly branch: number; +} + export interface WorkloadAnalysis { readonly id: string; readonly args: ReadonlyArray; + readonly runtime: "docker" | "native" | undefined; + readonly stack: boolean; readonly base: BuildAnalysis; readonly branch: BuildAnalysis; readonly deltaMs: number; @@ -374,6 +466,8 @@ export interface WorkloadAnalysis { /** Top phase deltas; empty unless the verdict is "slower" or "faster". */ readonly phaseRows: ReadonlyArray; readonly endpointChanges: ReadonlyArray; + /** Container verbs whose per-run call count differs between builds. */ + readonly containerChanges: ReadonlyArray; } /** Whether every launch of the workload exited 0 and every API call got a non-error response. */ @@ -402,10 +496,46 @@ function endpointChanges( .filter((change) => change.base !== change.branch); } +function containerChanges( + base: BuildAnalysis, + branch: BuildAnalysis, +): ReadonlyArray { + if (base.containers === undefined || branch.containers === undefined) return []; + const verbs = new Set([...Object.keys(base.containers), ...Object.keys(branch.containers)]); + return [...verbs] + .sort() + .map((verb) => ({ + verb, + base: base.containers?.[verb] ?? 0, + branch: branch.containers?.[verb] ?? 0, + })) + .filter((change) => change.base !== change.branch); +} + +/** Total container CLI calls per run, or undefined when container calls were not traced. */ +export function totalContainerCalls(build: BuildAnalysis): number | undefined { + return build.containers === undefined + ? undefined + : Object.values(build.containers).reduce((sum, count) => sum + count, 0); +} + /** Derives medians, comparisons and the timing verdict for one workload from its raw samples. */ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): WorkloadAnalysis { - const base = summarizeBuild(workload.base, traced); - const branch = summarizeBuild(workload.branch, traced); + const expectedTraces = (samples: number) => Math.min(workload.traceRuns ?? samples, samples); + // Stack backend container work runs in a detached owner process, so its spans never reach the CLI's trace. + const countContainers = workload.stack !== true; + const base = summarizeBuild( + workload.base, + traced, + expectedTraces(workload.base.length), + countContainers, + ); + const branch = summarizeBuild( + workload.branch, + traced, + expectedTraces(workload.branch.length), + countContainers, + ); const delta = deltaOf(base.timing.median, branch.timing.median); const differ = exitCodesDiffer(base.exitCodes, branch.exitCodes); const verdict = timingVerdict(base.timing, branch.timing, { @@ -419,6 +549,8 @@ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): Work return { id: workload.id, args: workload.args, + runtime: workload.runtime, + stack: workload.stack ?? false, base, branch, deltaMs: delta.ms, @@ -429,12 +561,16 @@ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): Work phaseRows: verdict === "slower" || verdict === "faster" ? (spans ?? []).slice(0, PHASE_ROWS) : [], endpointChanges: endpointChanges(base, branch), + containerChanges: containerChanges(base, branch), }; } export type FlagKind = | "new-api-calls" | "api-calls-dropped" + | "new-container-calls" + | "container-calls-dropped" + | "container-calls-changed" | "external-calls" | "non-deterministic" | "request-log-mismatch" @@ -466,6 +602,23 @@ export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray { if (baseCalls !== undefined && baseCalls > 0 && branchCalls === 0) { add("api-calls-dropped", `base makes ${baseCalls} API calls, branch makes 0`); } + const baseContainers = totalContainerCalls(analysis.base); + const branchContainers = totalContainerCalls(analysis.branch); + if (baseContainers !== undefined && branchContainers !== undefined) { + if (baseContainers === 0 && branchContainers > 0) { + add("new-container-calls", `base makes 0 container calls, branch makes ${branchContainers}`); + } else if (baseContainers > 0 && branchContainers === 0) { + add( + "container-calls-dropped", + `base makes ${baseContainers} container calls, branch makes 0`, + ); + } else if (analysis.containerChanges.length > 0) { + add( + "container-calls-changed", + `container calls changed from ${baseContainers} to ${branchContainers}`, + ); + } + } if (Math.min(analysis.base.samples, analysis.branch.samples) < VERDICT_MIN_SAMPLES) { add( "insufficient-samples", diff --git a/apps/cli/scripts/bench-stack.unit.test.ts b/apps/cli/scripts/bench-stack.unit.test.ts new file mode 100644 index 0000000000..d0c09b2461 --- /dev/null +++ b/apps/cli/scripts/bench-stack.unit.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { + analyzeWorkload, + collectFlags, + parseWorkloadManifest, + validateWorkloads, + type RunSample, + type WorkloadReport, +} from "./bench-report.ts"; + +const plain = (wallMs = 100): RunSample => ({ wallMs, exitCode: 0, tracedExitCode: 0 }); + +const times = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +describe("stack workload rules", () => { + const stackWorkload = { + id: "s", + args: ["stack", "status"], + cwdSetup: "init", + stack: true, + runtime: "native", + setup: [["stack", "start", "--runtime", "native"]], + teardown: [["stack", "destroy", "--yes"]], + } as const; + + it.effect("decodes the setup, scope, runtime, fixture and sizing fields", () => + Effect.gen(function* () { + const [parsed] = yield* parseWorkloadManifest( + `{"workloads":[{ + "id":"s","args":["stack","status"],"cwdSetup":"init","stack":true,"runtime":"native", + "setup":[["stack","start","--runtime","native"]], + "teardown":[["stack","destroy","--yes"]], + "setupScope":"block","fixture":"migrations","runs":10,"warmup":1,"traceRuns":2, + "timeoutSeconds":120,"estimatedSeconds":90 + }]}`, + ); + + expect(parsed).toMatchObject({ + setupScope: "block", + teardown: [["stack", "destroy", "--yes"]], + runtime: "native", + fixture: "migrations", + estimatedSeconds: 90, + }); + }), + ); + + it("accepts a stack workload whose stack start names its runtime", () => { + expect(validateWorkloads([stackWorkload], { apiUrl: undefined })).toEqual([]); + }); + + it.each([ + [ + "a stack workload with no runtime", + { ...stackWorkload, runtime: undefined }, + 'workload "s" is a stack workload and needs a runtime', + ], + [ + "the native runtime outside stack mode", + { ...stackWorkload, stack: undefined }, + 'workload "s" uses the native runtime, which only stack workloads support', + ], + [ + "a stack start without --runtime", + { ...stackWorkload, setup: [["stack", "start"]] }, + 'workload "s" runs stack start without --runtime native', + ], + [ + "a stack start on another runtime", + { ...stackWorkload, args: ["stack", "start", "--runtime", "docker"] }, + 'workload "s" runs stack start without --runtime native', + ], + [ + "a stack workload with no teardown", + { ...stackWorkload, teardown: undefined }, + 'workload "s" is a stack workload and needs a teardown that destroys its stack', + ], + ["a zero run count", { ...stackWorkload, runs: 0 }, 'workload "s" has an invalid runs'], + [ + "a fractional timeout", + { ...stackWorkload, timeoutSeconds: 1.5 }, + 'workload "s" has an invalid timeoutSeconds', + ], + [ + "a non-positive estimate", + { ...stackWorkload, estimatedSeconds: 0 }, + 'workload "s" has an invalid estimatedSeconds', + ], + ])("rejects %s", (_name, workload, problem) => { + expect(validateWorkloads([workload], { apiUrl: undefined })).toEqual([problem]); + }); +}); + +describe("container operations", () => { + const containerSample = (counts: Record): RunSample => ({ + ...plain(), + trace: { + skippedLines: 0, + spans: Object.fromEntries( + Object.entries(counts).map(([name, count]) => [name, { totalMs: count * 4, count }]), + ), + }, + }); + const containerWorkload = ( + base: Record, + branch: Record, + extra: Partial = {}, + ): WorkloadReport => ({ + id: "w", + args: ["status"], + base: times(5, () => containerSample(base)), + branch: times(5, () => containerSample(branch)), + ...extra, + }); + + it("sums spawn and run spans per verb and ignores other spans", () => { + const analysis = analyzeWorkload( + containerWorkload( + { + "ContainerCli.spawn (container inspect)": 2, + "ContainerCli.run (container inspect)": 1, + "cli.run": 1, + }, + { "ContainerCli.spawn (container inspect)": 3 }, + ), + true, + ); + + expect(analysis.base.containers).toEqual({ "container inspect": 3 }); + expect(analysis.branch.containers).toEqual({ "container inspect": 3 }); + expect(analysis.containerChanges).toEqual([]); + expect(collectFlags(analysis)).toEqual([]); + }); + + it("flags an extra call as a change", () => { + const analysis = analyzeWorkload( + containerWorkload( + { "ContainerCli.spawn (container inspect)": 3 }, + { "ContainerCli.spawn (container inspect)": 4 }, + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual(["container-calls-changed"]); + expect(analysis.containerChanges).toEqual([{ verb: "container inspect", base: 3, branch: 4 }]); + }); + + it("flags container calls that appear from zero and ones that drop to zero", () => { + const appeared = analyzeWorkload( + containerWorkload({ "cli.run": 1 }, { "ContainerCli.spawn (ps)": 2 }), + true, + ); + const dropped = analyzeWorkload( + containerWorkload({ "ContainerCli.run (ps)": 2 }, { "cli.run": 1 }), + true, + ); + + expect(collectFlags(appeared).map((flag) => flag.kind)).toEqual(["new-container-calls"]); + expect(collectFlags(dropped).map((flag) => flag.kind)).toEqual(["container-calls-dropped"]); + }); + + it("does not flag container calls when one build has no usable trace", () => { + const analysis = analyzeWorkload( + { + ...containerWorkload({ "ContainerCli.spawn (ps)": 2 }, {}), + branch: times(5, () => plain()), + }, + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).not.toContain( + "container-calls-dropped", + ); + }); + + it("leaves container calls uncounted for stack workloads and untraced runs", () => { + const stack = analyzeWorkload( + containerWorkload( + { "ContainerCli.spawn (ps)": 1 }, + { "ContainerCli.spawn (ps)": 2 }, + { stack: true }, + ), + true, + ); + const untraced = analyzeWorkload( + { + id: "w", + args: ["status"], + base: times(5, () => plain()), + branch: times(5, () => plain()), + }, + false, + ); + + expect(stack.base.containers).toBeUndefined(); + expect(collectFlags(stack)).toEqual([]); + expect(untraced.base.containers).toBeUndefined(); + }); +}); + +describe("traced run expectations", () => { + const tracedSample: RunSample = { + ...plain(), + trace: { skippedLines: 0, spans: { "cli.run": { totalMs: 50, count: 1 } } }, + }; + const withTraceRuns = (traceRuns: number): WorkloadReport => ({ + id: "w", + args: ["start"], + traceRuns, + base: [tracedSample, ...times(4, () => plain())], + branch: [tracedSample, ...times(4, () => plain())], + }); + + it("does not flag runs without a trace beyond the workload's traceRuns", () => { + expect(collectFlags(analyzeWorkload(withTraceRuns(1), true))).toEqual([]); + }); + + it("flags each build that is missing traces the workload expected", () => { + expect(collectFlags(analyzeWorkload(withTraceRuns(2), true)).map((flag) => flag.kind)).toEqual([ + "trace-incomplete", + "trace-incomplete", + ]); + }); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 21b94399ee..69a80f7a0e 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -1,8 +1,9 @@ /** * Renders a `bench-cli.ts --out` report as Markdown: the answer first, then a timing table, API - * calls, binary size, and collapsed details. Run as `bun scripts/bench-summary.ts --report ` - * with optional `--branch-build|--base-build|--replay ok|failed` step outcomes, `--head-sha`, - * `--base-ref`, `--merge-base`, and `--comment-out ` for the pull request + * calls, container operations (integration-heavy section), binary size, and collapsed details. Run + * as `bun scripts/bench-summary.ts --report ` with optional `--section + * lightweight|integration-heavy`, `--branch-build|--base-build|--replay ok|failed` step outcomes, + * `--head-sha`, `--base-ref`, `--merge-base`, and `--comment-out ` for the pull request * comment variant; the job summary goes to `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. */ import process from "node:process"; @@ -14,18 +15,29 @@ import { collectFlags, decodeBenchReport, VERDICT_MIN_PCT, + totalContainerCalls, type BenchReport, type Flag, type WorkloadAnalysis, } from "./bench-report.ts"; const USAGE = - "usage: bun scripts/bench-summary.ts --report [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha ] [--base-ref ] [--merge-base ] [--comment-out ]"; + "usage: bun scripts/bench-summary.ts --report [--section lightweight|integration-heavy] [--note ] [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha ] [--base-ref ] [--merge-base ] [--comment-out ]"; type Outcome = "ok" | "failed"; +/** A part of the benchmark comment; each is written by its own run. */ +export type Section = "lightweight" | "integration-heavy"; + +export const SECTION_NAMES: Record = { + lightweight: "Lightweight Commands", + "integration-heavy": "Integration Heavy Commands", +}; + export interface SummaryOptions { readonly report: string | undefined; + readonly section: Section; + readonly note: string | undefined; readonly branchBuild: Outcome; readonly baseBuild: Outcome; readonly replay: Outcome; @@ -43,6 +55,8 @@ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { args: [...argv], options: { report: { type: "string" }, + section: { type: "string", default: "lightweight" }, + note: { type: "string" }, "branch-build": { type: "string", default: "ok" }, "base-build": { type: "string", default: "ok" }, replay: { type: "string", default: "ok" }, @@ -57,8 +71,13 @@ export function parseSummaryArgs(argv: ReadonlyArray): SummaryOptions { throw new Error(`--${flag} must be "ok" or "failed"`); } } + if (values.section !== "lightweight" && values.section !== "integration-heavy") { + throw new Error('--section must be "lightweight" or "integration-heavy"'); + } const options: SummaryOptions = { report: values.report, + section: values.section, + note: values.note, branchBuild: values["branch-build"] === "failed" ? "failed" : "ok", baseBuild: values["base-build"] === "failed" ? "failed" : "ok", replay: values.replay === "failed" ? "failed" : "ok", @@ -137,6 +156,12 @@ function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { } case "api-calls-dropped": return `${name} no longer calls the Management API. It made ${callCount(analysis.base.api?.medianCalls)} before and now makes none.`; + case "new-container-calls": + return `${name} now runs container commands. It ran none before and now runs ${totalContainerCalls(analysis.branch) ?? "some"}.`; + case "container-calls-dropped": + return `${name} no longer runs container commands. It ran ${totalContainerCalls(analysis.base) ?? "some"} before and now runs none.`; + case "container-calls-changed": + return `${name} runs a different number of container commands: ${totalContainerCalls(analysis.base) ?? "some"} before, ${totalContainerCalls(analysis.branch) ?? "some"} now.`; case "external-calls": return `${name} made calls to hosts other than the local test server ${at}, so its timing may be skewed.`; case "non-deterministic": @@ -174,11 +199,22 @@ function changeText(analysis: WorkloadAnalysis): string { } } +/** How the workload runs: its container runtime, and `stack` when it uses the stack backend. */ +function runtimeLabel(analysis: WorkloadAnalysis): string { + const runtime = analysis.runtime ?? "docker"; + return analysis.stack ? `stack, ${runtime}` : runtime; +} + +const hasRuntimes = (analyses: ReadonlyArray) => + analyses.some((analysis) => analysis.runtime !== undefined || analysis.stack); + function timingTable(analyses: ReadonlyArray): string { + const withRuntime = hasRuntimes(analyses); return table( - ["Command", "Before", "After", "Change"], + ["Command", ...(withRuntime ? ["Runtime"] : []), "Before", "After", "Change"], analyses.map((analysis) => [ command(analysis), + ...(withRuntime ? [runtimeLabel(analysis)] : []), ms(analysis.base.timing.median), ms(analysis.branch.timing.median), changeText(analysis), @@ -186,6 +222,62 @@ function timingTable(analyses: ReadonlyArray): string { ); } +/** Commands whose container verbs are traced, with the change in each. */ +function containerOverview(analyses: ReadonlyArray): { + readonly line: string; +} { + const measured = analyses.filter( + (analysis) => + totalContainerCalls(analysis.base) !== undefined && + totalContainerCalls(analysis.branch) !== undefined, + ); + if (measured.length === 0) return { line: "Container operations were not counted." }; + const changed = measured.filter((analysis) => analysis.containerChanges.length > 0); + if (changed.length === 0) { + const any = measured.some( + (analysis) => + (totalContainerCalls(analysis.base) ?? 0) > 0 || + (totalContainerCalls(analysis.branch) ?? 0) > 0, + ); + return { + line: any + ? `Container operations: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}.` + : "Container operations: none recorded.", + }; + } + const lines = changed.map((analysis) => { + const verbs = analysis.containerChanges + .map((change) => `${code(change.verb)} ${change.base} -> ${change.branch}`) + .join(", "); + return `- ${command(analysis)} (${runtimeLabel(analysis)}): ${verbs}`; + }); + return { + line: [ + `Container operations changed for ${changed.length} ${plural(changed.length, "command", "commands")}:`, + ...lines, + ].join("\n"), + }; +} + +function containerDetailTable(analyses: ReadonlyArray): string { + const rows = analyses.flatMap((analysis) => { + const { base, branch } = analysis; + if (base.containers === undefined || branch.containers === undefined) return []; + const verbs = new Set([...Object.keys(base.containers), ...Object.keys(branch.containers)]); + return [...verbs] + .sort() + .map((verb) => [ + command(analysis), + runtimeLabel(analysis), + code(verb), + String(base.containers?.[verb] ?? 0), + String(branch.containers?.[verb] ?? 0), + ]); + }); + if (rows.length === 0) return "No container operations were recorded."; + return table(["Command", "Runtime", "Operation", "Before", "After"], rows); +} + interface ApiOverview { readonly line: string; /** Ids of the commands whose API calls changed. */ @@ -285,11 +377,12 @@ function phaseDetail(analyses: ReadonlyArray): string { return table(["Command", "Span", "Change", "Before -> after"], rows); } -export const BENCH_MARKER = ""; - export interface RenderContext { - /** `comment` adds the marker and the re-run footer; `summary` is the job summary. */ + /** A warning shown above the results, such as shards that did not finish. */ + readonly note?: string | undefined; + /** `comment` renders the section for the pull request comment; `summary` is the job summary. */ readonly target: "summary" | "comment"; + readonly section?: Section; readonly headSha?: string | undefined; readonly baseRef?: string | undefined; readonly mergeBase?: string | undefined; @@ -297,6 +390,17 @@ export interface RenderContext { const short = (sha: string) => sha.slice(0, 7); +/** "10 runs each", or the range when per-workload run counts differ. */ +function runsText(report: BenchReport): string { + const counts = report.workloads.flatMap((workload) => [ + workload.base.length, + workload.branch.length, + ]); + const low = counts.length === 0 ? report.runs : Math.min(...counts); + const high = counts.length === 0 ? report.runs : Math.max(...counts); + return low === high ? `${low} ${plural(low, "run", "runs")} each` : `${low}-${high} runs each`; +} + function contextLine(report: BenchReport | undefined, context: RenderContext): string { const parts: Array = []; if (context.headSha !== undefined && context.headSha !== "") { @@ -310,7 +414,7 @@ function contextLine(report: BenchReport | undefined, context: RenderContext): s }`; parts.push(`Commit ${code(short(context.headSha))}${against}`); } - if (report !== undefined) parts.push(`${report.runs} ${plural(report.runs, "run", "runs")} each`); + if (report !== undefined) parts.push(runsText(report)); parts.push("report only, never blocks merging"); return parts.join(" · "); } @@ -321,17 +425,15 @@ function frame( context: RenderContext, report?: BenchReport, ): string { - const footer = - context.target === "comment" ? ["Comment `/bench` to run it again on the latest commit."] : []; - return ( - [ - ...(context.target === "comment" ? [BENCH_MARKER] : []), - `## CLI benchmark: ${title}`, - contextLine(report, context), - ...body, - ...footer, - ].join("\n\n") + "\n" - ); + const section = context.section ?? "lightweight"; + const heading = + context.target === "comment" + ? `### ${SECTION_NAMES[section]}: ${title}` + : section === "lightweight" + ? `## CLI benchmark: ${title}` + : `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; + const note = context.note === undefined || context.note === "" ? [] : [`⚠️ ${context.note}`]; + return [heading, contextLine(report, context), ...note, ...body].join("\n\n") + "\n"; } /** The Markdown report, or `notice` in plain words when there is no report to compare. */ @@ -347,6 +449,8 @@ export function renderSummary( ); if (analyses.length === 0) return frame("no commands were benchmarked", [], context, report); const api = apiOverview(analyses); + const integration = (context.section ?? "lightweight") === "integration-heavy"; + const containers = integration ? containerOverview(analyses) : undefined; const moved = analyses .filter((analysis) => analysis.verdict === "slower" || analysis.verdict === "faster") .map((analysis) => analysis.id); @@ -360,13 +464,17 @@ export function renderSummary( ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis)}`), timingTable(analyses), api.line, + ...(containers === undefined ? [] : [containers.line]), sizeLine(report), [ - "
Details: timing ranges, API calls per command, slowest phases", + `
Details: timing ranges, API calls per command, ${integration ? "container operations, " : ""}slowest phases`, "#### Timing ranges", rangeTable(analyses), "#### API calls per command", apiDetailTable(analyses), + ...(integration + ? ["#### Container operations per command", containerDetailTable(analyses)] + : []), "#### Slowest phases", phaseDetail(analyses), "
", @@ -425,6 +533,8 @@ const main = Effect.gen(function* () { ? yield* loadReport(fs, options.report) : { report: undefined, notice: outcomeNotice ?? NO_REPORT }; const context = { + section: options.section, + note: options.note, headSha: options.headSha, baseRef: options.baseRef, mergeBase: options.mergeBase, diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 3d5e369c6a..de51ecf65a 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { BenchReport, RunSample } from "./bench-report.ts"; -import { BENCH_MARKER, parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; +import { parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; const run = (wallMs: number, calls: number, endpoints: Record): RunSample => ({ wallMs, @@ -128,18 +128,46 @@ describe("renderSummary", () => { expect(markdown).not.toContain("/bench"); }); - it("adds the marker, re-run hint only to the comment", () => { - const input = report([steady("version", ["--version"])]); - const context = { headSha: "abcdef1234" } as const; + it.each([ + { section: "lightweight", heading: "### Lightweight Commands: no changes found" }, + { section: "integration-heavy", heading: "### Integration Heavy Commands: no changes found" }, + ] as const)( + "titles a $section comment section with its name and leaves the markers and re-run note to the poster", + ({ section, heading }) => { + const input = report([steady("version", ["--version"])]); + + const comment = renderSummary(input, "unused", { target: "comment", section }); + + expect(comment.startsWith(`${heading}\n`)).toBe(true); + expect(comment).not.toContain(""); - expect(comment.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(comment).toContain("Comment `/bench` to run it again on the latest commit."); - expect(summary).not.toContain(BENCH_MARKER); - expect(summary).not.toContain("/bench"); + expect(markdown).toContain("3-5 runs each"); }); it("does not claim a missing timing verdict when commands were not comparable", () => { @@ -222,3 +250,86 @@ describe("parseSummaryArgs", () => { ).toMatchObject({ report: "r.json", replay: "ok", headSha: "abc", commentOut: "c.md" }); }); }); + +describe("renderSummary for integration-heavy commands", () => { + const withContainers = (counts: Record): RunSample => ({ + wallMs: 1000, + exitCode: 0, + tracedExitCode: 0, + trace: { + skippedLines: 0, + spans: Object.fromEntries( + Object.entries(counts).map(([verb, count]) => [ + `ContainerCli.spawn (${verb})`, + { totalMs: 5 * count, count }, + ]), + ), + }, + }); + const heavy = ( + id: string, + base: Record, + branch: Record, + extra: Partial = {}, + ) => ({ + id, + args: ["status"], + runtime: "docker" as const, + base: runs(5, () => withContainers(base)), + branch: runs(5, () => withContainers(branch)), + ...extra, + }); + const render = (workloads: BenchReport["workloads"]) => + renderSummary(report(workloads), "unused", { target: "summary", section: "integration-heavy" }); + + it("shows unchanged container operations with the runtime column", () => { + const markdown = render([ + heavy("status", { "container inspect": 3 }, { "container inspect": 3 }), + ]); + + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): no changes found"); + expect(markdown).toContain("| Command | Runtime | Before | After | Change |"); + expect(markdown).toContain("| `supabase status` | docker |"); + expect(markdown).toContain("Container operations: unchanged for the command."); + }); + + it("flags an extra container call per verb and counts the command once", () => { + const markdown = render([ + heavy("status", { "container inspect": 3 }, { "container inspect": 4 }), + ]); + + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): 1 thing to check"); + expect(markdown).toContain( + "⚠️ `supabase status` runs a different number of container commands: 3 before, 4 now.", + ); + expect(markdown).toContain("Container operations changed for 1 command:"); + expect(markdown).toContain("`container inspect` 3 -> 4"); + expect(markdown).toContain("| `supabase status` | docker | `container inspect` | 3 | 4 |"); + }); + + it("flags container commands that appear where there were none", () => { + const markdown = render([heavy("status", {}, { ps: 2 })]); + + expect(markdown).toContain( + "⚠️ `supabase status` now runs container commands. It ran none before and now runs 2.", + ); + }); + + it("does not count container operations for stack workloads, whose container work runs outside the trace", () => { + const markdown = render([ + heavy("stack-status", { ps: 1 }, { ps: 2 }, { stack: true, runtime: "native" }), + ]); + + expect(markdown).toContain("Container operations were not counted."); + expect(markdown).toContain("| `supabase status` | stack, native |"); + expect(markdown).not.toContain("container commands"); + }); + + it("leaves container operations out of the lightweight section", () => { + const markdown = renderSummary(report([heavy("status", { ps: 1 }, { ps: 1 })]), "unused", { + target: "summary", + }); + + expect(markdown).not.toContain("Container operations"); + }); +}); diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts index 403ce9664f..1b62dbe212 100644 --- a/apps/cli/scripts/bench-workloads.unit.test.ts +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -4,6 +4,8 @@ import { Effect, FileSystem } from "effect"; import { parseWorkloadManifest, validateWorkloads } from "./bench-report.ts"; const manifestPath = new URL("../bench/workloads.pr.json", import.meta.url).pathname; +const stackManifestPath = new URL("../bench/workloads.stack.json", import.meta.url).pathname; +const fixturesPath = new URL("../bench/fixtures/", import.meta.url).pathname; const scenariosDir = new URL("../../cli-e2e/fixtures/scenarios/", import.meta.url).pathname; const loadManifest = Effect.gen(function* () { @@ -11,6 +13,11 @@ const loadManifest = Effect.gen(function* () { return yield* parseWorkloadManifest(yield* fs.readFileString(manifestPath)); }); +const loadStackManifest = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(stackManifestPath)); +}); + describe("PR workload manifest", () => { it.effect("decodes and passes the structural rules when a replay server is configured", () => Effect.gen(function* () { @@ -37,3 +44,56 @@ describe("PR workload manifest", () => { }).pipe(Effect.provide(BunServices.layer)), ); }); + +describe("stack workload manifest", () => { + it.effect("decodes and passes the structural rules without a replay server", () => + Effect.gen(function* () { + const workloads = yield* loadStackManifest; + + expect(workloads.length).toBeGreaterThan(0); + expect(validateWorkloads(workloads, { apiUrl: undefined })).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("names only fixtures that exist and carry a migration, seed, or test", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workloads = yield* loadStackManifest; + + for (const workload of workloads) { + if (workload.fixture === undefined) continue; + const files = yield* fs.readDirectory(`${fixturesPath}${workload.fixture}`); + expect(files.length).toBeGreaterThan(0); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("sizes every workload for sharding and leaves nothing running after it", () => + Effect.gen(function* () { + const workloads = yield* loadStackManifest; + + for (const workload of workloads) { + expect(workload.estimatedSeconds, workload.id).toBeGreaterThan(0); + expect(workload.teardown?.length ?? 0, workload.id).toBeGreaterThan(0); + const stop = workload.stack === true ? ["stack", "destroy"] : ["stop", "--no-backup"]; + expect(workload.teardown?.at(-1)?.slice(0, stop.length), workload.id).toEqual(stop); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("starts a stack before any workload that measures a command against one", () => + Effect.gen(function* () { + const workloads = yield* loadStackManifest; + + for (const workload of workloads) { + const measuresStart = + workload.args.join(" ").startsWith("stack start") || + (workload.stack !== true && workload.args[0] === "start") || + (workload.stack !== true && workload.args.join(" ") === "db start"); + if (measuresStart) continue; + const start = workload.stack === true ? ["stack", "start"] : ["start"]; + expect(workload.setup?.[0]?.slice(0, start.length), workload.id).toEqual(start); + } + }).pipe(Effect.provide(BunServices.layer)), + ); +}); From 6eed9c204c5155689c477e567fb9be4c83f55d3c Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 16:59:42 +0530 Subject: [PATCH 03/11] ci(cli): add the stack benchmark workflow Runs from the Actions UI, the /bench full command, and a weekly schedule against the latest stable release. Plans shards, builds both binaries once, measures each shard on one runner, and reports to the job summary and the pull request comment. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/cli-bench/stack-plan.test.ts | 109 ++++ .github/scripts/cli-bench/stack-plan.ts | 196 +++++++ .github/workflows/cli-bench-stack.yml | 551 +++++++++++++++++++ 3 files changed, 856 insertions(+) create mode 100644 .github/scripts/cli-bench/stack-plan.test.ts create mode 100644 .github/scripts/cli-bench/stack-plan.ts create mode 100644 .github/workflows/cli-bench-stack.yml diff --git a/.github/scripts/cli-bench/stack-plan.test.ts b/.github/scripts/cli-bench/stack-plan.test.ts new file mode 100644 index 0000000000..b02255acb2 --- /dev/null +++ b/.github/scripts/cli-bench/stack-plan.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, test } from "bun:test"; +import { planStackRun, readManifest, type PlanDeps, type PlanInput } from "./stack-plan.ts"; + +const REPO = "supabase/cli"; +const manifest = readManifest( + JSON.stringify({ + workloads: [ + { id: "start", runtime: "docker", estimatedSeconds: 400 }, + { id: "db-reset", runtime: "docker", estimatedSeconds: 400 }, + { id: "stack-start-native", runtime: "native", estimatedSeconds: 100 }, + ], + }), +); + +const input = (overrides: Partial = {}): PlanInput => ({ + event: "workflow_dispatch", + workloads: "all", + runtime: "both", + base: "merge-base", + refName: "feature/x", + repository: REPO, + manifest, + ...overrides, +}); + +const deps = (overrides: Partial = {}): PlanDeps => ({ + latestRelease: async () => "v3.0.0", + openPullRequests: async () => [{ number: 12, headRepoFullName: REPO }], + ...overrides, +}); + +describe("planStackRun", () => { + test("measures everything against the latest stable release on a schedule, with no pull request", async () => { + const plan = await planStackRun( + input({ event: "schedule", workloads: "", runtime: "", base: "", refName: "develop" }), + deps(), + ); + + expect(plan).toMatchObject({ + workloads: "all", + runtime: "both", + baseKind: "release", + baseTag: "v3.0.0", + prNumber: "", + runLightweight: true, + runIntegration: true, + }); + }); + + test("attaches a merge-base run to the branch's single open pull request", async () => { + const plan = await planStackRun(input(), deps()); + + expect(plan).toMatchObject({ baseKind: "merge-base", baseTag: "", prNumber: "12" }); + }); + + test.each([ + ["no open pull request", [] as Array<{ number: number; headRepoFullName: string }>], + [ + "two open pull requests", + [ + { number: 1, headRepoFullName: REPO }, + { number: 2, headRepoFullName: REPO }, + ], + ], + ["a pull request from a fork", [{ number: 3, headRepoFullName: "someone/cli" }]], + ])("attaches no pull request when the branch has %s", async (_name, pulls) => { + const plan = await planStackRun(input(), deps({ openPullRequests: async () => pulls })); + + expect(plan.prNumber).toBe(""); + }); + + test("leaves the pull request comment alone for a run against a release tag", async () => { + const plan = await planStackRun(input({ base: "v3.0.0" }), deps()); + + expect(plan).toMatchObject({ baseKind: "release", baseTag: "v3.0.0", prNumber: "" }); + }); + + test.each(["main", "v3", "v3.0.0; rm -rf /", "$(id)", "../v3.0.0"])( + "rejects the base %p", + async (base) => { + await expect(planStackRun(input({ base }), deps())).rejects.toThrow("Unsupported base"); + }, + ); + + test.each([ + ["lightweight", { runLightweight: true, runIntegration: false, shards: 0 }], + ["integration-heavy", { runLightweight: false, runIntegration: true, shards: 3 }], + ["all", { runLightweight: true, runIntegration: true, shards: 3 }], + ])("plans shards for workloads=%s", async (workloads, expected) => { + const plan = await planStackRun(input({ workloads }), deps()); + + expect(plan.runLightweight).toBe(expected.runLightweight); + expect(plan.runIntegration).toBe(expected.runIntegration); + expect(plan.matrix.include).toHaveLength(expected.shards); + }); + + test("keeps only the requested runtime's shards", async () => { + const plan = await planStackRun(input({ runtime: "native" }), deps()); + + expect(plan.matrix.include).toEqual([ + expect.objectContaining({ runtime: "native", workloads: "stack-start-native" }), + ]); + }); + + test("rejects an unknown runtime or workloads choice", async () => { + await expect(planStackRun(input({ runtime: "podman" }), deps())).rejects.toThrow("runtime"); + await expect(planStackRun(input({ workloads: "none" }), deps())).rejects.toThrow("workloads"); + }); +}); diff --git a/.github/scripts/cli-bench/stack-plan.ts b/.github/scripts/cli-bench/stack-plan.ts new file mode 100644 index 0000000000..a9c6353b57 --- /dev/null +++ b/.github/scripts/cli-bench/stack-plan.ts @@ -0,0 +1,196 @@ +/** + * Planner for the stack benchmark workflow: resolves what a run measures and against which base, + * attaches the run to its pull request, and splits the workloads into shards. + * + * A scheduled run measures everything against the latest stable release. A dispatched run uses its + * inputs; its base is the merge base with `develop` unless a release tag is given. Only a + * merge-base run on a branch with exactly one open pull request into `develop` from this + * repository updates that pull request's comment. Needs read access only. + * Run in CI as: `bun .github/scripts/cli-bench/stack-plan.ts`. + */ +import { appendFileSync, readFileSync } from "node:fs"; + +import { packShards, type PackedWorkload } from "../../../apps/cli/scripts/bench-pack.ts"; +import { githubFetch, requireEnv } from "./github.ts"; + +export const WORKLOAD_CHOICES = ["all", "lightweight", "integration-heavy"] as const; +export const RUNTIME_CHOICES = ["both", "docker", "native"] as const; +export type WorkloadChoice = (typeof WORKLOAD_CHOICES)[number]; +export type RuntimeChoice = (typeof RUNTIME_CHOICES)[number]; + +const RELEASE_TAG = /^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; +const MANIFEST = "apps/cli/bench/workloads.stack.json"; + +export interface PlanInput { + event: string; + workloads: string; + runtime: string; + base: string; + refName: string; + repository: string; + manifest: ReadonlyArray; +} + +export interface PullRequestRef { + number: number; + headRepoFullName: string; +} + +export interface PlanDeps { + /** Tag of the latest stable release. */ + latestRelease: () => Promise; + /** Open pull requests into `develop` whose head is `refName`. */ + openPullRequests: (refName: string) => Promise>; +} + +export interface StackPlan { + workloads: WorkloadChoice; + runtime: RuntimeChoice; + baseKind: "merge-base" | "release"; + /** The release tag compared against; empty for a merge-base run. */ + baseTag: string; + /** Pull request whose comment this run updates; empty when none. */ + prNumber: string; + runLightweight: boolean; + runIntegration: boolean; + matrix: { include: ReadonlyArray> }; +} + +const isChoice = (choices: ReadonlyArray, value: string): value is T => + (choices as ReadonlyArray).includes(value); + +/** What a run measures, against what, and which shards run it. Throws on invalid inputs. */ +export async function planStackRun(input: PlanInput, deps: PlanDeps): Promise { + const scheduled = input.event === "schedule"; + const workloads = scheduled ? "all" : input.workloads; + const runtime = scheduled ? "both" : input.runtime; + if (!isChoice(WORKLOAD_CHOICES, workloads)) { + throw new Error(`Unsupported workloads "${workloads}".`); + } + if (!isChoice(RUNTIME_CHOICES, runtime)) throw new Error(`Unsupported runtime "${runtime}".`); + + let baseKind: StackPlan["baseKind"] = "merge-base"; + let baseTag = ""; + if (scheduled) { + baseKind = "release"; + baseTag = await deps.latestRelease(); + } else if (input.base !== "merge-base") { + baseKind = "release"; + baseTag = input.base; + } + if (baseKind === "release" && !RELEASE_TAG.test(baseTag)) { + throw new Error( + `Unsupported base "${baseTag}"; use merge-base or a release tag such as v3.0.0.`, + ); + } + + let prNumber = ""; + if (!scheduled && baseKind === "merge-base") { + const candidates = (await deps.openPullRequests(input.refName)).filter( + (pr) => pr.headRepoFullName === input.repository, + ); + if (candidates.length === 1) prNumber = String(candidates[0]?.number); + } + + const runIntegration = workloads === "all" || workloads === "integration-heavy"; + const wanted = (shardRuntime: string) => runtime === "both" || runtime === shardRuntime; + const include = runIntegration + ? packShards(input.manifest) + .filter((shard) => wanted(shard.runtime)) + .map((shard) => ({ ...shard, workloads: shard.workloads.join(",") })) + : []; + return { + workloads, + runtime, + baseKind, + baseTag, + prNumber, + runLightweight: workloads === "all" || workloads === "lightweight", + runIntegration, + matrix: { include }, + }; +} + +/** Reads the `id`, `runtime` and `estimatedSeconds` fields of a workload manifest. */ +export function readManifest(text: string): ReadonlyArray { + const parsed: unknown = JSON.parse(text); + const list = (parsed as { workloads?: unknown }).workloads; + if (!Array.isArray(list)) throw new Error("The workload manifest has no workloads list."); + return list.map((entry: Record) => ({ + id: String(entry.id), + ...(entry.runtime === "docker" || entry.runtime === "native" ? { runtime: entry.runtime } : {}), + ...(typeof entry.estimatedSeconds === "number" + ? { estimatedSeconds: entry.estimatedSeconds } + : {}), + })); +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const repository = requireEnv("GITHUB_REPOSITORY"); + const [owner = ""] = repository.split("/"); + const base = `https://api.github.com/repos/${repository}`; + + const plan = await planStackRun( + { + event: requireEnv("EVENT_NAME"), + workloads: process.env.INPUT_WORKLOADS ?? "all", + runtime: process.env.INPUT_RUNTIME ?? "both", + base: process.env.INPUT_BASE ?? "merge-base", + refName: requireEnv("REF_NAME"), + repository, + manifest: readManifest(readFileSync(MANIFEST, "utf8")), + }, + { + latestRelease: async () => { + const latest = (await (await githubFetch(`${base}/releases/latest`, token)).json()) as { + tag_name: string; + }; + return latest.tag_name; + }, + openPullRequests: async (refName) => { + const url = + `${base}/pulls?state=open&base=develop&per_page=100` + + `&head=${encodeURIComponent(`${owner}:${refName}`)}`; + const pulls = (await (await githubFetch(url, token)).json()) as Array<{ + number: number; + head: { repo: { full_name: string } | null }; + }>; + return pulls.map((pr) => ({ + number: pr.number, + headRepoFullName: pr.head.repo?.full_name ?? "", + })); + }, + }, + ); + + const outputs: Record = { + workloads: plan.workloads, + runtime: plan.runtime, + base_kind: plan.baseKind, + base_tag: plan.baseTag, + pr_number: plan.prNumber, + run_lightweight: String(plan.runLightweight), + run_integration: String(plan.runIntegration), + has_shards: String(plan.matrix.include.length > 0), + matrix: JSON.stringify(plan.matrix), + }; + console.log( + `CLI stack benchmark plan: workloads=${plan.workloads} runtime=${plan.runtime} ` + + `base=${plan.baseKind === "release" ? plan.baseTag : "merge-base"} pr=${plan.prNumber || "none"} ` + + `shards=${plan.matrix.include.length}`, + ); + appendFileSync( + requireEnv("GITHUB_OUTPUT"), + Object.entries(outputs) + .map(([key, value]) => `${key}=${value}\n`) + .join(""), + ); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/workflows/cli-bench-stack.yml b/.github/workflows/cli-bench-stack.yml new file mode 100644 index 0000000000..94746d88c7 --- /dev/null +++ b/.github/workflows/cli-bench-stack.yml @@ -0,0 +1,551 @@ +name: CLI benchmark (stack) +run-name: >- + ${{ github.event_name == 'schedule' && 'CLI benchmark stack (scheduled)' || + format('CLI benchmark stack ({0}): {1}', inputs.workloads, github.ref_name) }} + +# Report-only comparison of a CLI build with a base build for the workloads that need a running +# local stack or database. Starts only from the `/bench full` pull request command, the Actions UI +# (for any branch), or the weekly schedule; it never runs on a push. The base is the merge base +# with `develop` by default, a release tag when given, and the latest stable release on the +# schedule. Workloads are packed into shards, one runner at a time per workload, and each shard +# compares base and branch on its own machine. The run never blocks a pull request and is not a +# required check. +# See apps/cli/docs/tracing-monitoring.md. +on: + workflow_dispatch: + inputs: + workloads: + description: Which workloads to measure + type: choice + options: + - all + - lightweight + - integration-heavy + default: all + runtime: + description: Container runtime for the integration-heavy workloads + type: choice + options: + - both + - docker + - native + default: both + base: + description: merge-base, or a release tag such as v3.0.0 to compare against + type: string + default: merge-base + schedule: + - cron: "0 3 * * 0" + +permissions: {} + +# A second run for a branch waits for the first instead of cancelling it. +concurrency: + group: cli-bench-stack-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + # Decides what this run measures and against which base, splits the workloads into shards, and + # marks the pull request's Integration Heavy Commands section as running. + plan: + name: Plan + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + outputs: + workloads: ${{ steps.plan.outputs.workloads }} + runtime: ${{ steps.plan.outputs.runtime }} + base_kind: ${{ steps.plan.outputs.base_kind }} + base_tag: ${{ steps.plan.outputs.base_tag }} + pr_number: ${{ steps.plan.outputs.pr_number }} + run_lightweight: ${{ steps.plan.outputs.run_lightweight }} + run_integration: ${{ steps.plan.outputs.run_integration }} + has_shards: ${{ steps.plan.outputs.has_shards }} + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Plan + id: plan + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + INPUT_WORKLOADS: ${{ inputs.workloads }} + INPUT_RUNTIME: ${{ inputs.runtime }} + INPUT_BASE: ${{ inputs.base }} + run: bun .github/scripts/cli-bench/stack-plan.ts + - name: Mark the integration-heavy section as running + if: steps.plan.outputs.pr_number != '' && steps.plan.outputs.run_integration == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.plan.outputs.pr_number }} + MODE: running + SECTION: integration-heavy + HEAD_SHA: ${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts + + # Builds the branch binary, and the base binary from the merge base or the release asset, once + # for every shard. + build: + name: Build binaries + needs: plan + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + outputs: + branch_build: ${{ steps.outcomes.outputs.branch_build }} + base_build: ${{ steps.outcomes.outputs.base_build }} + merge_base: ${{ steps.base_build.outputs.merge_base }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Prepare benchmark directory + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + + - name: Build branch binary + id: branch_build + continue-on-error: true + shell: bash + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" + + - name: Build or download base binary + id: base_build + continue-on-error: true + if: steps.branch_build.outcome == 'success' + env: + HUSKY: "0" + BASE_KIND: ${{ needs.plan.outputs.base_kind }} + BASE_TAG: ${{ needs.plan.outputs.base_tag }} + RELEASE_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + shell: bash + run: | + mkdir -p "$BENCH_DIR/base" + if [ "$BASE_KIND" = release ]; then + echo "Base release: $BASE_TAG" + GH_TOKEN="$RELEASE_TOKEN" gh release download "$BASE_TAG" --repo "$GITHUB_REPOSITORY" \ + --pattern supabase_linux_amd64.tar.gz --dir "$RUNNER_TEMP/release" + tar -xzf "$RUNNER_TEMP/release/supabase_linux_amd64.tar.gz" -C "$BENCH_DIR/base" supabase + chmod +x "$BENCH_DIR/base/supabase" + exit 0 + fi + merge_base="$(git merge-base HEAD origin/develop)" + echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + # The setup action writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Record build outcomes + id: outcomes + if: always() + env: + BRANCH_OUTCOME: ${{ steps.branch_build.outcome }} + BASE_OUTCOME: ${{ steps.base_build.outcome }} + shell: bash + run: | + echo "branch_build=$([ "$BRANCH_OUTCOME" = success ] && echo ok || echo failed)" >> "$GITHUB_OUTPUT" + echo "base_build=$([ "$BASE_OUTCOME" = success ] && echo ok || echo failed)" >> "$GITHUB_OUTPUT" + + - name: Upload binaries + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-binaries + path: | + ${{ runner.temp }}/bench/base + ${{ runner.temp }}/bench/branch + retention-days: 1 + if-no-files-found: ignore + + # The pull request workloads against the replay server, for dispatches that ask for them (the + # `pull_request` run covers them otherwise). + lightweight: + name: Lightweight commands + needs: + - plan + - build + if: always() && needs.plan.result == 'success' && needs.plan.outputs.run_lightweight == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Download binaries + id: binaries + if: needs.build.outputs.branch_build == 'ok' && needs.build.outputs.base_build == 'ok' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-stack-binaries + path: ${{ runner.temp }}/bench + + - name: Start replay server + id: replay + continue-on-error: true + if: steps.binaries.outcome == 'success' + shell: bash + run: | + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + chmod +x "$RUNNER_TEMP/bench/base/supabase" "$RUNNER_TEMP/bench/branch/supabase" + bun apps/cli-e2e/scripts/replay-serve.ts > "$RUNNER_TEMP/bench/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$RUNNER_TEMP/bench/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$RUNNER_TEMP/bench/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + continue-on-error: true + if: steps.replay.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$RUNNER_TEMP/bench/report.json" | tee "$RUNNER_TEMP/bench/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() + env: + BRANCH_BUILD: ${{ needs.build.outputs.branch_build == 'ok' && 'ok' || 'failed' }} + BASE_BUILD: ${{ needs.build.outputs.base_build == 'ok' && 'ok' || 'failed' }} + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ needs.build.outputs.merge_base }} + BASE_REF: ${{ needs.plan.outputs.base_kind == 'release' && needs.plan.outputs.base_tag || 'develop' }} + HEAD_SHA: ${{ github.sha }} + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + bun apps/cli/scripts/bench-summary.ts \ + --section lightweight \ + --report "$RUNNER_TEMP/bench/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$RUNNER_TEMP/bench/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-report-lightweight + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-comment-lightweight + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + # One runner per shard and runtime. Base and branch alternate on the same machine; no other + # workload shares it. + shard: + name: Shard (${{ matrix.runtime }} ${{ matrix.shard }}/${{ matrix.shards }}) + needs: + - plan + - build + if: >- + needs.plan.outputs.has_shards == 'true' && + needs.build.outputs.branch_build == 'ok' && needs.build.outputs.base_build == 'ok' + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The image and artifact disks are read-only here: this job never commits them. + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + docker-images: ${{ matrix.runtime == 'docker' }} + native-artifacts: ${{ matrix.runtime == 'native' }} + + - name: Download binaries + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-stack-binaries + path: ${{ runner.temp }}/bench + + - name: Run benchmark + env: + RUNTIME: ${{ matrix.runtime }} + SHARD: ${{ matrix.shard }} + WORKLOADS: ${{ matrix.workloads }} + shell: bash + run: | + bench="$RUNNER_TEMP/bench" + chmod +x "$bench/base/supabase" "$bench/branch/supabase" + cache_args=() + if [ "$RUNTIME" = native ]; then + cache="${XDG_CACHE_HOME:-$HOME/.cache}/supabase-stack-artifacts" + mkdir -p "$cache" + cache_args=(--artifact-cache "$cache") + fi + bun apps/cli/scripts/bench-cli.ts \ + --base "$bench/base/supabase" \ + --branch "$bench/branch/supabase" \ + --workloads apps/cli/bench/workloads.stack.json \ + --runtime "$RUNTIME" \ + --only "$WORKLOADS" \ + "${cache_args[@]}" \ + --trace \ + --out "$bench/report-$RUNTIME-$SHARD.json" | tee "$bench/report-$RUNTIME-$SHARD.txt" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-report-${{ matrix.runtime }}-${{ matrix.shard }} + path: | + ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.json + ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.txt + retention-days: 14 + if-no-files-found: ignore + + # Releases /var/lib/docker so the image disk unmounts cleanly. + - name: Stop Docker + if: always() && matrix.runtime == 'docker' + run: sudo systemctl stop docker.socket docker.service + + # Combines the shard reports, writes the job summary and the comment section. + report: + name: Report + needs: + - plan + - build + - shard + if: always() && needs.plan.result == 'success' && needs.plan.outputs.run_integration == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Download shard reports + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cli-bench-stack-report-*-* + path: ${{ runner.temp }}/reports + + - name: Merge shard reports + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + mapfile -t files < <(find "$RUNNER_TEMP/reports" -name 'report-*.json' 2>/dev/null | sort) + if [ "${#files[@]}" -gt 0 ]; then + bun apps/cli/scripts/bench-merge.ts \ + --workloads apps/cli/bench/workloads.stack.json \ + --out "$RUNNER_TEMP/bench/report.json" "${files[@]}" + fi + + - name: Write job summary + env: + BRANCH_BUILD: ${{ needs.build.outputs.branch_build == 'ok' && 'ok' || 'failed' }} + BASE_BUILD: ${{ needs.build.outputs.base_build == 'ok' && 'ok' || 'failed' }} + SHARDS: ${{ needs.shard.result }} + MERGE_BASE: ${{ needs.build.outputs.merge_base }} + BASE_REF: ${{ needs.plan.outputs.base_kind == 'release' && needs.plan.outputs.base_tag || 'develop' }} + HEAD_SHA: ${{ github.sha }} + shell: bash + run: | + note="" + if [ "$SHARDS" = failure ] || [ "$SHARDS" = cancelled ]; then + note="Some shards did not finish, so their commands are missing." + fi + bun apps/cli/scripts/bench-summary.ts \ + --section integration-heavy \ + --report "$RUNNER_TEMP/bench/report.json" \ + --note "$note" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$RUNNER_TEMP/bench/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-merged-report + path: ${{ runner.temp }}/bench/report.json + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-stack-comment-integration-heavy + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + # The only jobs that write to the pull request. They run no benchmarked code and read the + # rendered markdown as data; a dispatch uses this file from the dispatched branch, which only + # people with write access can push or dispatch. + comment-lightweight: + name: Post lightweight section + needs: + - plan + - lightweight + if: always() && needs.plan.outputs.pr_number != '' && needs.plan.outputs.run_lightweight == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-stack-comment-lightweight + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ needs.plan.outputs.pr_number }} + MODE: upsert + SECTION: lightweight + MISSING_BODY: failure + HEAD_SHA: ${{ github.sha }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts + + comment: + name: Post integration-heavy section + needs: + - plan + - report + if: always() && needs.plan.outputs.pr_number != '' && needs.plan.outputs.run_integration == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-stack-comment-integration-heavy + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ needs.plan.outputs.pr_number }} + MODE: upsert + SECTION: integration-heavy + MISSING_BODY: failure + HEAD_SHA: ${{ github.sha }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts From 339eb13c07c9a552231a3afe023fbd48c966b491 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 16:59:42 +0530 Subject: [PATCH 04/11] docs(cli): document the stack benchmark Co-Authored-By: Claude Opus 5.5 --- apps/cli/docs/tracing-monitoring.md | 125 ++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 16 deletions(-) diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index fe78022672..82ae20c6fe 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -70,7 +70,8 @@ flag for more than one command, and use `--cwd-setup init` when a command, such run inside an initialized project. Pass `--json` for a machine-readable report, `--out ` to also write the raw per-run samples, and `--cpu-prof` to capture a Bun CPU profile of one untimed branch launch (`BUN_OPTIONS=--cpu-prof`), printing its `.cpuprofile` path. Launches set -`SUPABASE_NO_UPDATE_NOTIFIER=1`; pass `--update-check` to include the release check. Each launch's +`SUPABASE_NO_UPDATE_NOTIFIER=1`, `DO_NOT_TRACK=1`, and `SUPABASE_TELEMETRY_DISABLED=1`, so benchmark +runs never send product telemetry; pass `--update-check` to include the release check. Each launch's fresh `SUPABASE_HOME` has no cached release tag, so the check calls GitHub on every launch, except with `--cwd-setup init`, where the setup `init` caches the tag in the project before the measured command runs. The extra traced launches add machine load, so take wall-time conclusions from a run @@ -95,19 +96,23 @@ start replaces the comparison with a notice naming the step. The workloads use n prompts. The first push to a pull request from this repository benchmarks and posts one comment, tagged with -the hidden marker ``. Later pushes skip the benchmark, and the job summary says -so, while that comment reports a result; a comment that reports a failed run is benchmarked again -on the next push. Pull requests from forks have a read-only token, so they get the job summary only +the hidden marker ``. The comment has two sections, "Lightweight Commands" (this +run's workloads) and "Integration Heavy Commands" (the [stack benchmark](#stack-benchmark)); each +section is written by its own run and shows the commit it measured. Later pushes skip the benchmark, +and the job summary says so, while the Lightweight Commands section reports a result; a section that +reports a failed run is benchmarked again on the next push. Pull requests from forks have a read-only token, so they get the job summary only and benchmark every push. A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current head commit. The command re-runs that commit's `pull_request` workflow run, as "Re-run all jobs" in -the Actions UI does, and the re-run updates the comment in place, or creates it if it is missing. -The comment keeps a collapsed "Previous runs" list below the report with the commit, time (UTC), -headline, and link of each earlier run, newest first and up to 20, stored in a hidden marker in the -comment. Only the repository owner and people with write access can run it, and only on open, -non-draft pull requests into `develop` from this repository; the command must be the whole comment, -so `/bench full` and anything else is ignored. An eyes reaction on the comment means GitHub accepted +the Actions UI does, and the re-run updates the Lightweight Commands section in place, or creates the comment if it is +missing. The comment keeps a collapsed "Previous runs" list below the sections with the section, +commit, time (UTC), headline, and link of each result a section replaced, newest first and up to 20 +across both sections, stored in a hidden marker in the comment. One note under the sections tells +readers to comment `/bench` (lightweight benchmarks only) or `/bench full` to run it again. Only the +repository owner and people with write access can run either command, and only on open, non-draft +pull requests into `develop` from this repository; the command must be the whole comment, so +anything other than `/bench` or `/bench full` is ignored. An eyes reaction on the comment means GitHub accepted the re-run and a benchmark is running; the result arrives as the updated benchmark comment, whether it succeeded or failed. When no re-run can start, the pull request gets a reply saying why: a benchmark is already running for the commit, the run is older than 30 days, GitHub refused the @@ -117,8 +122,8 @@ Known gap: a pull request whose changes match none of the workflow's path filter more files than GitHub's path filter inspects (300), gets no run, so there is nothing for `/bench` to re-run and it replies that there is none. -Workflows triggered by comments run from the default branch, so `/bench` works only once the -workflow is on `develop`. The `/bench` job runs default branch code only: it never checks out or +Workflows triggered by comments run from the default branch, so `/bench` and `/bench full` work only +once the workflows are on `develop`. The `/bench` job runs default branch code only: it never checks out or runs pull request code, because a comment-triggered run shares the default branch's cache scope. Benchmarks always run in the `pull_request` run. `resolve` (read-only) decides whether a push benchmarks; `rerun` is the only job with `actions: write`. `bench` has read-only permissions, and @@ -220,16 +225,104 @@ Caveats: - `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided the choice but are not what the benchmark measures. -- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, and `test db` - need Docker; `db push`, `db lint`, `db dump`, `db diff`, `migration list`, `migration up`, - `migration repair`, and `db schema declarative sync` need a database (a nightly tier); - `link` probes `https://.`, which never reaches the replay server; +- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, `test db`, + `db push`, `db lint`, `db dump`, `db diff`, `migration list`, and `migration up` need Docker or a + database and run in the [stack benchmark](#stack-benchmark) instead; `migration repair` and + `db schema declarative sync` are not benchmarked; `link` probes `https://.`, which never reaches the replay server; `functions deploy` uses Docker by default and its API path has no success scenario; `login` is interactive unless a token is passed; `functions serve` needs Docker. - The first launch of a process is slower (credential loading); the warm-up runs absorb it. - Launches write a profile file, so numbers from runs with `--api-url` are not comparable with runs without it. +### Stack benchmark + +`.github/workflows/cli-bench-stack.yml` measures the workloads that need a running local stack or +database: `start`, `status`, `stop`, `db start`, `db reset`, `db query`, `test db`, `gen types` at +three schema sizes, `db diff` (cold and warm), `db dump`, `db lint`, `migration up`, `db push --local`, +and `migration list`, plus the `stack start|status|stop` lifecycle and `db query` and `db reset` +against the stack backend. Like the pull request benchmark it is report only and never blocks a +merge. It never runs on a push; it starts in three ways: + +- A maintainer comments exactly `/bench full` on an open, non-draft pull request into `develop` from + this repository (same permission rules as `/bench`). The command re-measures the Integration Heavy + Commands section by dispatching the workflow on the pull request's branch. It also refreshes the + Lightweight Commands section, by re-running that commit's `pull_request` run, only when the section + does not already show the head commit and no lightweight run is queued or running; with no run to + re-run, the dispatch measures those workloads itself. While an integration-heavy run for the pull + request is queued or running, the command replies with that run's link and does nothing else. The + pull request also gets a reply when the branch has no stack workflow file yet (merge `develop` + into it), when the pull request is a draft, from a fork, or not into `develop`, and when GitHub + refuses the dispatch (with the status and message). An eyes reaction means the dispatch was + accepted. +- From the Actions UI, on any branch, with inputs `workloads` (`all`, `lightweight`, or + `integration-heavy`), `runtime` (`both`, `docker`, or `native`), and `base` (`merge-base`, the + default, or a release tag such as `v3.0.0`). The job summary always has the report. The pull + request comment is updated too only when `base` is `merge-base` and the branch has exactly one + open pull request into `develop` from this repository. +- Weekly, on a schedule (Sunday 03:00 UTC) on `develop`, against the latest stable release resolved + at run time, so each report shows what changed since the last release. The report goes to the job + summary and the run's artifacts; nobody is notified. Slack notifications come later, once the data + proves valid and useful. To compare two releases by hand, dispatch the workflow on the newer tag + with `base` set to the older one. + +A dispatch run uses the workflow file and code of the branch it was started on. Because GitHub runs +dispatch, schedule, and comment-triggered workflows only from the default branch, none of these can +run before the workflow is on `develop`. + +The workflow plans, builds, measures, and reports in separate jobs. `plan` resolves the base, packs +the workloads into shards of roughly ten minutes each from each workload's `estimatedSeconds`, and +marks the Integration Heavy Commands section as running with a link to the run. `build` builds the +branch binary once, and the base binary from the merge base or the release asset. Each shard is one +runner per runtime; it alternates base and branch on that machine and runs no other workload at the +same time. `report` merges the shard reports into the job summary and the comment section. A failed +run shows the failure and a link to the run in its section. + +Runtimes are Docker (default-mode commands and `--runtime docker`) and the native backend +(`--runtime native`), both on Linux x64. Cold downloads are not measured: images and native +archives are fetched before the measured launches, and each shard shares one native artifact cache +across all launches through `--artifact-cache`, which links it in as `cache/stack` of every launch's +fresh `SUPABASE_HOME`. + +The workload manifest `apps/cli/bench/workloads.stack.json` adds these fields to the entries +described above: + +| Field | Meaning | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| `setup`, `teardown` | Lists of CLI arguments run untimed with the build under test. `teardown` always runs, even after a failure or timeout. | +| `setupScope` | `run` (default) repeats setup and teardown for every launch; `block` starts one stack per block of five launches. | +| `stack` | Sets `SUPABASE_EXPERIMENTAL_STACK=1` on every launch of the workload, the only variable a workload can set. | +| `runtime` | `docker` or `native`. A stack workload must pass the same `--runtime` to every `stack start`. | +| `fixture` | A directory under `apps/cli/bench/fixtures/` copied into the project's `supabase/` directory after `init`. | +| `runs`, `warmup`, `traceRuns` | Per-workload overrides; `--runs` and `--warmup` on the command line win. `traceRuns` is the traced launches per build. | +| `timeoutSeconds`, `estimatedSeconds` | Launch timeout (default 60) and the seconds one pass over both builds takes, which sizes the shards. | + +Setup, the measured launches, and teardown of one run (or block) share one project directory and +`SUPABASE_HOME`; every launch still gets its own telemetry and trace settings. Read-only workloads +use `block` scope; workloads that change state use `run`. Only one stack is ever up at a time per +runner. Stack mode supports only pg-delta for `db diff`, so the `db diff` workloads pass +`--use-pg-delta` in both modes. A workload that fails on the base build, such as one using a flag +the base does not have, shows as a base failure and is not compared. + +The Integration Heavy Commands report adds a container operations table: the number of container +CLI calls per run for each verb, such as `container inspect`, flagged when a count changes, appears, +or drops to zero. Container work in the stack backend runs outside the CLI's trace, so stack +workloads show no container counts. The fixtures under `apps/cli/bench/fixtures/` are generated by +`bun apps/cli/scripts/bench-fixtures.ts` and committed; changing one starts a new series of results. + +To run it locally, build two binaries as above and run a subset: + +```sh +bun apps/cli/scripts/bench-cli.ts --base --branch \ + --workloads apps/cli/bench/workloads.stack.json --runtime docker --only status,db-query \ + --trace --out /tmp/report.json +bun apps/cli/scripts/bench-summary.ts --section integration-heavy --report /tmp/report.json +``` + +Use `--runtime native` with `--artifact-cache ` for the native workloads. Refresh the list from +the same usage query as the pull request benchmark, keeping the commands that need a stack or +database instead of dropping them. + ## Grafana, Tempo, or Jaeger Run a local OpenTelemetry stack, for example Grafana's all-in-one image: From 1abdfced93066d7c7003cda7f86e242b3bc36d75 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 16:59:48 +0530 Subject: [PATCH 05/11] ci(cli): harden benchmark comment sanitizing and dispatch ref Co-Authored-By: Claude Opus 5.5 --- .github/scripts/cli-bench/history.test.ts | 11 +++++++---- .github/scripts/cli-bench/history.ts | 13 ++++++++----- .github/scripts/cli-bench/post-comment.test.ts | 6 ++++++ .github/scripts/cli-bench/post-comment.ts | 4 +++- .github/scripts/cli-bench/rerun.ts | 2 +- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts index f302079806..561080c211 100644 --- a/.github/scripts/cli-bench/history.test.ts +++ b/.github/scripts/cli-bench/history.test.ts @@ -120,6 +120,13 @@ describe("previousEntries", () => { }); describe("renderHistory and decodeHistory", () => { + test("reads an entry written before sections existed as a lightweight run", () => { + const legacy = { ...entry(1), section: undefined }; + const body = ``; + + expect(decodeHistory(body)).toEqual([entry(1)]); + }); + test("round-trips the entries through the hidden marker", () => { const entries = [entry(3), heavy(2), entry(1)]; @@ -148,10 +155,6 @@ describe("renderHistory and decodeHistory", () => { ["no marker", "plain comment"], ["not base64 json", ""], ["wrong shape", ``], - [ - "an entry without a section", - ``, - ], [ "a forged run link", ``, diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts index 82b695117e..c626f7ba44 100644 --- a/.github/scripts/cli-bench/history.ts +++ b/.github/scripts/cli-bench/history.ts @@ -85,11 +85,14 @@ export function decodeHistory(body: string): Array { if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return []; try { const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); - return Array.isArray(value) && - value.length <= MAX_PREVIOUS_RUNS + SECTIONS.length && - value.every(isEntry) - ? value - : []; + if (!Array.isArray(value) || value.length > MAX_PREVIOUS_RUNS + SECTIONS.length) return []; + // Entries written before comments had sections belong to the lightweight run. + const entries = value.map((entry: unknown) => + typeof entry === "object" && entry !== null && !("section" in entry) + ? { ...entry, section: "lightweight" } + : entry, + ); + return entries.every(isEntry) ? entries : []; } catch { return []; } diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index 2e5142013b..d44254d349 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -212,6 +212,12 @@ describe("sanitize", () => { expect(clean).toContain("
"); }); + test("keeps an escaped bracket from becoming a live link", () => { + const clean = sanitize("\\[x\\](//evil.example/p)"); + + expect(clean).toBe("\\\\\\[x\\\\\\](//evil.example/p)"); + }); + test("keeps a report from forging a section marker", () => { const forged = sanitize(""); diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts index 05fca5dd9c..1d15bd0f71 100644 --- a/.github/scripts/cli-bench/post-comment.ts +++ b/.github/scripts/cli-bench/post-comment.ts @@ -65,6 +65,7 @@ const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u /** Keeps mentions, links, images and raw HTML in untrusted markdown from rendering. */ export function sanitize(markdown: string): string { return markdown + .replaceAll("\\", "\\\\") .replaceAll("&", "&") .replaceAll(/<[^>\n]*>?/gu, (tag) => ALLOWED_TAGS.has(tag) ? tag : tag.replaceAll("<", "<"), @@ -184,7 +185,8 @@ export function composeBody(sections: Sections, entries: ReadonlyArray content: state.content.length <= share ? state.content - : state.content.slice(0, share - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE, + : state.content.slice(0, Math.max(0, share - TRUNCATION_NOTICE.length)) + + TRUNCATION_NOTICE, }; } return assemble(cut, []); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index ed31bb6f46..22e6796112 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -323,7 +323,7 @@ async function main(): Promise { await githubFetch(`${base}/actions/workflows/${STACK_WORKFLOW_FILE}/dispatches`, token, { method: "POST", body: JSON.stringify({ - ref: headRef, + ref: `refs/heads/${headRef}`, inputs: { workloads, runtime: "both", base: "merge-base" }, }), }); From 9b05727241b2be14facfe6395a2c0096558b3dde Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 17:28:54 +0530 Subject: [PATCH 06/11] ci(cli): rename the stack benchmark tier to full and drop unused plan outputs Co-Authored-By: Claude Opus 5.5 --- .../{stack-plan.test.ts => full-plan.test.ts} | 22 ++++---- .../cli-bench/{stack-plan.ts => full-plan.ts} | 18 +++--- .github/scripts/cli-bench/rerun.test.ts | 34 +++++------ .github/scripts/cli-bench/rerun.ts | 56 +++++++++---------- .github/scripts/cli-bench/resolve.test.ts | 4 +- ...cli-bench-stack.yml => cli-bench-full.yml} | 38 ++++++------- .github/workflows/cli-bench.yml | 4 +- ...rkloads.stack.json => workloads.full.json} | 0 apps/cli/docs/tracing-monitoring.md | 14 ++--- apps/cli/scripts/bench-pack.ts | 2 +- apps/cli/scripts/bench-workloads.unit.test.ts | 16 +++--- 11 files changed, 102 insertions(+), 106 deletions(-) rename .github/scripts/cli-bench/{stack-plan.test.ts => full-plan.test.ts} (78%) rename .github/scripts/cli-bench/{stack-plan.ts => full-plan.ts} (91%) rename .github/workflows/{cli-bench-stack.yml => cli-bench-full.yml} (94%) rename apps/cli/bench/{workloads.stack.json => workloads.full.json} (100%) diff --git a/.github/scripts/cli-bench/stack-plan.test.ts b/.github/scripts/cli-bench/full-plan.test.ts similarity index 78% rename from .github/scripts/cli-bench/stack-plan.test.ts rename to .github/scripts/cli-bench/full-plan.test.ts index b02255acb2..b11b214050 100644 --- a/.github/scripts/cli-bench/stack-plan.test.ts +++ b/.github/scripts/cli-bench/full-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { planStackRun, readManifest, type PlanDeps, type PlanInput } from "./stack-plan.ts"; +import { planFullRun, readManifest, type PlanDeps, type PlanInput } from "./full-plan.ts"; const REPO = "supabase/cli"; const manifest = readManifest( @@ -29,9 +29,9 @@ const deps = (overrides: Partial = {}): PlanDeps => ({ ...overrides, }); -describe("planStackRun", () => { +describe("planFullRun", () => { test("measures everything against the latest stable release on a schedule, with no pull request", async () => { - const plan = await planStackRun( + const plan = await planFullRun( input({ event: "schedule", workloads: "", runtime: "", base: "", refName: "develop" }), deps(), ); @@ -48,7 +48,7 @@ describe("planStackRun", () => { }); test("attaches a merge-base run to the branch's single open pull request", async () => { - const plan = await planStackRun(input(), deps()); + const plan = await planFullRun(input(), deps()); expect(plan).toMatchObject({ baseKind: "merge-base", baseTag: "", prNumber: "12" }); }); @@ -64,13 +64,13 @@ describe("planStackRun", () => { ], ["a pull request from a fork", [{ number: 3, headRepoFullName: "someone/cli" }]], ])("attaches no pull request when the branch has %s", async (_name, pulls) => { - const plan = await planStackRun(input(), deps({ openPullRequests: async () => pulls })); + const plan = await planFullRun(input(), deps({ openPullRequests: async () => pulls })); expect(plan.prNumber).toBe(""); }); test("leaves the pull request comment alone for a run against a release tag", async () => { - const plan = await planStackRun(input({ base: "v3.0.0" }), deps()); + const plan = await planFullRun(input({ base: "v3.0.0" }), deps()); expect(plan).toMatchObject({ baseKind: "release", baseTag: "v3.0.0", prNumber: "" }); }); @@ -78,7 +78,7 @@ describe("planStackRun", () => { test.each(["main", "v3", "v3.0.0; rm -rf /", "$(id)", "../v3.0.0"])( "rejects the base %p", async (base) => { - await expect(planStackRun(input({ base }), deps())).rejects.toThrow("Unsupported base"); + await expect(planFullRun(input({ base }), deps())).rejects.toThrow("Unsupported base"); }, ); @@ -87,7 +87,7 @@ describe("planStackRun", () => { ["integration-heavy", { runLightweight: false, runIntegration: true, shards: 3 }], ["all", { runLightweight: true, runIntegration: true, shards: 3 }], ])("plans shards for workloads=%s", async (workloads, expected) => { - const plan = await planStackRun(input({ workloads }), deps()); + const plan = await planFullRun(input({ workloads }), deps()); expect(plan.runLightweight).toBe(expected.runLightweight); expect(plan.runIntegration).toBe(expected.runIntegration); @@ -95,7 +95,7 @@ describe("planStackRun", () => { }); test("keeps only the requested runtime's shards", async () => { - const plan = await planStackRun(input({ runtime: "native" }), deps()); + const plan = await planFullRun(input({ runtime: "native" }), deps()); expect(plan.matrix.include).toEqual([ expect.objectContaining({ runtime: "native", workloads: "stack-start-native" }), @@ -103,7 +103,7 @@ describe("planStackRun", () => { }); test("rejects an unknown runtime or workloads choice", async () => { - await expect(planStackRun(input({ runtime: "podman" }), deps())).rejects.toThrow("runtime"); - await expect(planStackRun(input({ workloads: "none" }), deps())).rejects.toThrow("workloads"); + await expect(planFullRun(input({ runtime: "podman" }), deps())).rejects.toThrow("runtime"); + await expect(planFullRun(input({ workloads: "none" }), deps())).rejects.toThrow("workloads"); }); }); diff --git a/.github/scripts/cli-bench/stack-plan.ts b/.github/scripts/cli-bench/full-plan.ts similarity index 91% rename from .github/scripts/cli-bench/stack-plan.ts rename to .github/scripts/cli-bench/full-plan.ts index a9c6353b57..5c4b24ee9d 100644 --- a/.github/scripts/cli-bench/stack-plan.ts +++ b/.github/scripts/cli-bench/full-plan.ts @@ -1,12 +1,12 @@ /** - * Planner for the stack benchmark workflow: resolves what a run measures and against which base, + * Planner for the full benchmark workflow: resolves what a run measures and against which base, * attaches the run to its pull request, and splits the workloads into shards. * * A scheduled run measures everything against the latest stable release. A dispatched run uses its * inputs; its base is the merge base with `develop` unless a release tag is given. Only a * merge-base run on a branch with exactly one open pull request into `develop` from this * repository updates that pull request's comment. Needs read access only. - * Run in CI as: `bun .github/scripts/cli-bench/stack-plan.ts`. + * Run in CI as: `bun .github/scripts/cli-bench/full-plan.ts`. */ import { appendFileSync, readFileSync } from "node:fs"; @@ -19,7 +19,7 @@ export type WorkloadChoice = (typeof WORKLOAD_CHOICES)[number]; export type RuntimeChoice = (typeof RUNTIME_CHOICES)[number]; const RELEASE_TAG = /^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; -const MANIFEST = "apps/cli/bench/workloads.stack.json"; +const MANIFEST = "apps/cli/bench/workloads.full.json"; export interface PlanInput { event: string; @@ -43,7 +43,7 @@ export interface PlanDeps { openPullRequests: (refName: string) => Promise>; } -export interface StackPlan { +export interface FullPlan { workloads: WorkloadChoice; runtime: RuntimeChoice; baseKind: "merge-base" | "release"; @@ -60,7 +60,7 @@ const isChoice = (choices: ReadonlyArray, value: string): v (choices as ReadonlyArray).includes(value); /** What a run measures, against what, and which shards run it. Throws on invalid inputs. */ -export async function planStackRun(input: PlanInput, deps: PlanDeps): Promise { +export async function planFullRun(input: PlanInput, deps: PlanDeps): Promise { const scheduled = input.event === "schedule"; const workloads = scheduled ? "all" : input.workloads; const runtime = scheduled ? "both" : input.runtime; @@ -69,7 +69,7 @@ export async function planStackRun(input: PlanInput, deps: PlanDeps): Promise { const [owner = ""] = repository.split("/"); const base = `https://api.github.com/repos/${repository}`; - const plan = await planStackRun( + const plan = await planFullRun( { event: requireEnv("EVENT_NAME"), workloads: process.env.INPUT_WORKLOADS ?? "all", @@ -165,8 +165,6 @@ async function main(): Promise { ); const outputs: Record = { - workloads: plan.workloads, - runtime: plan.runtime, base_kind: plan.baseKind, base_tag: plan.baseTag, pr_number: plan.prNumber, @@ -176,7 +174,7 @@ async function main(): Promise { matrix: JSON.stringify(plan.matrix), }; console.log( - `CLI stack benchmark plan: workloads=${plan.workloads} runtime=${plan.runtime} ` + + `CLI full benchmark plan: workloads=${plan.workloads} runtime=${plan.runtime} ` + `base=${plan.baseKind === "release" ? plan.baseTag : "merge-base"} pr=${plan.prNumber || "none"} ` + `shards=${plan.matrix.include.length}`, ); diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts index 036d3ba5de..9f5e8c4490 100644 --- a/.github/scripts/cli-bench/rerun.test.ts +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -6,8 +6,8 @@ import { coversIntegrationHeavy, handleBenchCommand, RERUN_WINDOW_MS, - type StackRun, - type StackWorkloads, + type FullRun, + type FullWorkloads, type WorkflowRun, } from "./rerun.ts"; @@ -37,7 +37,7 @@ function setup( pr?: BenchPr; runs?: ReadonlyArray; rerunError?: Error; - stackRuns?: ReadonlyArray; + fullRuns?: ReadonlyArray; dispatchError?: Error; /** Short SHA the Lightweight Commands section measured. */ lightweightSha?: string; @@ -47,7 +47,7 @@ function setup( reactions: [] as Array, replies: [] as Array, reruns: [] as Array, - dispatches: [] as Array<{ ref: string; workloads: StackWorkloads }>, + dispatches: [] as Array<{ ref: string; workloads: FullWorkloads }>, }; const io: CommandIo = { fetchPr: async () => options.pr ?? pr(), @@ -57,8 +57,8 @@ function setup( if (options.rerunError !== undefined) throw options.rerunError; calls.reruns.push(id); }, - listStackRuns: async () => options.stackRuns ?? [], - dispatchStack: async (ref, workloads) => { + listFullRuns: async () => options.fullRuns ?? [], + dispatchFull: async (ref, workloads) => { if (options.dispatchError !== undefined) throw options.dispatchError; calls.dispatches.push({ ref, workloads }); }, @@ -198,10 +198,10 @@ describe("handleBenchCommand", () => { }); }); -const stackRun = (overrides: Partial = {}): StackRun => ({ +const stackRun = (overrides: Partial = {}): FullRun => ({ id: 31, status: "in_progress", - displayTitle: "CLI benchmark stack (integration-heavy): feature/x", + displayTitle: "CLI benchmark full (integration-heavy): feature/x", url: "https://github.com/supabase/cli/actions/runs/31", ...overrides, }); @@ -251,7 +251,7 @@ describe("handleBenchCommand with /bench full", () => { "its pull request run is older than 30 days", [run({ createdAt: new Date(NOW.getTime() - RERUN_WINDOW_MS - 1000).toISOString() })], ], - ])("measures the lightweight workloads in the stack run when %s", async (_name, runs) => { + ])("measures the lightweight workloads in the full run when %s", async (_name, runs) => { const { io, calls } = setup({ lightweightSha: undefined, runs }); const outcome = await handleBenchCommand(command("/bench full"), io); @@ -262,7 +262,7 @@ describe("handleBenchCommand with /bench full", () => { }); test("replies with the link of an integration-heavy run already in progress and starts nothing", async () => { - const { io, calls } = setup({ stackRuns: [stackRun()] }); + const { io, calls } = setup({ fullRuns: [stackRun()] }); const outcome = await handleBenchCommand(command("/bench full"), io); @@ -276,7 +276,7 @@ describe("handleBenchCommand with /bench full", () => { test("starts a run when the one in progress measures only the lightweight workloads", async () => { const { io, calls } = setup({ lightweightSha: "aaaaaaa", - stackRuns: [stackRun({ displayTitle: "CLI benchmark stack (lightweight): feature/x" })], + fullRuns: [stackRun({ displayTitle: "CLI benchmark full (lightweight): feature/x" })], }); await handleBenchCommand(command("/bench full"), io); @@ -287,7 +287,7 @@ describe("handleBenchCommand with /bench full", () => { test("starts a run when earlier ones have finished", async () => { const { io, calls } = setup({ lightweightSha: "aaaaaaa", - stackRuns: [stackRun({ status: "completed" })], + fullRuns: [stackRun({ status: "completed" })], }); await handleBenchCommand(command("/bench full"), io); @@ -347,7 +347,7 @@ describe("handleBenchCommand with /bench full", () => { expect(calls.dispatches).toEqual([]); }); - test("keeps a started stack run when re-running the lightweight run is refused", async () => { + test("keeps a started full run when re-running the lightweight run is refused", async () => { const { io, calls } = setup({ lightweightSha: "bbbbbbb", rerunError: new GithubRequestError(403, "nope", "u"), @@ -367,10 +367,10 @@ describe("handleBenchCommand with /bench full", () => { describe("coversIntegrationHeavy", () => { test.each([ - ["CLI benchmark stack (all): develop", true], - ["CLI benchmark stack (integration-heavy): feature/x", true], - ["CLI benchmark stack (lightweight): feature/x", false], - ["CLI benchmark stack (scheduled)", false], + ["CLI benchmark full (all): develop", true], + ["CLI benchmark full (integration-heavy): feature/x", true], + ["CLI benchmark full (lightweight): feature/x", false], + ["CLI benchmark full (scheduled)", false], ])("reads %p as %p", (title, expected) => { expect(coversIntegrationHeavy(title)).toBe(expected); }); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index 22e6796112..ff4342a398 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -5,7 +5,7 @@ * the repository owner or hold write access, and the pull request must be an open, ready one from * this repository into `develop`. `/bench` re-runs the existing `pull_request` run, which * benchmarks and updates the Lightweight Commands section. `/bench full` always re-measures the - * Integration Heavy Commands section by dispatching the stack benchmark workflow on the pull + * Integration Heavy Commands section by dispatching the full benchmark workflow on the pull * request's branch, and re-runs the lightweight part only when that section is behind the head * commit; the pull request gets a reply when nothing can run. * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. @@ -23,7 +23,7 @@ export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; /** The workflow file whose runs `/bench` re-runs. */ export const WORKFLOW_FILE = "cli-bench.yml"; /** The workflow `/bench full` dispatches. */ -export const STACK_WORKFLOW_FILE = "cli-bench-stack.yml"; +export const FULL_WORKFLOW_FILE = "cli-bench-full.yml"; /** GitHub refuses to re-run a workflow run older than this. */ export const RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; @@ -45,7 +45,7 @@ export interface WorkflowRun { createdAt: string; } -export interface StackRun { +export interface FullRun { id: number; status: string; /** The run name, which carries the `workloads` input. */ @@ -53,8 +53,8 @@ export interface StackRun { url: string; } -/** What a dispatched stack run measures. */ -export type StackWorkloads = "integration-heavy" | "all"; +/** What a dispatched full run measures. */ +export type FullWorkloads = "integration-heavy" | "all"; export interface CommandInput { prNumber: number; @@ -68,9 +68,9 @@ export interface CommandIo { /** Runs of the benchmark workflow triggered by `pull_request` for this commit, any order. */ listRuns: (headSha: string) => Promise>; rerun: (runId: number) => Promise; - /** Dispatched runs of the stack workflow on a branch, any order. */ - listStackRuns: (headRef: string) => Promise>; - dispatchStack: (headRef: string, workloads: StackWorkloads) => Promise; + /** Dispatched runs of the full workflow on a branch, any order. */ + listFullRuns: (headRef: string) => Promise>; + dispatchFull: (headRef: string, workloads: FullWorkloads) => Promise; /** Short SHA of the commit the comment's Lightweight Commands section measured, if it has one. */ lightweightSha: (prNumber: number) => Promise; reactToComment: (commentId: number) => Promise; @@ -81,7 +81,7 @@ export type CommandOutcome = | { kind: "ignored"; reason: string } | { kind: "replied"; reason: string } | { kind: "rerun"; runId: number } - | { kind: "dispatched"; workloads: StackWorkloads; rerunId: number | undefined }; + | { kind: "dispatched"; workloads: FullWorkloads; rerunId: number | undefined }; const short = (sha: string) => sha.slice(0, 7); @@ -106,22 +106,22 @@ export const replies = { notEligible: () => `\`${BENCH_FULL_COMMAND}\` runs only on pull requests from this repository into ` + `${BENCH_BASE_BRANCHES.join(" or ")}.`, - stackRunning: (url: string) => + fullRunning: (url: string) => `An integration-heavy benchmark is already queued or running for this pull request: ${url}. ` + "The comment updates when it finishes.", - noStackWorkflow: () => - "This branch does not have the stack benchmark workflow yet. Merge `develop` into the branch " + + noFullWorkflow: () => + "This branch does not have the full benchmark workflow yet. Merge `develop` into the branch " + `and comment \`${BENCH_FULL_COMMAND}\` again.`, - stackRefused: (status: number, detail: string) => + fullRefused: (status: number, detail: string) => `The integration-heavy benchmark could not be started: GitHub refused the request ` + `(${status} ${quoted(detail)}).`, }; -const STACK_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; +const FULL_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; -/** Whether a stack run's name says it measures the integration-heavy workloads. */ +/** Whether a full run's name says it measures the integration-heavy workloads. */ export function coversIntegrationHeavy(displayTitle: string): boolean { - const workloads = STACK_RUN_WORKLOADS.exec(displayTitle)?.[1]; + const workloads = FULL_RUN_WORKLOADS.exec(displayTitle)?.[1]; return workloads === "integration-heavy" || workloads === "all"; } @@ -200,29 +200,29 @@ export async function handleBenchCommand( return { kind: "rerun", runId: latest.id }; } - const active = (await io.listStackRuns(pr.headRef)).find( + const active = (await io.listFullRuns(pr.headRef)).find( (stackRun) => stackRun.status !== "completed" && coversIntegrationHeavy(stackRun.displayTitle), ); - if (active !== undefined) return reply(replies.stackRunning(active.url)); + if (active !== undefined) return reply(replies.fullRunning(active.url)); // The lightweight part refreshes only when its section is behind the head commit and no - // lightweight run is already in flight; with no run to re-run, the stack run measures it too. + // lightweight run is already in flight; with no run to re-run, the full run measures it too. const sectionSha = await io.lightweightSha(prNumber); const lightweightCurrent = sectionSha !== undefined && pr.headSha.startsWith(sectionSha); const lightweightInFlight = latest !== undefined && latest.status !== "completed"; const needsLightweight = !lightweightCurrent && !lightweightInFlight; const rerunnable = latest !== undefined && !expired ? latest : undefined; - const workloads: StackWorkloads = + const workloads: FullWorkloads = needsLightweight && rerunnable === undefined ? "all" : "integration-heavy"; try { - await io.dispatchStack(pr.headRef, workloads); + await io.dispatchFull(pr.headRef, workloads); } catch (error) { if (!(error instanceof GithubRequestError)) throw error; const missing = error.status === 404 || (error.status === 422 && /workflow_dispatch/u.test(error.detail)); return reply( - missing ? replies.noStackWorkflow() : replies.stackRefused(error.status, error.detail), + missing ? replies.noFullWorkflow() : replies.fullRefused(error.status, error.detail), ); } @@ -232,7 +232,7 @@ export async function handleBenchCommand( await io.rerun(rerunnable.id); rerunId = rerunnable.id; } catch (error) { - // The stack run is already under way; a refused lightweight re-run must not hide it. + // The full run is already under way; a refused lightweight re-run must not hide it. console.warn( `Could not re-run the lightweight benchmark run ${rerunnable.id}: ${String(error)}`, ); @@ -290,9 +290,9 @@ async function main(): Promise { rerun: async (runId) => { await githubFetch(`${base}/actions/runs/${runId}/rerun`, token, { method: "POST" }); }, - listStackRuns: async (headRef) => { + listFullRuns: async (headRef) => { const url = - `${base}/actions/workflows/${STACK_WORKFLOW_FILE}/runs` + + `${base}/actions/workflows/${FULL_WORKFLOW_FILE}/runs` + `?event=workflow_dispatch&branch=${encodeURIComponent(headRef)}&per_page=50`; const payload = await githubFetch(url, token).then( (response) => @@ -319,8 +319,8 @@ async function main(): Promise { url: run.html_url, })); }, - dispatchStack: async (headRef, workloads) => { - await githubFetch(`${base}/actions/workflows/${STACK_WORKFLOW_FILE}/dispatches`, token, { + dispatchFull: async (headRef, workloads) => { + await githubFetch(`${base}/actions/workflows/${FULL_WORKFLOW_FILE}/dispatches`, token, { method: "POST", body: JSON.stringify({ ref: `refs/heads/${headRef}`, @@ -363,7 +363,7 @@ async function main(): Promise { outcome.kind === "rerun" ? `Re-running benchmark run ${outcome.runId} for PR #${prNumber}.` : outcome.kind === "dispatched" - ? `Dispatched the stack benchmark (${outcome.workloads}) for PR #${prNumber}${ + ? `Dispatched the full benchmark (${outcome.workloads}) for PR #${prNumber}${ outcome.rerunId === undefined ? "" : ` and re-running benchmark run ${outcome.rerunId}` }.` : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 8fd2178c6b..5107fd64c7 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -29,7 +29,7 @@ describe("resolveBench", () => { }); test("benchmarks when the comment holds only an integration-heavy result", () => { - const stackOnly = { + const heavyOnly = { id: 5, body: composeBody({ "integration-heavy": { @@ -40,7 +40,7 @@ describe("resolveBench", () => { authorLogin: BOT, }; - expect(resolveBench({ runAttempt: 1, isFork: false, comments: [stackOnly] }).shouldRun).toBe( + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [heavyOnly] }).shouldRun).toBe( true, ); }); diff --git a/.github/workflows/cli-bench-stack.yml b/.github/workflows/cli-bench-full.yml similarity index 94% rename from .github/workflows/cli-bench-stack.yml rename to .github/workflows/cli-bench-full.yml index 94746d88c7..a1fca9169e 100644 --- a/.github/workflows/cli-bench-stack.yml +++ b/.github/workflows/cli-bench-full.yml @@ -1,7 +1,7 @@ -name: CLI benchmark (stack) +name: CLI benchmark (full) run-name: >- - ${{ github.event_name == 'schedule' && 'CLI benchmark stack (scheduled)' || - format('CLI benchmark stack ({0}): {1}', inputs.workloads, github.ref_name) }} + ${{ github.event_name == 'schedule' && 'CLI benchmark full (scheduled)' || + format('CLI benchmark full ({0}): {1}', inputs.workloads, github.ref_name) }} # Report-only comparison of a CLI build with a base build for the workloads that need a running # local stack or database. Starts only from the `/bench full` pull request command, the Actions UI @@ -41,7 +41,7 @@ permissions: {} # A second run for a branch waits for the first instead of cancelling it. concurrency: - group: cli-bench-stack-${{ github.ref_name }} + group: cli-bench-full-${{ github.ref_name }} cancel-in-progress: false jobs: @@ -55,8 +55,6 @@ jobs: contents: read pull-requests: write outputs: - workloads: ${{ steps.plan.outputs.workloads }} - runtime: ${{ steps.plan.outputs.runtime }} base_kind: ${{ steps.plan.outputs.base_kind }} base_tag: ${{ steps.plan.outputs.base_tag }} pr_number: ${{ steps.plan.outputs.pr_number }} @@ -82,7 +80,7 @@ jobs: INPUT_WORKLOADS: ${{ inputs.workloads }} INPUT_RUNTIME: ${{ inputs.runtime }} INPUT_BASE: ${{ inputs.base }} - run: bun .github/scripts/cli-bench/stack-plan.ts + run: bun .github/scripts/cli-bench/full-plan.ts - name: Mark the integration-heavy section as running if: steps.plan.outputs.pr_number != '' && steps.plan.outputs.run_integration == 'true' env: @@ -184,7 +182,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-binaries + name: cli-bench-full-binaries path: | ${{ runner.temp }}/bench/base ${{ runner.temp }}/bench/branch @@ -219,7 +217,7 @@ jobs: if: needs.build.outputs.branch_build == 'ok' && needs.build.outputs.base_build == 'ok' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-bench-stack-binaries + name: cli-bench-full-binaries path: ${{ runner.temp }}/bench - name: Start replay server @@ -290,7 +288,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-report-lightweight + name: cli-bench-full-report-lightweight path: | ${{ runner.temp }}/bench/report.json ${{ runner.temp }}/bench/report.txt @@ -301,7 +299,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-comment-lightweight + name: cli-bench-full-comment-lightweight path: ${{ runner.temp }}/bench/comment.md retention-days: 1 if-no-files-found: ignore @@ -341,7 +339,7 @@ jobs: - name: Download binaries uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-bench-stack-binaries + name: cli-bench-full-binaries path: ${{ runner.temp }}/bench - name: Run benchmark @@ -362,7 +360,7 @@ jobs: bun apps/cli/scripts/bench-cli.ts \ --base "$bench/base/supabase" \ --branch "$bench/branch/supabase" \ - --workloads apps/cli/bench/workloads.stack.json \ + --workloads apps/cli/bench/workloads.full.json \ --runtime "$RUNTIME" \ --only "$WORKLOADS" \ "${cache_args[@]}" \ @@ -373,7 +371,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-report-${{ matrix.runtime }}-${{ matrix.shard }} + name: cli-bench-full-report-${{ matrix.runtime }}-${{ matrix.shard }} path: | ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.json ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.txt @@ -412,7 +410,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: cli-bench-stack-report-*-* + pattern: cli-bench-full-report-*-* path: ${{ runner.temp }}/reports - name: Merge shard reports @@ -422,7 +420,7 @@ jobs: mapfile -t files < <(find "$RUNNER_TEMP/reports" -name 'report-*.json' 2>/dev/null | sort) if [ "${#files[@]}" -gt 0 ]; then bun apps/cli/scripts/bench-merge.ts \ - --workloads apps/cli/bench/workloads.stack.json \ + --workloads apps/cli/bench/workloads.full.json \ --out "$RUNNER_TEMP/bench/report.json" "${files[@]}" fi @@ -455,7 +453,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-merged-report + name: cli-bench-full-merged-report path: ${{ runner.temp }}/bench/report.json retention-days: 14 if-no-files-found: ignore @@ -464,7 +462,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-bench-stack-comment-integration-heavy + name: cli-bench-full-comment-integration-heavy path: ${{ runner.temp }}/bench/comment.md retention-days: 1 if-no-files-found: ignore @@ -496,7 +494,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-bench-stack-comment-lightweight + name: cli-bench-full-comment-lightweight path: ${{ runner.temp }}/bench-comment - name: Post comment env: @@ -535,7 +533,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: cli-bench-stack-comment-integration-heavy + name: cli-bench-full-comment-integration-heavy path: ${{ runner.temp }}/bench-comment - name: Post comment env: diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index cc4449680d..3bf1ce388f 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -10,7 +10,7 @@ run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.i # commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does # "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment's # Lightweight Commands section, keeping earlier runs in a collapsed list. A maintainer commenting -# exactly `/bench full` also dispatches cli-bench-stack.yml on the pull request's branch to +# exactly `/bench full` also dispatches cli-bench-full.yml on the pull request's branch to # re-measure the Integration Heavy Commands section. Pull requests from forks benchmark every push # and write a job summary only. A pull request that changes no benchmarked path, or more than the # 300 files GitHub's path filter inspects, has no run to re-run, and `/bench` says so. @@ -80,7 +80,7 @@ jobs: run: bun .github/scripts/cli-bench/resolve.ts # `/bench`: re-runs this workflow's run for the pull request's current head commit. `/bench full` - # also dispatches the stack benchmark workflow on the pull request's branch. The only job with + # also dispatches the full benchmark workflow on the pull request's branch. The only job with # `actions: write`; it runs default branch code and no pull request code. rerun: name: Re-run benchmark diff --git a/apps/cli/bench/workloads.stack.json b/apps/cli/bench/workloads.full.json similarity index 100% rename from apps/cli/bench/workloads.stack.json rename to apps/cli/bench/workloads.full.json diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 82ae20c6fe..6345203de8 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -97,7 +97,7 @@ prompts. The first push to a pull request from this repository benchmarks and posts one comment, tagged with the hidden marker ``. The comment has two sections, "Lightweight Commands" (this -run's workloads) and "Integration Heavy Commands" (the [stack benchmark](#stack-benchmark)); each +run's workloads) and "Integration Heavy Commands" (the [full benchmark](#full-benchmark)); each section is written by its own run and shows the commit it measured. Later pushes skip the benchmark, and the job summary says so, while the Lightweight Commands section reports a result; a section that reports a failed run is benchmarked again on the next push. Pull requests from forks have a read-only token, so they get the job summary only @@ -227,7 +227,7 @@ Caveats: the choice but are not what the benchmark measures. - Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, `test db`, `db push`, `db lint`, `db dump`, `db diff`, `migration list`, and `migration up` need Docker or a - database and run in the [stack benchmark](#stack-benchmark) instead; `migration repair` and + database and run in the [full benchmark](#full-benchmark) instead; `migration repair` and `db schema declarative sync` are not benchmarked; `link` probes `https://.`, which never reaches the replay server; `functions deploy` uses Docker by default and its API path has no success scenario; `login` is interactive unless a token is passed; `functions serve` needs Docker. @@ -235,9 +235,9 @@ Caveats: - Launches write a profile file, so numbers from runs with `--api-url` are not comparable with runs without it. -### Stack benchmark +### Full benchmark -`.github/workflows/cli-bench-stack.yml` measures the workloads that need a running local stack or +`.github/workflows/cli-bench-full.yml` measures the workloads that need a running local stack or database: `start`, `status`, `stop`, `db start`, `db reset`, `db query`, `test db`, `gen types` at three schema sizes, `db diff` (cold and warm), `db dump`, `db lint`, `migration up`, `db push --local`, and `migration list`, plus the `stack start|status|stop` lifecycle and `db query` and `db reset` @@ -251,7 +251,7 @@ merge. It never runs on a push; it starts in three ways: does not already show the head commit and no lightweight run is queued or running; with no run to re-run, the dispatch measures those workloads itself. While an integration-heavy run for the pull request is queued or running, the command replies with that run's link and does nothing else. The - pull request also gets a reply when the branch has no stack workflow file yet (merge `develop` + pull request also gets a reply when the branch has no full benchmark workflow file yet (merge `develop` into it), when the pull request is a draft, from a fork, or not into `develop`, and when GitHub refuses the dispatch (with the status and message). An eyes reaction means the dispatch was accepted. @@ -284,7 +284,7 @@ archives are fetched before the measured launches, and each shard shares one nat across all launches through `--artifact-cache`, which links it in as `cache/stack` of every launch's fresh `SUPABASE_HOME`. -The workload manifest `apps/cli/bench/workloads.stack.json` adds these fields to the entries +The workload manifest `apps/cli/bench/workloads.full.json` adds these fields to the entries described above: | Field | Meaning | @@ -314,7 +314,7 @@ To run it locally, build two binaries as above and run a subset: ```sh bun apps/cli/scripts/bench-cli.ts --base --branch \ - --workloads apps/cli/bench/workloads.stack.json --runtime docker --only status,db-query \ + --workloads apps/cli/bench/workloads.full.json --runtime docker --only status,db-query \ --trace --out /tmp/report.json bun apps/cli/scripts/bench-summary.ts --section integration-heavy --report /tmp/report.json ``` diff --git a/apps/cli/scripts/bench-pack.ts b/apps/cli/scripts/bench-pack.ts index 9d1cb28b62..6fab4e2435 100644 --- a/apps/cli/scripts/bench-pack.ts +++ b/apps/cli/scripts/bench-pack.ts @@ -1,5 +1,5 @@ /** - * Packs benchmark workloads into shards for the stack benchmark matrix. Each shard runs on one + * Packs benchmark workloads into shards for the full benchmark matrix. Each shard runs on one * runner, so base and branch of a workload always share a machine and no two workloads share one * at the same time. Free of runtime dependencies so the workflow's plan job can import it. */ diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts index 1b62dbe212..366c4da278 100644 --- a/apps/cli/scripts/bench-workloads.unit.test.ts +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -4,7 +4,7 @@ import { Effect, FileSystem } from "effect"; import { parseWorkloadManifest, validateWorkloads } from "./bench-report.ts"; const manifestPath = new URL("../bench/workloads.pr.json", import.meta.url).pathname; -const stackManifestPath = new URL("../bench/workloads.stack.json", import.meta.url).pathname; +const fullManifestPath = new URL("../bench/workloads.full.json", import.meta.url).pathname; const fixturesPath = new URL("../bench/fixtures/", import.meta.url).pathname; const scenariosDir = new URL("../../cli-e2e/fixtures/scenarios/", import.meta.url).pathname; @@ -13,9 +13,9 @@ const loadManifest = Effect.gen(function* () { return yield* parseWorkloadManifest(yield* fs.readFileString(manifestPath)); }); -const loadStackManifest = Effect.gen(function* () { +const loadFullManifest = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - return yield* parseWorkloadManifest(yield* fs.readFileString(stackManifestPath)); + return yield* parseWorkloadManifest(yield* fs.readFileString(fullManifestPath)); }); describe("PR workload manifest", () => { @@ -45,10 +45,10 @@ describe("PR workload manifest", () => { ); }); -describe("stack workload manifest", () => { +describe("full workload manifest", () => { it.effect("decodes and passes the structural rules without a replay server", () => Effect.gen(function* () { - const workloads = yield* loadStackManifest; + const workloads = yield* loadFullManifest; expect(workloads.length).toBeGreaterThan(0); expect(validateWorkloads(workloads, { apiUrl: undefined })).toEqual([]); @@ -58,7 +58,7 @@ describe("stack workload manifest", () => { it.effect("names only fixtures that exist and carry a migration, seed, or test", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const workloads = yield* loadStackManifest; + const workloads = yield* loadFullManifest; for (const workload of workloads) { if (workload.fixture === undefined) continue; @@ -70,7 +70,7 @@ describe("stack workload manifest", () => { it.effect("sizes every workload for sharding and leaves nothing running after it", () => Effect.gen(function* () { - const workloads = yield* loadStackManifest; + const workloads = yield* loadFullManifest; for (const workload of workloads) { expect(workload.estimatedSeconds, workload.id).toBeGreaterThan(0); @@ -83,7 +83,7 @@ describe("stack workload manifest", () => { it.effect("starts a stack before any workload that measures a command against one", () => Effect.gen(function* () { - const workloads = yield* loadStackManifest; + const workloads = yield* loadFullManifest; for (const workload of workloads) { const measuresStart = From 74397aa201cce29c313ad1bb5a4e41f4c1dbf463 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 17:32:41 +0530 Subject: [PATCH 07/11] ci(cli): address benchmark review findings Keep escaped table pipes through sanitizing, rebuild a comment another writer changed before writing it, keep each section's newest run and reallocate unused space when truncating, refresh a failed lightweight section on /bench full, skip launches when a setup step fails, and stage the migration fixture after setup for the migration workloads. Co-Authored-By: Claude Opus 5.5 --- .../scripts/cli-bench/post-comment.test.ts | 86 +++++++++++++++++-- .github/scripts/cli-bench/post-comment.ts | 61 ++++++++----- .github/scripts/cli-bench/rerun.test.ts | 20 +++++ .github/scripts/cli-bench/rerun.ts | 10 ++- apps/cli/bench/workloads.full.json | 4 +- apps/cli/docs/tracing-monitoring.md | 11 ++- .../cli/scripts/bench-cli.integration.test.ts | 54 ++++++++++++ apps/cli/scripts/bench-cli.ts | 43 +++++++--- apps/cli/scripts/bench-report.ts | 5 ++ 9 files changed, 247 insertions(+), 47 deletions(-) diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index d44254d349..c17e680780 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -31,9 +31,13 @@ const withSections = (id: number, body: string): ExistingComment => ({ function fakeThread(initial: Array = []) { const comments = [...initial]; let nextId = 100; - const hooks: { afterUpdate?: () => void; afterCreate?: () => void } = {}; + const hooks: { afterUpdate?: () => void; afterCreate?: () => void; afterList?: () => void } = {}; const io: CommentIo = { - list: async () => comments.map((comment) => ({ ...comment })), + list: async () => { + const snapshot = comments.map((comment) => ({ ...comment })); + hooks.afterList?.(); + return snapshot; + }, get: async (id) => comments.find((comment) => comment.id === id), create: async (body) => { const created = withSections(nextId++, body); @@ -123,13 +127,13 @@ describe("planComment", () => { }); describe("composeBody", () => { - test("puts the marker first exactly once and ends with the rerun note for both commands", () => { + test("puts the marker first and ends with the rerun note for both commands", () => { const body = composeBody({ - lightweight: { sha: "abc1234", content: `text ${BENCH_MARKER} more` }, + lightweight: { sha: "abc1234", content: report("new", "Lightweight") }, }); expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(body.split(BENCH_MARKER).length - 1).toBeGreaterThanOrEqual(1); + expect(body.split(BENCH_MARKER).length - 1).toBe(1); expect(body).toContain(RERUN_NOTE); expect(RERUN_NOTE).toBe( "Comment `/bench` (only includes lightweight benchmarks) or `/bench full` to run it again on the latest commit.", @@ -183,6 +187,32 @@ describe("composeBody", () => { expect(kept[0]).toEqual(history[0]); }); + test("keeps each section's newest run when the sections alone fill the body", () => { + const heavy = { ...run(9), section: "integration-heavy" as const }; + const body = composeBody( + { + lightweight: { sha: "abc1234", content: "x".repeat(200_000) }, + "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, + }, + [run(8), heavy, run(7)], + ); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(decodeHistory(body)).toEqual([run(8), heavy]); + }); + + test("lets a larger section use the space a smaller one leaves", () => { + const small = "s".repeat(1_000); + const body = composeBody({ + lightweight: { sha: "abc1234", content: small }, + "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, + }); + + const sections = parseSections(body); + expect(sections.lightweight?.content).toBe(small); + expect(sections["integration-heavy"]?.content.length).toBeGreaterThan(MAX_BODY_CHARS / 2); + }); + test("renders a previous run's headline through the sanitizer", () => { const hostile = { ...run(1), headline: "@team " }; @@ -192,7 +222,7 @@ describe("composeBody", () => { ); expect(body).not.toContain(" { const clean = sanitize(hostile); - expect(clean).not.toMatch(/@(?!​)/u); + expect(clean).not.toMatch(/@(?!\u200b)/u); expect(clean).not.toContain("@"); expect(clean).not.toMatch(/(? { expect(clean).toBe("\\\\\\[x\\\\\\](//evil.example/p)"); }); + test("keeps the escaped pipes of a table cell intact", () => { + expect(sanitize("| `supabase db query a\\|b` | 1 ms |")).toBe( + "| `supabase db query a\\|b` | 1 ms |", + ); + }); + test("keeps a report from forging a section marker", () => { const forged = sanitize(""); @@ -306,6 +342,42 @@ describe("writeSection", () => { ); }); + test("rebuilds from the comment when another writer changed it after the listing", async () => { + const thread = fakeThread([ + withSections( + 7, + composeBody({ lightweight: { sha: "abc1234", content: report("old", "Lightweight") } }), + ), + ]); + let interleaved = false; + thread.hooks.afterList = () => { + if (interleaved) return; + interleaved = true; + thread.comments[0]!.body = composeBody({ + lightweight: { sha: "abc1234", content: report("old", "Lightweight") }, + "integration-heavy": { sha: "def5678", content: report("other", "Integration Heavy") }, + }); + }; + + const result = await writeSection(thread.io, input({ markdown: report("new", "Lightweight") })); + + expect(result).toMatchObject({ kind: "written", attempts: 2 }); + const sections = parseSections(thread.comments[0]!.body); + expect(sections.lightweight?.content).toContain("new"); + expect(sections["integration-heavy"]?.content).toContain("other"); + }); + + test("keeps a report's own marker out of the comment", async () => { + const thread = fakeThread(); + + await writeSection( + thread.io, + input({ markdown: `${report("new", "Lightweight")} ${BENCH_MARKER}` }), + ); + + expect(thread.comments[0]!.body.split(BENCH_MARKER).length - 1).toBe(1); + }); + test("retries when a concurrent writer overwrites its section, and ends with both sections", async () => { const thread = fakeThread([ withSections( diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts index 1d15bd0f71..c4d1f338f0 100644 --- a/.github/scripts/cli-bench/post-comment.ts +++ b/.github/scripts/cli-bench/post-comment.ts @@ -58,7 +58,7 @@ export type CommentPlan = | { kind: "update"; id: number; previousBody: string }; const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); -const ZERO_WIDTH_SPACE = "​"; +const ZERO_WIDTH_SPACE = "\u200b"; export const MAX_FILE_BYTES = 1_000_000; const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; @@ -66,6 +66,7 @@ const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u export function sanitize(markdown: string): string { return markdown .replaceAll("\\", "\\\\") + .replaceAll("\\\\|", "\\|") .replaceAll("&", "&") .replaceAll(/<[^>\n]*>?/gu, (tag) => ALLOWED_TAGS.has(tag) ? tag : tag.replaceAll("<", "<"), @@ -162,34 +163,51 @@ function assemble(sections: Sections, entries: ReadonlyArray): string /** * The comment body: the marker, each section, the re-run note, then the run history. `entries` - * lists the newest run first; the oldest are dropped until the body fits, and then each section is - * cut to its share of the limit. + * lists the newest run first; the oldest previous runs are dropped until the body fits, then each + * section is cut to its share of the limit, with small sections keeping all of theirs. Each + * section's newest entry is always kept. */ export function composeBody(sections: Sections, entries: ReadonlyArray = []): string { - for (let kept = entries.length; kept >= 0; kept--) { - const body = assemble(sections, entries.slice(0, kept)); + const newest = new Set(); + const seen = new Set
(); + for (const entry of entries) { + if (seen.has(entry.section)) continue; + seen.add(entry.section); + newest.add(entry); + } + const older = entries.filter((entry) => !newest.has(entry)); + const withOlder = (count: number) => { + const kept = new Set(older.slice(0, count)); + return entries.filter((entry) => newest.has(entry) || kept.has(entry)); + }; + for (let count = older.length; count >= 0; count--) { + const body = assemble(sections, withOlder(count)); if (body.length <= MAX_BODY_CHARS) return body; } + const history = withOlder(0); const present = SECTIONS.filter((section) => sections[section] !== undefined); const overhead = assemble( Object.fromEntries(present.map((section) => [section, { sha: "0000000", content: "" }])), - [], + history, ).length; - const share = Math.floor((MAX_BODY_CHARS - overhead) / Math.max(present.length, 1)); + let remaining = MAX_BODY_CHARS - overhead; const cut: Sections = {}; - for (const section of present) { - const state = sections[section]; - if (state === undefined) continue; - cut[section] = { - sha: state.sha, - content: - state.content.length <= share - ? state.content - : state.content.slice(0, Math.max(0, share - TRUNCATION_NOTICE.length)) + - TRUNCATION_NOTICE, - }; - } - return assemble(cut, []); + const bySize = present + .flatMap((section) => { + const state = sections[section]; + return state === undefined ? [] : [{ section, state }]; + }) + .sort((a, b) => a.state.content.length - b.state.content.length); + bySize.forEach(({ section, state }, index) => { + const share = Math.floor(remaining / (bySize.length - index)); + const content = + state.content.length <= share + ? state.content + : state.content.slice(0, Math.max(0, share - TRUNCATION_NOTICE.length)) + TRUNCATION_NOTICE; + remaining -= content.length; + cut[section] = { sha: state.sha, content }; + }); + return assemble(cut, history); } /** The workflow bot's benchmark comment, if any; the oldest wins when concurrent writers made two. */ @@ -282,6 +300,9 @@ export async function writeSection(io: CommentIo, input: WriteInput): Promise { expect(coversIntegrationHeavy(title)).toBe(expected); }); }); + +describe("reportedLightweightSha", () => { + test("names the commit of a lightweight section that reports a result", () => { + const body = composeBody({ + lightweight: { sha: "abc1234", content: "### Lightweight Commands: no changes found" }, + }); + + expect(reportedLightweightSha(body)).toBe("abc1234"); + }); + + test("names no commit for a lightweight section whose run failed", () => { + const body = composeBody({ + lightweight: { sha: "abc1234", content: failureMarkdown("lightweight") }, + }); + + expect(reportedLightweightSha(body)).toBeUndefined(); + }); +}); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index ff4342a398..2df3a8e6d8 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -14,7 +14,7 @@ import { appendFileSync } from "node:fs"; import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; import { githubFetch, GithubRequestError, listIssueComments, requireEnv } from "./github.ts"; -import { findBenchComment, parseSections } from "./post-comment.ts"; +import { findBenchComment, parseSections, sectionFailed } from "./post-comment.ts"; export const BENCH_COMMAND = "/bench"; export const BENCH_FULL_COMMAND = "/bench full"; @@ -117,6 +117,12 @@ export const replies = { `(${status} ${quoted(detail)}).`, }; +/** The commit of a comment's lightweight section, unless that section reports a failed run. */ +export function reportedLightweightSha(body: string): string | undefined { + const section = parseSections(body).lightweight; + return section === undefined || sectionFailed(section.content) ? undefined : section.sha; +} + const FULL_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; /** Whether a full run's name says it measures the integration-heavy workloads. */ @@ -330,7 +336,7 @@ async function main(): Promise { }, lightweightSha: async (n) => { const existing = findBenchComment(await listIssueComments(token, base, n)); - return existing === undefined ? undefined : parseSections(existing.body).lightweight?.sha; + return existing === undefined ? undefined : reportedLightweightSha(existing.body); }, reactToComment: async (commentId) => { await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { diff --git a/apps/cli/bench/workloads.full.json b/apps/cli/bench/workloads.full.json index 21fc4a233d..90fef53a28 100644 --- a/apps/cli/bench/workloads.full.json +++ b/apps/cli/bench/workloads.full.json @@ -206,7 +206,7 @@ "setup": [["start"]], "teardown": [["stop", "--no-backup"]], "runtime": "docker", - "fixture": "migrations", + "fixtureAfterSetup": "migrations", "runs": 5, "warmup": 1, "traceRuns": 1, @@ -220,7 +220,7 @@ "setup": [["start"]], "teardown": [["stop", "--no-backup"]], "runtime": "docker", - "fixture": "migrations", + "fixtureAfterSetup": "migrations", "runs": 5, "warmup": 1, "traceRuns": 1, diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 6345203de8..082cecc453 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -266,9 +266,9 @@ merge. It never runs on a push; it starts in three ways: proves valid and useful. To compare two releases by hand, dispatch the workflow on the newer tag with `base` set to the older one. -A dispatch run uses the workflow file and code of the branch it was started on. Because GitHub runs -dispatch, schedule, and comment-triggered workflows only from the default branch, none of these can -run before the workflow is on `develop`. +A dispatch run uses the workflow file and code of the branch it was started on. GitHub registers +the workflow for dispatch, schedule, and comment-triggered runs only from the default branch, so +none of these can start before the workflow is on `develop`. The workflow plans, builds, measures, and reports in separate jobs. `plan` resolves the base, packs the workloads into shards of roughly ten minutes each from each workload's `estimatedSeconds`, and @@ -294,6 +294,7 @@ described above: | `stack` | Sets `SUPABASE_EXPERIMENTAL_STACK=1` on every launch of the workload, the only variable a workload can set. | | `runtime` | `docker` or `native`. A stack workload must pass the same `--runtime` to every `stack start`. | | `fixture` | A directory under `apps/cli/bench/fixtures/` copied into the project's `supabase/` directory after `init`. | +| `fixtureAfterSetup` | Like `fixture`, but copied after `setup`, so the launch finds changes that setup has not applied. Not for `block`. | | `runs`, `warmup`, `traceRuns` | Per-workload overrides; `--runs` and `--warmup` on the command line win. `traceRuns` is the traced launches per build. | | `timeoutSeconds`, `estimatedSeconds` | Launch timeout (default 60) and the seconds one pass over both builds takes, which sizes the shards. | @@ -302,7 +303,9 @@ Setup, the measured launches, and teardown of one run (or block) share one proje use `block` scope; workloads that change state use `run`. Only one stack is ever up at a time per runner. Stack mode supports only pg-delta for `db diff`, so the `db diff` workloads pass `--use-pg-delta` in both modes. A workload that fails on the base build, such as one using a flag -the base does not have, shows as a base failure and is not compared. +the base does not have, shows as a base failure and is not compared. A failing `setup` step skips +the launches of that run and records its exit code for each, so the workload is flagged instead of +measuring a half-prepared run. The Integration Heavy Commands report adds a container operations table: the number of container CLI calls per run for each verb, such as `container inspect`, flagged when a count changes, appears, diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts index 14bd58dd8e..c87c34d732 100644 --- a/apps/cli/scripts/bench-cli.integration.test.ts +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -54,6 +54,8 @@ LOG="$(dirname "$0")/calls.log" echo "$(basename "$0") $* stack=\${SUPABASE_EXPERIMENTAL_STACK:-} dnt=\${DO_NOT_TRACK:-} tel=\${SUPABASE_TELEMETRY_DISABLED:-} home=\${SUPABASE_HOME:-} cwd=$PWD" >> "$LOG" case "$1" in slow) exec /bin/sleep 30 ;; + fail) exit 3 ;; + boot) if [ -f supabase/seed.sql ]; then echo "fixture present at setup" >> "$LOG"; fi ;; check) if [ -f supabase/seed.sql ]; then echo "fixture present" >> "$LOG"; fi ;; cache) if [ -L "$SUPABASE_HOME/cache/stack" ]; then echo "cache linked to $(readlink "$SUPABASE_HOME/cache/stack")" >> "$LOG"; fi ;; esac @@ -295,6 +297,58 @@ describe("runWorkloadBench", () => { ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), ); + it.effect("copies fixtureAfterSetup only once setup has finished", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const builds = yield* makeBuilds; + const manifestDir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-manifest-" }); + yield* fs.makeDirectory(`${manifestDir}/fixtures/seeded`, { recursive: true }); + yield* fs.writeFileString(`${manifestDir}/fixtures/seeded/seed.sql`, "select 1;"); + + yield* runWorkloadBench( + baseOptions(builds.base, { + branch: builds.branch, + runs: 1, + warmup: 0, + workloadsFile: `${manifestDir}/workloads.json`, + }), + stackWorkload({ args: ["check"], setup: [["boot"]], fixtureAfterSetup: "seeded" }), + yield* Ref.make(false), + ); + + const log = yield* builds.calls; + expect(log).not.toContain("fixture present at setup"); + expect(log.match(/fixture present/gu)).toHaveLength(2); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("skips the launches and records a failing exit code when setup fails", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 2, warmup: 0 }), + stackWorkload({ setup: [["fail"], ["boot"]], teardown: [["halt"]] }), + yield* Ref.make(false), + ); + + const commands = parseCalls(yield* builds.calls).map((call) => call.command); + expect(commands).not.toContain("work"); + expect(commands).not.toContain("boot"); + expect(commands.filter((command) => command === "halt")).toHaveLength(4); + for (const samples of [report.base, report.branch]) { + expect(samples.map((sample) => sample.exitCode)).toEqual([3, 3]); + } + expect(collectFlags(analyzeWorkload(report, false)).map((flag) => flag.kind)).toContain( + "non-zero-exit", + ); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + it.effect( "traces only the workload's traceRuns launches per build and reports container calls from them", () => diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index ba34abe7e5..46fdfebb23 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -349,6 +349,20 @@ export interface Unit { readonly env: Record; } +/** Copies a fixture from the manifest's `fixtures/` directory into the project's `supabase/` directory. */ +const copyFixture = (options: BenchOptions, fixture: string, projectDir: string) => + Effect.gen(function* () { + if (options.workloadsFile === undefined) return; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // The fixtures sit outside any `supabase/` directory, which the repository ignores. + const target = path.join(projectDir, "supabase"); + yield* fs.makeDirectory(target, { recursive: true }); + yield* fs.copy(path.join(path.dirname(options.workloadsFile), "fixtures", fixture), target, { + overwrite: true, + }); + }); + /** * A project directory and `SUPABASE_HOME` shared by one unit's setup, launches and teardown, with * the replay profile written, the workload's `cwdSetup` applied, its fixture copied in, and the @@ -383,15 +397,8 @@ const prepareUnit = (options: BenchOptions, workload: Workload, binary: string) ); } } - if (workload.fixture !== undefined && options.workloadsFile !== undefined) { - // The fixtures sit outside any `supabase/` directory, which the repository ignores. - const target = path.join(projectDir, "supabase"); - yield* fs.makeDirectory(target, { recursive: true }); - yield* fs.copy( - path.join(path.dirname(options.workloadsFile), "fixtures", workload.fixture), - target, - { overwrite: true }, - ); + if (workload.fixture !== undefined) { + yield* copyFixture(options, workload.fixture, projectDir); } return { projectDir, env } satisfies Unit; }); @@ -415,6 +422,7 @@ const runStep = Effect.fn("BenchCli.runStep")(function* ( `warning: ${kind} step "${args.join(" ")}" exited ${result.exitCode}; the workload may not reflect a real stack`, ); } + return result.exitCode; }); /** Pulls images or downloads archives for each stack runtime, so no measured launch downloads anything. */ @@ -515,9 +523,20 @@ const runUnit = Effect.fn("BenchCli.runUnit")(function* (params: { discard: true, }).pipe(Effect.ignore), ); - yield* Effect.forEach(workload.setup ?? [], (args) => runStep("setup", binary, unit, args), { - discard: true, - }); + let setupExitCode = 0; + for (const args of workload.setup ?? []) { + setupExitCode = yield* runStep("setup", binary, unit, args); + if (setupExitCode !== 0) break; + } + if (setupExitCode === 0 && workload.fixtureAfterSetup !== undefined) { + yield* copyFixture(options, workload.fixtureAfterSetup, unit.projectDir); + } + // A failed setup leaves nothing real to measure, so every launch reports the failing exit code. + if (setupExitCode !== 0) { + return launches.map( + (kind): LaunchOutcome => ({ kind, wallMs: 0, exitCode: setupExitCode, trace: undefined }), + ); + } // Resets the replay server before each launch, so setup's own requests and a previous // launch's consumed scenario never reach the launch that follows. diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 918ad7280b..90c4e70ed6 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -161,6 +161,8 @@ const Workload = Schema.Struct({ runtime: Schema.optionalKey(Schema.Literals(["docker", "native"])), /** Directory under the manifest's `fixtures/` copied into the project's `supabase/` directory after `init`. */ fixture: Schema.optionalKey(Schema.NonEmptyString), + /** Directory under `fixtures/` copied into `supabase/` after setup, so the launch finds changes setup has not applied. */ + fixtureAfterSetup: Schema.optionalKey(Schema.NonEmptyString), runs: Schema.optionalKey(Schema.Finite), warmup: Schema.optionalKey(Schema.Finite), /** Traced launches per build; defaults to one per measured run. */ @@ -196,6 +198,9 @@ export function validateWorkloads( problems.push(`${name} has an invalid ${field}`); } } + if (workload.fixtureAfterSetup !== undefined && workload.setupScope === "block") { + problems.push(`${name} stages fixtureAfterSetup, which a block-scoped workload applies only once`); + } if (workload.estimatedSeconds !== undefined && !(workload.estimatedSeconds > 0)) { problems.push(`${name} has an invalid estimatedSeconds`); } From 691bf23133c68f461e4b595681c02cb5827c842f Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 17:43:57 +0530 Subject: [PATCH 08/11] chore(cli): format benchmark files Co-Authored-By: Claude Opus 5.5 --- apps/cli/scripts/bench-cli.ts | 9 ++++++--- apps/cli/scripts/bench-report.ts | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 46fdfebb23..f31142ad14 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -533,9 +533,12 @@ const runUnit = Effect.fn("BenchCli.runUnit")(function* (params: { } // A failed setup leaves nothing real to measure, so every launch reports the failing exit code. if (setupExitCode !== 0) { - return launches.map( - (kind): LaunchOutcome => ({ kind, wallMs: 0, exitCode: setupExitCode, trace: undefined }), - ); + return launches.map((kind): LaunchOutcome => ({ + kind, + wallMs: 0, + exitCode: setupExitCode, + trace: undefined, + })); } // Resets the replay server before each launch, so setup's own requests and a previous diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 90c4e70ed6..fcf1097a86 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -199,7 +199,9 @@ export function validateWorkloads( } } if (workload.fixtureAfterSetup !== undefined && workload.setupScope === "block") { - problems.push(`${name} stages fixtureAfterSetup, which a block-scoped workload applies only once`); + problems.push( + `${name} stages fixtureAfterSetup, which a block-scoped workload applies only once`, + ); } if (workload.estimatedSeconds !== undefined && !(workload.estimatedSeconds > 0)) { problems.push(`${name} has an invalid estimatedSeconds`); From c5bb8ba0b7fbe08036ea31e6bdeb97f6964acfe0 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Fri, 30 Oct 2026 17:57:37 +0530 Subject: [PATCH 09/11] ci(cli): post one benchmark comment per workflow The lightweight and integration-heavy benchmarks each keep their own pull request comment, identified by its own hidden marker and carrying its own history and re-run note. This removes the shared-comment section markers, cross-section size budgeting and the read-back retry loop. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/cli-bench/github.ts | 19 - .github/scripts/cli-bench/history.test.ts | 104 ++-- .github/scripts/cli-bench/history.ts | 97 +--- .../scripts/cli-bench/post-comment.test.ts | 511 ++++++++---------- .github/scripts/cli-bench/post-comment.ts | 384 +++++-------- .github/scripts/cli-bench/rerun.test.ts | 53 +- .github/scripts/cli-bench/rerun.ts | 28 +- .github/scripts/cli-bench/resolve.test.ts | 18 +- .github/scripts/cli-bench/resolve.ts | 14 +- .github/workflows/cli-bench-full.yml | 24 +- .github/workflows/cli-bench.yml | 14 +- apps/cli/docs/tracing-monitoring.md | 39 +- apps/cli/scripts/bench-summary.ts | 21 +- apps/cli/scripts/bench-summary.unit.test.ts | 32 +- 14 files changed, 538 insertions(+), 820 deletions(-) diff --git a/.github/scripts/cli-bench/github.ts b/.github/scripts/cli-bench/github.ts index ddd69aaa3a..4859009a68 100644 --- a/.github/scripts/cli-bench/github.ts +++ b/.github/scripts/cli-bench/github.ts @@ -67,22 +67,3 @@ export async function listIssueComments( if (batch.length < 100) return comments; } } - -/** One comment by id, or undefined when it no longer exists. */ -export async function getIssueComment( - token: string, - base: string, - id: number, -): Promise { - try { - const entry = (await (await githubFetch(`${base}/issues/comments/${id}`, token)).json()) as { - id: number; - body: string | null; - user: { login: string } | null; - }; - return { id: entry.id, body: entry.body ?? "", authorLogin: entry.user?.login ?? "" }; - } catch (error) { - if (error instanceof GithubRequestError && error.status === 404) return undefined; - throw error; - } -} diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts index 561080c211..9d001de2b2 100644 --- a/.github/scripts/cli-bench/history.test.ts +++ b/.github/scripts/cli-bench/history.test.ts @@ -5,33 +5,29 @@ import { makeEntry, MAX_PREVIOUS_RUNS, mergeHistory, - previousEntries, renderHistory, type RunEntry, } from "./history.ts"; const entry = (n: number, overrides: Partial = {}): RunEntry => ({ - section: "lightweight", sha: n.toString(16).padStart(7, "0"), at: "2026-10-28T09:05Z", headline: "no changes found", url: `https://github.com/o/r/actions/runs/${n}`, ...overrides, }); -const heavy = (n: number, overrides: Partial = {}) => - entry(n, { section: "integration-heavy", ...overrides }); describe("headlineOf", () => { - test.each([ - ["a comment section", "### Lightweight Commands: 2 things to check\n\nbody"], - ["an integration-heavy section", "### Integration Heavy Commands: 2 things to check\n\nbody"], - ["a job summary", "## CLI benchmark: 2 things to check\n\nbody"], - [ - "an integration-heavy job summary", - "## CLI benchmark (Integration Heavy Commands): 2 things to check", - ], - ])("reads the title of %s", (_name, markdown) => { - expect(headlineOf(markdown)).toBe("2 things to check"); + test("reads the title of the rendered report", () => { + expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( + "2 things to check", + ); + }); + + test("reads the title of the integration-heavy report", () => { + expect(headlineOf("## CLI benchmark (Integration Heavy Commands): 1 thing to check")).toBe( + "1 thing to check", + ); }); test("falls back when the report has no title", () => { @@ -42,23 +38,17 @@ describe("headlineOf", () => { describe("makeEntry", () => { test("shortens the commit and rounds the time down to the minute in UTC", () => { const made = makeEntry({ - section: "integration-heavy", sha: "0123456789abcdef0123456789abcdef01234567", now: new Date("2026-10-28T09:05:59Z"), headline: "no changes found", url: "https://github.com/o/r/actions/runs/9/attempts/2", }); - expect(made).toMatchObject({ - section: "integration-heavy", - sha: "0123456", - at: "2026-10-28T09:05Z", - }); + expect(made).toMatchObject({ sha: "0123456", at: "2026-10-28T09:05Z" }); }); test("refuses a run link that is not a workflow run", () => { const made = makeEntry({ - section: "lightweight", sha: "abcdef1", now: new Date(), headline: "h", @@ -77,78 +67,46 @@ describe("mergeHistory", () => { ]); }); - test("keeps each section's newest run plus at most 20 older ones across both sections", () => { - const previous = [ - ...Array.from({ length: 15 }, (_, i) => entry(100 - i)), - heavy(50), - ...Array.from({ length: 15 }, (_, i) => heavy(40 - i)), - ]; - - const merged = mergeHistory(previous, entry(200)); - - const older = merged.length - 2; - expect(older).toBe(MAX_PREVIOUS_RUNS); - expect(merged[0]).toEqual(entry(200)); - expect(merged.filter((e) => e.section === "integration-heavy")[0]).toEqual(heavy(50)); - }); - - test("keeps a section's newest run even when the other section has filled the list", () => { - const previous = [...Array.from({ length: 25 }, (_, i) => entry(100 - i)), heavy(5)]; - - const merged = mergeHistory(previous, entry(200)); - - expect(merged.at(-1)).toEqual(heavy(5)); - }); -}); - -describe("previousEntries", () => { - test("leaves out each section's newest entry, which is its current result", () => { - const entries = [entry(4), heavy(3), entry(2), heavy(1)]; + test("keeps the current run plus at most 20 earlier ones, dropping the oldest", () => { + const previous = Array.from({ length: 30 }, (_, i) => entry(i + 1)); - expect(previousEntries(entries)).toEqual([entry(2), heavy(1)]); - }); - - test("lists every entry of a section that is running, since it has no current result", () => { - const entries = [entry(4), heavy(3), entry(2), heavy(1)]; + const merged = mergeHistory(previous, entry(100)); - expect(previousEntries(entries, new Set(["integration-heavy"]))).toEqual([ - heavy(3), - entry(2), - heavy(1), - ]); + expect(merged).toHaveLength(MAX_PREVIOUS_RUNS + 1); + expect(merged[0]).toEqual(entry(100)); + expect(merged.at(-1)).toEqual(entry(MAX_PREVIOUS_RUNS)); }); }); describe("renderHistory and decodeHistory", () => { - test("reads an entry written before sections existed as a lightweight run", () => { - const legacy = { ...entry(1), section: undefined }; - const body = ``; - - expect(decodeHistory(body)).toEqual([entry(1)]); - }); - test("round-trips the entries through the hidden marker", () => { - const entries = [entry(3), heavy(2), entry(1)]; + const entries = [entry(3), entry(2), entry(1)]; expect(decodeHistory(`body\n${renderHistory(entries, (t) => t)}\n`)).toEqual(entries); }); - test("names the section of each replaced run and leaves current results out of the list", () => { + test("lists earlier runs newest first and leaves the current run out of the list", () => { const rendered = renderHistory( - [entry(4), heavy(3), heavy(2, { headline: "1 thing to check" })], + [entry(3), entry(2, { headline: "1 thing to check" })], (t) => t, ); expect(rendered).toContain("Previous runs"); expect(rendered).toContain( - "- Integration Heavy Commands · `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", + "- `0000002` · 2026-10-28 09:05 UTC · 1 thing to check · [run](https://github.com/o/r/actions/runs/2)", ); expect(rendered).not.toContain("0000003` ·"); - expect(rendered).not.toContain("0000004` ·"); }); - test("shows no list while every section has a single run", () => { - expect(renderHistory([entry(1), heavy(2)], (t) => t)).not.toContain("Previous runs"); + test("lists every run, the newest included, while a run is in progress", () => { + const rendered = renderHistory([entry(3), entry(2)], (t) => t, true); + + expect(rendered).toContain("`0000003` ·"); + expect(rendered).toContain("`0000002` ·"); + }); + + test("shows no list for a first run", () => { + expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs"); }); test.each([ @@ -161,7 +119,7 @@ describe("renderHistory and decodeHistory", () => { ], [ "too many entries", - ``, + ``, ], ])("treats %s as empty history", (_name, body) => { expect(decodeHistory(body)).toEqual([]); diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts index c626f7ba44..ce94f5c2b4 100644 --- a/.github/scripts/cli-bench/history.ts +++ b/.github/scripts/cli-bench/history.ts @@ -1,19 +1,11 @@ /** - * Run history kept inside the benchmark comment. Each section's newest result is the section - * itself; the results it replaced sit in a collapsed list, backed by a hidden marker so the next - * update can carry them on. The marker is read back only from the workflow bot's own comment, and - * anything that does not decode cleanly counts as no history. + * Run history kept inside one benchmark comment. The newest run is the comment itself; earlier + * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on. + * The marker is read back only from the workflow bot's own comment, and anything that does not + * decode cleanly counts as no history. */ -/** Parts of the comment, each written by its own run. */ -export type Section = "lightweight" | "integration-heavy"; -export const SECTIONS: ReadonlyArray
= ["lightweight", "integration-heavy"]; -export const SECTION_NAMES: Record = { - lightweight: "Lightweight Commands", - "integration-heavy": "Integration Heavy Commands", -}; - -/** Replaced results kept below the current ones. */ +/** Earlier runs kept below the current one. */ export const MAX_PREVIOUS_RUNS = 20; const HISTORY_MARKER = //u; const MAX_ENCODED_CHARS = 30_000; @@ -23,7 +15,6 @@ const AT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/u; const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; export interface RunEntry { - section: Section; /** Short commit SHA. */ sha: string; /** UTC time to the minute, `YYYY-MM-DDTHH:MMZ`. */ @@ -32,14 +23,10 @@ export interface RunEntry { url: string; } -const isSection = (value: unknown): value is Section => - typeof value === "string" && (SECTIONS as ReadonlyArray).includes(value); - const isEntry = (value: unknown): value is RunEntry => { if (typeof value !== "object" || value === null) return false; - const { section, sha, at, headline, url } = value as Record; + const { sha, at, headline, url } = value as Record; return ( - isSection(section) && typeof sha === "string" && SHA.test(sha) && typeof at === "string" && @@ -52,25 +39,20 @@ const isEntry = (value: unknown): value is RunEntry => { ); }; -/** The title a section heading carries after its name, flattened to one bounded line. */ +/** The title the renderer puts after the `## CLI benchmark` heading, flattened to one bounded line. */ export function headlineOf(markdown: string): string { - const title = - /^#{2,3} (?:CLI benchmark(?: \([^)\n]*\))?|Lightweight Commands|Integration Heavy Commands): (.+)$/mu.exec( - markdown, - )?.[1] ?? "unknown"; + const title = /^## CLI benchmark(?: \([^)\n]*\))?: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS); } /** An entry for a run, or undefined when the commit or run link is not in the expected shape. */ export function makeEntry(input: { - section: Section; sha: string; now: Date; headline: string; url: string; }): RunEntry | undefined { const entry = { - section: input.section, sha: input.sha.slice(0, 7), at: `${input.now.toISOString().slice(0, 16)}Z`, headline: input.headline, @@ -85,77 +67,40 @@ export function decodeHistory(body: string): Array { if (encoded === undefined || encoded.length > MAX_ENCODED_CHARS) return []; try { const value: unknown = JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); - if (!Array.isArray(value) || value.length > MAX_PREVIOUS_RUNS + SECTIONS.length) return []; - // Entries written before comments had sections belong to the lightweight run. - const entries = value.map((entry: unknown) => - typeof entry === "object" && entry !== null && !("section" in entry) - ? { ...entry, section: "lightweight" } - : entry, - ); - return entries.every(isEntry) ? entries : []; + return Array.isArray(value) && value.length <= MAX_PREVIOUS_RUNS + 1 && value.every(isEntry) + ? value + : []; } catch { return []; } } -/** - * The entries a comment shows as previous runs: everything but each section's newest entry, - * which is the section's current result. A running section has no current result, so all of its - * entries are previous. - */ -export function previousEntries( - entries: ReadonlyArray, - running: ReadonlySet
= new Set(), -): Array { - const seen = new Set
(); - return entries.filter((entry) => { - if (running.has(entry.section)) return true; - if (!seen.has(entry.section)) { - seen.add(entry.section); - return false; - } - return true; - }); -} - -/** - * The new result first, then the earlier entries, keeping every section's newest entry and at - * most {@link MAX_PREVIOUS_RUNS} older ones across both sections. - */ +/** The new run first, then the earlier ones, with at most {@link MAX_PREVIOUS_RUNS} behind it. */ export function mergeHistory( previous: ReadonlyArray, current: RunEntry, ): Array { - const all = [current, ...previous]; - const seen = new Set
(); - let older = 0; - return all.filter((entry) => { - if (!seen.has(entry.section)) { - seen.add(entry.section); - return true; - } - older += 1; - return older <= MAX_PREVIOUS_RUNS; - }); + return [current, ...previous].slice(0, MAX_PREVIOUS_RUNS + 1); } const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`; /** - * The hidden marker for `entries` (newest first), preceded by a collapsed list of the previous - * ones when there are any. `clean` makes a headline safe to render. + * The hidden marker for `entries` (current run first), preceded by a collapsed list of the + * earlier ones when there are any. While a run is in progress the comment has no current result, + * so every entry is listed. `clean` makes a headline safe to render. */ export function renderHistory( entries: ReadonlyArray, clean: (text: string) => string, - running: ReadonlySet
= new Set(), + running = false, ): string { const marker = ``; - const previous = previousEntries(entries, running); - if (previous.length === 0) return marker; - const lines = previous.map( + const earlier = running ? entries : entries.slice(1); + if (earlier.length === 0) return marker; + const lines = earlier.map( (entry) => - `- ${SECTION_NAMES[entry.section]} · \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, + `- \`${entry.sha}\` · ${day(entry.at)} · ${clean(entry.headline)} · [run](${entry.url})`, ); return ["
Previous runs", "", ...lines, "", "
", marker].join( "\n", diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index c17e680780..b3d14f248a 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -1,228 +1,152 @@ import { describe, expect, test } from "bun:test"; import { - BENCH_MARKER, + commentFailed, composeBody, failureMarkdown, - findBenchComment, + findComment, + MARKERS, MAX_BODY_CHARS, - MAX_WRITE_ATTEMPTS, - parseSections, - planComment, - RERUN_NOTE, + RERUN_NOTES, sanitize, - writeSection, + writeComment, type CommentIo, type ExistingComment, + type Kind, } from "./post-comment.ts"; -import { decodeHistory, type RunEntry } from "./history.ts"; +import { decodeHistory, MAX_PREVIOUS_RUNS, type RunEntry } from "./history.ts"; const BOT = "github-actions[bot]"; const RUN = "https://github.com/o/r/actions/runs/"; const NOW = new Date("2026-10-28T09:05:00Z"); const SHA = "0123456789abcdef"; -const withSections = (id: number, body: string): ExistingComment => ({ - id, - body, - authorLogin: BOT, -}); +const titleOf = (kind: Kind) => + kind === "lightweight" ? "## CLI benchmark" : "## CLI benchmark (Integration Heavy Commands)"; +const report = (kind: Kind, title: string) => `${titleOf(kind)}: ${title}\n\ndetails`; -/** An in-memory comment thread with hooks to interleave other writers' updates. */ +/** An in-memory comment thread. */ function fakeThread(initial: Array = []) { const comments = [...initial]; let nextId = 100; - const hooks: { afterUpdate?: () => void; afterCreate?: () => void; afterList?: () => void } = {}; const io: CommentIo = { - list: async () => { - const snapshot = comments.map((comment) => ({ ...comment })); - hooks.afterList?.(); - return snapshot; - }, - get: async (id) => comments.find((comment) => comment.id === id), + list: async () => comments.map((comment) => ({ ...comment })), create: async (body) => { - const created = withSections(nextId++, body); - comments.push(created); - hooks.afterCreate?.(); - return created.id; + comments.push({ id: nextId, body, authorLogin: BOT }); + return nextId++; }, update: async (id, body) => { const target = comments.find((comment) => comment.id === id); if (target !== undefined) target.body = body; - hooks.afterUpdate?.(); - }, - remove: async (id) => { - comments.splice( - comments.findIndex((comment) => comment.id === id), - 1, - ); }, }; - return { comments, io, hooks }; + return { comments, io }; } -const report = (title: string, section: "Lightweight" | "Integration Heavy") => - `### ${section} Commands: ${title}\n\ndetails`; - -describe("planComment", () => { - test("creates the comment when none carries the marker", () => { - expect( - planComment("create-if-missing", "lightweight", [{ id: 1, body: "hi", authorLogin: BOT }]), - ).toEqual({ kind: "create" }); - }); - - test("leaves a section that already reports a result alone on a later push", () => { - const existing = withSections( - 5, - composeBody({ lightweight: { sha: "abc1234", content: report("old", "Lightweight") } }), - ); +const bodyOf = (comments: Array, kind: Kind) => + findComment(kind, comments)?.body ?? ""; - expect(planComment("create-if-missing", "lightweight", [existing])).toEqual({ kind: "skip" }); - }); - - test("fills a section the comment does not have yet, even on a later push", () => { - const existing = withSections( - 5, - composeBody({ - "integration-heavy": { sha: "abc1234", content: report("old", "Integration Heavy") }, - }), - ); - - expect(planComment("create-if-missing", "lightweight", [existing])).toEqual({ - kind: "update", - id: 5, - previousBody: existing.body, - }); - }); - - test("replaces a first-run failure notice on a later push", () => { - const failed = withSections( - 5, - composeBody({ lightweight: { sha: "abc1234", content: failureMarkdown("lightweight") } }), - ); +const entry = (n: number): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `${RUN}${n}`, +}); - expect(planComment("create-if-missing", "lightweight", [failed]).kind).toBe("update"); +describe("findComment", () => { + test("keeps the lightweight marker the first benchmark comments already carry", () => { + expect(MARKERS.lightweight).toBe(""); + expect(MARKERS["integration-heavy"]).not.toBe(MARKERS.lightweight); }); - test.each(["upsert", "running"] as const)("updates a reported section for %s", (mode) => { - const existing = withSections( - 5, - composeBody({ lightweight: { sha: "abc1234", content: report("old", "Lightweight") } }), - ); + test("finds each comment by its own marker", () => { + const light = { id: 1, body: `${MARKERS.lightweight}\nx`, authorLogin: BOT }; + const full = { id: 2, body: `${MARKERS["integration-heavy"]}\nx`, authorLogin: BOT }; - expect(planComment(mode, "lightweight", [existing]).kind).toBe("update"); + expect(findComment("lightweight", [full, light])?.id).toBe(1); + expect(findComment("integration-heavy", [light, full])?.id).toBe(2); }); test("ignores a marker pasted by someone else", () => { - const forged = { id: 6, body: BENCH_MARKER, authorLogin: "mallory" }; + const forged = { id: 6, body: MARKERS.lightweight, authorLogin: "mallory" }; - expect(planComment("create-if-missing", "lightweight", [forged])).toEqual({ kind: "create" }); + expect(findComment("lightweight", [forged])).toBeUndefined(); }); - test("keeps the older comment when two writers each created one", () => { - const older = withSections(3, BENCH_MARKER); - const newer = withSections(9, BENCH_MARKER); + test("keeps the older comment when two exist", () => { + const older = { id: 3, body: MARKERS.lightweight, authorLogin: BOT }; + const newer = { id: 9, body: MARKERS.lightweight, authorLogin: BOT }; - expect(findBenchComment([newer, older])?.id).toBe(3); + expect(findComment("lightweight", [newer, older])?.id).toBe(3); }); }); describe("composeBody", () => { - test("puts the marker first and ends with the rerun note for both commands", () => { - const body = composeBody({ - lightweight: { sha: "abc1234", content: report("new", "Lightweight") }, - }); - - expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(body.split(BENCH_MARKER).length - 1).toBe(1); - expect(body).toContain(RERUN_NOTE); - expect(RERUN_NOTE).toBe( - "Comment `/bench` (only includes lightweight benchmarks) or `/bench full` to run it again on the latest commit.", + test("lightweight comment starts with its marker once and ends its text with its own note", () => { + const body = composeBody("lightweight", report("lightweight", "new")); + + expect(body.startsWith(`${MARKERS.lightweight}\n`)).toBe(true); + expect(body.split(MARKERS.lightweight).length - 1).toBe(1); + expect(body).toContain(`\n\n${RERUN_NOTES.lightweight}\n`); + expect(RERUN_NOTES.lightweight).toBe( + "Comment `/bench` to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to also run the integration-heavy benchmark, which posts in its own comment.", ); - expect(body).not.toContain("Comment `/bench` to run it again"); + expect(body).not.toContain(RERUN_NOTES["integration-heavy"]); }); - test("keeps each section's commit and content readable from the body", () => { - const body = composeBody({ - lightweight: { sha: "abc1234", content: report("a", "Lightweight") }, - "integration-heavy": { sha: "def5678", content: report("b", "Integration Heavy") }, - }); + test("integration-heavy comment starts with its marker once and ends its text with its own note", () => { + const body = composeBody("integration-heavy", report("integration-heavy", "new")); - expect(parseSections(body)).toEqual({ - lightweight: { sha: "abc1234", content: report("a", "Lightweight") }, - "integration-heavy": { sha: "def5678", content: report("b", "Integration Heavy") }, - }); + expect(body.startsWith(`${MARKERS["integration-heavy"]}\n`)).toBe(true); + expect(body.split(MARKERS["integration-heavy"]).length - 1).toBe(1); + expect(body).toContain(`\n\n${RERUN_NOTES["integration-heavy"]}\n`); + expect(RERUN_NOTES["integration-heavy"]).toBe( + "Comment `/bench full` to run this benchmark again on the latest commit, which also refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark.", + ); + expect(body).not.toContain(RERUN_NOTES.lightweight); }); - test("cuts oversized sections below GitHub's limit and says so", () => { - const body = composeBody({ - lightweight: { sha: "abc1234", content: "x".repeat(200_000) }, - "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, - }); + test("cuts an oversized report below GitHub's limit and says so", () => { + const body = composeBody("lightweight", "x".repeat(MAX_BODY_CHARS * 2), [entry(1)]); expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); - expect(body).toContain("truncated"); - expect(parseSections(body).lightweight).toBeDefined(); - expect(parseSections(body)["integration-heavy"]).toBeDefined(); - }); - - const run = (n: number): RunEntry => ({ - section: "lightweight", - sha: n.toString(16).padStart(7, "0"), - at: "2026-10-28T09:05Z", - headline: "no changes found", - url: `${RUN}${n}`, + expect(body).toContain("The report was truncated"); + expect(body.startsWith(MARKERS.lightweight)).toBe(true); + expect(decodeHistory(body)).toEqual([entry(1)]); }); - test("drops the oldest runs before cutting a section when the body is too long", () => { - const history = Array.from({ length: 21 }, (_, i) => run(21 - i)); - const content = "x".repeat(MAX_BODY_CHARS - 4_000); + test("drops the oldest runs before cutting the report when the body is too long", () => { + const entries = Array.from({ length: 21 }, (_, i) => ({ + ...entry(i + 1), + headline: "h".repeat(100), + })); + const content = "x".repeat(MAX_BODY_CHARS - 2_500); - const body = composeBody({ lightweight: { sha: "abc1234", content } }, history); + const body = composeBody("lightweight", content, entries); expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); - expect(body).not.toContain("truncated"); + expect(body).toContain(content); const kept = decodeHistory(body); - expect(kept.length).toBeGreaterThan(0); - expect(kept.length).toBeLessThan(history.length); - expect(kept[0]).toEqual(history[0]); + expect(kept.length).toBeGreaterThanOrEqual(1); + expect(kept.length).toBeLessThan(entries.length); + expect(kept).toEqual(entries.slice(0, kept.length)); }); - test("keeps each section's newest run when the sections alone fill the body", () => { - const heavy = { ...run(9), section: "integration-heavy" as const }; - const body = composeBody( - { - lightweight: { sha: "abc1234", content: "x".repeat(200_000) }, - "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, - }, - [run(8), heavy, run(7)], - ); - - expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); - expect(decodeHistory(body)).toEqual([run(8), heavy]); - }); + test("keeps the newest run when the report alone fills the body", () => { + const entries = [entry(3), entry(2), entry(1)]; - test("lets a larger section use the space a smaller one leaves", () => { - const small = "s".repeat(1_000); - const body = composeBody({ - lightweight: { sha: "abc1234", content: small }, - "integration-heavy": { sha: "abc1234", content: "y".repeat(200_000) }, - }); + const body = composeBody("integration-heavy", "x".repeat(MAX_BODY_CHARS), entries); - const sections = parseSections(body); - expect(sections.lightweight?.content).toBe(small); - expect(sections["integration-heavy"]?.content.length).toBeGreaterThan(MAX_BODY_CHARS / 2); + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(decodeHistory(body)).toEqual([entry(3)]); }); test("renders a previous run's headline through the sanitizer", () => { - const hostile = { ...run(1), headline: "@team " }; + const hostile = { ...entry(1), headline: "@team [x](https://evil.test)" }; - const body = composeBody( - { lightweight: { sha: "abc1234", content: report("new", "Lightweight") } }, - [run(3), run(2), hostile], - ); + const body = composeBody("lightweight", "report", [entry(2), hostile]); - expect(body).not.toContain(" { const clean = sanitize(hostile); - expect(clean).not.toMatch(/@(?!\u200b)/u); + expect(clean).not.toMatch(/@(?!​)/u); expect(clean).not.toContain("@"); expect(clean).not.toMatch(/(? { ); }); - test("keeps a report from forging a section marker", () => { - const forged = sanitize(""); - - expect(forged).not.toContain("`)).not.toContain(""; -/** The one note under every comment, naming both commands. */ -export const RERUN_NOTE = - "Comment `/bench` (only includes lightweight benchmarks) or `/bench full` to run it again on the latest commit."; +/** The two comments a pull request can have. */ +export type Kind = "lightweight" | "integration-heavy"; + +/** Hidden tags identifying each comment; the lightweight one is the original benchmark marker. */ +export const MARKERS: Record = { + lightweight: "", + "integration-heavy": "", +}; +/** The line under each comment telling readers which command re-runs what. */ +export const RERUN_NOTES: Record = { + lightweight: + "Comment `/bench` to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to also run the integration-heavy benchmark, which posts in its own comment.", + "integration-heavy": + "Comment `/bench full` to run this benchmark again on the latest commit, which also refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark.", +}; +const TITLES: Record = { + lightweight: "## CLI benchmark", + "integration-heavy": "## CLI benchmark (Integration Heavy Commands)", +}; /** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ const WORKFLOW_BOT_LOGIN = "github-actions[bot]"; /** GitHub rejects comment bodies longer than 65536 characters. */ export const MAX_BODY_CHARS = 65_000; const TRUNCATION_NOTICE = "\n\n
\n\n_The report was truncated; see the workflow run for the rest._\n"; -/** Writes attempted before giving up on a section that keeps being overwritten. */ -export const MAX_WRITE_ATTEMPTS = 3; export type Mode = "create-if-missing" | "upsert" | "running"; export type ExistingComment = IssueComment; -export type CommentPlan = - | { kind: "skip" } - | { kind: "create" } - | { kind: "update"; id: number; previousBody: string }; - const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); const ZERO_WIDTH_SPACE = "\u200b"; export const MAX_FILE_BYTES = 1_000_000; @@ -78,252 +77,152 @@ export function sanitize(markdown: string): string { .replaceAll("]", "\\]"); } -export interface SectionState { - /** Short SHA of the commit the section measured. */ - sha: string; - /** The section's Markdown, from its heading to its run link. */ - content: string; -} -export type Sections = Partial>; - -const startMarker = (section: Section, sha: string) => - ``; -const endMarker = (section: Section) => ``; - -/** The sections recorded in a comment body. */ -export function parseSections(body: string): Sections { - const sections: Sections = {}; - for (const section of SECTIONS) { - const match = new RegExp( - `\\n([\\s\\S]*?)\\n${endMarker(section)}`, - "u", - ).exec(body); - if (match?.[1] !== undefined && match[2] !== undefined) { - sections[section] = { sha: match[1], content: match[2] }; - } - } - return sections; -} - -/** Whether a section's heading says the run produced no comparison. */ -export const sectionFailed = (content: string) => /^### [^:\n]+: could not run$/mu.test(content); -const sectionRunning = (content: string) => /^### [^:\n]+: running$/mu.test(content); - const short = (sha: string) => sha.slice(0, 7); +const runLink = (runUrl: string | undefined) => + runUrl !== undefined && RUN_URL.test(runUrl) ? ` [Workflow run](${runUrl})` : ""; + +/** Whether a comment reports a run that produced no comparison. */ +export const commentFailed = (body: string) => + /^## CLI benchmark[^\n]*: could not run$/mu.test(body); /** The placeholder shown while a run is in progress. */ export function runningContent( - section: Section, + kind: Kind, sha: string, now: Date, runUrl: string | undefined, ): string { - const link = runUrl !== undefined && RUN_URL.test(runUrl) ? ` [Workflow run](${runUrl})` : ""; return ( - `### ${SECTION_NAMES[section]}: running\n\n` + - `Started ${now.toISOString().slice(0, 16).replace("T", " ")} UTC for commit \`${short(sha)}\`.${link}` + `${TITLES[kind]}: running\n\n` + + `Started ${now.toISOString().slice(0, 16).replace("T", " ")} UTC for commit \`${short(sha)}\`.${runLink(runUrl)}` ); } -/** The section for a run that produced no report. */ -export function failureMarkdown(section: Section): string { - return `### ${SECTION_NAMES[section]}: could not run\n\nThe benchmark did not finish. See the workflow run.`; +/** The report for a run that produced none. */ +export function failureMarkdown(kind: Kind, sha: string): string { + return `${TITLES[kind]}: could not run\n\nThe benchmark did not finish for commit \`${short(sha)}\`. See the workflow run.`; } /** The sanitized report, then the trusted run link. */ -export function sectionContent(markdown: string, runUrl: string | undefined): string { - const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); +export function reportContent(markdown: string, runUrl: string | undefined): string { + const stripped = Object.values(MARKERS).reduce( + (text, marker) => text.replaceAll(marker, ""), + markdown, + ); const link = runUrl !== undefined && RUN_URL.test(runUrl) ? `\n\n[Workflow run](${runUrl})` : ""; - return `${content}${link}`; + return `${sanitize(stripped).trim()}${link}`; } -function assemble(sections: Sections, entries: ReadonlyArray): string { - const running = new Set( - SECTIONS.filter((section) => { - const state = sections[section]; - return state !== undefined && sectionRunning(state.content); - }), - ); - const blocks = SECTIONS.flatMap((section) => { - const state = sections[section]; - return state === undefined - ? [] - : [`${startMarker(section, state.sha)}\n${state.content}\n${endMarker(section)}`]; - }); +/** + * The short SHA of the commit a comment's newest result measured, or undefined when the comment + * reports a failed or running benchmark or has no recorded result. + */ +export function measuredSha(body: string): string | undefined { + if (commentFailed(body) || /^## CLI benchmark[^\n]*: running$/mu.test(body)) return undefined; + return decodeHistory(body)[0]?.sha; +} + +function assemble( + kind: Kind, + content: string, + entries: ReadonlyArray, + running: boolean, +): string { return ( - [ - BENCH_MARKER, - "## CLI benchmark", - ...blocks, - RERUN_NOTE, - renderHistory(entries, sanitize, running), - ].join("\n\n") + "\n" + [MARKERS[kind], content, RERUN_NOTES[kind], renderHistory(entries, sanitize, running)].join( + "\n\n", + ) + "\n" ); } /** - * The comment body: the marker, each section, the re-run note, then the run history. `entries` - * lists the newest run first; the oldest previous runs are dropped until the body fits, then each - * section is cut to its share of the limit, with small sections keeping all of theirs. Each - * section's newest entry is always kept. + * The comment body: the marker, the report, the re-run note, then the run history. `entries` + * lists the newest run first; the oldest are dropped until the body fits, always keeping the + * newest, and only then is the report cut. */ -export function composeBody(sections: Sections, entries: ReadonlyArray = []): string { - const newest = new Set(); - const seen = new Set
(); - for (const entry of entries) { - if (seen.has(entry.section)) continue; - seen.add(entry.section); - newest.add(entry); - } - const older = entries.filter((entry) => !newest.has(entry)); - const withOlder = (count: number) => { - const kept = new Set(older.slice(0, count)); - return entries.filter((entry) => newest.has(entry) || kept.has(entry)); - }; - for (let count = older.length; count >= 0; count--) { - const body = assemble(sections, withOlder(count)); +export function composeBody( + kind: Kind, + content: string, + entries: ReadonlyArray = [], + running = false, +): string { + const shown = running ? entries.slice(0, MAX_PREVIOUS_RUNS) : entries; + for (let kept = shown.length; kept >= 1; kept--) { + const body = assemble(kind, content, shown.slice(0, kept), running); if (body.length <= MAX_BODY_CHARS) return body; } - const history = withOlder(0); - const present = SECTIONS.filter((section) => sections[section] !== undefined); - const overhead = assemble( - Object.fromEntries(present.map((section) => [section, { sha: "0000000", content: "" }])), - history, - ).length; - let remaining = MAX_BODY_CHARS - overhead; - const cut: Sections = {}; - const bySize = present - .flatMap((section) => { - const state = sections[section]; - return state === undefined ? [] : [{ section, state }]; - }) - .sort((a, b) => a.state.content.length - b.state.content.length); - bySize.forEach(({ section, state }, index) => { - const share = Math.floor(remaining / (bySize.length - index)); - const content = - state.content.length <= share - ? state.content - : state.content.slice(0, Math.max(0, share - TRUNCATION_NOTICE.length)) + TRUNCATION_NOTICE; - remaining -= content.length; - cut[section] = { sha: state.sha, content }; - }); - return assemble(cut, history); + const newest = shown.slice(0, 1); + const limit = MAX_BODY_CHARS - assemble(kind, "", newest, running).length; + const fitted = + content.length <= limit + ? content + : content.slice(0, Math.max(0, limit - TRUNCATION_NOTICE.length)) + TRUNCATION_NOTICE; + return assemble(kind, fitted, newest, running); } -/** The workflow bot's benchmark comment, if any; the oldest wins when concurrent writers made two. */ -export function findBenchComment( +/** The workflow bot's comment for `kind`, if any; the oldest wins should two exist. */ +export function findComment( + kind: Kind, comments: ReadonlyArray, ): ExistingComment | undefined { return [...comments] .filter( (comment) => - comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), + comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(MARKERS[kind]), ) .sort((a, b) => a.id - b.id)[0]; } -/** What to do with the pull request's benchmark comment for `section`. */ -export function planComment( - mode: Mode, - section: Section, - comments: ReadonlyArray, -): CommentPlan { - const existing = findBenchComment(comments); - if (existing === undefined) return { kind: "create" }; - const state = parseSections(existing.body)[section]; - // A first run that could not compare anything must not block the next push's real result. - const replaceable = - mode !== "create-if-missing" || state === undefined || sectionFailed(state.content); - return replaceable - ? { kind: "update", id: existing.id, previousBody: existing.body } - : { kind: "skip" }; -} - export interface CommentIo { list: () => Promise>; - get: (id: number) => Promise; create: (body: string) => Promise; update: (id: number, body: string) => Promise; - remove: (id: number) => Promise; } export interface WriteInput { mode: Mode; - section: Section; + kind: Kind; /** The commit the run measured. */ sha: string; - /** The rendered section; absent for `running`. */ + /** The rendered report; absent for `running`. */ markdown: string | undefined; runUrl: string | undefined; now: Date; } -export type WriteResult = - | { kind: "skipped" } - | { kind: "written"; id: number; created: boolean; attempts: number } - | { kind: "unverified"; attempts: number }; - -/** - * Writes `section` into the comment and verifies it by reading the comment back, retrying up to - * {@link MAX_WRITE_ATTEMPTS} times when a concurrent writer overwrote it. A comment created while - * another writer created one too is removed in favour of the older one. - */ -export async function writeSection(io: CommentIo, input: WriteInput): Promise { - const { mode, section } = input; - const content = - mode === "running" || input.markdown === undefined - ? runningContent(section, input.sha, input.now, input.runUrl) - : sectionContent(input.markdown, input.runUrl); - const entry = - mode === "running" || input.markdown === undefined - ? undefined - : makeEntry({ - section, - sha: input.sha, - now: input.now, - headline: headlineOf(input.markdown), - url: input.runUrl ?? "", - }); - - for (let attempt = 1; attempt <= MAX_WRITE_ATTEMPTS; attempt++) { - const plan = planComment(mode, section, await io.list()); - if (plan.kind === "skip") return { kind: "skipped" }; - const earlier = plan.kind === "update" ? plan.previousBody : ""; - const sections = parseSections(earlier); - sections[section] = { sha: short(input.sha), content }; - const history = decodeHistory(earlier); - const body = composeBody( - sections, - entry === undefined ? history : mergeHistory(history, entry), - ); +export type WriteResult = { kind: "skipped" } | { kind: "written"; id: number; created: boolean }; - let id: number; - if (plan.kind === "update") { - id = plan.id; - // Another writer's change since the listing would be lost by this write; rebuild from it. - const latest = await io.get(id); - if (latest?.body !== plan.previousBody) continue; - await io.update(id, body); - } else { - id = await io.create(body); - } - - const written = await io.get(id); - // Compare with the section as composed, which may have been truncated to fit. - const intended = parseSections(body)[section]; - const actual = written === undefined ? undefined : parseSections(written.body)[section]; - const landed = - intended !== undefined && actual?.sha === intended.sha && actual.content === intended.content; - if (plan.kind === "create") { - const canonical = findBenchComment(await io.list()); - if (canonical !== undefined && canonical.id !== id) { - await io.remove(id); - continue; - } - } - if (landed) return { kind: "written", id, created: plan.kind === "create", attempts: attempt }; +/** Writes the comment for `input.kind`, carrying its history on; no other comment is read or changed. */ +export async function writeComment(io: CommentIo, input: WriteInput): Promise { + const { mode, kind } = input; + const existing = findComment(kind, await io.list()); + if (mode === "create-if-missing" && existing !== undefined && !commentFailed(existing.body)) { + return { kind: "skipped" }; } - return { kind: "unverified", attempts: MAX_WRITE_ATTEMPTS }; + const running = mode === "running" || input.markdown === undefined; + const history = decodeHistory(existing?.body ?? ""); + const entry = running + ? undefined + : makeEntry({ + sha: input.sha, + now: input.now, + headline: headlineOf(input.markdown ?? ""), + url: input.runUrl ?? "", + }); + const content = running + ? runningContent(kind, input.sha, input.now, input.runUrl) + : reportContent(input.markdown ?? "", input.runUrl); + const body = composeBody( + kind, + content, + entry === undefined ? history : mergeHistory(history, entry), + running, + ); + if (existing === undefined) { + return { kind: "written", id: await io.create(body), created: true }; + } + await io.update(existing.id, body); + return { kind: "written", id: existing.id, created: false }; } async function main(): Promise { @@ -334,10 +233,11 @@ async function main(): Promise { if (mode !== "create-if-missing" && mode !== "upsert" && mode !== "running") { throw new Error(`Unsupported MODE "${mode}".`); } - const section = process.env.SECTION ?? "lightweight"; - if (section !== "lightweight" && section !== "integration-heavy") { - throw new Error(`Unsupported SECTION "${section}".`); + const kind = process.env.KIND ?? "lightweight"; + if (kind !== "lightweight" && kind !== "integration-heavy") { + throw new Error(`Unsupported KIND "${kind}".`); } + const sha = requireEnv("HEAD_SHA"); let markdown: string | undefined; if (mode !== "running") { @@ -353,7 +253,7 @@ async function main(): Promise { console.log("The benchmark job produced no comment."); return; } - markdown = failureMarkdown(section); + markdown = failureMarkdown(kind, sha); } else { if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { throw new Error("The benchmark comment file is not a regular file under the size limit."); @@ -364,7 +264,6 @@ async function main(): Promise { const io: CommentIo = { list: () => listIssueComments(token, base, pr), - get: (id) => getIssueComment(token, base, id), create: async (body) => { const created = (await ( await githubFetch(`${base}/issues/${pr}/comments`, token, { @@ -380,30 +279,21 @@ async function main(): Promise { body: JSON.stringify({ body }), }); }, - remove: async (id) => { - await githubFetch(`${base}/issues/comments/${id}`, token, { method: "DELETE" }); - }, }; - const result = await writeSection(io, { + const result = await writeComment(io, { mode, - section, - sha: requireEnv("HEAD_SHA"), + kind, + sha, markdown, runUrl: process.env.RUN_URL, now: new Date(), }); - if (result.kind === "skipped") { - console.log(`PR #${pr} already has a ${section} result; leaving it untouched.`); - } else if (result.kind === "written") { - console.log( - `${result.created ? "Created" : "Updated"} benchmark comment ${result.id} on PR #${pr} (${section}).`, - ); - } else { - throw new Error( - `The ${section} section did not land after ${result.attempts} attempts; another writer kept replacing it.`, - ); - } + console.log( + result.kind === "skipped" + ? `PR #${pr} already has a ${kind} result; leaving it untouched.` + : `${result.created ? "Created" : "Updated"} ${kind} benchmark comment ${result.id} on PR #${pr}.`, + ); } if (import.meta.main) { diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts index b76b1e0ae5..26e9ba2670 100644 --- a/.github/scripts/cli-bench/rerun.test.ts +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -1,12 +1,12 @@ import { describe, expect, test } from "bun:test"; import { GithubRequestError } from "./github.ts"; -import { composeBody, failureMarkdown } from "./post-comment.ts"; +import { makeEntry } from "./history.ts"; +import { composeBody, failureMarkdown, measuredSha, runningContent } from "./post-comment.ts"; import { type BenchPr, type CommandIo, coversIntegrationHeavy, handleBenchCommand, - reportedLightweightSha, RERUN_WINDOW_MS, type FullRun, type FullWorkloads, @@ -41,7 +41,7 @@ function setup( rerunError?: Error; fullRuns?: ReadonlyArray; dispatchError?: Error; - /** Short SHA the Lightweight Commands section measured. */ + /** Short SHA the lightweight comment measured. */ lightweightSha?: string; } = {}, ) { @@ -209,7 +209,7 @@ const stackRun = (overrides: Partial = {}): FullRun => ({ }); describe("handleBenchCommand with /bench full", () => { - test("dispatches the integration-heavy workloads on the branch and leaves a current lightweight section alone", async () => { + test("dispatches the integration-heavy workloads on the branch and leaves a current lightweight comment alone", async () => { const { io, calls } = setup({ lightweightSha: "aaaaaaa" }); const outcome = await handleBenchCommand(command("/bench full\n"), io); @@ -225,7 +225,7 @@ describe("handleBenchCommand with /bench full", () => { expect(calls.replies).toEqual([]); }); - test("also re-runs the lightweight run when its section is behind the head commit", async () => { + test("also re-runs the lightweight run when its comment is behind the head commit", async () => { const { io, calls } = setup({ lightweightSha: "bbbbbbb", runs: [run({ id: 4 })] }); const outcome = await handleBenchCommand(command("/bench full"), io); @@ -378,20 +378,41 @@ describe("coversIntegrationHeavy", () => { }); }); -describe("reportedLightweightSha", () => { - test("names the commit of a lightweight section that reports a result", () => { - const body = composeBody({ - lightweight: { sha: "abc1234", content: "### Lightweight Commands: no changes found" }, - }); +describe("measuredSha", () => { + const run = "https://github.com/o/r/actions/runs/7"; + const recorded = (sha: string, headline: string) => + makeEntry({ sha, now: NOW, headline, url: run }); + const entries = (...list: Array>) => + list.flatMap((item) => (item === undefined ? [] : [item])); + + test("names the commit of a lightweight comment that reports a result", () => { + const body = composeBody( + "lightweight", + "## CLI benchmark: no changes found", + entries(recorded("abc1234", "no changes found")), + ); + + expect(measuredSha(body)).toBe("abc1234"); + }); - expect(reportedLightweightSha(body)).toBe("abc1234"); + test("names no commit when the run failed", () => { + const body = composeBody( + "lightweight", + failureMarkdown("lightweight", "abc1234def"), + entries(recorded("abc1234", "could not run")), + ); + + expect(measuredSha(body)).toBeUndefined(); }); - test("names no commit for a lightweight section whose run failed", () => { - const body = composeBody({ - lightweight: { sha: "abc1234", content: failureMarkdown("lightweight") }, - }); + test("names no commit while a run is in progress", () => { + const body = composeBody( + "integration-heavy", + runningContent("integration-heavy", "abc1234def", NOW, run), + entries(recorded("1111111", "no changes found")), + true, + ); - expect(reportedLightweightSha(body)).toBeUndefined(); + expect(measuredSha(body)).toBeUndefined(); }); }); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index 2df3a8e6d8..2a5f7443dc 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -4,17 +4,17 @@ * Runs default branch code and never checks out or runs pull request code. The commenter must be * the repository owner or hold write access, and the pull request must be an open, ready one from * this repository into `develop`. `/bench` re-runs the existing `pull_request` run, which - * benchmarks and updates the Lightweight Commands section. `/bench full` always re-measures the - * Integration Heavy Commands section by dispatching the full benchmark workflow on the pull - * request's branch, and re-runs the lightweight part only when that section is behind the head - * commit; the pull request gets a reply when nothing can run. + * benchmarks and updates the lightweight comment. `/bench full` always re-measures the + * integration-heavy benchmark, which has its own comment, by dispatching the full benchmark + * workflow on the pull request's branch, and re-runs the lightweight part only when its comment + * is behind the head commit; the pull request gets a reply when nothing can run. * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. */ import { appendFileSync } from "node:fs"; import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; import { githubFetch, GithubRequestError, listIssueComments, requireEnv } from "./github.ts"; -import { findBenchComment, parseSections, sectionFailed } from "./post-comment.ts"; +import { findComment, measuredSha } from "./post-comment.ts"; export const BENCH_COMMAND = "/bench"; export const BENCH_FULL_COMMAND = "/bench full"; @@ -71,7 +71,7 @@ export interface CommandIo { /** Dispatched runs of the full workflow on a branch, any order. */ listFullRuns: (headRef: string) => Promise>; dispatchFull: (headRef: string, workloads: FullWorkloads) => Promise; - /** Short SHA of the commit the comment's Lightweight Commands section measured, if it has one. */ + /** Short SHA of the commit the lightweight comment measured, if it reports a result. */ lightweightSha: (prNumber: number) => Promise; reactToComment: (commentId: number) => Promise; replyToPr: (prNumber: number, body: string) => Promise; @@ -117,12 +117,6 @@ export const replies = { `(${status} ${quoted(detail)}).`, }; -/** The commit of a comment's lightweight section, unless that section reports a failed run. */ -export function reportedLightweightSha(body: string): string | undefined { - const section = parseSections(body).lightweight; - return section === undefined || sectionFailed(section.content) ? undefined : section.sha; -} - const FULL_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; /** Whether a full run's name says it measures the integration-heavy workloads. */ @@ -211,10 +205,10 @@ export async function handleBenchCommand( ); if (active !== undefined) return reply(replies.fullRunning(active.url)); - // The lightweight part refreshes only when its section is behind the head commit and no + // The lightweight part refreshes only when its comment is behind the head commit and no // lightweight run is already in flight; with no run to re-run, the full run measures it too. - const sectionSha = await io.lightweightSha(prNumber); - const lightweightCurrent = sectionSha !== undefined && pr.headSha.startsWith(sectionSha); + const commentSha = await io.lightweightSha(prNumber); + const lightweightCurrent = commentSha !== undefined && pr.headSha.startsWith(commentSha); const lightweightInFlight = latest !== undefined && latest.status !== "completed"; const needsLightweight = !lightweightCurrent && !lightweightInFlight; const rerunnable = latest !== undefined && !expired ? latest : undefined; @@ -335,8 +329,8 @@ async function main(): Promise { }); }, lightweightSha: async (n) => { - const existing = findBenchComment(await listIssueComments(token, base, n)); - return existing === undefined ? undefined : reportedLightweightSha(existing.body); + const existing = findComment("lightweight", await listIssueComments(token, base, n)); + return existing === undefined ? undefined : measuredSha(existing.body); }, reactToComment: async (commentId) => { await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 5107fd64c7..4d94b3a1bf 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -5,10 +5,10 @@ import { resolveBench, SKIP_REASON } from "./resolve.ts"; const BOT = "github-actions[bot]"; const lightweight = (content: string) => ({ id: 5, - body: composeBody({ lightweight: { sha: "abc1234", content } }), + body: composeBody("lightweight", content), authorLogin: BOT, }); -const reported = lightweight("### Lightweight Commands: no changes found"); +const reported = lightweight("## CLI benchmark: no changes found"); describe("resolveBench", () => { test("benchmarks the first push of a pull request that has no comment yet", () => { @@ -23,20 +23,18 @@ describe("resolveBench", () => { }); test("benchmarks again when the comment reports a failed run", () => { - const failed = lightweight(failureMarkdown("lightweight")); + const failed = lightweight(failureMarkdown("lightweight", "abc1234")); expect(resolveBench({ runAttempt: 1, isFork: false, comments: [failed] }).shouldRun).toBe(true); }); - test("benchmarks when the comment holds only an integration-heavy result", () => { + test("benchmarks when only the integration-heavy comment exists", () => { const heavyOnly = { id: 5, - body: composeBody({ - "integration-heavy": { - sha: "abc1234", - content: "### Integration Heavy Commands: no changes found", - }, - }), + body: composeBody( + "integration-heavy", + "## CLI benchmark (Integration Heavy Commands): no changes found", + ), authorLogin: BOT, }; diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index b59c278ba7..574b811ac6 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -1,15 +1,16 @@ /** * CLI benchmark resolver for `pull_request` runs: decides whether the benchmark job runs. * - * The first run on a pull request benchmarks and comments. Later pushes skip while the comment's - * Lightweight Commands section exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first - * run, or a fork pull request, which never has a comment, benchmarks. Needs read access only. + * The first run on a pull request benchmarks and comments. Later pushes skip while the + * lightweight comment exists and reports a result; a re-run (`run_attempt` above 1), a missing + * comment, a failed first run, or a fork pull request, which never has a comment, benchmarks. + * Needs read access only. * Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. */ import { appendFileSync } from "node:fs"; import { listIssueComments, requireEnv, type IssueComment } from "./github.ts"; -import { findBenchComment, parseSections, sectionFailed } from "./post-comment.ts"; +import { commentFailed, findComment } from "./post-comment.ts"; export interface ResolveInput { runAttempt: number; @@ -29,9 +30,8 @@ export const SKIP_REASON = /** Skips only a first attempt on a pull request whose benchmark comment reports a result. */ export function resolveBench(input: ResolveInput): ResolveResult { if (input.isFork || input.runAttempt > 1) return { shouldRun: true }; - const existing = findBenchComment(input.comments); - const section = existing === undefined ? undefined : parseSections(existing.body).lightweight; - if (section === undefined || sectionFailed(section.content)) return { shouldRun: true }; + const existing = findComment("lightweight", input.comments); + if (existing === undefined || commentFailed(existing.body)) return { shouldRun: true }; return { shouldRun: false, skipReason: SKIP_REASON }; } diff --git a/.github/workflows/cli-bench-full.yml b/.github/workflows/cli-bench-full.yml index a1fca9169e..550f2aa8a8 100644 --- a/.github/workflows/cli-bench-full.yml +++ b/.github/workflows/cli-bench-full.yml @@ -39,14 +39,16 @@ on: permissions: {} -# A second run for a branch waits for the first instead of cancelling it. +# A second run for a branch waits for the first instead of cancelling it, which also takes turns +# between the writers of the integration-heavy comment; a run still waiting when a newer one +# queues never started, so it had written nothing. concurrency: group: cli-bench-full-${{ github.ref_name }} cancel-in-progress: false jobs: # Decides what this run measures and against which base, splits the workloads into shards, and - # marks the pull request's Integration Heavy Commands section as running. + # marks the pull request's integration-heavy comment as running. plan: name: Plan runs-on: ubuntu-latest @@ -81,14 +83,14 @@ jobs: INPUT_RUNTIME: ${{ inputs.runtime }} INPUT_BASE: ${{ inputs.base }} run: bun .github/scripts/cli-bench/full-plan.ts - - name: Mark the integration-heavy section as running + - name: Mark the integration-heavy comment as running if: steps.plan.outputs.pr_number != '' && steps.plan.outputs.run_integration == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ steps.plan.outputs.pr_number }} MODE: running - SECTION: integration-heavy + KIND: integration-heavy HEAD_SHA: ${{ github.sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} run: bun .github/scripts/cli-bench/post-comment.ts @@ -383,7 +385,7 @@ jobs: if: always() && matrix.runtime == 'docker' run: sudo systemctl stop docker.socket docker.service - # Combines the shard reports, writes the job summary and the comment section. + # Combines the shard reports, writes the job summary and the comment. report: name: Report needs: @@ -471,7 +473,7 @@ jobs: # rendered markdown as data; a dispatch uses this file from the dispatched branch, which only # people with write access can push or dispatch. comment-lightweight: - name: Post lightweight section + name: Post lightweight comment needs: - plan - lightweight @@ -479,6 +481,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 continue-on-error: true + # Shares its turn with the lightweight benchmark workflow, the other writer of this comment. + concurrency: + group: cli-bench-comment-lightweight-${{ needs.plan.outputs.pr_number }} + cancel-in-progress: false permissions: contents: read pull-requests: write @@ -502,7 +508,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ needs.plan.outputs.pr_number }} MODE: upsert - SECTION: lightweight + KIND: lightweight MISSING_BODY: failure HEAD_SHA: ${{ github.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md @@ -510,7 +516,7 @@ jobs: run: bun .github/scripts/cli-bench/post-comment.ts comment: - name: Post integration-heavy section + name: Post integration-heavy comment needs: - plan - report @@ -541,7 +547,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ needs.plan.outputs.pr_number }} MODE: upsert - SECTION: integration-heavy + KIND: integration-heavy MISSING_BODY: failure HEAD_SHA: ${{ github.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 3bf1ce388f..491ead06b0 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -8,10 +8,9 @@ run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.i # The first push to a pull request from this repository benchmarks and posts one comment tagged # ``; later pushes skip while that comment reports a result. A maintainer # commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does -# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment's -# Lightweight Commands section, keeping earlier runs in a collapsed list. A maintainer commenting -# exactly `/bench full` also dispatches cli-bench-full.yml on the pull request's branch to -# re-measure the Integration Heavy Commands section. Pull requests from forks benchmark every push +# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates that comment, keeping +# earlier runs in a collapsed list. A maintainer commenting exactly `/bench full` also dispatches +# cli-bench-full.yml on the pull request's branch, which keeps its own comment. Pull requests from forks benchmark every push # and write a job summary only. A pull request that changes no benchmarked path, or more than the # 300 files GitHub's path filter inspects, has no run to re-run, and `/bench` says so. # See apps/cli/docs/tracing-monitoring.md. @@ -268,6 +267,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 continue-on-error: true + # Writers of the lightweight comment, here and in cli-bench-full.yml, take turns; a waiting + # writer is never cancelled by a newer one that has not started. + concurrency: + group: cli-bench-comment-lightweight-${{ github.event.pull_request.number }} + cancel-in-progress: false # The only pull_request job that can write to the pull request. It checks out the merge commit, # which its author can already run with this token by editing the workflow. It never builds or # runs the benchmarked binaries and reads the rendered markdown as data. @@ -294,7 +298,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} MODE: ${{ github.run_attempt > 1 && 'upsert' || 'create-if-missing' }} - SECTION: lightweight + KIND: lightweight HEAD_SHA: ${{ github.event.pull_request.head.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 082cecc453..5f6391c062 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -96,20 +96,25 @@ start replaces the comparison with a notice naming the step. The workloads use n prompts. The first push to a pull request from this repository benchmarks and posts one comment, tagged with -the hidden marker ``. The comment has two sections, "Lightweight Commands" (this -run's workloads) and "Integration Heavy Commands" (the [full benchmark](#full-benchmark)); each -section is written by its own run and shows the commit it measured. Later pushes skip the benchmark, -and the job summary says so, while the Lightweight Commands section reports a result; a section that -reports a failed run is benchmarked again on the next push. Pull requests from forks have a read-only token, so they get the job summary only -and benchmark every push. +the hidden marker ``. The comment shows the commit it measured. Later pushes +skip the benchmark, and the job summary says so, while the comment reports a result; a comment that +reports a failed run is benchmarked again on the next push. The [full benchmark](#full-benchmark) +keeps a separate comment, tagged ``, so each workflow writes only its own +comment. Pull requests from forks have a read-only token, so they get the job summary only and +benchmark every push. A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current head commit. The command re-runs that commit's `pull_request` workflow run, as "Re-run all jobs" in -the Actions UI does, and the re-run updates the Lightweight Commands section in place, or creates the comment if it is -missing. The comment keeps a collapsed "Previous runs" list below the sections with the section, -commit, time (UTC), headline, and link of each result a section replaced, newest first and up to 20 -across both sections, stored in a hidden marker in the comment. One note under the sections tells -readers to comment `/bench` (lightweight benchmarks only) or `/bench full` to run it again. Only the +the Actions UI does, and the re-run updates the comment in place, or creates it if it is missing. +Each comment keeps a collapsed "Previous runs" list with the commit, time (UTC), headline, and link +of each result it replaced, newest first and up to 20, stored in a hidden marker in the comment. +Each comment ends with a note for its own benchmark. The lightweight comment says: "Comment `/bench` +to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to +also run the integration-heavy benchmark, which posts in its own comment." The integration-heavy +comment says: "Comment `/bench full` to run this benchmark again on the latest commit, which also +refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark." +The writers of one comment take turns through a concurrency group, and a comment that outgrows +GitHub's size limit drops its oldest previous runs first and then cuts its report. Only the repository owner and people with write access can run either command, and only on open, non-draft pull requests into `develop` from this repository; the command must be the whole comment, so anything other than `/bench` or `/bench full` is ignored. An eyes reaction on the comment means GitHub accepted @@ -245,9 +250,9 @@ against the stack backend. Like the pull request benchmark it is report only and merge. It never runs on a push; it starts in three ways: - A maintainer comments exactly `/bench full` on an open, non-draft pull request into `develop` from - this repository (same permission rules as `/bench`). The command re-measures the Integration Heavy - Commands section by dispatching the workflow on the pull request's branch. It also refreshes the - Lightweight Commands section, by re-running that commit's `pull_request` run, only when the section + this repository (same permission rules as `/bench`). The command re-measures the integration-heavy + benchmark by dispatching the workflow on the pull request's branch. It also refreshes the + lightweight comment, by re-running that commit's `pull_request` run, only when that comment does not already show the head commit and no lightweight run is queued or running; with no run to re-run, the dispatch measures those workloads itself. While an integration-heavy run for the pull request is queued or running, the command replies with that run's link and does nothing else. The @@ -272,11 +277,11 @@ none of these can start before the workflow is on `develop`. The workflow plans, builds, measures, and reports in separate jobs. `plan` resolves the base, packs the workloads into shards of roughly ten minutes each from each workload's `estimatedSeconds`, and -marks the Integration Heavy Commands section as running with a link to the run. `build` builds the +writes "running" with the start time and a link to the run into the integration-heavy comment, creating it if it is missing. `build` builds the branch binary once, and the base binary from the merge base or the release asset. Each shard is one runner per runtime; it alternates base and branch on that machine and runs no other workload at the -same time. `report` merges the shard reports into the job summary and the comment section. A failed -run shows the failure and a link to the run in its section. +same time. `report` merges the shard reports into the job summary and the integration-heavy comment. A failed +run shows the failure and a link to the run in that comment. Runtimes are Docker (default-mode commands and `--runtime docker`) and the native backend (`--runtime native`), both on Linux x64. Cold downloads are not measured: images and native diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 69a80f7a0e..39adcd486c 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -26,7 +26,7 @@ const USAGE = type Outcome = "ok" | "failed"; -/** A part of the benchmark comment; each is written by its own run. */ +/** The benchmark a report covers; each has its own pull request comment. */ export type Section = "lightweight" | "integration-heavy"; export const SECTION_NAMES: Record = { @@ -380,8 +380,6 @@ function phaseDetail(analyses: ReadonlyArray): string { export interface RenderContext { /** A warning shown above the results, such as shards that did not finish. */ readonly note?: string | undefined; - /** `comment` renders the section for the pull request comment; `summary` is the job summary. */ - readonly target: "summary" | "comment"; readonly section?: Section; readonly headSha?: string | undefined; readonly baseRef?: string | undefined; @@ -427,11 +425,9 @@ function frame( ): string { const section = context.section ?? "lightweight"; const heading = - context.target === "comment" - ? `### ${SECTION_NAMES[section]}: ${title}` - : section === "lightweight" - ? `## CLI benchmark: ${title}` - : `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; + section === "lightweight" + ? `## CLI benchmark: ${title}` + : `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; const note = context.note === undefined || context.note === "" ? [] : [`⚠️ ${context.note}`]; return [heading, contextLine(report, context), ...note, ...body].join("\n\n") + "\n"; } @@ -440,7 +436,7 @@ function frame( export function renderSummary( report: BenchReport | undefined, notice: string, - context: RenderContext = { target: "summary" }, + context: RenderContext = {}, ): string { if (report === undefined) return frame("could not run", [notice], context); const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); @@ -539,7 +535,7 @@ const main = Effect.gen(function* () { baseRef: options.baseRef, mergeBase: options.mergeBase, }; - const markdown = renderSummary(loaded.report, loaded.notice, { target: "summary", ...context }); + const markdown = renderSummary(loaded.report, loaded.notice, context); const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); if (Option.isSome(summaryFile) && summaryFile.value !== "") { yield* fs.writeFileString(summaryFile.value, markdown, { flag: "a" }); @@ -548,10 +544,7 @@ const main = Effect.gen(function* () { } if (options.commentOut !== undefined) { - yield* fs.writeFileString( - options.commentOut, - renderSummary(loaded.report, loaded.notice, { target: "comment", ...context }), - ); + yield* fs.writeFileString(options.commentOut, markdown); } }); diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index de51ecf65a..c2bebb2e15 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -50,7 +50,6 @@ describe("renderSummary", () => { branchSizeBytes: 140269744, }; const markdown = renderSummary(sized, "unused", { - target: "summary", headSha: "abcdef1234567890", baseRef: "develop", mergeBase: "1234567890abcdef", @@ -119,9 +118,7 @@ describe("renderSummary", () => { }); it("names the failed step and stays a notice when there is no report", () => { - const markdown = renderSummary(undefined, "The build of this commit failed.", { - target: "summary", - }); + const markdown = renderSummary(undefined, "The build of this commit failed.", {}); expect(markdown).toContain("## CLI benchmark: could not run"); expect(markdown).toContain("The build of this commit failed."); @@ -129,14 +126,17 @@ describe("renderSummary", () => { }); it.each([ - { section: "lightweight", heading: "### Lightweight Commands: no changes found" }, - { section: "integration-heavy", heading: "### Integration Heavy Commands: no changes found" }, + { section: "lightweight", heading: "## CLI benchmark: no changes found" }, + { + section: "integration-heavy", + heading: "## CLI benchmark (Integration Heavy Commands): no changes found", + }, ] as const)( - "titles a $section comment section with its name and leaves the markers and re-run note to the poster", + "titles a $section report with its name and leaves the markers and re-run note to the poster", ({ section, heading }) => { const input = report([steady("version", ["--version"])]); - const comment = renderSummary(input, "unused", { target: "comment", section }); + const comment = renderSummary(input, "unused", { section }); expect(comment.startsWith(`${heading}\n`)).toBe(true); expect(comment).not.toContain("\n\n## CLI benchmark (Lightweight Commands): 2 things to check\n\nbody", + ), + ).toBe("2 things to check"); + }); + + test("reads the title of a lightweight report under the previous heading", () => { expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( "2 things to check", ); diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index b3d14f248a..981acab6d4 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -21,7 +21,9 @@ const NOW = new Date("2026-10-28T09:05:00Z"); const SHA = "0123456789abcdef"; const titleOf = (kind: Kind) => - kind === "lightweight" ? "## CLI benchmark" : "## CLI benchmark (Integration Heavy Commands)"; + kind === "lightweight" + ? "## CLI benchmark (Lightweight Commands)" + : "## CLI benchmark (Integration Heavy Commands)"; const report = (kind: Kind, title: string) => `${titleOf(kind)}: ${title}\n\ndetails`; /** An in-memory comment thread. */ diff --git a/.github/scripts/cli-bench/post-comment.ts b/.github/scripts/cli-bench/post-comment.ts index f6342497be..d61ec82dbb 100644 --- a/.github/scripts/cli-bench/post-comment.ts +++ b/.github/scripts/cli-bench/post-comment.ts @@ -42,7 +42,7 @@ export const RERUN_NOTES: Record = { "Comment `/bench full` to run this benchmark again on the latest commit, which also refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark.", }; const TITLES: Record = { - lightweight: "## CLI benchmark", + lightweight: "## CLI benchmark (Lightweight Commands)", "integration-heavy": "## CLI benchmark (Integration Heavy Commands)", }; /** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts index 26e9ba2670..77ab816da0 100644 --- a/.github/scripts/cli-bench/rerun.test.ts +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -386,6 +386,16 @@ describe("measuredSha", () => { list.flatMap((item) => (item === undefined ? [] : [item])); test("names the commit of a lightweight comment that reports a result", () => { + const body = composeBody( + "lightweight", + "## CLI benchmark (Lightweight Commands): no changes found", + entries(recorded("abc1234", "no changes found")), + ); + + expect(measuredSha(body)).toBe("abc1234"); + }); + + test("names the commit of a lightweight comment still under the previous heading", () => { const body = composeBody( "lightweight", "## CLI benchmark: no changes found", diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 4d94b3a1bf..5a5e74ec24 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -8,7 +8,7 @@ const lightweight = (content: string) => ({ body: composeBody("lightweight", content), authorLogin: BOT, }); -const reported = lightweight("## CLI benchmark: no changes found"); +const reported = lightweight("## CLI benchmark (Lightweight Commands): no changes found"); describe("resolveBench", () => { test("benchmarks the first push of a pull request that has no comment yet", () => { @@ -22,6 +22,18 @@ describe("resolveBench", () => { expect(SKIP_REASON).toContain("`/bench`"); }); + test("skips a later push while a comment under the previous heading reports a result", () => { + const old = lightweight("## CLI benchmark: no changes found"); + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [old] }).shouldRun).toBe(false); + }); + + test("benchmarks again when a comment under the previous heading reports a failed run", () => { + const old = lightweight("## CLI benchmark: could not run"); + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [old] }).shouldRun).toBe(true); + }); + test("benchmarks again when the comment reports a failed run", () => { const failed = lightweight(failureMarkdown("lightweight", "abc1234")); diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 5f6391c062..35abf87fbe 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -100,7 +100,9 @@ the hidden marker ``. The comment shows the commit it measu skip the benchmark, and the job summary says so, while the comment reports a result; a comment that reports a failed run is benchmarked again on the next push. The [full benchmark](#full-benchmark) keeps a separate comment, tagged ``, so each workflow writes only its own -comment. Pull requests from forks have a read-only token, so they get the job summary only and +comment. The lightweight comment is headed `## CLI benchmark (Lightweight Commands): ` and the +full one `## CLI benchmark (Integration Heavy Commands): <title>`; comments posted under the +earlier `## CLI benchmark: <title>` heading are still recognized and take the new heading on their next update. Pull requests from forks have a read-only token, so they get the job summary only and benchmark every push. A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 39adcd486c..6456c9d7ff 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -424,10 +424,7 @@ function frame( report?: BenchReport, ): string { const section = context.section ?? "lightweight"; - const heading = - section === "lightweight" - ? `## CLI benchmark: ${title}` - : `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; + const heading = `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; const note = context.note === undefined || context.note === "" ? [] : [`⚠️ ${context.note}`]; return [heading, contextLine(report, context), ...note, ...body].join("\n\n") + "\n"; } diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index c2bebb2e15..3ce6abe22c 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -55,7 +55,7 @@ describe("renderSummary", () => { mergeBase: "1234567890abcdef", }); - expect(markdown).toContain("## CLI benchmark: no changes found"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): no changes found"); expect(markdown).toContain( "Commit `abcdef1` against `develop` at `1234567` · 5 runs each · report only, never blocks merging", ); @@ -82,7 +82,7 @@ describe("renderSummary", () => { it("explains a new API call in plain words and counts each command once", () => { const markdown = renderSummary(report([migrationNew]), "unused"); - expect(markdown).toContain("## CLI benchmark: 1 thing to check"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): 1 thing to check"); expect(markdown).toContain( "⚠️ `supabase migration new bench` now calls the Management API. It made no calls before and now makes 1 (`GET /v1/projects/abc`).", ); @@ -98,7 +98,9 @@ describe("renderSummary", () => { it("says nothing was benchmarked instead of a clean pass when there are no commands", () => { const markdown = renderSummary(report([]), "unused"); - expect(markdown).toContain("## CLI benchmark: no commands were benchmarked"); + expect(markdown).toContain( + "## CLI benchmark (Lightweight Commands): no commands were benchmarked", + ); expect(markdown).not.toContain("no changes found"); }); @@ -120,13 +122,16 @@ describe("renderSummary", () => { it("names the failed step and stays a notice when there is no report", () => { const markdown = renderSummary(undefined, "The build of this commit failed.", {}); - expect(markdown).toContain("## CLI benchmark: could not run"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): could not run"); expect(markdown).toContain("The build of this commit failed."); expect(markdown).not.toContain("/bench"); }); it.each([ - { section: "lightweight", heading: "## CLI benchmark: no changes found" }, + { + section: "lightweight", + heading: "## CLI benchmark (Lightweight Commands): no changes found", + }, { section: "integration-heavy", heading: "## CLI benchmark (Integration Heavy Commands): no changes found", From c1855b45905d34468fb29993e5fb82ac7516d632 Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha <prashansa.kulshrestha@supabase.io> Date: Fri, 30 Oct 2026 19:39:59 +0530 Subject: [PATCH 11/11] ci(cli): count API calls in the full benchmark Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .github/workflows/cli-bench-full.yml | 30 +++++++++- apps/cli/docs/tracing-monitoring.md | 13 ++++- apps/cli/scripts/bench-merge.ts | 2 + apps/cli/scripts/bench-merge.unit.test.ts | 49 ++++++++++++++++ apps/cli/scripts/bench-summary.ts | 30 ++++++++-- apps/cli/scripts/bench-summary.unit.test.ts | 64 +++++++++++++++++++++ 6 files changed, 178 insertions(+), 10 deletions(-) create mode 100644 apps/cli/scripts/bench-merge.unit.test.ts diff --git a/.github/workflows/cli-bench-full.yml b/.github/workflows/cli-bench-full.yml index 550f2aa8a8..6528fbc7d4 100644 --- a/.github/workflows/cli-bench-full.yml +++ b/.github/workflows/cli-bench-full.yml @@ -344,6 +344,27 @@ jobs: name: cli-bench-full-binaries path: ${{ runner.temp }}/bench + # A shard that cannot start the replay server fails, and the report job names it as a shard + # that did not finish. + - name: Start replay server + shell: bash + run: | + bench="$RUNNER_TEMP/bench" + chmod +x "$bench/base/supabase" "$bench/branch/supabase" + bun apps/cli-e2e/scripts/replay-serve.ts > "$bench/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$bench/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$bench/replay.log" >&2 + exit 1 + - name: Run benchmark env: RUNTIME: ${{ matrix.runtime }} @@ -352,7 +373,6 @@ jobs: shell: bash run: | bench="$RUNNER_TEMP/bench" - chmod +x "$bench/base/supabase" "$bench/branch/supabase" cache_args=() if [ "$RUNTIME" = native ]; then cache="${XDG_CACHE_HOME:-$HOME/.cache}/supabase-stack-artifacts" @@ -365,10 +385,16 @@ jobs: --workloads apps/cli/bench/workloads.full.json \ --runtime "$RUNTIME" \ --only "$WORKLOADS" \ + --api-url "$REPLAY_URL" \ "${cache_args[@]}" \ --trace \ --out "$bench/report-$RUNTIME-$SHARD.json" | tee "$bench/report-$RUNTIME-$SHARD.txt" + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + - name: Upload report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -438,7 +464,7 @@ jobs: run: | note="" if [ "$SHARDS" = failure ] || [ "$SHARDS" = cancelled ]; then - note="Some shards did not finish, so their commands are missing." + note="Some shards did not finish (for example, the local test server failed to start), so their commands are missing." fi bun apps/cli/scripts/bench-summary.ts \ --section integration-heavy \ diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index 35abf87fbe..cdc4b92766 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -320,16 +320,25 @@ or drops to zero. Container work in the stack backend runs outside the CLI's tra workloads show no container counts. The fixtures under `apps/cli/bench/fixtures/` are generated by `bun apps/cli/scripts/bench-fixtures.ts` and committed; changing one starts a new series of results. +Each shard also starts the replay server, as the pull request benchmark does, and passes it with +`--api-url`, so the report counts Management API calls for every command. These commands have no +recorded scenario and are expected to make no API calls, so the expected count is zero and the +report says "none, before or after"; a call that first appears on the branch is flagged, and a call +that gets an error response from the empty server is flagged as unexpected. Traffic to hosts other +than the replay server, such as a local stack's readiness checks, is not an API call and does not +raise a flag. A shard whose replay server fails to start fails, and the report notes that its +commands are missing. + To run it locally, build two binaries as above and run a subset: ```sh bun apps/cli/scripts/bench-cli.ts --base <base-binary> --branch <branch-binary> \ --workloads apps/cli/bench/workloads.full.json --runtime docker --only status,db-query \ - --trace --out /tmp/report.json + --api-url <replay-server-url> --trace --out /tmp/report.json bun apps/cli/scripts/bench-summary.ts --section integration-heavy --report /tmp/report.json ``` -Use `--runtime native` with `--artifact-cache <dir>` for the native workloads. Refresh the list from +Start the replay server with `bun apps/cli-e2e/scripts/replay-serve.ts` for the URL, or drop `--api-url` to skip API counts. Use `--runtime native` with `--artifact-cache <dir>` for the native workloads. Refresh the list from the same usage query as the pull request benchmark, keeping the commands that need a stack or database instead of dropping them. diff --git a/apps/cli/scripts/bench-merge.ts b/apps/cli/scripts/bench-merge.ts index 2a7b67e74e..f0659fa0f1 100644 --- a/apps/cli/scripts/bench-merge.ts +++ b/apps/cli/scripts/bench-merge.ts @@ -34,8 +34,10 @@ export function mergeReports( const cpuProfilePath = reports.find( (report) => report.cpuProfilePath !== undefined, )?.cpuProfilePath; + const apiUrl = reports.find((report) => report.apiUrl !== undefined)?.apiUrl; return { ...first, + ...(apiUrl === undefined ? {} : { apiUrl }), workloads: [...byId.values()].sort((a, b) => rank(a.id) - rank(b.id)), ...(cpuProfilePath === undefined ? {} : { cpuProfilePath }), }; diff --git a/apps/cli/scripts/bench-merge.unit.test.ts b/apps/cli/scripts/bench-merge.unit.test.ts new file mode 100644 index 0000000000..2d7b899744 --- /dev/null +++ b/apps/cli/scripts/bench-merge.unit.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import type { BenchReport, RunSample } from "./bench-report.ts"; +import { mergeReports } from "./bench-merge.ts"; + +const sample = (calls: number): RunSample => ({ + wallMs: 10, + exitCode: 0, + trace: { + skippedLines: 0, + spans: {}, + api: { + apiCalls: calls, + externalCalls: 2, + errorStatuses: 0, + endpoints: {}, + requestLogCount: calls, + }, + }, +}); + +const shard = (apiUrl: string | undefined, ids: ReadonlyArray<string>, calls = 0): BenchReport => ({ + base: "/base", + branch: "/branch", + runs: 1, + warmup: 0, + trace: true, + ...(apiUrl === undefined ? {} : { apiUrl }), + workloads: ids.map((id) => ({ id, args: [id], base: [sample(calls)], branch: [sample(calls)] })), +}); + +describe("mergeReports", () => { + it("keeps each shard's API counts, including a count of zero, in manifest order", () => { + const merged = mergeReports( + [shard("http://127.0.0.1:1", ["status"], 0), shard("http://127.0.0.1:2", ["db-query"], 3)], + ["db-query", "status"], + ); + + expect(merged?.workloads.map((workload) => workload.id)).toEqual(["db-query", "status"]); + expect(merged?.workloads.map((workload) => workload.branch[0]?.trace?.api?.apiCalls)).toEqual([ + 3, 0, + ]); + }); + + it("carries an API URL even when the first shard recorded none", () => { + const merged = mergeReports([shard(undefined, ["a"]), shard("http://127.0.0.1:2", ["b"])], []); + + expect(merged?.apiUrl).toBe("http://127.0.0.1:2"); + }); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 6456c9d7ff..a3f06d2ce9 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -140,7 +140,7 @@ function table(header: ReadonlyArray<string>, rows: ReadonlyArray<ReadonlyArray< const when = (build: Flag["build"]) => (build === "base" ? "before" : "after"); /** One plain-words sentence for a flag. */ -function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { +function describeFlag(flag: Flag, analysis: WorkloadAnalysis, integration: boolean): string { const name = command(analysis); const at = when(flag.build); switch (flag.kind) { @@ -169,6 +169,9 @@ function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { case "request-log-mismatch": return `The API calls counted for ${name} ${at} do not match what the local test server saw, so the counts may be wrong.`; case "error-status": + if (integration) { + return `${name} called the Management API ${at} and got an error response from the local test server, which has no recorded scenario for these commands; they are expected to make no API calls.`; + } return `${name} got error responses from the local test server ${at}, so it did not follow the recorded scenario.`; case "trace-incomplete": return `Some runs of ${name} ${at} produced no usable trace, so call counts and phase timings may be incomplete.`; @@ -284,7 +287,7 @@ interface ApiOverview { readonly changedIds: ReadonlyArray<string>; } -function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>): ApiOverview { +function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>, expectNone: boolean): ApiOverview { const measured = analyses.filter( (analysis) => analysis.base.api !== undefined && analysis.branch.api !== undefined, ); @@ -297,6 +300,18 @@ function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>): ApiOverview { const missing = analyses.length - measured.length; const suffix = missing === 0 ? "" : ` (not measured for ${missing})`; if (changed.length === 0) { + const idle = + expectNone && + measured.every( + (analysis) => + analysis.base.api?.medianCalls === 0 && analysis.branch.api?.medianCalls === 0, + ); + if (idle) { + return { + line: `API calls: none, before or after, for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, + changedIds: [], + }; + } return { line: `API calls: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, changedIds: [], @@ -437,12 +452,15 @@ export function renderSummary( ): string { if (report === undefined) return frame("could not run", [notice], context); const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); + const integration = (context.section ?? "lightweight") === "integration-heavy"; + // A local stack's own readiness and health traffic always leaves the test server's host. const flags = analyses.flatMap((analysis) => - collectFlags(analysis).map((flag) => ({ flag, analysis })), + collectFlags(analysis) + .filter((flag) => !(integration && flag.kind === "external-calls")) + .map((flag) => ({ flag, analysis })), ); if (analyses.length === 0) return frame("no commands were benchmarked", [], context, report); - const api = apiOverview(analyses); - const integration = (context.section ?? "lightweight") === "integration-heavy"; + const api = apiOverview(analyses, integration); const containers = integration ? containerOverview(analyses) : undefined; const moved = analyses .filter((analysis) => analysis.verdict === "slower" || analysis.verdict === "faster") @@ -454,7 +472,7 @@ export function renderSummary( return frame( title, [ - ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis)}`), + ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis, integration)}`), timingTable(analyses), api.line, ...(containers === undefined ? [] : [containers.line]), diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 3ce6abe22c..ba5c6719c0 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -320,6 +320,70 @@ describe("renderSummary for integration-heavy commands", () => { expect(markdown).not.toContain("container commands"); }); + const counted = (calls: number, extra: Partial<RunSample> = {}, external = 0): RunSample => ({ + ...run(30, calls, calls === 0 ? {} : { "GET /v1/projects": calls }), + ...extra, + trace: { + skippedLines: 0, + spans: { "cli.run": { totalMs: 15, count: 1 } }, + api: { + apiCalls: calls, + externalCalls: external, + errorStatuses: 0, + endpoints: calls === 0 ? {} : { "GET /v1/projects": calls }, + requestLogCount: calls, + }, + }, + }); + const withApi = (id: string, base: RunSample, branch: RunSample) => ({ + id, + args: ["status"], + stack: true, + runtime: "native" as const, + base: runs(5, () => base), + branch: runs(5, () => branch), + }); + + it("reports a command that made no API calls as counted, not as not counted", () => { + const markdown = render([withApi("status", counted(0), counted(0))]); + + expect(markdown).toContain("API calls: none, before or after, for the command."); + expect(markdown).not.toContain("API calls were not counted."); + expect(markdown).toContain("| `supabase status` | 0 | 0 |"); + }); + + it("says API calls were not counted when no launch was traced against the test server", () => { + const untraced = { wallMs: 30, exitCode: 0 }; + const markdown = render([withApi("status", untraced, untraced)]); + + expect(markdown).toContain("API calls were not counted."); + }); + + it("flags a Management API call that first appears on the branch", () => { + const markdown = render([withApi("status", counted(0), counted(1))]); + + expect(markdown).toContain("now calls the Management API"); + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): 1 thing to check"); + }); + + it("does not flag a local stack's calls to other hosts", () => { + const markdown = render([withApi("status", counted(0, {}, 12), counted(0, {}, 12))]); + + expect(markdown).toContain("no changes found"); + expect(markdown).not.toContain("hosts other than"); + }); + + it("flags an error response from the empty test server in plain words", () => { + const failing = counted(1); + const errored: RunSample = { + ...failing, + trace: { ...failing.trace!, api: { ...failing.trace!.api!, errorStatuses: 1 } }, + }; + const markdown = render([withApi("status", counted(0), errored)]); + + expect(markdown).toContain("expected to make no API calls"); + }); + it("leaves container operations out of the lightweight section", () => { const markdown = renderSummary(report([heavy("status", { ps: 1 }, { ps: 1 })]), "unused", {});