diff --git a/.github/scripts/cli-bench/full-plan.test.ts b/.github/scripts/cli-bench/full-plan.test.ts new file mode 100644 index 0000000000..b11b214050 --- /dev/null +++ b/.github/scripts/cli-bench/full-plan.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, test } from "bun:test"; +import { planFullRun, readManifest, type PlanDeps, type PlanInput } from "./full-plan.ts"; + +const REPO = "supabase/cli"; +const manifest = readManifest( + JSON.stringify({ + workloads: [ + { id: "start", runtime: "docker", estimatedSeconds: 400 }, + { id: "db-reset", runtime: "docker", estimatedSeconds: 400 }, + { id: "stack-start-native", runtime: "native", estimatedSeconds: 100 }, + ], + }), +); + +const input = (overrides: Partial = {}): PlanInput => ({ + event: "workflow_dispatch", + workloads: "all", + runtime: "both", + base: "merge-base", + refName: "feature/x", + repository: REPO, + manifest, + ...overrides, +}); + +const deps = (overrides: Partial = {}): PlanDeps => ({ + latestRelease: async () => "v3.0.0", + openPullRequests: async () => [{ number: 12, headRepoFullName: REPO }], + ...overrides, +}); + +describe("planFullRun", () => { + test("measures everything against the latest stable release on a schedule, with no pull request", async () => { + const plan = await planFullRun( + input({ event: "schedule", workloads: "", runtime: "", base: "", refName: "develop" }), + deps(), + ); + + expect(plan).toMatchObject({ + workloads: "all", + runtime: "both", + baseKind: "release", + baseTag: "v3.0.0", + prNumber: "", + runLightweight: true, + runIntegration: true, + }); + }); + + test("attaches a merge-base run to the branch's single open pull request", async () => { + const plan = await planFullRun(input(), deps()); + + expect(plan).toMatchObject({ baseKind: "merge-base", baseTag: "", prNumber: "12" }); + }); + + test.each([ + ["no open pull request", [] as Array<{ number: number; headRepoFullName: string }>], + [ + "two open pull requests", + [ + { number: 1, headRepoFullName: REPO }, + { number: 2, headRepoFullName: REPO }, + ], + ], + ["a pull request from a fork", [{ number: 3, headRepoFullName: "someone/cli" }]], + ])("attaches no pull request when the branch has %s", async (_name, pulls) => { + const plan = await planFullRun(input(), deps({ openPullRequests: async () => pulls })); + + expect(plan.prNumber).toBe(""); + }); + + test("leaves the pull request comment alone for a run against a release tag", async () => { + const plan = await planFullRun(input({ base: "v3.0.0" }), deps()); + + expect(plan).toMatchObject({ baseKind: "release", baseTag: "v3.0.0", prNumber: "" }); + }); + + test.each(["main", "v3", "v3.0.0; rm -rf /", "$(id)", "../v3.0.0"])( + "rejects the base %p", + async (base) => { + await expect(planFullRun(input({ base }), deps())).rejects.toThrow("Unsupported base"); + }, + ); + + test.each([ + ["lightweight", { runLightweight: true, runIntegration: false, shards: 0 }], + ["integration-heavy", { runLightweight: false, runIntegration: true, shards: 3 }], + ["all", { runLightweight: true, runIntegration: true, shards: 3 }], + ])("plans shards for workloads=%s", async (workloads, expected) => { + const plan = await planFullRun(input({ workloads }), deps()); + + expect(plan.runLightweight).toBe(expected.runLightweight); + expect(plan.runIntegration).toBe(expected.runIntegration); + expect(plan.matrix.include).toHaveLength(expected.shards); + }); + + test("keeps only the requested runtime's shards", async () => { + const plan = await planFullRun(input({ runtime: "native" }), deps()); + + expect(plan.matrix.include).toEqual([ + expect.objectContaining({ runtime: "native", workloads: "stack-start-native" }), + ]); + }); + + test("rejects an unknown runtime or workloads choice", async () => { + await expect(planFullRun(input({ runtime: "podman" }), deps())).rejects.toThrow("runtime"); + await expect(planFullRun(input({ workloads: "none" }), deps())).rejects.toThrow("workloads"); + }); +}); diff --git a/.github/scripts/cli-bench/full-plan.ts b/.github/scripts/cli-bench/full-plan.ts new file mode 100644 index 0000000000..5c4b24ee9d --- /dev/null +++ b/.github/scripts/cli-bench/full-plan.ts @@ -0,0 +1,194 @@ +/** + * Planner for the full benchmark workflow: resolves what a run measures and against which base, + * attaches the run to its pull request, and splits the workloads into shards. + * + * A scheduled run measures everything against the latest stable release. A dispatched run uses its + * inputs; its base is the merge base with `develop` unless a release tag is given. Only a + * merge-base run on a branch with exactly one open pull request into `develop` from this + * repository updates that pull request's comment. Needs read access only. + * Run in CI as: `bun .github/scripts/cli-bench/full-plan.ts`. + */ +import { appendFileSync, readFileSync } from "node:fs"; + +import { packShards, type PackedWorkload } from "../../../apps/cli/scripts/bench-pack.ts"; +import { githubFetch, requireEnv } from "./github.ts"; + +export const WORKLOAD_CHOICES = ["all", "lightweight", "integration-heavy"] as const; +export const RUNTIME_CHOICES = ["both", "docker", "native"] as const; +export type WorkloadChoice = (typeof WORKLOAD_CHOICES)[number]; +export type RuntimeChoice = (typeof RUNTIME_CHOICES)[number]; + +const RELEASE_TAG = /^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; +const MANIFEST = "apps/cli/bench/workloads.full.json"; + +export interface PlanInput { + event: string; + workloads: string; + runtime: string; + base: string; + refName: string; + repository: string; + manifest: ReadonlyArray; +} + +export interface PullRequestRef { + number: number; + headRepoFullName: string; +} + +export interface PlanDeps { + /** Tag of the latest stable release. */ + latestRelease: () => Promise; + /** Open pull requests into `develop` whose head is `refName`. */ + openPullRequests: (refName: string) => Promise>; +} + +export interface FullPlan { + workloads: WorkloadChoice; + runtime: RuntimeChoice; + baseKind: "merge-base" | "release"; + /** The release tag compared against; empty for a merge-base run. */ + baseTag: string; + /** Pull request whose comment this run updates; empty when none. */ + prNumber: string; + runLightweight: boolean; + runIntegration: boolean; + matrix: { include: ReadonlyArray> }; +} + +const isChoice = (choices: ReadonlyArray, value: string): value is T => + (choices as ReadonlyArray).includes(value); + +/** What a run measures, against what, and which shards run it. Throws on invalid inputs. */ +export async function planFullRun(input: PlanInput, deps: PlanDeps): Promise { + const scheduled = input.event === "schedule"; + const workloads = scheduled ? "all" : input.workloads; + const runtime = scheduled ? "both" : input.runtime; + if (!isChoice(WORKLOAD_CHOICES, workloads)) { + throw new Error(`Unsupported workloads "${workloads}".`); + } + if (!isChoice(RUNTIME_CHOICES, runtime)) throw new Error(`Unsupported runtime "${runtime}".`); + + let baseKind: FullPlan["baseKind"] = "merge-base"; + let baseTag = ""; + if (scheduled) { + baseKind = "release"; + baseTag = await deps.latestRelease(); + } else if (input.base !== "merge-base") { + baseKind = "release"; + baseTag = input.base; + } + if (baseKind === "release" && !RELEASE_TAG.test(baseTag)) { + throw new Error( + `Unsupported base "${baseTag}"; use merge-base or a release tag such as v3.0.0.`, + ); + } + + let prNumber = ""; + if (!scheduled && baseKind === "merge-base") { + const candidates = (await deps.openPullRequests(input.refName)).filter( + (pr) => pr.headRepoFullName === input.repository, + ); + if (candidates.length === 1) prNumber = String(candidates[0]?.number); + } + + const runIntegration = workloads === "all" || workloads === "integration-heavy"; + const wanted = (shardRuntime: string) => runtime === "both" || runtime === shardRuntime; + const include = runIntegration + ? packShards(input.manifest) + .filter((shard) => wanted(shard.runtime)) + .map((shard) => ({ ...shard, workloads: shard.workloads.join(",") })) + : []; + return { + workloads, + runtime, + baseKind, + baseTag, + prNumber, + runLightweight: workloads === "all" || workloads === "lightweight", + runIntegration, + matrix: { include }, + }; +} + +/** Reads the `id`, `runtime` and `estimatedSeconds` fields of a workload manifest. */ +export function readManifest(text: string): ReadonlyArray { + const parsed: unknown = JSON.parse(text); + const list = (parsed as { workloads?: unknown }).workloads; + if (!Array.isArray(list)) throw new Error("The workload manifest has no workloads list."); + return list.map((entry: Record) => ({ + id: String(entry.id), + ...(entry.runtime === "docker" || entry.runtime === "native" ? { runtime: entry.runtime } : {}), + ...(typeof entry.estimatedSeconds === "number" + ? { estimatedSeconds: entry.estimatedSeconds } + : {}), + })); +} + +async function main(): Promise { + const token = requireEnv("GITHUB_TOKEN"); + const repository = requireEnv("GITHUB_REPOSITORY"); + const [owner = ""] = repository.split("/"); + const base = `https://api.github.com/repos/${repository}`; + + const plan = await planFullRun( + { + event: requireEnv("EVENT_NAME"), + workloads: process.env.INPUT_WORKLOADS ?? "all", + runtime: process.env.INPUT_RUNTIME ?? "both", + base: process.env.INPUT_BASE ?? "merge-base", + refName: requireEnv("REF_NAME"), + repository, + manifest: readManifest(readFileSync(MANIFEST, "utf8")), + }, + { + latestRelease: async () => { + const latest = (await (await githubFetch(`${base}/releases/latest`, token)).json()) as { + tag_name: string; + }; + return latest.tag_name; + }, + openPullRequests: async (refName) => { + const url = + `${base}/pulls?state=open&base=develop&per_page=100` + + `&head=${encodeURIComponent(`${owner}:${refName}`)}`; + const pulls = (await (await githubFetch(url, token)).json()) as Array<{ + number: number; + head: { repo: { full_name: string } | null }; + }>; + return pulls.map((pr) => ({ + number: pr.number, + headRepoFullName: pr.head.repo?.full_name ?? "", + })); + }, + }, + ); + + const outputs: Record = { + base_kind: plan.baseKind, + base_tag: plan.baseTag, + pr_number: plan.prNumber, + run_lightweight: String(plan.runLightweight), + run_integration: String(plan.runIntegration), + has_shards: String(plan.matrix.include.length > 0), + matrix: JSON.stringify(plan.matrix), + }; + console.log( + `CLI full benchmark plan: workloads=${plan.workloads} runtime=${plan.runtime} ` + + `base=${plan.baseKind === "release" ? plan.baseTag : "merge-base"} pr=${plan.prNumber || "none"} ` + + `shards=${plan.matrix.include.length}`, + ); + appendFileSync( + requireEnv("GITHUB_OUTPUT"), + Object.entries(outputs) + .map(([key, value]) => `${key}=${value}\n`) + .join(""), + ); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.github/scripts/cli-bench/history.test.ts b/.github/scripts/cli-bench/history.test.ts index 0d096bce41..90b044a605 100644 --- a/.github/scripts/cli-bench/history.test.ts +++ b/.github/scripts/cli-bench/history.test.ts @@ -19,11 +19,25 @@ const entry = (n: number, overrides: Partial = {}): RunEntry => ({ describe("headlineOf", () => { test("reads the title of the rendered report", () => { + expect( + headlineOf( + "\n\n## CLI benchmark (Lightweight Commands): 2 things to check\n\nbody", + ), + ).toBe("2 things to check"); + }); + + test("reads the title of a lightweight report under the previous heading", () => { expect(headlineOf("\n\n## CLI benchmark: 2 things to check\n\nbody")).toBe( "2 things to check", ); }); + test("reads the title of the integration-heavy report", () => { + expect(headlineOf("## CLI benchmark (Integration Heavy Commands): 1 thing to check")).toBe( + "1 thing to check", + ); + }); + test("falls back when the report has no title", () => { expect(headlineOf("nothing here")).toBe("unknown"); }); @@ -92,6 +106,13 @@ describe("renderHistory and decodeHistory", () => { expect(rendered).not.toContain("0000003` ·"); }); + test("lists every run, the newest included, while a run is in progress", () => { + const rendered = renderHistory([entry(3), entry(2)], (t) => t, true); + + expect(rendered).toContain("`0000003` ·"); + expect(rendered).toContain("`0000002` ·"); + }); + test("shows no list for a first run", () => { expect(renderHistory([entry(1)], (t) => t)).not.toContain("Previous runs"); }); diff --git a/.github/scripts/cli-bench/history.ts b/.github/scripts/cli-bench/history.ts index 9602383119..ce94f5c2b4 100644 --- a/.github/scripts/cli-bench/history.ts +++ b/.github/scripts/cli-bench/history.ts @@ -1,5 +1,5 @@ /** - * Run history kept inside the benchmark comment. The newest run is the comment itself; earlier + * Run history kept inside one benchmark comment. The newest run is the comment itself; earlier * runs sit in a collapsed list, backed by a hidden marker so the next update can carry them on. * The marker is read back only from the workflow bot's own comment, and anything that does not * decode cleanly counts as no history. @@ -39,9 +39,9 @@ const isEntry = (value: unknown): value is RunEntry => { ); }; -/** The title the renderer puts after `## CLI benchmark:`, flattened to one bounded line. */ +/** The title the renderer puts after the `## CLI benchmark` heading, flattened to one bounded line. */ export function headlineOf(markdown: string): string { - const title = /^## CLI benchmark: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; + const title = /^## CLI benchmark(?: \([^)\n]*\))?: (.+)$/mu.exec(markdown)?.[1] ?? "unknown"; return title.replaceAll(/\s+/gu, " ").trim().slice(0, MAX_HEADLINE_CHARS); } @@ -87,14 +87,16 @@ const day = (at: string) => `${at.slice(0, 10)} ${at.slice(11, 16)} UTC`; /** * The hidden marker for `entries` (current run first), preceded by a collapsed list of the - * earlier ones when there are any. `clean` makes a headline safe to render. + * earlier ones when there are any. While a run is in progress the comment has no current result, + * so every entry is listed. `clean` makes a headline safe to render. */ export function renderHistory( entries: ReadonlyArray, clean: (text: string) => string, + running = false, ): string { const marker = ``; - const earlier = entries.slice(1); + const earlier = running ? entries : entries.slice(1); if (earlier.length === 0) return marker; const lines = earlier.map( (entry) => diff --git a/.github/scripts/cli-bench/post-comment.test.ts b/.github/scripts/cli-bench/post-comment.test.ts index fb890a8952..981acab6d4 100644 --- a/.github/scripts/cli-bench/post-comment.test.ts +++ b/.github/scripts/cli-bench/post-comment.test.ts @@ -1,112 +1,154 @@ import { describe, expect, test } from "bun:test"; import { - BENCH_MARKER, - FAILURE_TITLE, + commentFailed, + composeBody, + failureMarkdown, + findComment, + MARKERS, MAX_BODY_CHARS, - planComment, - prepareBody, + RERUN_NOTES, sanitize, + writeComment, + type CommentIo, + type ExistingComment, + type Kind, } from "./post-comment.ts"; -import { decodeHistory, type RunEntry } from "./history.ts"; +import { decodeHistory, MAX_PREVIOUS_RUNS, type RunEntry } from "./history.ts"; const BOT = "github-actions[bot]"; +const RUN = "https://github.com/o/r/actions/runs/"; +const NOW = new Date("2026-10-28T09:05:00Z"); +const SHA = "0123456789abcdef"; -describe("planComment", () => { - test("creates the comment when none carries the marker", () => { - expect(planComment("create-if-missing", [{ id: 1, body: "hi", authorLogin: BOT }])).toEqual({ - kind: "create", - }); - }); +const titleOf = (kind: Kind) => + kind === "lightweight" + ? "## CLI benchmark (Lightweight Commands)" + : "## CLI benchmark (Integration Heavy Commands)"; +const report = (kind: Kind, title: string) => `${titleOf(kind)}: ${title}\n\ndetails`; - test("leaves an existing marker comment alone on a later push", () => { - const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; +/** An in-memory comment thread. */ +function fakeThread(initial: Array = []) { + const comments = [...initial]; + let nextId = 100; + const io: CommentIo = { + list: async () => comments.map((comment) => ({ ...comment })), + create: async (body) => { + comments.push({ id: nextId, body, authorLogin: BOT }); + return nextId++; + }, + update: async (id, body) => { + const target = comments.find((comment) => comment.id === id); + if (target !== undefined) target.body = body; + }, + }; + return { comments, io }; +} - expect(planComment("create-if-missing", [existing])).toEqual({ kind: "skip" }); - }); +const bodyOf = (comments: Array, kind: Kind) => + findComment(kind, comments)?.body ?? ""; - test("replaces a first-run failure notice on a later push", () => { - const failed = { - id: 5, - body: `${BENCH_MARKER}\n${FAILURE_TITLE}\nbuild failed`, - authorLogin: BOT, - }; +const entry = (n: number): RunEntry => ({ + sha: n.toString(16).padStart(7, "0"), + at: "2026-10-28T09:05Z", + headline: "no changes found", + url: `${RUN}${n}`, +}); - expect(planComment("create-if-missing", [failed])).toEqual({ - kind: "update", - id: 5, - previousBody: failed.body, - }); +describe("findComment", () => { + test("keeps the lightweight marker the first benchmark comments already carry", () => { + expect(MARKERS.lightweight).toBe(""); + expect(MARKERS["integration-heavy"]).not.toBe(MARKERS.lightweight); }); - test("updates the marker comment in place for a re-run", () => { - const existing = { id: 5, body: `${BENCH_MARKER}\nold`, authorLogin: BOT }; + test("finds each comment by its own marker", () => { + const light = { id: 1, body: `${MARKERS.lightweight}\nx`, authorLogin: BOT }; + const full = { id: 2, body: `${MARKERS["integration-heavy"]}\nx`, authorLogin: BOT }; - expect(planComment("upsert", [existing])).toEqual({ - kind: "update", - id: 5, - previousBody: existing.body, - }); + expect(findComment("lightweight", [full, light])?.id).toBe(1); + expect(findComment("integration-heavy", [light, full])?.id).toBe(2); }); test("ignores a marker pasted by someone else", () => { - const forged = { id: 6, body: BENCH_MARKER, authorLogin: "mallory" }; + const forged = { id: 6, body: MARKERS.lightweight, authorLogin: "mallory" }; - expect(planComment("create-if-missing", [forged])).toEqual({ kind: "create" }); + expect(findComment("lightweight", [forged])).toBeUndefined(); }); -}); -describe("prepareBody", () => { - test("puts the marker first exactly once", () => { - const body = prepareBody(`text ${BENCH_MARKER} more`); + test("keeps the older comment when two exist", () => { + const older = { id: 3, body: MARKERS.lightweight, authorLogin: BOT }; + const newer = { id: 9, body: MARKERS.lightweight, authorLogin: BOT }; - expect(body.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(body.split(BENCH_MARKER)).toHaveLength(2); + expect(findComment("lightweight", [newer, older])?.id).toBe(3); }); +}); - test("cuts an oversized report below GitHub's limit and says so", () => { - const body = prepareBody("x".repeat(200_000)); +describe("composeBody", () => { + test("lightweight comment starts with its marker once and ends its text with its own note", () => { + const body = composeBody("lightweight", report("lightweight", "new")); - expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); - expect(body).toContain("truncated"); + expect(body.startsWith(`${MARKERS.lightweight}\n`)).toBe(true); + expect(body.split(MARKERS.lightweight).length - 1).toBe(1); + expect(body).toContain(`\n\n${RERUN_NOTES.lightweight}\n`); + expect(RERUN_NOTES.lightweight).toBe( + "Comment `/bench` to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to also run the integration-heavy benchmark, which posts in its own comment.", + ); + expect(body).not.toContain(RERUN_NOTES["integration-heavy"]); }); -}); -describe("prepareBody history", () => { - const run = (n: number): RunEntry => ({ - sha: n.toString(16).padStart(7, "0"), - at: "2026-10-28T09:05Z", - headline: "no changes found", - url: `https://github.com/o/r/actions/runs/${n}`, + test("integration-heavy comment starts with its marker once and ends its text with its own note", () => { + const body = composeBody("integration-heavy", report("integration-heavy", "new")); + + expect(body.startsWith(`${MARKERS["integration-heavy"]}\n`)).toBe(true); + expect(body.split(MARKERS["integration-heavy"]).length - 1).toBe(1); + expect(body).toContain(`\n\n${RERUN_NOTES["integration-heavy"]}\n`); + expect(RERUN_NOTES["integration-heavy"]).toBe( + "Comment `/bench full` to run this benchmark again on the latest commit, which also refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark.", + ); + expect(body).not.toContain(RERUN_NOTES.lightweight); }); - test("lists earlier runs after the run link and carries them in the marker", () => { - const body = prepareBody("report", "https://github.com/o/r/actions/runs/3", [run(3), run(2)]); + test("cuts an oversized report below GitHub's limit and says so", () => { + const body = composeBody("lightweight", "x".repeat(MAX_BODY_CHARS * 2), [entry(1)]); - expect(body.indexOf("[Workflow run]")).toBeLessThan(body.indexOf("Previous runs")); - expect(decodeHistory(body)).toEqual([run(3), run(2)]); + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(body).toContain("The report was truncated"); + expect(body.startsWith(MARKERS.lightweight)).toBe(true); + expect(decodeHistory(body)).toEqual([entry(1)]); }); test("drops the oldest runs before cutting the report when the body is too long", () => { - const history = Array.from({ length: 21 }, (_, i) => run(21 - i)); - const report = "x".repeat(MAX_BODY_CHARS - 4_000); + const entries = Array.from({ length: 21 }, (_, i) => ({ + ...entry(i + 1), + headline: "h".repeat(100), + })); + const content = "x".repeat(MAX_BODY_CHARS - 2_500); - const body = prepareBody(report, "https://github.com/o/r/actions/runs/21", history); + const body = composeBody("lightweight", content, entries); expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); - expect(body).not.toContain("truncated"); + expect(body).toContain(content); const kept = decodeHistory(body); - expect(kept.length).toBeGreaterThan(0); - expect(kept.length).toBeLessThan(history.length); - expect(kept[0]).toEqual(history[0]); + expect(kept.length).toBeGreaterThanOrEqual(1); + expect(kept.length).toBeLessThan(entries.length); + expect(kept).toEqual(entries.slice(0, kept.length)); + }); + + test("keeps the newest run when the report alone fills the body", () => { + const entries = [entry(3), entry(2), entry(1)]; + + const body = composeBody("integration-heavy", "x".repeat(MAX_BODY_CHARS), entries); + + expect(body.length).toBeLessThanOrEqual(MAX_BODY_CHARS); + expect(decodeHistory(body)).toEqual([entry(3)]); }); - test("renders an earlier run's headline through the sanitizer", () => { - const hostile = { ...run(1), headline: "@team " }; + test("renders a previous run's headline through the sanitizer", () => { + const hostile = { ...entry(1), headline: "@team [x](https://evil.test)" }; - const body = prepareBody("report", undefined, [run(2), hostile]); + const body = composeBody("lightweight", "report", [entry(2), hostile]); - expect(body).not.toContain(" { const clean = sanitize(hostile); - expect(clean).not.toMatch(/@(?!\u200b)/u); + expect(clean).not.toMatch(/@(?!​)/u); expect(clean).not.toContain("@"); expect(clean).not.toMatch(/(? { expect(clean).toContain("
"); }); - test("pins the failure title and marker literals the renderer emits", () => { - expect(FAILURE_TITLE).toBe("## CLI benchmark: could not run"); - expect(BENCH_MARKER).toBe(""); + test("keeps an escaped bracket from becoming a live link", () => { + const clean = sanitize("\\[x\\](//evil.example/p)"); + + expect(clean).toBe("\\\\\\[x\\\\\\](//evil.example/p)"); + }); + + test("keeps the escaped pipes of a table cell intact", () => { + expect(sanitize("| `supabase db query a\\|b` | 1 ms |")).toBe( + "| `supabase db query a\\|b` | 1 ms |", + ); + }); + + test("keeps a report from forging a marker", () => { + expect(sanitize(`${MARKERS.lightweight} `)).not.toContain(""; +/** The two comments a pull request can have. */ +export type Kind = "lightweight" | "integration-heavy"; + +/** Hidden tags identifying each comment; the lightweight one is the original benchmark marker. */ +export const MARKERS: Record = { + lightweight: "", + "integration-heavy": "", +}; +/** The line under each comment telling readers which command re-runs what. */ +export const RERUN_NOTES: Record = { + lightweight: + "Comment `/bench` to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to also run the integration-heavy benchmark, which posts in its own comment.", + "integration-heavy": + "Comment `/bench full` to run this benchmark again on the latest commit, which also refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark.", +}; +const TITLES: Record = { + lightweight: "## CLI benchmark (Lightweight Commands)", + "integration-heavy": "## CLI benchmark (Integration Heavy Commands)", +}; /** Login of comments posted with the workflow's `GITHUB_TOKEN`. */ const WORKFLOW_BOT_LOGIN = "github-actions[bot]"; /** GitHub rejects comment bodies longer than 65536 characters. */ @@ -25,24 +52,20 @@ export const MAX_BODY_CHARS = 65_000; const TRUNCATION_NOTICE = "\n\n
\n\n_The report was truncated; see the workflow run for the rest._\n"; -export type Mode = "create-if-missing" | "upsert"; +export type Mode = "create-if-missing" | "upsert" | "running"; export type ExistingComment = IssueComment; -export type CommentPlan = - | { kind: "skip" } - | { kind: "create" } - | { kind: "update"; id: number; previousBody: string }; - const ALLOWED_TAGS = new Set(["
", "
", "", "", "
"]); const ZERO_WIDTH_SPACE = "\u200b"; -/** Title the renderer gives a run that produced no comparison. */ -export const FAILURE_TITLE = "## CLI benchmark: could not run"; export const MAX_FILE_BYTES = 1_000_000; +const RUN_URL = /^https:\/\/[^\s)<>]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?$/u; /** Keeps mentions, links, images and raw HTML in untrusted markdown from rendering. */ export function sanitize(markdown: string): string { return markdown + .replaceAll("\\", "\\\\") + .replaceAll("\\\\|", "\\|") .replaceAll("&", "&") .replaceAll(/<[^>\n]*>?/gu, (tag) => ALLOWED_TAGS.has(tag) ? tag : tag.replaceAll("<", "<"), @@ -54,49 +77,152 @@ export function sanitize(markdown: string): string { .replaceAll("]", "\\]"); } +const short = (sha: string) => sha.slice(0, 7); +const runLink = (runUrl: string | undefined) => + runUrl !== undefined && RUN_URL.test(runUrl) ? ` [Workflow run](${runUrl})` : ""; + +/** Whether a comment reports a run that produced no comparison. */ +export const commentFailed = (body: string) => + /^## CLI benchmark[^\n]*: could not run$/mu.test(body); + +/** The placeholder shown while a run is in progress. */ +export function runningContent( + kind: Kind, + sha: string, + now: Date, + runUrl: string | undefined, +): string { + return ( + `${TITLES[kind]}: running\n\n` + + `Started ${now.toISOString().slice(0, 16).replace("T", " ")} UTC for commit \`${short(sha)}\`.${runLink(runUrl)}` + ); +} + +/** The report for a run that produced none. */ +export function failureMarkdown(kind: Kind, sha: string): string { + return `${TITLES[kind]}: could not run\n\nThe benchmark did not finish for commit \`${short(sha)}\`. See the workflow run.`; +} + +/** The sanitized report, then the trusted run link. */ +export function reportContent(markdown: string, runUrl: string | undefined): string { + const stripped = Object.values(MARKERS).reduce( + (text, marker) => text.replaceAll(marker, ""), + markdown, + ); + const link = runUrl !== undefined && RUN_URL.test(runUrl) ? `\n\n[Workflow run](${runUrl})` : ""; + return `${sanitize(stripped).trim()}${link}`; +} + +/** + * The short SHA of the commit a comment's newest result measured, or undefined when the comment + * reports a failed or running benchmark or has no recorded result. + */ +export function measuredSha(body: string): string | undefined { + if (commentFailed(body) || /^## CLI benchmark[^\n]*: running$/mu.test(body)) return undefined; + return decodeHistory(body)[0]?.sha; +} + +function assemble( + kind: Kind, + content: string, + entries: ReadonlyArray, + running: boolean, +): string { + return ( + [MARKERS[kind], content, RERUN_NOTES[kind], renderHistory(entries, sanitize, running)].join( + "\n\n", + ) + "\n" + ); +} + /** - * The comment body: the marker first, the sanitized report, the trusted run link, then the run - * history. `history` lists the current run first; the oldest runs are dropped until the body fits. + * The comment body: the marker, the report, the re-run note, then the run history. `entries` + * lists the newest run first; the oldest are dropped until the body fits, always keeping the + * newest, and only then is the report cut. */ -export function prepareBody( - markdown: string, - runUrl?: string, - history: ReadonlyArray = [], +export function composeBody( + kind: Kind, + content: string, + entries: ReadonlyArray = [], + running = false, ): string { - const link = runUrl === undefined || runUrl === "" ? "" : `\n\n[Workflow run](${runUrl})\n`; - const content = sanitize(markdown.replaceAll(BENCH_MARKER, "")).trim(); - const body = `${BENCH_MARKER}\n${content}\n`; - for (let kept = history.length; kept >= 1; kept--) { - const tail = `${link}\n${renderHistory(history.slice(0, kept), sanitize)}\n`; - if (body.length + tail.length <= MAX_BODY_CHARS) return body + tail; + const shown = running ? entries.slice(0, MAX_PREVIOUS_RUNS) : entries; + for (let kept = shown.length; kept >= 1; kept--) { + const body = assemble(kind, content, shown.slice(0, kept), running); + if (body.length <= MAX_BODY_CHARS) return body; } - const tail = `${link}\n${history.length === 0 ? "" : `${renderHistory(history.slice(0, 1), sanitize)}\n`}`; - const limit = MAX_BODY_CHARS - tail.length; + const newest = shown.slice(0, 1); + const limit = MAX_BODY_CHARS - assemble(kind, "", newest, running).length; const fitted = - body.length <= limit - ? body - : body.slice(0, limit - TRUNCATION_NOTICE.length) + TRUNCATION_NOTICE; - return fitted + tail; + content.length <= limit + ? content + : content.slice(0, Math.max(0, limit - TRUNCATION_NOTICE.length)) + TRUNCATION_NOTICE; + return assemble(kind, fitted, newest, running); } -/** The workflow bot's benchmark comment, if any. Only its marker counts. */ -export function findBenchComment( +/** The workflow bot's comment for `kind`, if any; the oldest wins should two exist. */ +export function findComment( + kind: Kind, comments: ReadonlyArray, ): ExistingComment | undefined { - return comments.find( - (comment) => comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(BENCH_MARKER), - ); + return [...comments] + .filter( + (comment) => + comment.authorLogin === WORKFLOW_BOT_LOGIN && comment.body.includes(MARKERS[kind]), + ) + .sort((a, b) => a.id - b.id)[0]; +} + +export interface CommentIo { + list: () => Promise>; + create: (body: string) => Promise; + update: (id: number, body: string) => Promise; +} + +export interface WriteInput { + mode: Mode; + kind: Kind; + /** The commit the run measured. */ + sha: string; + /** The rendered report; absent for `running`. */ + markdown: string | undefined; + runUrl: string | undefined; + now: Date; } -/** What to do with the pull request's benchmark comment. */ -export function planComment(mode: Mode, comments: ReadonlyArray): CommentPlan { - const existing = findBenchComment(comments); - if (existing === undefined) return { kind: "create" }; - // A first run that could not compare anything must not block the next push's real result. - const replaceable = mode === "upsert" || existing.body.includes(FAILURE_TITLE); - return replaceable - ? { kind: "update", id: existing.id, previousBody: existing.body } - : { kind: "skip" }; +export type WriteResult = { kind: "skipped" } | { kind: "written"; id: number; created: boolean }; + +/** Writes the comment for `input.kind`, carrying its history on; no other comment is read or changed. */ +export async function writeComment(io: CommentIo, input: WriteInput): Promise { + const { mode, kind } = input; + const existing = findComment(kind, await io.list()); + if (mode === "create-if-missing" && existing !== undefined && !commentFailed(existing.body)) { + return { kind: "skipped" }; + } + const running = mode === "running" || input.markdown === undefined; + const history = decodeHistory(existing?.body ?? ""); + const entry = running + ? undefined + : makeEntry({ + sha: input.sha, + now: input.now, + headline: headlineOf(input.markdown ?? ""), + url: input.runUrl ?? "", + }); + const content = running + ? runningContent(kind, input.sha, input.now, input.runUrl) + : reportContent(input.markdown ?? "", input.runUrl); + const body = composeBody( + kind, + content, + entry === undefined ? history : mergeHistory(history, entry), + running, + ); + if (existing === undefined) { + return { kind: "written", id: await io.create(body), created: true }; + } + await io.update(existing.id, body); + return { kind: "written", id: existing.id, created: false }; } async function main(): Promise { @@ -104,45 +230,70 @@ async function main(): Promise { const base = `https://api.github.com/repos/${requireEnv("GITHUB_REPOSITORY")}`; const pr = Number(requireEnv("PR_NUMBER")); const mode = requireEnv("MODE"); - if (mode !== "create-if-missing" && mode !== "upsert") { + if (mode !== "create-if-missing" && mode !== "upsert" && mode !== "running") { throw new Error(`Unsupported MODE "${mode}".`); } - const file = requireEnv("BODY_FILE"); - const stat = lstatSync(file); - if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { - throw new Error("The benchmark comment file is not a regular file under the size limit."); + const kind = process.env.KIND ?? "lightweight"; + if (kind !== "lightweight" && kind !== "integration-heavy") { + throw new Error(`Unsupported KIND "${kind}".`); } - const markdown = readFileSync(file, "utf8"); - const runUrl = process.env.RUN_URL; - const entry = makeEntry({ - sha: requireEnv("HEAD_SHA"), - now: new Date(), - headline: headlineOf(markdown), - url: runUrl ?? "", - }); + const sha = requireEnv("HEAD_SHA"); + + let markdown: string | undefined; + if (mode !== "running") { + const file = requireEnv("BODY_FILE"); + let stat: ReturnType | undefined; + try { + stat = lstatSync(file); + } catch { + stat = undefined; + } + if (stat === undefined) { + if (process.env.MISSING_BODY !== "failure") { + console.log("The benchmark job produced no comment."); + return; + } + markdown = failureMarkdown(kind, sha); + } else { + if (!stat.isFile() || stat.size > MAX_FILE_BYTES) { + throw new Error("The benchmark comment file is not a regular file under the size limit."); + } + markdown = readFileSync(file, "utf8"); + } + } + + const io: CommentIo = { + list: () => listIssueComments(token, base, pr), + create: async (body) => { + const created = (await ( + await githubFetch(`${base}/issues/${pr}/comments`, token, { + method: "POST", + body: JSON.stringify({ body }), + }) + ).json()) as { id: number }; + return created.id; + }, + update: async (id, body) => { + await githubFetch(`${base}/issues/comments/${id}`, token, { + method: "PATCH", + body: JSON.stringify({ body }), + }); + }, + }; - const plan = planComment(mode, await listIssueComments(token, base, pr)); - const earlier = plan.kind === "update" ? decodeHistory(plan.previousBody) : []; - const body = prepareBody( + const result = await writeComment(io, { + mode, + kind, + sha, markdown, - runUrl, - entry === undefined ? [] : mergeHistory(earlier, entry), + runUrl: process.env.RUN_URL, + now: new Date(), + }); + console.log( + result.kind === "skipped" + ? `PR #${pr} already has a ${kind} result; leaving it untouched.` + : `${result.created ? "Created" : "Updated"} ${kind} benchmark comment ${result.id} on PR #${pr}.`, ); - if (plan.kind === "skip") { - console.log(`PR #${pr} already has a benchmark comment; leaving it untouched.`); - } else if (plan.kind === "update") { - await githubFetch(`${base}/issues/comments/${plan.id}`, token, { - method: "PATCH", - body: JSON.stringify({ body }), - }); - console.log(`Updated benchmark comment ${plan.id} on PR #${pr}.`); - } else { - await githubFetch(`${base}/issues/${pr}/comments`, token, { - method: "POST", - body: JSON.stringify({ body }), - }); - console.log(`Created benchmark comment on PR #${pr}.`); - } } if (import.meta.main) { diff --git a/.github/scripts/cli-bench/rerun.test.ts b/.github/scripts/cli-bench/rerun.test.ts index fa37cad648..77ab816da0 100644 --- a/.github/scripts/cli-bench/rerun.test.ts +++ b/.github/scripts/cli-bench/rerun.test.ts @@ -1,10 +1,15 @@ import { describe, expect, test } from "bun:test"; import { GithubRequestError } from "./github.ts"; +import { makeEntry } from "./history.ts"; +import { composeBody, failureMarkdown, measuredSha, runningContent } from "./post-comment.ts"; import { type BenchPr, type CommandIo, + coversIntegrationHeavy, handleBenchCommand, RERUN_WINDOW_MS, + type FullRun, + type FullWorkloads, type WorkflowRun, } from "./rerun.ts"; @@ -15,6 +20,7 @@ const pr = (overrides: Partial = {}): BenchPr => ({ state: "open", draft: false, headSha: HEAD, + headRef: "feature/x", headRepoFullName: REPO, baseRepoFullName: REPO, baseRef: "develop", @@ -33,12 +39,17 @@ function setup( pr?: BenchPr; runs?: ReadonlyArray; rerunError?: Error; + fullRuns?: ReadonlyArray; + dispatchError?: Error; + /** Short SHA the lightweight comment measured. */ + lightweightSha?: string; } = {}, ) { const calls = { reactions: [] as Array, replies: [] as Array, reruns: [] as Array, + dispatches: [] as Array<{ ref: string; workloads: FullWorkloads }>, }; const io: CommandIo = { fetchPr: async () => options.pr ?? pr(), @@ -48,6 +59,12 @@ function setup( if (options.rerunError !== undefined) throw options.rerunError; calls.reruns.push(id); }, + listFullRuns: async () => options.fullRuns ?? [], + dispatchFull: async (ref, workloads) => { + if (options.dispatchError !== undefined) throw options.dispatchError; + calls.dispatches.push({ ref, workloads }); + }, + lightweightSha: async () => options.lightweightSha, reactToComment: async (id) => { calls.reactions.push(id); }, @@ -78,7 +95,7 @@ describe("handleBenchCommand", () => { expect(calls.replies).toEqual([]); }); - test.each(["/bench full", "/benchmark", "please /bench", "/Bench"])( + test.each(["/bench fully", "/benchmark", "please /bench", "/Bench", "/bench full now"])( "ignores %p without a reply", async (body) => { const { io, calls } = setup(); @@ -86,7 +103,7 @@ describe("handleBenchCommand", () => { const outcome = await handleBenchCommand(command(body), io); expect(outcome.kind).toBe("ignored"); - expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [], dispatches: [] }); }, ); @@ -116,7 +133,7 @@ describe("handleBenchCommand", () => { const outcome = await handleBenchCommand(command("/bench"), io); expect(outcome.kind).toBe("ignored"); - expect(calls).toEqual({ reactions: [], replies: [], reruns: [] }); + expect(calls).toEqual({ reactions: [], replies: [], reruns: [], dispatches: [] }); }); test("tells the pull request there is no run when none exists for the head commit", async () => { @@ -182,3 +199,230 @@ describe("handleBenchCommand", () => { expect(calls.reruns).toEqual([]); }); }); + +const stackRun = (overrides: Partial = {}): FullRun => ({ + id: 31, + status: "in_progress", + displayTitle: "CLI benchmark full (integration-heavy): feature/x", + url: "https://github.com/supabase/cli/actions/runs/31", + ...overrides, +}); + +describe("handleBenchCommand with /bench full", () => { + test("dispatches the integration-heavy workloads on the branch and leaves a current lightweight comment alone", async () => { + const { io, calls } = setup({ lightweightSha: "aaaaaaa" }); + + const outcome = await handleBenchCommand(command("/bench full\n"), io); + + expect(outcome).toEqual({ + kind: "dispatched", + workloads: "integration-heavy", + rerunId: undefined, + }); + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "integration-heavy" }]); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([99]); + expect(calls.replies).toEqual([]); + }); + + test("also re-runs the lightweight run when its comment is behind the head commit", async () => { + const { io, calls } = setup({ lightweightSha: "bbbbbbb", runs: [run({ id: 4 })] }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ kind: "dispatched", workloads: "integration-heavy", rerunId: 4 }); + expect(calls.reruns).toEqual([4]); + }); + + test("leaves the lightweight part to a pull request run that is already in flight", async () => { + const { io, calls } = setup({ + lightweightSha: "bbbbbbb", + runs: [run({ status: "queued" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "integration-heavy" }]); + expect(calls.reruns).toEqual([]); + expect(calls.replies).toEqual([]); + }); + + test.each([ + ["no pull request run exists for the head commit", [] as Array], + [ + "its pull request run is older than 30 days", + [run({ createdAt: new Date(NOW.getTime() - RERUN_WINDOW_MS - 1000).toISOString() })], + ], + ])("measures the lightweight workloads in the full run when %s", async (_name, runs) => { + const { io, calls } = setup({ lightweightSha: undefined, runs }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ kind: "dispatched", workloads: "all", rerunId: undefined }); + expect(calls.dispatches).toEqual([{ ref: "feature/x", workloads: "all" }]); + expect(calls.reruns).toEqual([]); + }); + + test("replies with the link of an integration-heavy run already in progress and starts nothing", async () => { + const { io, calls } = setup({ fullRuns: [stackRun()] }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("https://github.com/supabase/cli/actions/runs/31"); + expect(calls.dispatches).toEqual([]); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("starts a run when the one in progress measures only the lightweight workloads", async () => { + const { io, calls } = setup({ + lightweightSha: "aaaaaaa", + fullRuns: [stackRun({ displayTitle: "CLI benchmark full (lightweight): feature/x" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toHaveLength(1); + }); + + test("starts a run when earlier ones have finished", async () => { + const { io, calls } = setup({ + lightweightSha: "aaaaaaa", + fullRuns: [stackRun({ status: "completed" })], + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.dispatches).toHaveLength(1); + }); + + test.each([ + ["a missing workflow file", new GithubRequestError(404, "Not Found", "u")], + [ + "a branch whose workflow cannot be dispatched", + new GithubRequestError(422, "Workflow does not have 'workflow_dispatch' trigger", "u"), + ], + ])("tells the author to merge develop on %s", async (_name, dispatchError) => { + const { io, calls } = setup({ dispatchError }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome.kind).toBe("replied"); + expect(calls.replies[0]).toContain("Merge `develop` into the branch"); + expect(calls.reruns).toEqual([]); + expect(calls.reactions).toEqual([]); + }); + + test("surfaces GitHub's refusal of the dispatch with its status and message", async () => { + const { io, calls } = setup({ + dispatchError: new GithubRequestError(403, "Resource not accessible by integration", "u"), + }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.replies[0]).toContain("403"); + expect(calls.replies[0]).toContain("Resource not accessible by integration"); + expect(calls.reactions).toEqual([]); + }); + + test.each([ + ["a draft", pr({ draft: true }), "draft"], + ["a fork", pr({ headRepoFullName: "someone/cli" }), "from this repository"], + ["a pull request into another branch", pr({ baseRef: "release" }), "into develop"], + ])("replies that %s is not benchmarked and starts nothing", async (_name, head, text) => { + const { io, calls } = setup({ pr: head }); + + await handleBenchCommand(command("/bench full"), io); + + expect(calls.replies[0]).toContain(text); + expect(calls.dispatches).toEqual([]); + }); + + test("ignores a commenter without write permission without a reply", async () => { + const { io, calls } = setup({ permissions: { maintainer: "read" } }); + + const outcome = await handleBenchCommand(command("/bench full", "MEMBER"), io); + + expect(outcome.kind).toBe("ignored"); + expect(calls.replies).toEqual([]); + expect(calls.dispatches).toEqual([]); + }); + + test("keeps a started full run when re-running the lightweight run is refused", async () => { + const { io, calls } = setup({ + lightweightSha: "bbbbbbb", + rerunError: new GithubRequestError(403, "nope", "u"), + }); + + const outcome = await handleBenchCommand(command("/bench full"), io); + + expect(outcome).toEqual({ + kind: "dispatched", + workloads: "integration-heavy", + rerunId: undefined, + }); + expect(calls.dispatches).toHaveLength(1); + expect(calls.reactions).toEqual([99]); + }); +}); + +describe("coversIntegrationHeavy", () => { + test.each([ + ["CLI benchmark full (all): develop", true], + ["CLI benchmark full (integration-heavy): feature/x", true], + ["CLI benchmark full (lightweight): feature/x", false], + ["CLI benchmark full (scheduled)", false], + ])("reads %p as %p", (title, expected) => { + expect(coversIntegrationHeavy(title)).toBe(expected); + }); +}); + +describe("measuredSha", () => { + const run = "https://github.com/o/r/actions/runs/7"; + const recorded = (sha: string, headline: string) => + makeEntry({ sha, now: NOW, headline, url: run }); + const entries = (...list: Array>) => + list.flatMap((item) => (item === undefined ? [] : [item])); + + test("names the commit of a lightweight comment that reports a result", () => { + const body = composeBody( + "lightweight", + "## CLI benchmark (Lightweight Commands): no changes found", + entries(recorded("abc1234", "no changes found")), + ); + + expect(measuredSha(body)).toBe("abc1234"); + }); + + test("names the commit of a lightweight comment still under the previous heading", () => { + const body = composeBody( + "lightweight", + "## CLI benchmark: no changes found", + entries(recorded("abc1234", "no changes found")), + ); + + expect(measuredSha(body)).toBe("abc1234"); + }); + + test("names no commit when the run failed", () => { + const body = composeBody( + "lightweight", + failureMarkdown("lightweight", "abc1234def"), + entries(recorded("abc1234", "could not run")), + ); + + expect(measuredSha(body)).toBeUndefined(); + }); + + test("names no commit while a run is in progress", () => { + const body = composeBody( + "integration-heavy", + runningContent("integration-heavy", "abc1234def", NOW, run), + entries(recorded("1111111", "no changes found")), + true, + ); + + expect(measuredSha(body)).toBeUndefined(); + }); +}); diff --git a/.github/scripts/cli-bench/rerun.ts b/.github/scripts/cli-bench/rerun.ts index b0020185b8..2a5f7443dc 100644 --- a/.github/scripts/cli-bench/rerun.ts +++ b/.github/scripts/cli-bench/rerun.ts @@ -1,22 +1,29 @@ /** - * `/bench` handler: re-runs the benchmark workflow run of a pull request's current head commit. + * `/bench` and `/bench full` handler. * * Runs default branch code and never checks out or runs pull request code. The commenter must be * the repository owner or hold write access, and the pull request must be an open, ready one from - * this repository into `develop`. The command re-runs the existing `pull_request` run, which - * benchmarks and updates the comment; the pull request gets a reply when no run can be re-run. + * this repository into `develop`. `/bench` re-runs the existing `pull_request` run, which + * benchmarks and updates the lightweight comment. `/bench full` always re-measures the + * integration-heavy benchmark, which has its own comment, by dispatching the full benchmark + * workflow on the pull request's branch, and re-runs the lightweight part only when its comment + * is behind the head commit; the pull request gets a reply when nothing can run. * Run in CI as: `bun .github/scripts/cli-bench/rerun.ts`. */ import { appendFileSync } from "node:fs"; import { fetchAuthorPermission, WRITE_PERMISSIONS } from "../contribution-gate.ts"; -import { githubFetch, GithubRequestError, requireEnv } from "./github.ts"; +import { githubFetch, GithubRequestError, listIssueComments, requireEnv } from "./github.ts"; +import { findComment, measuredSha } from "./post-comment.ts"; export const BENCH_COMMAND = "/bench"; +export const BENCH_FULL_COMMAND = "/bench full"; /** Base branches the benchmark runs for; keep in sync with the workflow's `branches:` filter. */ export const BENCH_BASE_BRANCHES: ReadonlyArray = ["develop"]; /** The workflow file whose runs `/bench` re-runs. */ export const WORKFLOW_FILE = "cli-bench.yml"; +/** The workflow `/bench full` dispatches. */ +export const FULL_WORKFLOW_FILE = "cli-bench-full.yml"; /** GitHub refuses to re-run a workflow run older than this. */ export const RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; @@ -24,6 +31,8 @@ export interface BenchPr { state: "open" | "closed"; draft: boolean; headSha: string; + /** Name of the head branch. */ + headRef: string; /** `owner/name` of the head repository, empty when it was deleted. */ headRepoFullName: string; baseRepoFullName: string; @@ -36,6 +45,17 @@ export interface WorkflowRun { createdAt: string; } +export interface FullRun { + id: number; + status: string; + /** The run name, which carries the `workloads` input. */ + displayTitle: string; + url: string; +} + +/** What a dispatched full run measures. */ +export type FullWorkloads = "integration-heavy" | "all"; + export interface CommandInput { prNumber: number; comment: { id: number; authorLogin: string; authorAssociation: string; body: string }; @@ -48,6 +68,11 @@ export interface CommandIo { /** Runs of the benchmark workflow triggered by `pull_request` for this commit, any order. */ listRuns: (headSha: string) => Promise>; rerun: (runId: number) => Promise; + /** Dispatched runs of the full workflow on a branch, any order. */ + listFullRuns: (headRef: string) => Promise>; + dispatchFull: (headRef: string, workloads: FullWorkloads) => Promise; + /** Short SHA of the commit the lightweight comment measured, if it reports a result. */ + lightweightSha: (prNumber: number) => Promise; reactToComment: (commentId: number) => Promise; replyToPr: (prNumber: number, body: string) => Promise; } @@ -55,7 +80,8 @@ export interface CommandIo { export type CommandOutcome = | { kind: "ignored"; reason: string } | { kind: "replied"; reason: string } - | { kind: "rerun"; runId: number }; + | { kind: "rerun"; runId: number } + | { kind: "dispatched"; workloads: FullWorkloads; rerunId: number | undefined }; const short = (sha: string) => sha.slice(0, 7); @@ -77,18 +103,46 @@ export const replies = { refused: (sha: string, status: number, detail: string) => `The benchmark could not be re-run for commit \`${short(sha)}\`: GitHub refused the request ` + `(${status} ${quoted(detail)}).`, + notEligible: () => + `\`${BENCH_FULL_COMMAND}\` runs only on pull requests from this repository into ` + + `${BENCH_BASE_BRANCHES.join(" or ")}.`, + fullRunning: (url: string) => + `An integration-heavy benchmark is already queued or running for this pull request: ${url}. ` + + "The comment updates when it finishes.", + noFullWorkflow: () => + "This branch does not have the full benchmark workflow yet. Merge `develop` into the branch " + + `and comment \`${BENCH_FULL_COMMAND}\` again.`, + fullRefused: (status: number, detail: string) => + `The integration-heavy benchmark could not be started: GitHub refused the request ` + + `(${status} ${quoted(detail)}).`, }; -/** Handles one `/bench` comment; a rejected command is ignored without a reply. */ +const FULL_RUN_WORKLOADS = /\((lightweight|integration-heavy|all)\)/u; + +/** Whether a full run's name says it measures the integration-heavy workloads. */ +export function coversIntegrationHeavy(displayTitle: string): boolean { + const workloads = FULL_RUN_WORKLOADS.exec(displayTitle)?.[1]; + return workloads === "integration-heavy" || workloads === "all"; +} + +const isCommand = (body: string) => body.trim() === BENCH_COMMAND; +const isFullCommand = (body: string) => body.trim() === BENCH_FULL_COMMAND; + +/** Handles one `/bench` or `/bench full` comment; a rejected command is ignored without a reply. */ export async function handleBenchCommand( input: CommandInput, io: CommandIo, ): Promise { const { comment, prNumber } = input; - // The workflow's `if:` only pre-filters on `startsWith`, so `/bench full` must be rejected here. - if (comment.body.trim() !== BENCH_COMMAND) { - return { kind: "ignored", reason: `Comment is not exactly ${BENCH_COMMAND}.` }; + const full = isFullCommand(comment.body); + // The workflow's `if:` only pre-filters on `startsWith`, so any other text must be rejected here. + if (!full && !isCommand(comment.body)) { + return { + kind: "ignored", + reason: `Comment is not exactly ${BENCH_COMMAND} or ${BENCH_FULL_COMMAND}.`, + }; } + const command = full ? BENCH_FULL_COMMAND : BENCH_COMMAND; // Only the repository OWNER skips the lookup: MEMBER and COLLABORATOR do not imply push access. const permission = await io.fetchPermission(comment.authorLogin); const authorized = @@ -98,55 +152,103 @@ export async function handleBenchCommand( return { kind: "ignored", reason: - `Commenter @${comment.authorLogin} is not authorized to run ${BENCH_COMMAND} ` + + `Commenter @${comment.authorLogin} is not authorized to run ${command} ` + `(author_association=${comment.authorAssociation}, permission=${permission ?? "n/a"}).`, }; } const pr = await io.fetchPr(prNumber); if (pr.state === "closed") return { kind: "ignored", reason: `PR #${prNumber} is closed.` }; - if (pr.headRepoFullName !== pr.baseRepoFullName) { - return { - kind: "ignored", - reason: `${BENCH_COMMAND} runs only on pull requests from this repository.`, - }; - } - if (!BENCH_BASE_BRANCHES.includes(pr.baseRef)) { - return { - kind: "ignored", - reason: `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, - }; - } const reply = async (text: string): Promise => { await io.replyToPr(prNumber, text); return { kind: "replied", reason: text }; }; + const eligible = + pr.headRepoFullName === pr.baseRepoFullName && BENCH_BASE_BRANCHES.includes(pr.baseRef); + if (!eligible) { + return full + ? reply(replies.notEligible()) + : { + kind: "ignored", + reason: + pr.headRepoFullName !== pr.baseRepoFullName + ? `${BENCH_COMMAND} runs only on pull requests from this repository.` + : `${BENCH_COMMAND} runs only on pull requests into ${BENCH_BASE_BRANCHES.join(" or ")}.`, + }; + } if (pr.draft) return reply(replies.draft()); const latest = [...(await io.listRuns(pr.headSha))].sort((a, b) => b.createdAt.localeCompare(a.createdAt), )[0]; - if (latest === undefined) return reply(replies.noRun(pr.headSha)); - if (latest.status !== "completed") return reply(replies.running(pr.headSha)); - if (input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS) { - return reply(replies.expired(pr.headSha)); + const expired = + latest !== undefined && input.now.getTime() - Date.parse(latest.createdAt) > RERUN_WINDOW_MS; + + if (!full) { + if (latest === undefined) return reply(replies.noRun(pr.headSha)); + if (latest.status !== "completed") return reply(replies.running(pr.headSha)); + if (expired) return reply(replies.expired(pr.headSha)); + try { + await io.rerun(latest.id); + } catch (error) { + if (error instanceof GithubRequestError) { + return reply(replies.refused(pr.headSha, error.status, error.detail)); + } + throw error; + } + await react(io, comment.id); + return { kind: "rerun", runId: latest.id }; } + const active = (await io.listFullRuns(pr.headRef)).find( + (stackRun) => stackRun.status !== "completed" && coversIntegrationHeavy(stackRun.displayTitle), + ); + if (active !== undefined) return reply(replies.fullRunning(active.url)); + + // The lightweight part refreshes only when its comment is behind the head commit and no + // lightweight run is already in flight; with no run to re-run, the full run measures it too. + const commentSha = await io.lightweightSha(prNumber); + const lightweightCurrent = commentSha !== undefined && pr.headSha.startsWith(commentSha); + const lightweightInFlight = latest !== undefined && latest.status !== "completed"; + const needsLightweight = !lightweightCurrent && !lightweightInFlight; + const rerunnable = latest !== undefined && !expired ? latest : undefined; + const workloads: FullWorkloads = + needsLightweight && rerunnable === undefined ? "all" : "integration-heavy"; + try { - await io.rerun(latest.id); + await io.dispatchFull(pr.headRef, workloads); } catch (error) { - if (error instanceof GithubRequestError) { - return reply(replies.refused(pr.headSha, error.status, error.detail)); + if (!(error instanceof GithubRequestError)) throw error; + const missing = + error.status === 404 || (error.status === 422 && /workflow_dispatch/u.test(error.detail)); + return reply( + missing ? replies.noFullWorkflow() : replies.fullRefused(error.status, error.detail), + ); + } + + let rerunId: number | undefined; + if (needsLightweight && rerunnable !== undefined) { + try { + await io.rerun(rerunnable.id); + rerunId = rerunnable.id; + } catch (error) { + // The full run is already under way; a refused lightweight re-run must not hide it. + console.warn( + `Could not re-run the lightweight benchmark run ${rerunnable.id}: ${String(error)}`, + ); } - throw error; } - // Cosmetic: a failed reaction must never hide an accepted re-run. + await react(io, comment.id); + return { kind: "dispatched", workloads, rerunId }; +} + +/** Cosmetic: a failed reaction must never hide an accepted command. */ +async function react(io: CommandIo, commentId: number): Promise { try { - await io.reactToComment(comment.id); + await io.reactToComment(commentId); } catch (error) { - console.warn(`Could not react to comment ${comment.id}: ${String(error)}`); + console.warn(`Could not react to comment ${commentId}: ${String(error)}`); } - return { kind: "rerun", runId: latest.id }; } async function main(): Promise { @@ -160,13 +262,14 @@ async function main(): Promise { const value = (await (await githubFetch(`${base}/pulls/${n}`, token)).json()) as { state: "open" | "closed"; draft: boolean; - head: { sha: string; repo: { full_name: string } | null }; + head: { sha: string; ref: string; repo: { full_name: string } | null }; base: { ref: string; repo: { full_name: string } }; }; return { state: value.state, draft: value.draft, headSha: value.head.sha, + headRef: value.head.ref, headRepoFullName: value.head.repo?.full_name ?? "", baseRepoFullName: value.base.repo.full_name, baseRef: value.base.ref, @@ -187,6 +290,48 @@ async function main(): Promise { rerun: async (runId) => { await githubFetch(`${base}/actions/runs/${runId}/rerun`, token, { method: "POST" }); }, + listFullRuns: async (headRef) => { + const url = + `${base}/actions/workflows/${FULL_WORKFLOW_FILE}/runs` + + `?event=workflow_dispatch&branch=${encodeURIComponent(headRef)}&per_page=50`; + const payload = await githubFetch(url, token).then( + (response) => + response.json() as Promise<{ + workflow_runs: Array<{ + id: number; + status: string; + display_title: string; + html_url: string; + }>; + }>, + (error: unknown) => { + // A branch without the workflow file has no runs of it. + if (error instanceof GithubRequestError && error.status === 404) { + return { workflow_runs: [] }; + } + throw error; + }, + ); + return payload.workflow_runs.map((run) => ({ + id: run.id, + status: run.status, + displayTitle: run.display_title, + url: run.html_url, + })); + }, + dispatchFull: async (headRef, workloads) => { + await githubFetch(`${base}/actions/workflows/${FULL_WORKFLOW_FILE}/dispatches`, token, { + method: "POST", + body: JSON.stringify({ + ref: `refs/heads/${headRef}`, + inputs: { workloads, runtime: "both", base: "merge-base" }, + }), + }); + }, + lightweightSha: async (n) => { + const existing = findComment("lightweight", await listIssueComments(token, base, n)); + return existing === undefined ? undefined : measuredSha(existing.body); + }, reactToComment: async (commentId) => { await githubFetch(`${base}/issues/comments/${commentId}/reactions`, token, { method: "POST", @@ -217,7 +362,11 @@ async function main(): Promise { const message = outcome.kind === "rerun" ? `Re-running benchmark run ${outcome.runId} for PR #${prNumber}.` - : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; + : outcome.kind === "dispatched" + ? `Dispatched the full benchmark (${outcome.workloads}) for PR #${prNumber}${ + outcome.rerunId === undefined ? "" : ` and re-running benchmark run ${outcome.rerunId}` + }.` + : `${outcome.kind === "replied" ? "Replied on" : "Ignored command on"} PR #${prNumber}: ${outcome.reason}`; console.log(message); const summaryFile = process.env.GITHUB_STEP_SUMMARY; if (summaryFile) appendFileSync(summaryFile, `${message}\n`); diff --git a/.github/scripts/cli-bench/resolve.test.ts b/.github/scripts/cli-bench/resolve.test.ts index 7ac55287b2..5a5e74ec24 100644 --- a/.github/scripts/cli-bench/resolve.test.ts +++ b/.github/scripts/cli-bench/resolve.test.ts @@ -1,13 +1,14 @@ import { describe, expect, test } from "bun:test"; -import { BENCH_MARKER, FAILURE_TITLE } from "./post-comment.ts"; +import { composeBody, failureMarkdown } from "./post-comment.ts"; import { resolveBench, SKIP_REASON } from "./resolve.ts"; const BOT = "github-actions[bot]"; -const reported = { +const lightweight = (content: string) => ({ id: 5, - body: `${BENCH_MARKER}\n## CLI benchmark: no changes found`, + body: composeBody("lightweight", content), authorLogin: BOT, -}; +}); +const reported = lightweight("## CLI benchmark (Lightweight Commands): no changes found"); describe("resolveBench", () => { test("benchmarks the first push of a pull request that has no comment yet", () => { @@ -21,12 +22,39 @@ describe("resolveBench", () => { expect(SKIP_REASON).toContain("`/bench`"); }); + test("skips a later push while a comment under the previous heading reports a result", () => { + const old = lightweight("## CLI benchmark: no changes found"); + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [old] }).shouldRun).toBe(false); + }); + + test("benchmarks again when a comment under the previous heading reports a failed run", () => { + const old = lightweight("## CLI benchmark: could not run"); + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [old] }).shouldRun).toBe(true); + }); + test("benchmarks again when the comment reports a failed run", () => { - const failed = { ...reported, body: `${BENCH_MARKER}\n${FAILURE_TITLE}` }; + const failed = lightweight(failureMarkdown("lightweight", "abc1234")); expect(resolveBench({ runAttempt: 1, isFork: false, comments: [failed] }).shouldRun).toBe(true); }); + test("benchmarks when only the integration-heavy comment exists", () => { + const heavyOnly = { + id: 5, + body: composeBody( + "integration-heavy", + "## CLI benchmark (Integration Heavy Commands): no changes found", + ), + authorLogin: BOT, + }; + + expect(resolveBench({ runAttempt: 1, isFork: false, comments: [heavyOnly] }).shouldRun).toBe( + true, + ); + }); + test("benchmarks every re-run even when the comment reports a result", () => { expect(resolveBench({ runAttempt: 2, isFork: false, comments: [reported] }).shouldRun).toBe( true, diff --git a/.github/scripts/cli-bench/resolve.ts b/.github/scripts/cli-bench/resolve.ts index 03882d2450..574b811ac6 100644 --- a/.github/scripts/cli-bench/resolve.ts +++ b/.github/scripts/cli-bench/resolve.ts @@ -1,15 +1,16 @@ /** * CLI benchmark resolver for `pull_request` runs: decides whether the benchmark job runs. * - * The first run on a pull request benchmarks and comments. Later pushes skip while the comment - * exists and reports a result; a re-run (`run_attempt` above 1), a missing comment, a failed first - * run, or a fork pull request, which never has a comment, benchmarks. Needs read access only. + * The first run on a pull request benchmarks and comments. Later pushes skip while the + * lightweight comment exists and reports a result; a re-run (`run_attempt` above 1), a missing + * comment, a failed first run, or a fork pull request, which never has a comment, benchmarks. + * Needs read access only. * Run in CI as: `bun .github/scripts/cli-bench/resolve.ts`. */ import { appendFileSync } from "node:fs"; import { listIssueComments, requireEnv, type IssueComment } from "./github.ts"; -import { FAILURE_TITLE, findBenchComment } from "./post-comment.ts"; +import { commentFailed, findComment } from "./post-comment.ts"; export interface ResolveInput { runAttempt: number; @@ -29,8 +30,8 @@ export const SKIP_REASON = /** Skips only a first attempt on a pull request whose benchmark comment reports a result. */ export function resolveBench(input: ResolveInput): ResolveResult { if (input.isFork || input.runAttempt > 1) return { shouldRun: true }; - const existing = findBenchComment(input.comments); - if (existing === undefined || existing.body.includes(FAILURE_TITLE)) return { shouldRun: true }; + const existing = findComment("lightweight", input.comments); + if (existing === undefined || commentFailed(existing.body)) return { shouldRun: true }; return { shouldRun: false, skipReason: SKIP_REASON }; } diff --git a/.github/workflows/cli-bench-full.yml b/.github/workflows/cli-bench-full.yml new file mode 100644 index 0000000000..6528fbc7d4 --- /dev/null +++ b/.github/workflows/cli-bench-full.yml @@ -0,0 +1,581 @@ +name: CLI benchmark (full) +run-name: >- + ${{ github.event_name == 'schedule' && 'CLI benchmark full (scheduled)' || + format('CLI benchmark full ({0}): {1}', inputs.workloads, github.ref_name) }} + +# Report-only comparison of a CLI build with a base build for the workloads that need a running +# local stack or database. Starts only from the `/bench full` pull request command, the Actions UI +# (for any branch), or the weekly schedule; it never runs on a push. The base is the merge base +# with `develop` by default, a release tag when given, and the latest stable release on the +# schedule. Workloads are packed into shards, one runner at a time per workload, and each shard +# compares base and branch on its own machine. The run never blocks a pull request and is not a +# required check. +# See apps/cli/docs/tracing-monitoring.md. +on: + workflow_dispatch: + inputs: + workloads: + description: Which workloads to measure + type: choice + options: + - all + - lightweight + - integration-heavy + default: all + runtime: + description: Container runtime for the integration-heavy workloads + type: choice + options: + - both + - docker + - native + default: both + base: + description: merge-base, or a release tag such as v3.0.0 to compare against + type: string + default: merge-base + schedule: + - cron: "0 3 * * 0" + +permissions: {} + +# A second run for a branch waits for the first instead of cancelling it, which also takes turns +# between the writers of the integration-heavy comment; a run still waiting when a newer one +# queues never started, so it had written nothing. +concurrency: + group: cli-bench-full-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + # Decides what this run measures and against which base, splits the workloads into shards, and + # marks the pull request's integration-heavy comment as running. + plan: + name: Plan + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + outputs: + base_kind: ${{ steps.plan.outputs.base_kind }} + base_tag: ${{ steps.plan.outputs.base_tag }} + pr_number: ${{ steps.plan.outputs.pr_number }} + run_lightweight: ${{ steps.plan.outputs.run_lightweight }} + run_integration: ${{ steps.plan.outputs.run_integration }} + has_shards: ${{ steps.plan.outputs.has_shards }} + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Plan + id: plan + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + INPUT_WORKLOADS: ${{ inputs.workloads }} + INPUT_RUNTIME: ${{ inputs.runtime }} + INPUT_BASE: ${{ inputs.base }} + run: bun .github/scripts/cli-bench/full-plan.ts + - name: Mark the integration-heavy comment as running + if: steps.plan.outputs.pr_number != '' && steps.plan.outputs.run_integration == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.plan.outputs.pr_number }} + MODE: running + KIND: integration-heavy + HEAD_SHA: ${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts + + # Builds the branch binary, and the base binary from the merge base or the release asset, once + # for every shard. + build: + name: Build binaries + needs: plan + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + outputs: + branch_build: ${{ steps.outcomes.outputs.branch_build }} + base_build: ${{ steps.outcomes.outputs.base_build }} + merge_base: ${{ steps.base_build.outputs.merge_base }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Prepare benchmark directory + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + + - name: Build branch binary + id: branch_build + continue-on-error: true + shell: bash + run: | + pnpm --filter supabase run build:binary + mkdir -p "$BENCH_DIR/branch" + cp apps/cli/dist/supabase "$BENCH_DIR/branch/supabase" + + - name: Build or download base binary + id: base_build + continue-on-error: true + if: steps.branch_build.outcome == 'success' + env: + HUSKY: "0" + BASE_KIND: ${{ needs.plan.outputs.base_kind }} + BASE_TAG: ${{ needs.plan.outputs.base_tag }} + RELEASE_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + shell: bash + run: | + mkdir -p "$BENCH_DIR/base" + if [ "$BASE_KIND" = release ]; then + echo "Base release: $BASE_TAG" + GH_TOKEN="$RELEASE_TOKEN" gh release download "$BASE_TAG" --repo "$GITHUB_REPOSITORY" \ + --pattern supabase_linux_amd64.tar.gz --dir "$RUNNER_TEMP/release" + tar -xzf "$RUNNER_TEMP/release/supabase_linux_amd64.tar.gz" -C "$BENCH_DIR/base" supabase + chmod +x "$BENCH_DIR/base/supabase" + exit 0 + fi + merge_base="$(git merge-base HEAD origin/develop)" + echo "Merge base: $merge_base" + echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" + git worktree add "$RUNNER_TEMP/base-src" "$merge_base" + cd "$RUNNER_TEMP/base-src" + # The setup action writes this file only when the Dependency Firewall token is available. + if [ -f "$RUNNER_TEMP/dependency-firewall.npmrc" ]; then + export NPM_CONFIG_USERCONFIG="$RUNNER_TEMP/dependency-firewall.npmrc" + fi + pnpm install --frozen-lockfile + pnpm --filter supabase run build:binary + cp apps/cli/dist/supabase "$BENCH_DIR/base/supabase" + + - name: Record build outcomes + id: outcomes + if: always() + env: + BRANCH_OUTCOME: ${{ steps.branch_build.outcome }} + BASE_OUTCOME: ${{ steps.base_build.outcome }} + shell: bash + run: | + echo "branch_build=$([ "$BRANCH_OUTCOME" = success ] && echo ok || echo failed)" >> "$GITHUB_OUTPUT" + echo "base_build=$([ "$BASE_OUTCOME" = success ] && echo ok || echo failed)" >> "$GITHUB_OUTPUT" + + - name: Upload binaries + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-binaries + path: | + ${{ runner.temp }}/bench/base + ${{ runner.temp }}/bench/branch + retention-days: 1 + if-no-files-found: ignore + + # The pull request workloads against the replay server, for dispatches that ask for them (the + # `pull_request` run covers them otherwise). + lightweight: + name: Lightweight commands + needs: + - plan + - build + if: always() && needs.plan.result == 'success' && needs.plan.outputs.run_lightweight == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Download binaries + id: binaries + if: needs.build.outputs.branch_build == 'ok' && needs.build.outputs.base_build == 'ok' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-full-binaries + path: ${{ runner.temp }}/bench + + - name: Start replay server + id: replay + continue-on-error: true + if: steps.binaries.outcome == 'success' + shell: bash + run: | + echo "BENCH_DIR=$RUNNER_TEMP/bench" >> "$GITHUB_ENV" + chmod +x "$RUNNER_TEMP/bench/base/supabase" "$RUNNER_TEMP/bench/branch/supabase" + bun apps/cli-e2e/scripts/replay-serve.ts > "$RUNNER_TEMP/bench/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$RUNNER_TEMP/bench/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$RUNNER_TEMP/bench/replay.log" >&2 + exit 1 + + - name: Run benchmark + id: bench + continue-on-error: true + if: steps.replay.outcome == 'success' + shell: bash + run: | + bun apps/cli/scripts/bench-cli.ts \ + --base "$BENCH_DIR/base/supabase" \ + --branch "$BENCH_DIR/branch/supabase" \ + --workloads apps/cli/bench/workloads.pr.json \ + --api-url "$REPLAY_URL" \ + --trace \ + --out "$RUNNER_TEMP/bench/report.json" | tee "$RUNNER_TEMP/bench/report.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Write job summary + if: always() + env: + BRANCH_BUILD: ${{ needs.build.outputs.branch_build == 'ok' && 'ok' || 'failed' }} + BASE_BUILD: ${{ needs.build.outputs.base_build == 'ok' && 'ok' || 'failed' }} + REPLAY: ${{ steps.replay.outcome == 'success' && 'ok' || 'failed' }} + MERGE_BASE: ${{ needs.build.outputs.merge_base }} + BASE_REF: ${{ needs.plan.outputs.base_kind == 'release' && needs.plan.outputs.base_tag || 'develop' }} + HEAD_SHA: ${{ github.sha }} + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + bun apps/cli/scripts/bench-summary.ts \ + --section lightweight \ + --report "$RUNNER_TEMP/bench/report.json" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --replay "$REPLAY" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$RUNNER_TEMP/bench/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-report-lightweight + path: | + ${{ runner.temp }}/bench/report.json + ${{ runner.temp }}/bench/report.txt + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-comment-lightweight + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + # One runner per shard and runtime. Base and branch alternate on the same machine; no other + # workload shares it. + shard: + name: Shard (${{ matrix.runtime }} ${{ matrix.shard }}/${{ matrix.shards }}) + needs: + - plan + - build + if: >- + needs.plan.outputs.has_shards == 'true' && + needs.build.outputs.branch_build == 'ok' && needs.build.outputs.base_build == 'ok' + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The image and artifact disks are read-only here: this job never commits them. + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + sticky-disks: "true" + docker-images: ${{ matrix.runtime == 'docker' }} + native-artifacts: ${{ matrix.runtime == 'native' }} + + - name: Download binaries + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-full-binaries + path: ${{ runner.temp }}/bench + + # A shard that cannot start the replay server fails, and the report job names it as a shard + # that did not finish. + - name: Start replay server + shell: bash + run: | + bench="$RUNNER_TEMP/bench" + chmod +x "$bench/base/supabase" "$bench/branch/supabase" + bun apps/cli-e2e/scripts/replay-serve.ts > "$bench/replay.log" 2>&1 & + echo "REPLAY_PID=$!" >> "$GITHUB_ENV" + for _ in $(seq 1 60); do + url="$(grep -o 'http://[^"]*' "$bench/replay.log" | head -n 1 || true)" + if [ -n "$url" ]; then + echo "REPLAY_URL=$url" >> "$GITHUB_ENV" + exit 0 + fi + sleep 0.5 + done + echo "replay server did not report a URL" >&2 + cat "$bench/replay.log" >&2 + exit 1 + + - name: Run benchmark + env: + RUNTIME: ${{ matrix.runtime }} + SHARD: ${{ matrix.shard }} + WORKLOADS: ${{ matrix.workloads }} + shell: bash + run: | + bench="$RUNNER_TEMP/bench" + cache_args=() + if [ "$RUNTIME" = native ]; then + cache="${XDG_CACHE_HOME:-$HOME/.cache}/supabase-stack-artifacts" + mkdir -p "$cache" + cache_args=(--artifact-cache "$cache") + fi + bun apps/cli/scripts/bench-cli.ts \ + --base "$bench/base/supabase" \ + --branch "$bench/branch/supabase" \ + --workloads apps/cli/bench/workloads.full.json \ + --runtime "$RUNTIME" \ + --only "$WORKLOADS" \ + --api-url "$REPLAY_URL" \ + "${cache_args[@]}" \ + --trace \ + --out "$bench/report-$RUNTIME-$SHARD.json" | tee "$bench/report-$RUNTIME-$SHARD.txt" + + - name: Stop replay server + if: always() && env.REPLAY_PID != '' + shell: bash + run: kill "$REPLAY_PID" || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-report-${{ matrix.runtime }}-${{ matrix.shard }} + path: | + ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.json + ${{ runner.temp }}/bench/report-${{ matrix.runtime }}-${{ matrix.shard }}.txt + retention-days: 14 + if-no-files-found: ignore + + # Releases /var/lib/docker so the image disk unmounts cleanly. + - name: Stop Docker + if: always() && matrix.runtime == 'docker' + run: sudo systemctl stop docker.socket docker.service + + # Combines the shard reports, writes the job summary and the comment. + report: + name: Report + needs: + - plan + - build + - shard + if: always() && needs.plan.result == 'success' && needs.plan.outputs.run_integration == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Download shard reports + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cli-bench-full-report-*-* + path: ${{ runner.temp }}/reports + + - name: Merge shard reports + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/bench" + mapfile -t files < <(find "$RUNNER_TEMP/reports" -name 'report-*.json' 2>/dev/null | sort) + if [ "${#files[@]}" -gt 0 ]; then + bun apps/cli/scripts/bench-merge.ts \ + --workloads apps/cli/bench/workloads.full.json \ + --out "$RUNNER_TEMP/bench/report.json" "${files[@]}" + fi + + - name: Write job summary + env: + BRANCH_BUILD: ${{ needs.build.outputs.branch_build == 'ok' && 'ok' || 'failed' }} + BASE_BUILD: ${{ needs.build.outputs.base_build == 'ok' && 'ok' || 'failed' }} + SHARDS: ${{ needs.shard.result }} + MERGE_BASE: ${{ needs.build.outputs.merge_base }} + BASE_REF: ${{ needs.plan.outputs.base_kind == 'release' && needs.plan.outputs.base_tag || 'develop' }} + HEAD_SHA: ${{ github.sha }} + shell: bash + run: | + note="" + if [ "$SHARDS" = failure ] || [ "$SHARDS" = cancelled ]; then + note="Some shards did not finish (for example, the local test server failed to start), so their commands are missing." + fi + bun apps/cli/scripts/bench-summary.ts \ + --section integration-heavy \ + --report "$RUNNER_TEMP/bench/report.json" \ + --note "$note" \ + --branch-build "$BRANCH_BUILD" \ + --base-build "$BASE_BUILD" \ + --head-sha "$HEAD_SHA" \ + --base-ref "$BASE_REF" \ + --merge-base "$MERGE_BASE" \ + --comment-out "$RUNNER_TEMP/bench/comment.md" + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-merged-report + path: ${{ runner.temp }}/bench/report.json + retention-days: 14 + if-no-files-found: ignore + + - name: Upload comment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-bench-full-comment-integration-heavy + path: ${{ runner.temp }}/bench/comment.md + retention-days: 1 + if-no-files-found: ignore + + # The only jobs that write to the pull request. They run no benchmarked code and read the + # rendered markdown as data; a dispatch uses this file from the dispatched branch, which only + # people with write access can push or dispatch. + comment-lightweight: + name: Post lightweight comment + needs: + - plan + - lightweight + if: always() && needs.plan.outputs.pr_number != '' && needs.plan.outputs.run_lightweight == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + # Shares its turn with the lightweight benchmark workflow, the other writer of this comment. + concurrency: + group: cli-bench-comment-lightweight-${{ needs.plan.outputs.pr_number }} + cancel-in-progress: false + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-full-comment-lightweight + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ needs.plan.outputs.pr_number }} + MODE: upsert + KIND: lightweight + MISSING_BODY: failure + HEAD_SHA: ${{ github.sha }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts + + comment: + name: Post integration-heavy comment + needs: + - plan + - report + if: always() && needs.plan.outputs.pr_number != '' && needs.plan.outputs.run_integration == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: ".bun-version" + no-cache: true + - name: Download comment + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cli-bench-full-comment-integration-heavy + path: ${{ runner.temp }}/bench-comment + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ needs.plan.outputs.pr_number }} + MODE: upsert + KIND: integration-heavy + MISSING_BODY: failure + HEAD_SHA: ${{ github.sha }} + BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: bun .github/scripts/cli-bench/post-comment.ts diff --git a/.github/workflows/cli-bench.yml b/.github/workflows/cli-bench.yml index 73f76a6ac5..491ead06b0 100644 --- a/.github/workflows/cli-bench.yml +++ b/.github/workflows/cli-bench.yml @@ -8,10 +8,11 @@ run-name: "CLI benchmark #${{ github.event.pull_request.number || github.event.i # The first push to a pull request from this repository benchmarks and posts one comment tagged # ``; later pushes skip while that comment reports a result. A maintainer # commenting exactly `/bench` re-runs the run for the pull request's current head commit, as does -# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates the comment, keeping -# earlier runs in a collapsed list. Pull requests from forks benchmark every push and write a job -# summary only. A pull request that changes no benchmarked path, or more than the 300 files -# GitHub's path filter inspects, has no run to re-run, and `/bench` says so. +# "Re-run all jobs" in the Actions UI: a re-run benchmarks again and updates that comment, keeping +# earlier runs in a collapsed list. A maintainer commenting exactly `/bench full` also dispatches +# cli-bench-full.yml on the pull request's branch, which keeps its own comment. Pull requests from forks benchmark every push +# and write a job summary only. A pull request that changes no benchmarked path, or more than the +# 300 files GitHub's path filter inspects, has no run to re-run, and `/bench` says so. # See apps/cli/docs/tracing-monitoring.md. on: pull_request: @@ -30,7 +31,8 @@ on: - "turbo.json" - ".github/workflows/cli-bench.yml" - ".github/scripts/cli-bench/**" - # `/bench` runs default branch code only: it never checks out or runs pull request code. + # `/bench` and `/bench full` run default branch code only: they never check out or run pull + # request code. # jactionlint ignore=dangerous-triggers issue_comment: types: @@ -76,8 +78,9 @@ jobs: RUN_ATTEMPT: ${{ github.run_attempt }} run: bun .github/scripts/cli-bench/resolve.ts - # `/bench`: re-runs this workflow's run for the pull request's current head commit. The only job - # with `actions: write`; it runs default branch code and no pull request code. + # `/bench`: re-runs this workflow's run for the pull request's current head commit. `/bench full` + # also dispatches the full benchmark workflow on the pull request's branch. The only job with + # `actions: write`; it runs default branch code and no pull request code. rerun: name: Re-run benchmark # Cheap, non-authoritative pre-filter; rerun.ts does the exact command match and the @@ -264,6 +267,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 continue-on-error: true + # Writers of the lightweight comment, here and in cli-bench-full.yml, take turns; a waiting + # writer is never cancelled by a newer one that has not started. + concurrency: + group: cli-bench-comment-lightweight-${{ github.event.pull_request.number }} + cancel-in-progress: false # The only pull_request job that can write to the pull request. It checks out the merge commit, # which its author can already run with this token by editing the workflow. It never builds or # runs the benchmarked binaries and reads the rendered markdown as data. @@ -290,6 +298,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} MODE: ${{ github.run_attempt > 1 && 'upsert' || 'create-if-missing' }} + KIND: lightweight HEAD_SHA: ${{ github.event.pull_request.head.sha }} BODY_FILE: ${{ runner.temp }}/bench-comment/comment.md RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql new file mode 100644 index 0000000000..91673d12aa --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000001_bench_01.sql @@ -0,0 +1,7 @@ +create table public.bench_items ( + id bigint generated always as identity primary key, + label text not null, + amount numeric not null, + created_at timestamptz not null default now() +); +create index bench_items_label_idx on public.bench_items (label); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql new file mode 100644 index 0000000000..a50e4ce1d3 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000002_bench_02.sql @@ -0,0 +1,6 @@ +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_002_item_idx on public.bench_table_002 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql new file mode 100644 index 0000000000..5b68d67841 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000003_bench_03.sql @@ -0,0 +1,6 @@ +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_003_item_idx on public.bench_table_003 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql new file mode 100644 index 0000000000..feff4efd96 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000004_bench_04.sql @@ -0,0 +1,6 @@ +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_004_item_idx on public.bench_table_004 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql new file mode 100644 index 0000000000..c1431ae655 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000005_bench_05.sql @@ -0,0 +1,6 @@ +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_005_item_idx on public.bench_table_005 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql new file mode 100644 index 0000000000..8a954b1805 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000006_bench_06.sql @@ -0,0 +1,6 @@ +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_006_item_idx on public.bench_table_006 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql new file mode 100644 index 0000000000..4cee582f77 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000007_bench_07.sql @@ -0,0 +1,6 @@ +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_007_item_idx on public.bench_table_007 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql new file mode 100644 index 0000000000..f6198057b0 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000008_bench_08.sql @@ -0,0 +1,6 @@ +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_008_item_idx on public.bench_table_008 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql new file mode 100644 index 0000000000..a9ec7f6083 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000009_bench_09.sql @@ -0,0 +1,6 @@ +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_009_item_idx on public.bench_table_009 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql new file mode 100644 index 0000000000..1e9819111b --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000010_bench_10.sql @@ -0,0 +1,6 @@ +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_010_item_idx on public.bench_table_010 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql new file mode 100644 index 0000000000..19837f3ddf --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000011_bench_11.sql @@ -0,0 +1,6 @@ +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_011_item_idx on public.bench_table_011 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql new file mode 100644 index 0000000000..5877718517 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000012_bench_12.sql @@ -0,0 +1,6 @@ +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_012_item_idx on public.bench_table_012 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql new file mode 100644 index 0000000000..4db2608229 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000013_bench_13.sql @@ -0,0 +1,6 @@ +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_013_item_idx on public.bench_table_013 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql new file mode 100644 index 0000000000..7b32a4dcc7 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000014_bench_14.sql @@ -0,0 +1,6 @@ +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_014_item_idx on public.bench_table_014 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql new file mode 100644 index 0000000000..68f9dd68d8 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000015_bench_15.sql @@ -0,0 +1,6 @@ +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_015_item_idx on public.bench_table_015 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql new file mode 100644 index 0000000000..2c56106853 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000016_bench_16.sql @@ -0,0 +1,6 @@ +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_016_item_idx on public.bench_table_016 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql new file mode 100644 index 0000000000..9064dd0f34 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000017_bench_17.sql @@ -0,0 +1,6 @@ +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_017_item_idx on public.bench_table_017 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql new file mode 100644 index 0000000000..285e5f0b42 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000018_bench_18.sql @@ -0,0 +1,6 @@ +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_018_item_idx on public.bench_table_018 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql new file mode 100644 index 0000000000..f899cc45ac --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000019_bench_19.sql @@ -0,0 +1,6 @@ +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_019_item_idx on public.bench_table_019 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql b/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql new file mode 100644 index 0000000000..5cc1eef527 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/migrations/20260101000020_bench_20.sql @@ -0,0 +1,6 @@ +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index bench_table_020_item_idx on public.bench_table_020 (item_id); diff --git a/apps/cli/bench/fixtures/migrations/seed.sql b/apps/cli/bench/fixtures/migrations/seed.sql new file mode 100644 index 0000000000..a557250dd3 --- /dev/null +++ b/apps/cli/bench/fixtures/migrations/seed.sql @@ -0,0 +1,2 @@ +insert into public.bench_items (label, amount) +select 'item-' || n, (n % 97) * 1.5 from generate_series(1, 1000) as n; diff --git a/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql b/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql new file mode 100644 index 0000000000..93dd159907 --- /dev/null +++ b/apps/cli/bench/fixtures/pgtap/migrations/20260101000001_bench_checks.sql @@ -0,0 +1,4 @@ +create table public.bench_checks ( + id bigint generated always as identity primary key, + label text not null +); diff --git a/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql b/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql new file mode 100644 index 0000000000..4b1b9687f0 --- /dev/null +++ b/apps/cli/bench/fixtures/pgtap/tests/database/bench.test.sql @@ -0,0 +1,8 @@ +begin; +select plan(4); +select has_table('public', 'bench_checks', 'bench_checks exists'); +select has_column('public', 'bench_checks', 'label', 'label column exists'); +select col_not_null('public', 'bench_checks', 'label', 'label is required'); +select col_is_pk('public', 'bench_checks', 'id', 'id is the primary key'); +select * from finish(); +rollback; diff --git a/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..1623748429 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-large/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,2779 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; + +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_011_owner_idx on public.bench_table_011 (owner_id); +create index bench_table_011_status_idx on public.bench_table_011 (status, created_at); +alter table public.bench_table_011 enable row level security; +create policy bench_table_011_read on public.bench_table_011 for select using (true); + +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_012_owner_idx on public.bench_table_012 (owner_id); +create index bench_table_012_status_idx on public.bench_table_012 (status, created_at); +alter table public.bench_table_012 enable row level security; +create policy bench_table_012_read on public.bench_table_012 for select using (true); + +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_013_owner_idx on public.bench_table_013 (owner_id); +create index bench_table_013_status_idx on public.bench_table_013 (status, created_at); +alter table public.bench_table_013 enable row level security; +create policy bench_table_013_read on public.bench_table_013 for select using (true); + +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_014_owner_idx on public.bench_table_014 (owner_id); +create index bench_table_014_status_idx on public.bench_table_014 (status, created_at); +alter table public.bench_table_014 enable row level security; +create policy bench_table_014_read on public.bench_table_014 for select using (true); + +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_015_owner_idx on public.bench_table_015 (owner_id); +create index bench_table_015_status_idx on public.bench_table_015 (status, created_at); +alter table public.bench_table_015 enable row level security; +create policy bench_table_015_read on public.bench_table_015 for select using (true); + +create view public.bench_table_015_summary as + select status, count(*) as total from public.bench_table_015 group by status; + +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_016_owner_idx on public.bench_table_016 (owner_id); +create index bench_table_016_status_idx on public.bench_table_016 (status, created_at); +alter table public.bench_table_016 enable row level security; +create policy bench_table_016_read on public.bench_table_016 for select using (true); + +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_017_owner_idx on public.bench_table_017 (owner_id); +create index bench_table_017_status_idx on public.bench_table_017 (status, created_at); +alter table public.bench_table_017 enable row level security; +create policy bench_table_017_read on public.bench_table_017 for select using (true); + +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_018_owner_idx on public.bench_table_018 (owner_id); +create index bench_table_018_status_idx on public.bench_table_018 (status, created_at); +alter table public.bench_table_018 enable row level security; +create policy bench_table_018_read on public.bench_table_018 for select using (true); + +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_019_owner_idx on public.bench_table_019 (owner_id); +create index bench_table_019_status_idx on public.bench_table_019 (status, created_at); +alter table public.bench_table_019 enable row level security; +create policy bench_table_019_read on public.bench_table_019 for select using (true); + +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_020_owner_idx on public.bench_table_020 (owner_id); +create index bench_table_020_status_idx on public.bench_table_020 (status, created_at); +alter table public.bench_table_020 enable row level security; +create policy bench_table_020_read on public.bench_table_020 for select using (true); + +create view public.bench_table_020_summary as + select status, count(*) as total from public.bench_table_020 group by status; + +create function public.bench_table_020_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_020 $$; + +create table public.bench_table_021 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_021_owner_idx on public.bench_table_021 (owner_id); +create index bench_table_021_status_idx on public.bench_table_021 (status, created_at); +alter table public.bench_table_021 enable row level security; +create policy bench_table_021_read on public.bench_table_021 for select using (true); + +create table public.bench_table_022 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_022_owner_idx on public.bench_table_022 (owner_id); +create index bench_table_022_status_idx on public.bench_table_022 (status, created_at); +alter table public.bench_table_022 enable row level security; +create policy bench_table_022_read on public.bench_table_022 for select using (true); + +create table public.bench_table_023 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_023_owner_idx on public.bench_table_023 (owner_id); +create index bench_table_023_status_idx on public.bench_table_023 (status, created_at); +alter table public.bench_table_023 enable row level security; +create policy bench_table_023_read on public.bench_table_023 for select using (true); + +create table public.bench_table_024 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_024_owner_idx on public.bench_table_024 (owner_id); +create index bench_table_024_status_idx on public.bench_table_024 (status, created_at); +alter table public.bench_table_024 enable row level security; +create policy bench_table_024_read on public.bench_table_024 for select using (true); + +create table public.bench_table_025 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_025_owner_idx on public.bench_table_025 (owner_id); +create index bench_table_025_status_idx on public.bench_table_025 (status, created_at); +alter table public.bench_table_025 enable row level security; +create policy bench_table_025_read on public.bench_table_025 for select using (true); + +create view public.bench_table_025_summary as + select status, count(*) as total from public.bench_table_025 group by status; + +create table public.bench_table_026 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_026_owner_idx on public.bench_table_026 (owner_id); +create index bench_table_026_status_idx on public.bench_table_026 (status, created_at); +alter table public.bench_table_026 enable row level security; +create policy bench_table_026_read on public.bench_table_026 for select using (true); + +create table public.bench_table_027 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_027_owner_idx on public.bench_table_027 (owner_id); +create index bench_table_027_status_idx on public.bench_table_027 (status, created_at); +alter table public.bench_table_027 enable row level security; +create policy bench_table_027_read on public.bench_table_027 for select using (true); + +create table public.bench_table_028 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_028_owner_idx on public.bench_table_028 (owner_id); +create index bench_table_028_status_idx on public.bench_table_028 (status, created_at); +alter table public.bench_table_028 enable row level security; +create policy bench_table_028_read on public.bench_table_028 for select using (true); + +create table public.bench_table_029 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_029_owner_idx on public.bench_table_029 (owner_id); +create index bench_table_029_status_idx on public.bench_table_029 (status, created_at); +alter table public.bench_table_029 enable row level security; +create policy bench_table_029_read on public.bench_table_029 for select using (true); + +create table public.bench_table_030 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_030_owner_idx on public.bench_table_030 (owner_id); +create index bench_table_030_status_idx on public.bench_table_030 (status, created_at); +alter table public.bench_table_030 enable row level security; +create policy bench_table_030_read on public.bench_table_030 for select using (true); + +create view public.bench_table_030_summary as + select status, count(*) as total from public.bench_table_030 group by status; + +create function public.bench_table_030_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_030 $$; + +create table public.bench_table_031 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_031_owner_idx on public.bench_table_031 (owner_id); +create index bench_table_031_status_idx on public.bench_table_031 (status, created_at); +alter table public.bench_table_031 enable row level security; +create policy bench_table_031_read on public.bench_table_031 for select using (true); + +create table public.bench_table_032 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_032_owner_idx on public.bench_table_032 (owner_id); +create index bench_table_032_status_idx on public.bench_table_032 (status, created_at); +alter table public.bench_table_032 enable row level security; +create policy bench_table_032_read on public.bench_table_032 for select using (true); + +create table public.bench_table_033 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_033_owner_idx on public.bench_table_033 (owner_id); +create index bench_table_033_status_idx on public.bench_table_033 (status, created_at); +alter table public.bench_table_033 enable row level security; +create policy bench_table_033_read on public.bench_table_033 for select using (true); + +create table public.bench_table_034 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_034_owner_idx on public.bench_table_034 (owner_id); +create index bench_table_034_status_idx on public.bench_table_034 (status, created_at); +alter table public.bench_table_034 enable row level security; +create policy bench_table_034_read on public.bench_table_034 for select using (true); + +create table public.bench_table_035 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_035_owner_idx on public.bench_table_035 (owner_id); +create index bench_table_035_status_idx on public.bench_table_035 (status, created_at); +alter table public.bench_table_035 enable row level security; +create policy bench_table_035_read on public.bench_table_035 for select using (true); + +create view public.bench_table_035_summary as + select status, count(*) as total from public.bench_table_035 group by status; + +create table public.bench_table_036 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_036_owner_idx on public.bench_table_036 (owner_id); +create index bench_table_036_status_idx on public.bench_table_036 (status, created_at); +alter table public.bench_table_036 enable row level security; +create policy bench_table_036_read on public.bench_table_036 for select using (true); + +create table public.bench_table_037 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_037_owner_idx on public.bench_table_037 (owner_id); +create index bench_table_037_status_idx on public.bench_table_037 (status, created_at); +alter table public.bench_table_037 enable row level security; +create policy bench_table_037_read on public.bench_table_037 for select using (true); + +create table public.bench_table_038 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_038_owner_idx on public.bench_table_038 (owner_id); +create index bench_table_038_status_idx on public.bench_table_038 (status, created_at); +alter table public.bench_table_038 enable row level security; +create policy bench_table_038_read on public.bench_table_038 for select using (true); + +create table public.bench_table_039 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_039_owner_idx on public.bench_table_039 (owner_id); +create index bench_table_039_status_idx on public.bench_table_039 (status, created_at); +alter table public.bench_table_039 enable row level security; +create policy bench_table_039_read on public.bench_table_039 for select using (true); + +create table public.bench_table_040 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_040_owner_idx on public.bench_table_040 (owner_id); +create index bench_table_040_status_idx on public.bench_table_040 (status, created_at); +alter table public.bench_table_040 enable row level security; +create policy bench_table_040_read on public.bench_table_040 for select using (true); + +create view public.bench_table_040_summary as + select status, count(*) as total from public.bench_table_040 group by status; + +create function public.bench_table_040_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_040 $$; + +create table public.bench_table_041 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_041_owner_idx on public.bench_table_041 (owner_id); +create index bench_table_041_status_idx on public.bench_table_041 (status, created_at); +alter table public.bench_table_041 enable row level security; +create policy bench_table_041_read on public.bench_table_041 for select using (true); + +create table public.bench_table_042 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_042_owner_idx on public.bench_table_042 (owner_id); +create index bench_table_042_status_idx on public.bench_table_042 (status, created_at); +alter table public.bench_table_042 enable row level security; +create policy bench_table_042_read on public.bench_table_042 for select using (true); + +create table public.bench_table_043 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_043_owner_idx on public.bench_table_043 (owner_id); +create index bench_table_043_status_idx on public.bench_table_043 (status, created_at); +alter table public.bench_table_043 enable row level security; +create policy bench_table_043_read on public.bench_table_043 for select using (true); + +create table public.bench_table_044 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_044_owner_idx on public.bench_table_044 (owner_id); +create index bench_table_044_status_idx on public.bench_table_044 (status, created_at); +alter table public.bench_table_044 enable row level security; +create policy bench_table_044_read on public.bench_table_044 for select using (true); + +create table public.bench_table_045 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_045_owner_idx on public.bench_table_045 (owner_id); +create index bench_table_045_status_idx on public.bench_table_045 (status, created_at); +alter table public.bench_table_045 enable row level security; +create policy bench_table_045_read on public.bench_table_045 for select using (true); + +create view public.bench_table_045_summary as + select status, count(*) as total from public.bench_table_045 group by status; + +create table public.bench_table_046 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_046_owner_idx on public.bench_table_046 (owner_id); +create index bench_table_046_status_idx on public.bench_table_046 (status, created_at); +alter table public.bench_table_046 enable row level security; +create policy bench_table_046_read on public.bench_table_046 for select using (true); + +create table public.bench_table_047 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_047_owner_idx on public.bench_table_047 (owner_id); +create index bench_table_047_status_idx on public.bench_table_047 (status, created_at); +alter table public.bench_table_047 enable row level security; +create policy bench_table_047_read on public.bench_table_047 for select using (true); + +create table public.bench_table_048 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_048_owner_idx on public.bench_table_048 (owner_id); +create index bench_table_048_status_idx on public.bench_table_048 (status, created_at); +alter table public.bench_table_048 enable row level security; +create policy bench_table_048_read on public.bench_table_048 for select using (true); + +create table public.bench_table_049 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_049_owner_idx on public.bench_table_049 (owner_id); +create index bench_table_049_status_idx on public.bench_table_049 (status, created_at); +alter table public.bench_table_049 enable row level security; +create policy bench_table_049_read on public.bench_table_049 for select using (true); + +create table public.bench_table_050 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_050_owner_idx on public.bench_table_050 (owner_id); +create index bench_table_050_status_idx on public.bench_table_050 (status, created_at); +alter table public.bench_table_050 enable row level security; +create policy bench_table_050_read on public.bench_table_050 for select using (true); + +create view public.bench_table_050_summary as + select status, count(*) as total from public.bench_table_050 group by status; + +create function public.bench_table_050_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_050 $$; + +create table public.bench_table_051 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_051_owner_idx on public.bench_table_051 (owner_id); +create index bench_table_051_status_idx on public.bench_table_051 (status, created_at); +alter table public.bench_table_051 enable row level security; +create policy bench_table_051_read on public.bench_table_051 for select using (true); + +create table public.bench_table_052 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_052_owner_idx on public.bench_table_052 (owner_id); +create index bench_table_052_status_idx on public.bench_table_052 (status, created_at); +alter table public.bench_table_052 enable row level security; +create policy bench_table_052_read on public.bench_table_052 for select using (true); + +create table public.bench_table_053 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_053_owner_idx on public.bench_table_053 (owner_id); +create index bench_table_053_status_idx on public.bench_table_053 (status, created_at); +alter table public.bench_table_053 enable row level security; +create policy bench_table_053_read on public.bench_table_053 for select using (true); + +create table public.bench_table_054 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_054_owner_idx on public.bench_table_054 (owner_id); +create index bench_table_054_status_idx on public.bench_table_054 (status, created_at); +alter table public.bench_table_054 enable row level security; +create policy bench_table_054_read on public.bench_table_054 for select using (true); + +create table public.bench_table_055 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_055_owner_idx on public.bench_table_055 (owner_id); +create index bench_table_055_status_idx on public.bench_table_055 (status, created_at); +alter table public.bench_table_055 enable row level security; +create policy bench_table_055_read on public.bench_table_055 for select using (true); + +create view public.bench_table_055_summary as + select status, count(*) as total from public.bench_table_055 group by status; + +create table public.bench_table_056 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_056_owner_idx on public.bench_table_056 (owner_id); +create index bench_table_056_status_idx on public.bench_table_056 (status, created_at); +alter table public.bench_table_056 enable row level security; +create policy bench_table_056_read on public.bench_table_056 for select using (true); + +create table public.bench_table_057 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_057_owner_idx on public.bench_table_057 (owner_id); +create index bench_table_057_status_idx on public.bench_table_057 (status, created_at); +alter table public.bench_table_057 enable row level security; +create policy bench_table_057_read on public.bench_table_057 for select using (true); + +create table public.bench_table_058 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_058_owner_idx on public.bench_table_058 (owner_id); +create index bench_table_058_status_idx on public.bench_table_058 (status, created_at); +alter table public.bench_table_058 enable row level security; +create policy bench_table_058_read on public.bench_table_058 for select using (true); + +create table public.bench_table_059 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_059_owner_idx on public.bench_table_059 (owner_id); +create index bench_table_059_status_idx on public.bench_table_059 (status, created_at); +alter table public.bench_table_059 enable row level security; +create policy bench_table_059_read on public.bench_table_059 for select using (true); + +create table public.bench_table_060 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_060_owner_idx on public.bench_table_060 (owner_id); +create index bench_table_060_status_idx on public.bench_table_060 (status, created_at); +alter table public.bench_table_060 enable row level security; +create policy bench_table_060_read on public.bench_table_060 for select using (true); + +create view public.bench_table_060_summary as + select status, count(*) as total from public.bench_table_060 group by status; + +create function public.bench_table_060_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_060 $$; + +create table public.bench_table_061 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_061_owner_idx on public.bench_table_061 (owner_id); +create index bench_table_061_status_idx on public.bench_table_061 (status, created_at); +alter table public.bench_table_061 enable row level security; +create policy bench_table_061_read on public.bench_table_061 for select using (true); + +create table public.bench_table_062 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_062_owner_idx on public.bench_table_062 (owner_id); +create index bench_table_062_status_idx on public.bench_table_062 (status, created_at); +alter table public.bench_table_062 enable row level security; +create policy bench_table_062_read on public.bench_table_062 for select using (true); + +create table public.bench_table_063 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_063_owner_idx on public.bench_table_063 (owner_id); +create index bench_table_063_status_idx on public.bench_table_063 (status, created_at); +alter table public.bench_table_063 enable row level security; +create policy bench_table_063_read on public.bench_table_063 for select using (true); + +create table public.bench_table_064 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_064_owner_idx on public.bench_table_064 (owner_id); +create index bench_table_064_status_idx on public.bench_table_064 (status, created_at); +alter table public.bench_table_064 enable row level security; +create policy bench_table_064_read on public.bench_table_064 for select using (true); + +create table public.bench_table_065 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_065_owner_idx on public.bench_table_065 (owner_id); +create index bench_table_065_status_idx on public.bench_table_065 (status, created_at); +alter table public.bench_table_065 enable row level security; +create policy bench_table_065_read on public.bench_table_065 for select using (true); + +create view public.bench_table_065_summary as + select status, count(*) as total from public.bench_table_065 group by status; + +create table public.bench_table_066 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_066_owner_idx on public.bench_table_066 (owner_id); +create index bench_table_066_status_idx on public.bench_table_066 (status, created_at); +alter table public.bench_table_066 enable row level security; +create policy bench_table_066_read on public.bench_table_066 for select using (true); + +create table public.bench_table_067 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_067_owner_idx on public.bench_table_067 (owner_id); +create index bench_table_067_status_idx on public.bench_table_067 (status, created_at); +alter table public.bench_table_067 enable row level security; +create policy bench_table_067_read on public.bench_table_067 for select using (true); + +create table public.bench_table_068 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_068_owner_idx on public.bench_table_068 (owner_id); +create index bench_table_068_status_idx on public.bench_table_068 (status, created_at); +alter table public.bench_table_068 enable row level security; +create policy bench_table_068_read on public.bench_table_068 for select using (true); + +create table public.bench_table_069 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_069_owner_idx on public.bench_table_069 (owner_id); +create index bench_table_069_status_idx on public.bench_table_069 (status, created_at); +alter table public.bench_table_069 enable row level security; +create policy bench_table_069_read on public.bench_table_069 for select using (true); + +create table public.bench_table_070 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_070_owner_idx on public.bench_table_070 (owner_id); +create index bench_table_070_status_idx on public.bench_table_070 (status, created_at); +alter table public.bench_table_070 enable row level security; +create policy bench_table_070_read on public.bench_table_070 for select using (true); + +create view public.bench_table_070_summary as + select status, count(*) as total from public.bench_table_070 group by status; + +create function public.bench_table_070_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_070 $$; + +create table public.bench_table_071 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_071_owner_idx on public.bench_table_071 (owner_id); +create index bench_table_071_status_idx on public.bench_table_071 (status, created_at); +alter table public.bench_table_071 enable row level security; +create policy bench_table_071_read on public.bench_table_071 for select using (true); + +create table public.bench_table_072 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_072_owner_idx on public.bench_table_072 (owner_id); +create index bench_table_072_status_idx on public.bench_table_072 (status, created_at); +alter table public.bench_table_072 enable row level security; +create policy bench_table_072_read on public.bench_table_072 for select using (true); + +create table public.bench_table_073 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_073_owner_idx on public.bench_table_073 (owner_id); +create index bench_table_073_status_idx on public.bench_table_073 (status, created_at); +alter table public.bench_table_073 enable row level security; +create policy bench_table_073_read on public.bench_table_073 for select using (true); + +create table public.bench_table_074 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_074_owner_idx on public.bench_table_074 (owner_id); +create index bench_table_074_status_idx on public.bench_table_074 (status, created_at); +alter table public.bench_table_074 enable row level security; +create policy bench_table_074_read on public.bench_table_074 for select using (true); + +create table public.bench_table_075 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_075_owner_idx on public.bench_table_075 (owner_id); +create index bench_table_075_status_idx on public.bench_table_075 (status, created_at); +alter table public.bench_table_075 enable row level security; +create policy bench_table_075_read on public.bench_table_075 for select using (true); + +create view public.bench_table_075_summary as + select status, count(*) as total from public.bench_table_075 group by status; + +create table public.bench_table_076 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_076_owner_idx on public.bench_table_076 (owner_id); +create index bench_table_076_status_idx on public.bench_table_076 (status, created_at); +alter table public.bench_table_076 enable row level security; +create policy bench_table_076_read on public.bench_table_076 for select using (true); + +create table public.bench_table_077 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_077_owner_idx on public.bench_table_077 (owner_id); +create index bench_table_077_status_idx on public.bench_table_077 (status, created_at); +alter table public.bench_table_077 enable row level security; +create policy bench_table_077_read on public.bench_table_077 for select using (true); + +create table public.bench_table_078 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_078_owner_idx on public.bench_table_078 (owner_id); +create index bench_table_078_status_idx on public.bench_table_078 (status, created_at); +alter table public.bench_table_078 enable row level security; +create policy bench_table_078_read on public.bench_table_078 for select using (true); + +create table public.bench_table_079 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_079_owner_idx on public.bench_table_079 (owner_id); +create index bench_table_079_status_idx on public.bench_table_079 (status, created_at); +alter table public.bench_table_079 enable row level security; +create policy bench_table_079_read on public.bench_table_079 for select using (true); + +create table public.bench_table_080 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_080_owner_idx on public.bench_table_080 (owner_id); +create index bench_table_080_status_idx on public.bench_table_080 (status, created_at); +alter table public.bench_table_080 enable row level security; +create policy bench_table_080_read on public.bench_table_080 for select using (true); + +create view public.bench_table_080_summary as + select status, count(*) as total from public.bench_table_080 group by status; + +create function public.bench_table_080_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_080 $$; + +create table public.bench_table_081 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_081_owner_idx on public.bench_table_081 (owner_id); +create index bench_table_081_status_idx on public.bench_table_081 (status, created_at); +alter table public.bench_table_081 enable row level security; +create policy bench_table_081_read on public.bench_table_081 for select using (true); + +create table public.bench_table_082 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_082_owner_idx on public.bench_table_082 (owner_id); +create index bench_table_082_status_idx on public.bench_table_082 (status, created_at); +alter table public.bench_table_082 enable row level security; +create policy bench_table_082_read on public.bench_table_082 for select using (true); + +create table public.bench_table_083 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_083_owner_idx on public.bench_table_083 (owner_id); +create index bench_table_083_status_idx on public.bench_table_083 (status, created_at); +alter table public.bench_table_083 enable row level security; +create policy bench_table_083_read on public.bench_table_083 for select using (true); + +create table public.bench_table_084 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_084_owner_idx on public.bench_table_084 (owner_id); +create index bench_table_084_status_idx on public.bench_table_084 (status, created_at); +alter table public.bench_table_084 enable row level security; +create policy bench_table_084_read on public.bench_table_084 for select using (true); + +create table public.bench_table_085 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_085_owner_idx on public.bench_table_085 (owner_id); +create index bench_table_085_status_idx on public.bench_table_085 (status, created_at); +alter table public.bench_table_085 enable row level security; +create policy bench_table_085_read on public.bench_table_085 for select using (true); + +create view public.bench_table_085_summary as + select status, count(*) as total from public.bench_table_085 group by status; + +create table public.bench_table_086 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_086_owner_idx on public.bench_table_086 (owner_id); +create index bench_table_086_status_idx on public.bench_table_086 (status, created_at); +alter table public.bench_table_086 enable row level security; +create policy bench_table_086_read on public.bench_table_086 for select using (true); + +create table public.bench_table_087 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_087_owner_idx on public.bench_table_087 (owner_id); +create index bench_table_087_status_idx on public.bench_table_087 (status, created_at); +alter table public.bench_table_087 enable row level security; +create policy bench_table_087_read on public.bench_table_087 for select using (true); + +create table public.bench_table_088 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_088_owner_idx on public.bench_table_088 (owner_id); +create index bench_table_088_status_idx on public.bench_table_088 (status, created_at); +alter table public.bench_table_088 enable row level security; +create policy bench_table_088_read on public.bench_table_088 for select using (true); + +create table public.bench_table_089 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_089_owner_idx on public.bench_table_089 (owner_id); +create index bench_table_089_status_idx on public.bench_table_089 (status, created_at); +alter table public.bench_table_089 enable row level security; +create policy bench_table_089_read on public.bench_table_089 for select using (true); + +create table public.bench_table_090 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_090_owner_idx on public.bench_table_090 (owner_id); +create index bench_table_090_status_idx on public.bench_table_090 (status, created_at); +alter table public.bench_table_090 enable row level security; +create policy bench_table_090_read on public.bench_table_090 for select using (true); + +create view public.bench_table_090_summary as + select status, count(*) as total from public.bench_table_090 group by status; + +create function public.bench_table_090_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_090 $$; + +create table public.bench_table_091 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_091_owner_idx on public.bench_table_091 (owner_id); +create index bench_table_091_status_idx on public.bench_table_091 (status, created_at); +alter table public.bench_table_091 enable row level security; +create policy bench_table_091_read on public.bench_table_091 for select using (true); + +create table public.bench_table_092 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_092_owner_idx on public.bench_table_092 (owner_id); +create index bench_table_092_status_idx on public.bench_table_092 (status, created_at); +alter table public.bench_table_092 enable row level security; +create policy bench_table_092_read on public.bench_table_092 for select using (true); + +create table public.bench_table_093 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_093_owner_idx on public.bench_table_093 (owner_id); +create index bench_table_093_status_idx on public.bench_table_093 (status, created_at); +alter table public.bench_table_093 enable row level security; +create policy bench_table_093_read on public.bench_table_093 for select using (true); + +create table public.bench_table_094 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_094_owner_idx on public.bench_table_094 (owner_id); +create index bench_table_094_status_idx on public.bench_table_094 (status, created_at); +alter table public.bench_table_094 enable row level security; +create policy bench_table_094_read on public.bench_table_094 for select using (true); + +create table public.bench_table_095 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_095_owner_idx on public.bench_table_095 (owner_id); +create index bench_table_095_status_idx on public.bench_table_095 (status, created_at); +alter table public.bench_table_095 enable row level security; +create policy bench_table_095_read on public.bench_table_095 for select using (true); + +create view public.bench_table_095_summary as + select status, count(*) as total from public.bench_table_095 group by status; + +create table public.bench_table_096 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_096_owner_idx on public.bench_table_096 (owner_id); +create index bench_table_096_status_idx on public.bench_table_096 (status, created_at); +alter table public.bench_table_096 enable row level security; +create policy bench_table_096_read on public.bench_table_096 for select using (true); + +create table public.bench_table_097 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_097_owner_idx on public.bench_table_097 (owner_id); +create index bench_table_097_status_idx on public.bench_table_097 (status, created_at); +alter table public.bench_table_097 enable row level security; +create policy bench_table_097_read on public.bench_table_097 for select using (true); + +create table public.bench_table_098 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_098_owner_idx on public.bench_table_098 (owner_id); +create index bench_table_098_status_idx on public.bench_table_098 (status, created_at); +alter table public.bench_table_098 enable row level security; +create policy bench_table_098_read on public.bench_table_098 for select using (true); + +create table public.bench_table_099 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_099_owner_idx on public.bench_table_099 (owner_id); +create index bench_table_099_status_idx on public.bench_table_099 (status, created_at); +alter table public.bench_table_099 enable row level security; +create policy bench_table_099_read on public.bench_table_099 for select using (true); + +create table public.bench_table_100 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_100_owner_idx on public.bench_table_100 (owner_id); +create index bench_table_100_status_idx on public.bench_table_100 (status, created_at); +alter table public.bench_table_100 enable row level security; +create policy bench_table_100_read on public.bench_table_100 for select using (true); + +create view public.bench_table_100_summary as + select status, count(*) as total from public.bench_table_100 group by status; + +create function public.bench_table_100_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_100 $$; + +create table public.bench_table_101 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_101_owner_idx on public.bench_table_101 (owner_id); +create index bench_table_101_status_idx on public.bench_table_101 (status, created_at); +alter table public.bench_table_101 enable row level security; +create policy bench_table_101_read on public.bench_table_101 for select using (true); + +create table public.bench_table_102 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_102_owner_idx on public.bench_table_102 (owner_id); +create index bench_table_102_status_idx on public.bench_table_102 (status, created_at); +alter table public.bench_table_102 enable row level security; +create policy bench_table_102_read on public.bench_table_102 for select using (true); + +create table public.bench_table_103 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_103_owner_idx on public.bench_table_103 (owner_id); +create index bench_table_103_status_idx on public.bench_table_103 (status, created_at); +alter table public.bench_table_103 enable row level security; +create policy bench_table_103_read on public.bench_table_103 for select using (true); + +create table public.bench_table_104 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_104_owner_idx on public.bench_table_104 (owner_id); +create index bench_table_104_status_idx on public.bench_table_104 (status, created_at); +alter table public.bench_table_104 enable row level security; +create policy bench_table_104_read on public.bench_table_104 for select using (true); + +create table public.bench_table_105 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_105_owner_idx on public.bench_table_105 (owner_id); +create index bench_table_105_status_idx on public.bench_table_105 (status, created_at); +alter table public.bench_table_105 enable row level security; +create policy bench_table_105_read on public.bench_table_105 for select using (true); + +create view public.bench_table_105_summary as + select status, count(*) as total from public.bench_table_105 group by status; + +create table public.bench_table_106 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_106_owner_idx on public.bench_table_106 (owner_id); +create index bench_table_106_status_idx on public.bench_table_106 (status, created_at); +alter table public.bench_table_106 enable row level security; +create policy bench_table_106_read on public.bench_table_106 for select using (true); + +create table public.bench_table_107 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_107_owner_idx on public.bench_table_107 (owner_id); +create index bench_table_107_status_idx on public.bench_table_107 (status, created_at); +alter table public.bench_table_107 enable row level security; +create policy bench_table_107_read on public.bench_table_107 for select using (true); + +create table public.bench_table_108 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_108_owner_idx on public.bench_table_108 (owner_id); +create index bench_table_108_status_idx on public.bench_table_108 (status, created_at); +alter table public.bench_table_108 enable row level security; +create policy bench_table_108_read on public.bench_table_108 for select using (true); + +create table public.bench_table_109 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_109_owner_idx on public.bench_table_109 (owner_id); +create index bench_table_109_status_idx on public.bench_table_109 (status, created_at); +alter table public.bench_table_109 enable row level security; +create policy bench_table_109_read on public.bench_table_109 for select using (true); + +create table public.bench_table_110 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_110_owner_idx on public.bench_table_110 (owner_id); +create index bench_table_110_status_idx on public.bench_table_110 (status, created_at); +alter table public.bench_table_110 enable row level security; +create policy bench_table_110_read on public.bench_table_110 for select using (true); + +create view public.bench_table_110_summary as + select status, count(*) as total from public.bench_table_110 group by status; + +create function public.bench_table_110_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_110 $$; + +create table public.bench_table_111 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_111_owner_idx on public.bench_table_111 (owner_id); +create index bench_table_111_status_idx on public.bench_table_111 (status, created_at); +alter table public.bench_table_111 enable row level security; +create policy bench_table_111_read on public.bench_table_111 for select using (true); + +create table public.bench_table_112 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_112_owner_idx on public.bench_table_112 (owner_id); +create index bench_table_112_status_idx on public.bench_table_112 (status, created_at); +alter table public.bench_table_112 enable row level security; +create policy bench_table_112_read on public.bench_table_112 for select using (true); + +create table public.bench_table_113 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_113_owner_idx on public.bench_table_113 (owner_id); +create index bench_table_113_status_idx on public.bench_table_113 (status, created_at); +alter table public.bench_table_113 enable row level security; +create policy bench_table_113_read on public.bench_table_113 for select using (true); + +create table public.bench_table_114 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_114_owner_idx on public.bench_table_114 (owner_id); +create index bench_table_114_status_idx on public.bench_table_114 (status, created_at); +alter table public.bench_table_114 enable row level security; +create policy bench_table_114_read on public.bench_table_114 for select using (true); + +create table public.bench_table_115 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_115_owner_idx on public.bench_table_115 (owner_id); +create index bench_table_115_status_idx on public.bench_table_115 (status, created_at); +alter table public.bench_table_115 enable row level security; +create policy bench_table_115_read on public.bench_table_115 for select using (true); + +create view public.bench_table_115_summary as + select status, count(*) as total from public.bench_table_115 group by status; + +create table public.bench_table_116 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_116_owner_idx on public.bench_table_116 (owner_id); +create index bench_table_116_status_idx on public.bench_table_116 (status, created_at); +alter table public.bench_table_116 enable row level security; +create policy bench_table_116_read on public.bench_table_116 for select using (true); + +create table public.bench_table_117 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_117_owner_idx on public.bench_table_117 (owner_id); +create index bench_table_117_status_idx on public.bench_table_117 (status, created_at); +alter table public.bench_table_117 enable row level security; +create policy bench_table_117_read on public.bench_table_117 for select using (true); + +create table public.bench_table_118 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_118_owner_idx on public.bench_table_118 (owner_id); +create index bench_table_118_status_idx on public.bench_table_118 (status, created_at); +alter table public.bench_table_118 enable row level security; +create policy bench_table_118_read on public.bench_table_118 for select using (true); + +create table public.bench_table_119 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_119_owner_idx on public.bench_table_119 (owner_id); +create index bench_table_119_status_idx on public.bench_table_119 (status, created_at); +alter table public.bench_table_119 enable row level security; +create policy bench_table_119_read on public.bench_table_119 for select using (true); + +create table public.bench_table_120 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_120_owner_idx on public.bench_table_120 (owner_id); +create index bench_table_120_status_idx on public.bench_table_120 (status, created_at); +alter table public.bench_table_120 enable row level security; +create policy bench_table_120_read on public.bench_table_120 for select using (true); + +create view public.bench_table_120_summary as + select status, count(*) as total from public.bench_table_120 group by status; + +create function public.bench_table_120_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_120 $$; + +create table public.bench_table_121 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_121_owner_idx on public.bench_table_121 (owner_id); +create index bench_table_121_status_idx on public.bench_table_121 (status, created_at); +alter table public.bench_table_121 enable row level security; +create policy bench_table_121_read on public.bench_table_121 for select using (true); + +create table public.bench_table_122 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_122_owner_idx on public.bench_table_122 (owner_id); +create index bench_table_122_status_idx on public.bench_table_122 (status, created_at); +alter table public.bench_table_122 enable row level security; +create policy bench_table_122_read on public.bench_table_122 for select using (true); + +create table public.bench_table_123 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_123_owner_idx on public.bench_table_123 (owner_id); +create index bench_table_123_status_idx on public.bench_table_123 (status, created_at); +alter table public.bench_table_123 enable row level security; +create policy bench_table_123_read on public.bench_table_123 for select using (true); + +create table public.bench_table_124 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_124_owner_idx on public.bench_table_124 (owner_id); +create index bench_table_124_status_idx on public.bench_table_124 (status, created_at); +alter table public.bench_table_124 enable row level security; +create policy bench_table_124_read on public.bench_table_124 for select using (true); + +create table public.bench_table_125 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_125_owner_idx on public.bench_table_125 (owner_id); +create index bench_table_125_status_idx on public.bench_table_125 (status, created_at); +alter table public.bench_table_125 enable row level security; +create policy bench_table_125_read on public.bench_table_125 for select using (true); + +create view public.bench_table_125_summary as + select status, count(*) as total from public.bench_table_125 group by status; + +create table public.bench_table_126 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_126_owner_idx on public.bench_table_126 (owner_id); +create index bench_table_126_status_idx on public.bench_table_126 (status, created_at); +alter table public.bench_table_126 enable row level security; +create policy bench_table_126_read on public.bench_table_126 for select using (true); + +create table public.bench_table_127 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_127_owner_idx on public.bench_table_127 (owner_id); +create index bench_table_127_status_idx on public.bench_table_127 (status, created_at); +alter table public.bench_table_127 enable row level security; +create policy bench_table_127_read on public.bench_table_127 for select using (true); + +create table public.bench_table_128 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_128_owner_idx on public.bench_table_128 (owner_id); +create index bench_table_128_status_idx on public.bench_table_128 (status, created_at); +alter table public.bench_table_128 enable row level security; +create policy bench_table_128_read on public.bench_table_128 for select using (true); + +create table public.bench_table_129 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_129_owner_idx on public.bench_table_129 (owner_id); +create index bench_table_129_status_idx on public.bench_table_129 (status, created_at); +alter table public.bench_table_129 enable row level security; +create policy bench_table_129_read on public.bench_table_129 for select using (true); + +create table public.bench_table_130 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_130_owner_idx on public.bench_table_130 (owner_id); +create index bench_table_130_status_idx on public.bench_table_130 (status, created_at); +alter table public.bench_table_130 enable row level security; +create policy bench_table_130_read on public.bench_table_130 for select using (true); + +create view public.bench_table_130_summary as + select status, count(*) as total from public.bench_table_130 group by status; + +create function public.bench_table_130_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_130 $$; + +create table public.bench_table_131 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_131_owner_idx on public.bench_table_131 (owner_id); +create index bench_table_131_status_idx on public.bench_table_131 (status, created_at); +alter table public.bench_table_131 enable row level security; +create policy bench_table_131_read on public.bench_table_131 for select using (true); + +create table public.bench_table_132 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_132_owner_idx on public.bench_table_132 (owner_id); +create index bench_table_132_status_idx on public.bench_table_132 (status, created_at); +alter table public.bench_table_132 enable row level security; +create policy bench_table_132_read on public.bench_table_132 for select using (true); + +create table public.bench_table_133 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_133_owner_idx on public.bench_table_133 (owner_id); +create index bench_table_133_status_idx on public.bench_table_133 (status, created_at); +alter table public.bench_table_133 enable row level security; +create policy bench_table_133_read on public.bench_table_133 for select using (true); + +create table public.bench_table_134 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_134_owner_idx on public.bench_table_134 (owner_id); +create index bench_table_134_status_idx on public.bench_table_134 (status, created_at); +alter table public.bench_table_134 enable row level security; +create policy bench_table_134_read on public.bench_table_134 for select using (true); + +create table public.bench_table_135 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_135_owner_idx on public.bench_table_135 (owner_id); +create index bench_table_135_status_idx on public.bench_table_135 (status, created_at); +alter table public.bench_table_135 enable row level security; +create policy bench_table_135_read on public.bench_table_135 for select using (true); + +create view public.bench_table_135_summary as + select status, count(*) as total from public.bench_table_135 group by status; + +create table public.bench_table_136 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_136_owner_idx on public.bench_table_136 (owner_id); +create index bench_table_136_status_idx on public.bench_table_136 (status, created_at); +alter table public.bench_table_136 enable row level security; +create policy bench_table_136_read on public.bench_table_136 for select using (true); + +create table public.bench_table_137 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_137_owner_idx on public.bench_table_137 (owner_id); +create index bench_table_137_status_idx on public.bench_table_137 (status, created_at); +alter table public.bench_table_137 enable row level security; +create policy bench_table_137_read on public.bench_table_137 for select using (true); + +create table public.bench_table_138 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_138_owner_idx on public.bench_table_138 (owner_id); +create index bench_table_138_status_idx on public.bench_table_138 (status, created_at); +alter table public.bench_table_138 enable row level security; +create policy bench_table_138_read on public.bench_table_138 for select using (true); + +create table public.bench_table_139 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_139_owner_idx on public.bench_table_139 (owner_id); +create index bench_table_139_status_idx on public.bench_table_139 (status, created_at); +alter table public.bench_table_139 enable row level security; +create policy bench_table_139_read on public.bench_table_139 for select using (true); + +create table public.bench_table_140 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_140_owner_idx on public.bench_table_140 (owner_id); +create index bench_table_140_status_idx on public.bench_table_140 (status, created_at); +alter table public.bench_table_140 enable row level security; +create policy bench_table_140_read on public.bench_table_140 for select using (true); + +create view public.bench_table_140_summary as + select status, count(*) as total from public.bench_table_140 group by status; + +create function public.bench_table_140_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_140 $$; + +create table public.bench_table_141 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_141_owner_idx on public.bench_table_141 (owner_id); +create index bench_table_141_status_idx on public.bench_table_141 (status, created_at); +alter table public.bench_table_141 enable row level security; +create policy bench_table_141_read on public.bench_table_141 for select using (true); + +create table public.bench_table_142 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_142_owner_idx on public.bench_table_142 (owner_id); +create index bench_table_142_status_idx on public.bench_table_142 (status, created_at); +alter table public.bench_table_142 enable row level security; +create policy bench_table_142_read on public.bench_table_142 for select using (true); + +create table public.bench_table_143 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_143_owner_idx on public.bench_table_143 (owner_id); +create index bench_table_143_status_idx on public.bench_table_143 (status, created_at); +alter table public.bench_table_143 enable row level security; +create policy bench_table_143_read on public.bench_table_143 for select using (true); + +create table public.bench_table_144 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_144_owner_idx on public.bench_table_144 (owner_id); +create index bench_table_144_status_idx on public.bench_table_144 (status, created_at); +alter table public.bench_table_144 enable row level security; +create policy bench_table_144_read on public.bench_table_144 for select using (true); + +create table public.bench_table_145 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_145_owner_idx on public.bench_table_145 (owner_id); +create index bench_table_145_status_idx on public.bench_table_145 (status, created_at); +alter table public.bench_table_145 enable row level security; +create policy bench_table_145_read on public.bench_table_145 for select using (true); + +create view public.bench_table_145_summary as + select status, count(*) as total from public.bench_table_145 group by status; + +create table public.bench_table_146 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_146_owner_idx on public.bench_table_146 (owner_id); +create index bench_table_146_status_idx on public.bench_table_146 (status, created_at); +alter table public.bench_table_146 enable row level security; +create policy bench_table_146_read on public.bench_table_146 for select using (true); + +create table public.bench_table_147 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_147_owner_idx on public.bench_table_147 (owner_id); +create index bench_table_147_status_idx on public.bench_table_147 (status, created_at); +alter table public.bench_table_147 enable row level security; +create policy bench_table_147_read on public.bench_table_147 for select using (true); + +create table public.bench_table_148 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_148_owner_idx on public.bench_table_148 (owner_id); +create index bench_table_148_status_idx on public.bench_table_148 (status, created_at); +alter table public.bench_table_148 enable row level security; +create policy bench_table_148_read on public.bench_table_148 for select using (true); + +create table public.bench_table_149 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_149_owner_idx on public.bench_table_149 (owner_id); +create index bench_table_149_status_idx on public.bench_table_149 (status, created_at); +alter table public.bench_table_149 enable row level security; +create policy bench_table_149_read on public.bench_table_149 for select using (true); + +create table public.bench_table_150 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_150_owner_idx on public.bench_table_150 (owner_id); +create index bench_table_150_status_idx on public.bench_table_150 (status, created_at); +alter table public.bench_table_150 enable row level security; +create policy bench_table_150_read on public.bench_table_150 for select using (true); + +create view public.bench_table_150_summary as + select status, count(*) as total from public.bench_table_150 group by status; + +create function public.bench_table_150_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_150 $$; + +create table public.bench_table_151 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_151_owner_idx on public.bench_table_151 (owner_id); +create index bench_table_151_status_idx on public.bench_table_151 (status, created_at); +alter table public.bench_table_151 enable row level security; +create policy bench_table_151_read on public.bench_table_151 for select using (true); + +create table public.bench_table_152 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_152_owner_idx on public.bench_table_152 (owner_id); +create index bench_table_152_status_idx on public.bench_table_152 (status, created_at); +alter table public.bench_table_152 enable row level security; +create policy bench_table_152_read on public.bench_table_152 for select using (true); + +create table public.bench_table_153 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_153_owner_idx on public.bench_table_153 (owner_id); +create index bench_table_153_status_idx on public.bench_table_153 (status, created_at); +alter table public.bench_table_153 enable row level security; +create policy bench_table_153_read on public.bench_table_153 for select using (true); + +create table public.bench_table_154 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_154_owner_idx on public.bench_table_154 (owner_id); +create index bench_table_154_status_idx on public.bench_table_154 (status, created_at); +alter table public.bench_table_154 enable row level security; +create policy bench_table_154_read on public.bench_table_154 for select using (true); + +create table public.bench_table_155 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_155_owner_idx on public.bench_table_155 (owner_id); +create index bench_table_155_status_idx on public.bench_table_155 (status, created_at); +alter table public.bench_table_155 enable row level security; +create policy bench_table_155_read on public.bench_table_155 for select using (true); + +create view public.bench_table_155_summary as + select status, count(*) as total from public.bench_table_155 group by status; + +create table public.bench_table_156 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_156_owner_idx on public.bench_table_156 (owner_id); +create index bench_table_156_status_idx on public.bench_table_156 (status, created_at); +alter table public.bench_table_156 enable row level security; +create policy bench_table_156_read on public.bench_table_156 for select using (true); + +create table public.bench_table_157 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_157_owner_idx on public.bench_table_157 (owner_id); +create index bench_table_157_status_idx on public.bench_table_157 (status, created_at); +alter table public.bench_table_157 enable row level security; +create policy bench_table_157_read on public.bench_table_157 for select using (true); + +create table public.bench_table_158 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_158_owner_idx on public.bench_table_158 (owner_id); +create index bench_table_158_status_idx on public.bench_table_158 (status, created_at); +alter table public.bench_table_158 enable row level security; +create policy bench_table_158_read on public.bench_table_158 for select using (true); + +create table public.bench_table_159 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_159_owner_idx on public.bench_table_159 (owner_id); +create index bench_table_159_status_idx on public.bench_table_159 (status, created_at); +alter table public.bench_table_159 enable row level security; +create policy bench_table_159_read on public.bench_table_159 for select using (true); + +create table public.bench_table_160 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_160_owner_idx on public.bench_table_160 (owner_id); +create index bench_table_160_status_idx on public.bench_table_160 (status, created_at); +alter table public.bench_table_160 enable row level security; +create policy bench_table_160_read on public.bench_table_160 for select using (true); + +create view public.bench_table_160_summary as + select status, count(*) as total from public.bench_table_160 group by status; + +create function public.bench_table_160_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_160 $$; + +create table public.bench_table_161 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_161_owner_idx on public.bench_table_161 (owner_id); +create index bench_table_161_status_idx on public.bench_table_161 (status, created_at); +alter table public.bench_table_161 enable row level security; +create policy bench_table_161_read on public.bench_table_161 for select using (true); + +create table public.bench_table_162 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_162_owner_idx on public.bench_table_162 (owner_id); +create index bench_table_162_status_idx on public.bench_table_162 (status, created_at); +alter table public.bench_table_162 enable row level security; +create policy bench_table_162_read on public.bench_table_162 for select using (true); + +create table public.bench_table_163 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_163_owner_idx on public.bench_table_163 (owner_id); +create index bench_table_163_status_idx on public.bench_table_163 (status, created_at); +alter table public.bench_table_163 enable row level security; +create policy bench_table_163_read on public.bench_table_163 for select using (true); + +create table public.bench_table_164 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_164_owner_idx on public.bench_table_164 (owner_id); +create index bench_table_164_status_idx on public.bench_table_164 (status, created_at); +alter table public.bench_table_164 enable row level security; +create policy bench_table_164_read on public.bench_table_164 for select using (true); + +create table public.bench_table_165 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_165_owner_idx on public.bench_table_165 (owner_id); +create index bench_table_165_status_idx on public.bench_table_165 (status, created_at); +alter table public.bench_table_165 enable row level security; +create policy bench_table_165_read on public.bench_table_165 for select using (true); + +create view public.bench_table_165_summary as + select status, count(*) as total from public.bench_table_165 group by status; + +create table public.bench_table_166 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_166_owner_idx on public.bench_table_166 (owner_id); +create index bench_table_166_status_idx on public.bench_table_166 (status, created_at); +alter table public.bench_table_166 enable row level security; +create policy bench_table_166_read on public.bench_table_166 for select using (true); + +create table public.bench_table_167 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_167_owner_idx on public.bench_table_167 (owner_id); +create index bench_table_167_status_idx on public.bench_table_167 (status, created_at); +alter table public.bench_table_167 enable row level security; +create policy bench_table_167_read on public.bench_table_167 for select using (true); + +create table public.bench_table_168 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_168_owner_idx on public.bench_table_168 (owner_id); +create index bench_table_168_status_idx on public.bench_table_168 (status, created_at); +alter table public.bench_table_168 enable row level security; +create policy bench_table_168_read on public.bench_table_168 for select using (true); + +create table public.bench_table_169 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_169_owner_idx on public.bench_table_169 (owner_id); +create index bench_table_169_status_idx on public.bench_table_169 (status, created_at); +alter table public.bench_table_169 enable row level security; +create policy bench_table_169_read on public.bench_table_169 for select using (true); + +create table public.bench_table_170 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_170_owner_idx on public.bench_table_170 (owner_id); +create index bench_table_170_status_idx on public.bench_table_170 (status, created_at); +alter table public.bench_table_170 enable row level security; +create policy bench_table_170_read on public.bench_table_170 for select using (true); + +create view public.bench_table_170_summary as + select status, count(*) as total from public.bench_table_170 group by status; + +create function public.bench_table_170_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_170 $$; + +create table public.bench_table_171 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_171_owner_idx on public.bench_table_171 (owner_id); +create index bench_table_171_status_idx on public.bench_table_171 (status, created_at); +alter table public.bench_table_171 enable row level security; +create policy bench_table_171_read on public.bench_table_171 for select using (true); + +create table public.bench_table_172 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_172_owner_idx on public.bench_table_172 (owner_id); +create index bench_table_172_status_idx on public.bench_table_172 (status, created_at); +alter table public.bench_table_172 enable row level security; +create policy bench_table_172_read on public.bench_table_172 for select using (true); + +create table public.bench_table_173 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_173_owner_idx on public.bench_table_173 (owner_id); +create index bench_table_173_status_idx on public.bench_table_173 (status, created_at); +alter table public.bench_table_173 enable row level security; +create policy bench_table_173_read on public.bench_table_173 for select using (true); + +create table public.bench_table_174 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_174_owner_idx on public.bench_table_174 (owner_id); +create index bench_table_174_status_idx on public.bench_table_174 (status, created_at); +alter table public.bench_table_174 enable row level security; +create policy bench_table_174_read on public.bench_table_174 for select using (true); + +create table public.bench_table_175 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_175_owner_idx on public.bench_table_175 (owner_id); +create index bench_table_175_status_idx on public.bench_table_175 (status, created_at); +alter table public.bench_table_175 enable row level security; +create policy bench_table_175_read on public.bench_table_175 for select using (true); + +create view public.bench_table_175_summary as + select status, count(*) as total from public.bench_table_175 group by status; + +create table public.bench_table_176 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_176_owner_idx on public.bench_table_176 (owner_id); +create index bench_table_176_status_idx on public.bench_table_176 (status, created_at); +alter table public.bench_table_176 enable row level security; +create policy bench_table_176_read on public.bench_table_176 for select using (true); + +create table public.bench_table_177 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_177_owner_idx on public.bench_table_177 (owner_id); +create index bench_table_177_status_idx on public.bench_table_177 (status, created_at); +alter table public.bench_table_177 enable row level security; +create policy bench_table_177_read on public.bench_table_177 for select using (true); + +create table public.bench_table_178 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_178_owner_idx on public.bench_table_178 (owner_id); +create index bench_table_178_status_idx on public.bench_table_178 (status, created_at); +alter table public.bench_table_178 enable row level security; +create policy bench_table_178_read on public.bench_table_178 for select using (true); + +create table public.bench_table_179 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_179_owner_idx on public.bench_table_179 (owner_id); +create index bench_table_179_status_idx on public.bench_table_179 (status, created_at); +alter table public.bench_table_179 enable row level security; +create policy bench_table_179_read on public.bench_table_179 for select using (true); + +create table public.bench_table_180 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_180_owner_idx on public.bench_table_180 (owner_id); +create index bench_table_180_status_idx on public.bench_table_180 (status, created_at); +alter table public.bench_table_180 enable row level security; +create policy bench_table_180_read on public.bench_table_180 for select using (true); + +create view public.bench_table_180_summary as + select status, count(*) as total from public.bench_table_180 group by status; + +create function public.bench_table_180_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_180 $$; + +create table public.bench_table_181 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_181_owner_idx on public.bench_table_181 (owner_id); +create index bench_table_181_status_idx on public.bench_table_181 (status, created_at); +alter table public.bench_table_181 enable row level security; +create policy bench_table_181_read on public.bench_table_181 for select using (true); + +create table public.bench_table_182 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_182_owner_idx on public.bench_table_182 (owner_id); +create index bench_table_182_status_idx on public.bench_table_182 (status, created_at); +alter table public.bench_table_182 enable row level security; +create policy bench_table_182_read on public.bench_table_182 for select using (true); + +create table public.bench_table_183 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_183_owner_idx on public.bench_table_183 (owner_id); +create index bench_table_183_status_idx on public.bench_table_183 (status, created_at); +alter table public.bench_table_183 enable row level security; +create policy bench_table_183_read on public.bench_table_183 for select using (true); + +create table public.bench_table_184 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_184_owner_idx on public.bench_table_184 (owner_id); +create index bench_table_184_status_idx on public.bench_table_184 (status, created_at); +alter table public.bench_table_184 enable row level security; +create policy bench_table_184_read on public.bench_table_184 for select using (true); + +create table public.bench_table_185 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_185_owner_idx on public.bench_table_185 (owner_id); +create index bench_table_185_status_idx on public.bench_table_185 (status, created_at); +alter table public.bench_table_185 enable row level security; +create policy bench_table_185_read on public.bench_table_185 for select using (true); + +create view public.bench_table_185_summary as + select status, count(*) as total from public.bench_table_185 group by status; + +create table public.bench_table_186 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_186_owner_idx on public.bench_table_186 (owner_id); +create index bench_table_186_status_idx on public.bench_table_186 (status, created_at); +alter table public.bench_table_186 enable row level security; +create policy bench_table_186_read on public.bench_table_186 for select using (true); + +create table public.bench_table_187 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_187_owner_idx on public.bench_table_187 (owner_id); +create index bench_table_187_status_idx on public.bench_table_187 (status, created_at); +alter table public.bench_table_187 enable row level security; +create policy bench_table_187_read on public.bench_table_187 for select using (true); + +create table public.bench_table_188 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_188_owner_idx on public.bench_table_188 (owner_id); +create index bench_table_188_status_idx on public.bench_table_188 (status, created_at); +alter table public.bench_table_188 enable row level security; +create policy bench_table_188_read on public.bench_table_188 for select using (true); + +create table public.bench_table_189 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_189_owner_idx on public.bench_table_189 (owner_id); +create index bench_table_189_status_idx on public.bench_table_189 (status, created_at); +alter table public.bench_table_189 enable row level security; +create policy bench_table_189_read on public.bench_table_189 for select using (true); + +create table public.bench_table_190 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_190_owner_idx on public.bench_table_190 (owner_id); +create index bench_table_190_status_idx on public.bench_table_190 (status, created_at); +alter table public.bench_table_190 enable row level security; +create policy bench_table_190_read on public.bench_table_190 for select using (true); + +create view public.bench_table_190_summary as + select status, count(*) as total from public.bench_table_190 group by status; + +create function public.bench_table_190_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_190 $$; + +create table public.bench_table_191 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_191_owner_idx on public.bench_table_191 (owner_id); +create index bench_table_191_status_idx on public.bench_table_191 (status, created_at); +alter table public.bench_table_191 enable row level security; +create policy bench_table_191_read on public.bench_table_191 for select using (true); + +create table public.bench_table_192 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_192_owner_idx on public.bench_table_192 (owner_id); +create index bench_table_192_status_idx on public.bench_table_192 (status, created_at); +alter table public.bench_table_192 enable row level security; +create policy bench_table_192_read on public.bench_table_192 for select using (true); + +create table public.bench_table_193 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_193_owner_idx on public.bench_table_193 (owner_id); +create index bench_table_193_status_idx on public.bench_table_193 (status, created_at); +alter table public.bench_table_193 enable row level security; +create policy bench_table_193_read on public.bench_table_193 for select using (true); + +create table public.bench_table_194 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_194_owner_idx on public.bench_table_194 (owner_id); +create index bench_table_194_status_idx on public.bench_table_194 (status, created_at); +alter table public.bench_table_194 enable row level security; +create policy bench_table_194_read on public.bench_table_194 for select using (true); + +create table public.bench_table_195 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_195_owner_idx on public.bench_table_195 (owner_id); +create index bench_table_195_status_idx on public.bench_table_195 (status, created_at); +alter table public.bench_table_195 enable row level security; +create policy bench_table_195_read on public.bench_table_195 for select using (true); + +create view public.bench_table_195_summary as + select status, count(*) as total from public.bench_table_195 group by status; + +create table public.bench_table_196 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_196_owner_idx on public.bench_table_196 (owner_id); +create index bench_table_196_status_idx on public.bench_table_196 (status, created_at); +alter table public.bench_table_196 enable row level security; +create policy bench_table_196_read on public.bench_table_196 for select using (true); + +create table public.bench_table_197 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_197_owner_idx on public.bench_table_197 (owner_id); +create index bench_table_197_status_idx on public.bench_table_197 (status, created_at); +alter table public.bench_table_197 enable row level security; +create policy bench_table_197_read on public.bench_table_197 for select using (true); + +create table public.bench_table_198 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_198_owner_idx on public.bench_table_198 (owner_id); +create index bench_table_198_status_idx on public.bench_table_198 (status, created_at); +alter table public.bench_table_198 enable row level security; +create policy bench_table_198_read on public.bench_table_198 for select using (true); + +create table public.bench_table_199 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_199_owner_idx on public.bench_table_199 (owner_id); +create index bench_table_199_status_idx on public.bench_table_199 (status, created_at); +alter table public.bench_table_199 enable row level security; +create policy bench_table_199_read on public.bench_table_199 for select using (true); + +create table public.bench_table_200 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_200_owner_idx on public.bench_table_200 (owner_id); +create index bench_table_200_status_idx on public.bench_table_200 (status, created_at); +alter table public.bench_table_200 enable row level security; +create policy bench_table_200_read on public.bench_table_200 for select using (true); + +create view public.bench_table_200_summary as + select status, count(*) as total from public.bench_table_200 group by status; + +create function public.bench_table_200_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_200 $$; diff --git a/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..24049cf329 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-medium/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,694 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; + +create table public.bench_table_011 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_011_owner_idx on public.bench_table_011 (owner_id); +create index bench_table_011_status_idx on public.bench_table_011 (status, created_at); +alter table public.bench_table_011 enable row level security; +create policy bench_table_011_read on public.bench_table_011 for select using (true); + +create table public.bench_table_012 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_012_owner_idx on public.bench_table_012 (owner_id); +create index bench_table_012_status_idx on public.bench_table_012 (status, created_at); +alter table public.bench_table_012 enable row level security; +create policy bench_table_012_read on public.bench_table_012 for select using (true); + +create table public.bench_table_013 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_013_owner_idx on public.bench_table_013 (owner_id); +create index bench_table_013_status_idx on public.bench_table_013 (status, created_at); +alter table public.bench_table_013 enable row level security; +create policy bench_table_013_read on public.bench_table_013 for select using (true); + +create table public.bench_table_014 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_014_owner_idx on public.bench_table_014 (owner_id); +create index bench_table_014_status_idx on public.bench_table_014 (status, created_at); +alter table public.bench_table_014 enable row level security; +create policy bench_table_014_read on public.bench_table_014 for select using (true); + +create table public.bench_table_015 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_015_owner_idx on public.bench_table_015 (owner_id); +create index bench_table_015_status_idx on public.bench_table_015 (status, created_at); +alter table public.bench_table_015 enable row level security; +create policy bench_table_015_read on public.bench_table_015 for select using (true); + +create view public.bench_table_015_summary as + select status, count(*) as total from public.bench_table_015 group by status; + +create table public.bench_table_016 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_016_owner_idx on public.bench_table_016 (owner_id); +create index bench_table_016_status_idx on public.bench_table_016 (status, created_at); +alter table public.bench_table_016 enable row level security; +create policy bench_table_016_read on public.bench_table_016 for select using (true); + +create table public.bench_table_017 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_017_owner_idx on public.bench_table_017 (owner_id); +create index bench_table_017_status_idx on public.bench_table_017 (status, created_at); +alter table public.bench_table_017 enable row level security; +create policy bench_table_017_read on public.bench_table_017 for select using (true); + +create table public.bench_table_018 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_018_owner_idx on public.bench_table_018 (owner_id); +create index bench_table_018_status_idx on public.bench_table_018 (status, created_at); +alter table public.bench_table_018 enable row level security; +create policy bench_table_018_read on public.bench_table_018 for select using (true); + +create table public.bench_table_019 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_019_owner_idx on public.bench_table_019 (owner_id); +create index bench_table_019_status_idx on public.bench_table_019 (status, created_at); +alter table public.bench_table_019 enable row level security; +create policy bench_table_019_read on public.bench_table_019 for select using (true); + +create table public.bench_table_020 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_020_owner_idx on public.bench_table_020 (owner_id); +create index bench_table_020_status_idx on public.bench_table_020 (status, created_at); +alter table public.bench_table_020 enable row level security; +create policy bench_table_020_read on public.bench_table_020 for select using (true); + +create view public.bench_table_020_summary as + select status, count(*) as total from public.bench_table_020 group by status; + +create function public.bench_table_020_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_020 $$; + +create table public.bench_table_021 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_021_owner_idx on public.bench_table_021 (owner_id); +create index bench_table_021_status_idx on public.bench_table_021 (status, created_at); +alter table public.bench_table_021 enable row level security; +create policy bench_table_021_read on public.bench_table_021 for select using (true); + +create table public.bench_table_022 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_022_owner_idx on public.bench_table_022 (owner_id); +create index bench_table_022_status_idx on public.bench_table_022 (status, created_at); +alter table public.bench_table_022 enable row level security; +create policy bench_table_022_read on public.bench_table_022 for select using (true); + +create table public.bench_table_023 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_023_owner_idx on public.bench_table_023 (owner_id); +create index bench_table_023_status_idx on public.bench_table_023 (status, created_at); +alter table public.bench_table_023 enable row level security; +create policy bench_table_023_read on public.bench_table_023 for select using (true); + +create table public.bench_table_024 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_024_owner_idx on public.bench_table_024 (owner_id); +create index bench_table_024_status_idx on public.bench_table_024 (status, created_at); +alter table public.bench_table_024 enable row level security; +create policy bench_table_024_read on public.bench_table_024 for select using (true); + +create table public.bench_table_025 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_025_owner_idx on public.bench_table_025 (owner_id); +create index bench_table_025_status_idx on public.bench_table_025 (status, created_at); +alter table public.bench_table_025 enable row level security; +create policy bench_table_025_read on public.bench_table_025 for select using (true); + +create view public.bench_table_025_summary as + select status, count(*) as total from public.bench_table_025 group by status; + +create table public.bench_table_026 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_026_owner_idx on public.bench_table_026 (owner_id); +create index bench_table_026_status_idx on public.bench_table_026 (status, created_at); +alter table public.bench_table_026 enable row level security; +create policy bench_table_026_read on public.bench_table_026 for select using (true); + +create table public.bench_table_027 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_027_owner_idx on public.bench_table_027 (owner_id); +create index bench_table_027_status_idx on public.bench_table_027 (status, created_at); +alter table public.bench_table_027 enable row level security; +create policy bench_table_027_read on public.bench_table_027 for select using (true); + +create table public.bench_table_028 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_028_owner_idx on public.bench_table_028 (owner_id); +create index bench_table_028_status_idx on public.bench_table_028 (status, created_at); +alter table public.bench_table_028 enable row level security; +create policy bench_table_028_read on public.bench_table_028 for select using (true); + +create table public.bench_table_029 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_029_owner_idx on public.bench_table_029 (owner_id); +create index bench_table_029_status_idx on public.bench_table_029 (status, created_at); +alter table public.bench_table_029 enable row level security; +create policy bench_table_029_read on public.bench_table_029 for select using (true); + +create table public.bench_table_030 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_030_owner_idx on public.bench_table_030 (owner_id); +create index bench_table_030_status_idx on public.bench_table_030 (status, created_at); +alter table public.bench_table_030 enable row level security; +create policy bench_table_030_read on public.bench_table_030 for select using (true); + +create view public.bench_table_030_summary as + select status, count(*) as total from public.bench_table_030 group by status; + +create function public.bench_table_030_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_030 $$; + +create table public.bench_table_031 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_031_owner_idx on public.bench_table_031 (owner_id); +create index bench_table_031_status_idx on public.bench_table_031 (status, created_at); +alter table public.bench_table_031 enable row level security; +create policy bench_table_031_read on public.bench_table_031 for select using (true); + +create table public.bench_table_032 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_032_owner_idx on public.bench_table_032 (owner_id); +create index bench_table_032_status_idx on public.bench_table_032 (status, created_at); +alter table public.bench_table_032 enable row level security; +create policy bench_table_032_read on public.bench_table_032 for select using (true); + +create table public.bench_table_033 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_033_owner_idx on public.bench_table_033 (owner_id); +create index bench_table_033_status_idx on public.bench_table_033 (status, created_at); +alter table public.bench_table_033 enable row level security; +create policy bench_table_033_read on public.bench_table_033 for select using (true); + +create table public.bench_table_034 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_034_owner_idx on public.bench_table_034 (owner_id); +create index bench_table_034_status_idx on public.bench_table_034 (status, created_at); +alter table public.bench_table_034 enable row level security; +create policy bench_table_034_read on public.bench_table_034 for select using (true); + +create table public.bench_table_035 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_035_owner_idx on public.bench_table_035 (owner_id); +create index bench_table_035_status_idx on public.bench_table_035 (status, created_at); +alter table public.bench_table_035 enable row level security; +create policy bench_table_035_read on public.bench_table_035 for select using (true); + +create view public.bench_table_035_summary as + select status, count(*) as total from public.bench_table_035 group by status; + +create table public.bench_table_036 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_036_owner_idx on public.bench_table_036 (owner_id); +create index bench_table_036_status_idx on public.bench_table_036 (status, created_at); +alter table public.bench_table_036 enable row level security; +create policy bench_table_036_read on public.bench_table_036 for select using (true); + +create table public.bench_table_037 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_037_owner_idx on public.bench_table_037 (owner_id); +create index bench_table_037_status_idx on public.bench_table_037 (status, created_at); +alter table public.bench_table_037 enable row level security; +create policy bench_table_037_read on public.bench_table_037 for select using (true); + +create table public.bench_table_038 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_038_owner_idx on public.bench_table_038 (owner_id); +create index bench_table_038_status_idx on public.bench_table_038 (status, created_at); +alter table public.bench_table_038 enable row level security; +create policy bench_table_038_read on public.bench_table_038 for select using (true); + +create table public.bench_table_039 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_039_owner_idx on public.bench_table_039 (owner_id); +create index bench_table_039_status_idx on public.bench_table_039 (status, created_at); +alter table public.bench_table_039 enable row level security; +create policy bench_table_039_read on public.bench_table_039 for select using (true); + +create table public.bench_table_040 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_040_owner_idx on public.bench_table_040 (owner_id); +create index bench_table_040_status_idx on public.bench_table_040 (status, created_at); +alter table public.bench_table_040 enable row level security; +create policy bench_table_040_read on public.bench_table_040 for select using (true); + +create view public.bench_table_040_summary as + select status, count(*) as total from public.bench_table_040 group by status; + +create function public.bench_table_040_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_040 $$; + +create table public.bench_table_041 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_041_owner_idx on public.bench_table_041 (owner_id); +create index bench_table_041_status_idx on public.bench_table_041 (status, created_at); +alter table public.bench_table_041 enable row level security; +create policy bench_table_041_read on public.bench_table_041 for select using (true); + +create table public.bench_table_042 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_042_owner_idx on public.bench_table_042 (owner_id); +create index bench_table_042_status_idx on public.bench_table_042 (status, created_at); +alter table public.bench_table_042 enable row level security; +create policy bench_table_042_read on public.bench_table_042 for select using (true); + +create table public.bench_table_043 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_043_owner_idx on public.bench_table_043 (owner_id); +create index bench_table_043_status_idx on public.bench_table_043 (status, created_at); +alter table public.bench_table_043 enable row level security; +create policy bench_table_043_read on public.bench_table_043 for select using (true); + +create table public.bench_table_044 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_044_owner_idx on public.bench_table_044 (owner_id); +create index bench_table_044_status_idx on public.bench_table_044 (status, created_at); +alter table public.bench_table_044 enable row level security; +create policy bench_table_044_read on public.bench_table_044 for select using (true); + +create table public.bench_table_045 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_045_owner_idx on public.bench_table_045 (owner_id); +create index bench_table_045_status_idx on public.bench_table_045 (status, created_at); +alter table public.bench_table_045 enable row level security; +create policy bench_table_045_read on public.bench_table_045 for select using (true); + +create view public.bench_table_045_summary as + select status, count(*) as total from public.bench_table_045 group by status; + +create table public.bench_table_046 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_046_owner_idx on public.bench_table_046 (owner_id); +create index bench_table_046_status_idx on public.bench_table_046 (status, created_at); +alter table public.bench_table_046 enable row level security; +create policy bench_table_046_read on public.bench_table_046 for select using (true); + +create table public.bench_table_047 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_047_owner_idx on public.bench_table_047 (owner_id); +create index bench_table_047_status_idx on public.bench_table_047 (status, created_at); +alter table public.bench_table_047 enable row level security; +create policy bench_table_047_read on public.bench_table_047 for select using (true); + +create table public.bench_table_048 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_048_owner_idx on public.bench_table_048 (owner_id); +create index bench_table_048_status_idx on public.bench_table_048 (status, created_at); +alter table public.bench_table_048 enable row level security; +create policy bench_table_048_read on public.bench_table_048 for select using (true); + +create table public.bench_table_049 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_049_owner_idx on public.bench_table_049 (owner_id); +create index bench_table_049_status_idx on public.bench_table_049 (status, created_at); +alter table public.bench_table_049 enable row level security; +create policy bench_table_049_read on public.bench_table_049 for select using (true); + +create table public.bench_table_050 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_050_owner_idx on public.bench_table_050 (owner_id); +create index bench_table_050_status_idx on public.bench_table_050 (status, created_at); +alter table public.bench_table_050 enable row level security; +create policy bench_table_050_read on public.bench_table_050 for select using (true); + +create view public.bench_table_050_summary as + select status, count(*) as total from public.bench_table_050 group by status; + +create function public.bench_table_050_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_050 $$; diff --git a/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql b/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql new file mode 100644 index 0000000000..2ddae3c330 --- /dev/null +++ b/apps/cli/bench/fixtures/schema-small/migrations/20260101000001_bench_schema.sql @@ -0,0 +1,138 @@ +create table public.bench_table_001 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_001_owner_idx on public.bench_table_001 (owner_id); +create index bench_table_001_status_idx on public.bench_table_001 (status, created_at); +alter table public.bench_table_001 enable row level security; +create policy bench_table_001_read on public.bench_table_001 for select using (true); + +create table public.bench_table_002 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_002_owner_idx on public.bench_table_002 (owner_id); +create index bench_table_002_status_idx on public.bench_table_002 (status, created_at); +alter table public.bench_table_002 enable row level security; +create policy bench_table_002_read on public.bench_table_002 for select using (true); + +create table public.bench_table_003 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_003_owner_idx on public.bench_table_003 (owner_id); +create index bench_table_003_status_idx on public.bench_table_003 (status, created_at); +alter table public.bench_table_003 enable row level security; +create policy bench_table_003_read on public.bench_table_003 for select using (true); + +create table public.bench_table_004 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_004_owner_idx on public.bench_table_004 (owner_id); +create index bench_table_004_status_idx on public.bench_table_004 (status, created_at); +alter table public.bench_table_004 enable row level security; +create policy bench_table_004_read on public.bench_table_004 for select using (true); + +create table public.bench_table_005 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_005_owner_idx on public.bench_table_005 (owner_id); +create index bench_table_005_status_idx on public.bench_table_005 (status, created_at); +alter table public.bench_table_005 enable row level security; +create policy bench_table_005_read on public.bench_table_005 for select using (true); + +create view public.bench_table_005_summary as + select status, count(*) as total from public.bench_table_005 group by status; + +create table public.bench_table_006 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_006_owner_idx on public.bench_table_006 (owner_id); +create index bench_table_006_status_idx on public.bench_table_006 (status, created_at); +alter table public.bench_table_006 enable row level security; +create policy bench_table_006_read on public.bench_table_006 for select using (true); + +create table public.bench_table_007 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_007_owner_idx on public.bench_table_007 (owner_id); +create index bench_table_007_status_idx on public.bench_table_007 (status, created_at); +alter table public.bench_table_007 enable row level security; +create policy bench_table_007_read on public.bench_table_007 for select using (true); + +create table public.bench_table_008 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_008_owner_idx on public.bench_table_008 (owner_id); +create index bench_table_008_status_idx on public.bench_table_008 (status, created_at); +alter table public.bench_table_008 enable row level security; +create policy bench_table_008_read on public.bench_table_008 for select using (true); + +create table public.bench_table_009 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_009_owner_idx on public.bench_table_009 (owner_id); +create index bench_table_009_status_idx on public.bench_table_009 (status, created_at); +alter table public.bench_table_009 enable row level security; +create policy bench_table_009_read on public.bench_table_009 for select using (true); + +create table public.bench_table_010 ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index bench_table_010_owner_idx on public.bench_table_010 (owner_id); +create index bench_table_010_status_idx on public.bench_table_010 (status, created_at); +alter table public.bench_table_010 enable row level security; +create policy bench_table_010_read on public.bench_table_010 for select using (true); + +create view public.bench_table_010_summary as + select status, count(*) as total from public.bench_table_010 group by status; + +create function public.bench_table_010_count() returns bigint + language sql stable as $$ select count(*) from public.bench_table_010 $$; diff --git a/apps/cli/bench/workloads.full.json b/apps/cli/bench/workloads.full.json new file mode 100644 index 0000000000..90fef53a28 --- /dev/null +++ b/apps/cli/bench/workloads.full.json @@ -0,0 +1,385 @@ +{ + "workloads": [ + { + "id": "start", + "args": ["start"], + "cwdSetup": "init", + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 290 + }, + { + "id": "stop", + "args": ["stop", "--no-backup"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "db-start", + "args": ["db", "start"], + "cwdSetup": "init", + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 170 + }, + { + "id": "status", + "args": ["status"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "db-query", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "db-reset", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 530 + }, + { + "id": "test-db", + "args": ["test", "db"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "pgtap", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 60 + }, + { + "id": "gen-types-small", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-small", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "gen-types-medium", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "gen-types-large", + "args": ["gen", "types", "--lang", "typescript", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-large", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 100 + }, + { + "id": "db-diff-cold", + "args": ["db", "diff", "--use-pg-delta"], + "cwdSetup": "init", + "setup": [ + ["start"], + ["db", "reset"], + ["db", "query", "--local", "create table public.bench_drift (id int)"] + ], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 620 + }, + { + "id": "db-diff-warm", + "args": ["db", "diff", "--use-pg-delta"], + "cwdSetup": "init", + "setup": [ + ["start"], + ["db", "reset"], + ["db", "query", "--local", "create table public.bench_drift (id int)"], + ["db", "diff", "--use-pg-delta"] + ], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 660 + }, + { + "id": "db-dump", + "args": ["db", "dump", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 100 + }, + { + "id": "db-lint", + "args": ["db", "lint", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "schema-medium", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "migration-up", + "args": ["migration", "up", "--local"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixtureAfterSetup": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "db-push-local", + "args": ["db", "push", "--local"], + "cwdSetup": "init", + "setup": [["start"]], + "teardown": [["stop", "--no-backup"]], + "runtime": "docker", + "fixtureAfterSetup": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 300 + }, + { + "id": "migration-list", + "args": ["migration", "list", "--local"], + "cwdSetup": "init", + "setup": [["start"], ["migration", "up", "--local"]], + "teardown": [["stop", "--no-backup"]], + "setupScope": "block", + "runtime": "docker", + "fixture": "migrations", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 90 + }, + { + "id": "stack-start-docker", + "args": ["stack", "start", "--runtime", "docker", "--eager"], + "cwdSetup": "init", + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 190 + }, + { + "id": "stack-status-docker", + "args": ["stack", "status"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-stop-docker", + "args": ["stack", "stop"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 200 + }, + { + "id": "stack-db-query-docker", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "docker", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-db-reset-docker", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "docker", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "docker", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 370 + }, + { + "id": "stack-start-native", + "args": ["stack", "start", "--runtime", "native", "--eager"], + "cwdSetup": "init", + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 290 + }, + { + "id": "stack-status-native", + "args": ["stack", "status"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "native", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-stop-native", + "args": ["stack", "stop"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 120, + "estimatedSeconds": 190 + }, + { + "id": "stack-db-query-native", + "args": ["db", "query", "--local", "select 1"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "setupScope": "block", + "stack": true, + "runtime": "native", + "runs": 10, + "warmup": 1, + "traceRuns": 2, + "estimatedSeconds": 60 + }, + { + "id": "stack-db-reset-native", + "args": ["db", "reset"], + "cwdSetup": "init", + "setup": [["stack", "start", "--runtime", "native", "--eager"]], + "teardown": [["stack", "destroy", "--yes"]], + "stack": true, + "runtime": "native", + "fixture": "migrations", + "runs": 5, + "warmup": 1, + "traceRuns": 1, + "timeoutSeconds": 300, + "estimatedSeconds": 350 + } + ] +} diff --git a/apps/cli/docs/tracing-monitoring.md b/apps/cli/docs/tracing-monitoring.md index fe78022672..cdc4b92766 100644 --- a/apps/cli/docs/tracing-monitoring.md +++ b/apps/cli/docs/tracing-monitoring.md @@ -70,7 +70,8 @@ flag for more than one command, and use `--cwd-setup init` when a command, such run inside an initialized project. Pass `--json` for a machine-readable report, `--out ` to also write the raw per-run samples, and `--cpu-prof` to capture a Bun CPU profile of one untimed branch launch (`BUN_OPTIONS=--cpu-prof`), printing its `.cpuprofile` path. Launches set -`SUPABASE_NO_UPDATE_NOTIFIER=1`; pass `--update-check` to include the release check. Each launch's +`SUPABASE_NO_UPDATE_NOTIFIER=1`, `DO_NOT_TRACK=1`, and `SUPABASE_TELEMETRY_DISABLED=1`, so benchmark +runs never send product telemetry; pass `--update-check` to include the release check. Each launch's fresh `SUPABASE_HOME` has no cached release tag, so the check calls GitHub on every launch, except with `--cwd-setup init`, where the setup `init` caches the tag in the project before the measured command runs. The extra traced launches add machine load, so take wall-time conclusions from a run @@ -95,19 +96,30 @@ start replaces the comparison with a notice naming the step. The workloads use n prompts. The first push to a pull request from this repository benchmarks and posts one comment, tagged with -the hidden marker ``. Later pushes skip the benchmark, and the job summary says -so, while that comment reports a result; a comment that reports a failed run is benchmarked again -on the next push. Pull requests from forks have a read-only token, so they get the job summary only -and benchmark every push. +the hidden marker ``. The comment shows the commit it measured. Later pushes +skip the benchmark, and the job summary says so, while the comment reports a result; a comment that +reports a failed run is benchmarked again on the next push. The [full benchmark](#full-benchmark) +keeps a separate comment, tagged ``, so each workflow writes only its own +comment. The lightweight comment is headed `## CLI benchmark (Lightweight Commands): ` and the +full one `## CLI benchmark (Integration Heavy Commands): <title>`; comments posted under the +earlier `## CLI benchmark: <title>` heading are still recognized and take the new heading on their next update. Pull requests from forks have a read-only token, so they get the job summary only and +benchmark every push. A maintainer can comment exactly `/bench` to run the benchmark again on the pull request's current head commit. The command re-runs that commit's `pull_request` workflow run, as "Re-run all jobs" in the Actions UI does, and the re-run updates the comment in place, or creates it if it is missing. -The comment keeps a collapsed "Previous runs" list below the report with the commit, time (UTC), -headline, and link of each earlier run, newest first and up to 20, stored in a hidden marker in the -comment. Only the repository owner and people with write access can run it, and only on open, -non-draft pull requests into `develop` from this repository; the command must be the whole comment, -so `/bench full` and anything else is ignored. An eyes reaction on the comment means GitHub accepted +Each comment keeps a collapsed "Previous runs" list with the commit, time (UTC), headline, and link +of each result it replaced, newest first and up to 20, stored in a hidden marker in the comment. +Each comment ends with a note for its own benchmark. The lightweight comment says: "Comment `/bench` +to run the lightweight benchmark (this comment) again on the latest commit, or `/bench full` to +also run the integration-heavy benchmark, which posts in its own comment." The integration-heavy +comment says: "Comment `/bench full` to run this benchmark again on the latest commit, which also +refreshes the lightweight comment if it is behind. `/bench` runs only the lightweight benchmark." +The writers of one comment take turns through a concurrency group, and a comment that outgrows +GitHub's size limit drops its oldest previous runs first and then cuts its report. Only the +repository owner and people with write access can run either command, and only on open, non-draft +pull requests into `develop` from this repository; the command must be the whole comment, so +anything other than `/bench` or `/bench full` is ignored. An eyes reaction on the comment means GitHub accepted the re-run and a benchmark is running; the result arrives as the updated benchmark comment, whether it succeeded or failed. When no re-run can start, the pull request gets a reply saying why: a benchmark is already running for the commit, the run is older than 30 days, GitHub refused the @@ -117,8 +129,8 @@ Known gap: a pull request whose changes match none of the workflow's path filter more files than GitHub's path filter inspects (300), gets no run, so there is nothing for `/bench` to re-run and it replies that there is none. -Workflows triggered by comments run from the default branch, so `/bench` works only once the -workflow is on `develop`. The `/bench` job runs default branch code only: it never checks out or +Workflows triggered by comments run from the default branch, so `/bench` and `/bench full` work only +once the workflows are on `develop`. The `/bench` job runs default branch code only: it never checks out or runs pull request code, because a comment-triggered run shares the default branch's cache scope. Benchmarks always run in the `pull_request` run. `resolve` (read-only) decides whether a push benchmarks; `rerun` is the only job with `actions: write`. `bench` has read-only permissions, and @@ -220,16 +232,116 @@ Caveats: - `device_id` counts machines, not people, short-lived CI runners inflate it, and a large share of runs come from agents. Telemetry durations include network, Docker, and wait time, so they guided the choice but are not what the benchmark measures. -- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, and `test db` - need Docker; `db push`, `db lint`, `db dump`, `db diff`, `migration list`, `migration up`, - `migration repair`, and `db schema declarative sync` need a database (a nightly tier); - `link` probes `https://<ref>.<project_host>`, which never reaches the replay server; +- Excluded high-usage commands: `start`, `status`, `stop`, `db start`, `db reset`, `test db`, + `db push`, `db lint`, `db dump`, `db diff`, `migration list`, and `migration up` need Docker or a + database and run in the [full benchmark](#full-benchmark) instead; `migration repair` and + `db schema declarative sync` are not benchmarked; `link` probes `https://<ref>.<project_host>`, which never reaches the replay server; `functions deploy` uses Docker by default and its API path has no success scenario; `login` is interactive unless a token is passed; `functions serve` needs Docker. - The first launch of a process is slower (credential loading); the warm-up runs absorb it. - Launches write a profile file, so numbers from runs with `--api-url` are not comparable with runs without it. +### Full benchmark + +`.github/workflows/cli-bench-full.yml` measures the workloads that need a running local stack or +database: `start`, `status`, `stop`, `db start`, `db reset`, `db query`, `test db`, `gen types` at +three schema sizes, `db diff` (cold and warm), `db dump`, `db lint`, `migration up`, `db push --local`, +and `migration list`, plus the `stack start|status|stop` lifecycle and `db query` and `db reset` +against the stack backend. Like the pull request benchmark it is report only and never blocks a +merge. It never runs on a push; it starts in three ways: + +- A maintainer comments exactly `/bench full` on an open, non-draft pull request into `develop` from + this repository (same permission rules as `/bench`). The command re-measures the integration-heavy + benchmark by dispatching the workflow on the pull request's branch. It also refreshes the + lightweight comment, by re-running that commit's `pull_request` run, only when that comment + does not already show the head commit and no lightweight run is queued or running; with no run to + re-run, the dispatch measures those workloads itself. While an integration-heavy run for the pull + request is queued or running, the command replies with that run's link and does nothing else. The + pull request also gets a reply when the branch has no full benchmark workflow file yet (merge `develop` + into it), when the pull request is a draft, from a fork, or not into `develop`, and when GitHub + refuses the dispatch (with the status and message). An eyes reaction means the dispatch was + accepted. +- From the Actions UI, on any branch, with inputs `workloads` (`all`, `lightweight`, or + `integration-heavy`), `runtime` (`both`, `docker`, or `native`), and `base` (`merge-base`, the + default, or a release tag such as `v3.0.0`). The job summary always has the report. The pull + request comment is updated too only when `base` is `merge-base` and the branch has exactly one + open pull request into `develop` from this repository. +- Weekly, on a schedule (Sunday 03:00 UTC) on `develop`, against the latest stable release resolved + at run time, so each report shows what changed since the last release. The report goes to the job + summary and the run's artifacts; nobody is notified. Slack notifications come later, once the data + proves valid and useful. To compare two releases by hand, dispatch the workflow on the newer tag + with `base` set to the older one. + +A dispatch run uses the workflow file and code of the branch it was started on. GitHub registers +the workflow for dispatch, schedule, and comment-triggered runs only from the default branch, so +none of these can start before the workflow is on `develop`. + +The workflow plans, builds, measures, and reports in separate jobs. `plan` resolves the base, packs +the workloads into shards of roughly ten minutes each from each workload's `estimatedSeconds`, and +writes "running" with the start time and a link to the run into the integration-heavy comment, creating it if it is missing. `build` builds the +branch binary once, and the base binary from the merge base or the release asset. Each shard is one +runner per runtime; it alternates base and branch on that machine and runs no other workload at the +same time. `report` merges the shard reports into the job summary and the integration-heavy comment. A failed +run shows the failure and a link to the run in that comment. + +Runtimes are Docker (default-mode commands and `--runtime docker`) and the native backend +(`--runtime native`), both on Linux x64. Cold downloads are not measured: images and native +archives are fetched before the measured launches, and each shard shares one native artifact cache +across all launches through `--artifact-cache`, which links it in as `cache/stack` of every launch's +fresh `SUPABASE_HOME`. + +The workload manifest `apps/cli/bench/workloads.full.json` adds these fields to the entries +described above: + +| Field | Meaning | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| `setup`, `teardown` | Lists of CLI arguments run untimed with the build under test. `teardown` always runs, even after a failure or timeout. | +| `setupScope` | `run` (default) repeats setup and teardown for every launch; `block` starts one stack per block of five launches. | +| `stack` | Sets `SUPABASE_EXPERIMENTAL_STACK=1` on every launch of the workload, the only variable a workload can set. | +| `runtime` | `docker` or `native`. A stack workload must pass the same `--runtime` to every `stack start`. | +| `fixture` | A directory under `apps/cli/bench/fixtures/` copied into the project's `supabase/` directory after `init`. | +| `fixtureAfterSetup` | Like `fixture`, but copied after `setup`, so the launch finds changes that setup has not applied. Not for `block`. | +| `runs`, `warmup`, `traceRuns` | Per-workload overrides; `--runs` and `--warmup` on the command line win. `traceRuns` is the traced launches per build. | +| `timeoutSeconds`, `estimatedSeconds` | Launch timeout (default 60) and the seconds one pass over both builds takes, which sizes the shards. | + +Setup, the measured launches, and teardown of one run (or block) share one project directory and +`SUPABASE_HOME`; every launch still gets its own telemetry and trace settings. Read-only workloads +use `block` scope; workloads that change state use `run`. Only one stack is ever up at a time per +runner. Stack mode supports only pg-delta for `db diff`, so the `db diff` workloads pass +`--use-pg-delta` in both modes. A workload that fails on the base build, such as one using a flag +the base does not have, shows as a base failure and is not compared. A failing `setup` step skips +the launches of that run and records its exit code for each, so the workload is flagged instead of +measuring a half-prepared run. + +The Integration Heavy Commands report adds a container operations table: the number of container +CLI calls per run for each verb, such as `container inspect`, flagged when a count changes, appears, +or drops to zero. Container work in the stack backend runs outside the CLI's trace, so stack +workloads show no container counts. The fixtures under `apps/cli/bench/fixtures/` are generated by +`bun apps/cli/scripts/bench-fixtures.ts` and committed; changing one starts a new series of results. + +Each shard also starts the replay server, as the pull request benchmark does, and passes it with +`--api-url`, so the report counts Management API calls for every command. These commands have no +recorded scenario and are expected to make no API calls, so the expected count is zero and the +report says "none, before or after"; a call that first appears on the branch is flagged, and a call +that gets an error response from the empty server is flagged as unexpected. Traffic to hosts other +than the replay server, such as a local stack's readiness checks, is not an API call and does not +raise a flag. A shard whose replay server fails to start fails, and the report notes that its +commands are missing. + +To run it locally, build two binaries as above and run a subset: + +```sh +bun apps/cli/scripts/bench-cli.ts --base <base-binary> --branch <branch-binary> \ + --workloads apps/cli/bench/workloads.full.json --runtime docker --only status,db-query \ + --api-url <replay-server-url> --trace --out /tmp/report.json +bun apps/cli/scripts/bench-summary.ts --section integration-heavy --report /tmp/report.json +``` + +Start the replay server with `bun apps/cli-e2e/scripts/replay-serve.ts` for the URL, or drop `--api-url` to skip API counts. Use `--runtime native` with `--artifact-cache <dir>` for the native workloads. Refresh the list from +the same usage query as the pull request benchmark, keeping the commands that need a stack or +database instead of dropping them. + ## Grafana, Tempo, or Jaeger Run a local OpenTelemetry stack, for example Grafana's all-in-one image: diff --git a/apps/cli/scripts/bench-cli.integration.test.ts b/apps/cli/scripts/bench-cli.integration.test.ts index c3a1ebb411..54460239e5 100644 --- a/apps/cli/scripts/bench-cli.integration.test.ts +++ b/apps/cli/scripts/bench-cli.integration.test.ts @@ -5,13 +5,37 @@ import { FetchHttpClient } from "effect/http"; import { startReplayServer } from "../../cli-e2e/src/server/replay-server.ts"; import { LAUNCH_TIMEOUT_EXIT_CODE, + prepareArtifacts, runLaunch, runWorkloadBench, type BenchOptions, } from "./bench-cli.ts"; +import { analyzeWorkload, collectFlags, type Workload } from "./bench-report.ts"; const fixturesDir = new URL("../../cli-e2e/fixtures", import.meta.url).pathname; +const baseOptions = (binary: string, overrides: Partial<BenchOptions> = {}): BenchOptions => ({ + base: binary, + branch: binary, + runs: 2, + warmup: 1, + runsExplicit: false, + warmupExplicit: false, + commands: [], + workloadsFile: undefined, + only: undefined, + runtime: undefined, + artifactCache: undefined, + cwdSetupInit: false, + apiUrl: undefined, + trace: false, + json: false, + out: undefined, + cpuProf: false, + updateCheck: false, + ...overrides, +}); + // Stands in for a CLI build: every launch makes one request to the profile's API and, when asked // to trace, records it as an HTTP client span. const FAKE_CLI = `#!/bin/sh @@ -23,6 +47,76 @@ if [ -n "$SUPABASE_TRACE_FILE" ]; then fi `; +// Stands in for a stack-aware CLI build. It logs every call with the launch environment, hangs on +// `slow`, and reports one container call per launch when traced. +const LOGGING_CLI = `#!/bin/sh +LOG="$(dirname "$0")/calls.log" +echo "$(basename "$0") $* stack=\${SUPABASE_EXPERIMENTAL_STACK:-} dnt=\${DO_NOT_TRACK:-} tel=\${SUPABASE_TELEMETRY_DISABLED:-} home=\${SUPABASE_HOME:-} cwd=$PWD" >> "$LOG" +case "$1" in + slow) exec /bin/sleep 30 ;; + fail) exit 3 ;; + boot) if [ -f supabase/seed.sql ]; then echo "fixture present at setup" >> "$LOG"; fi ;; + check) if [ -f supabase/seed.sql ]; then echo "fixture present" >> "$LOG"; fi ;; + cache) if [ -L "$SUPABASE_HOME/cache/stack" ]; then echo "cache linked to $(readlink "$SUPABASE_HOME/cache/stack")" >> "$LOG"; fi ;; +esac +if [ -n "$SUPABASE_TRACE_FILE" ]; then + printf '{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"t","spanId":"s","name":"ContainerCli.spawn","startTimeUnixNano":"0","endTimeUnixNano":"1000000","status":{"code":1},"attributes":[{"key":"process.subcommand","value":{"stringValue":"container inspect"}}]}]}]}]}\\n' >> "$SUPABASE_TRACE_FILE" +fi +exit 0 +`; + +interface Call { + readonly build: string; + readonly command: string; + readonly stack: string; + readonly dnt: string; + readonly tel: string; + readonly home: string; + readonly cwd: string; +} + +const parseCalls = (log: string): ReadonlyArray<Call> => + log.split("\n").flatMap((line) => { + const match = /^(\S+) (.*) stack=(\S*) dnt=(\S*) tel=(\S*) home=(\S*) cwd=(\S*)$/u.exec(line); + return match === null + ? [] + : [ + { + build: match[1]!, + command: match[2]!, + stack: match[3]!, + dnt: match[4]!, + tel: match[5]!, + home: match[6]!, + cwd: match[7]!, + }, + ]; + }); + +/** Two stand-in builds sharing one call log, so the order of launches across builds is visible. */ +const makeBuilds = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-logging-cli-" }); + for (const name of ["base", "branch"]) { + yield* fs.writeFileString(`${dir}/${name}`, LOGGING_CLI); + yield* fs.chmod(`${dir}/${name}`, 0o755); + } + const calls = Effect.map( + fs.readFileString(`${dir}/calls.log`).pipe(Effect.orElseSucceed(() => "")), + (text) => text, + ); + return { dir, base: `${dir}/base`, branch: `${dir}/branch`, calls }; +}); + +const stackWorkload = (overrides: Partial<Workload> = {}): Workload => ({ + id: "w", + args: ["work"], + cwdSetup: "none", + stack: true, + runtime: "docker", + ...overrides, +}); + describe("runWorkloadBench", () => { it.effect("counts only the measured launch's API calls, not setup or earlier launches", () => Effect.scoped( @@ -37,24 +131,9 @@ describe("runWorkloadBench", () => { const binary = `${dir}/supabase`; yield* fs.writeFileString(binary, FAKE_CLI); yield* fs.chmod(binary, 0o755); - const options: BenchOptions = { - base: binary, - branch: binary, - runs: 2, - warmup: 1, - commands: [], - workloadsFile: undefined, - cwdSetupInit: false, - apiUrl: server.url, - trace: true, - json: false, - out: undefined, - cpuProf: false, - updateCheck: false, - }; const { report } = yield* runWorkloadBench( - options, + baseOptions(binary, { apiUrl: server.url, trace: true }), { id: "w", args: ["projects", "list"], @@ -75,6 +154,255 @@ describe("runWorkloadBench", () => { }), ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), ); + + it.effect( + "runs setup, the launch and teardown in one directory and home per run-scoped unit", + () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 1 }), + stackWorkload({ + setup: [["boot"]], + teardown: [["halt"]], + }), + yield* Ref.make(false), + ); + + const calls = parseCalls(yield* builds.calls); + expect(report.base).toHaveLength(1); + expect(report.branch).toHaveLength(1); + // Warmup and measured iteration, base and branch each: four units of three calls. + expect(calls.map((call) => call.command)).toEqual( + Array.from({ length: 4 }, () => ["boot", "work", "halt"]).flat(), + ); + for (let unit = 0; unit < 4; unit++) { + const [setup, launch, teardown] = calls.slice(unit * 3, unit * 3 + 3); + expect(launch?.home).toBe(setup?.home); + expect(teardown?.home).toBe(setup?.home); + expect(launch?.cwd).toBe(setup?.cwd); + } + expect(new Set(calls.map((call) => call.home)).size).toBe(4); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("starts one stack per block of launches and alternates the builds between blocks", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 6, warmup: 1 }), + stackWorkload({ setupScope: "block", setup: [["boot"]], teardown: [["halt"]] }), + yield* Ref.make(false), + ); + + const calls = parseCalls(yield* builds.calls); + expect(report.base).toHaveLength(6); + expect(report.branch).toHaveLength(6); + const boots = calls.filter((call) => call.command === "boot"); + expect(boots.map((call) => call.build)).toEqual(["base", "branch", "branch", "base"]); + expect(calls.filter((call) => call.command === "halt")).toHaveLength(4); + // One warmup plus five measured launches share the first block's stack. + const firstBlock = calls.filter((call) => call.home === boots[0]?.home); + expect(firstBlock.map((call) => call.command)).toEqual([ + "boot", + ...Array.from({ length: 6 }, () => "work"), + "halt", + ]); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("opens stack mode only for stack workloads and never lets telemetry through", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + const run = (workload: Workload) => + runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 0 }), + workload, + Ref.make(false).pipe(Effect.runSync), + ); + + yield* run(stackWorkload({ id: "stack", args: ["on"], setup: [["boot"]] })); + yield* run({ id: "plain", args: ["off"], cwdSetup: "none" }); + + const calls = parseCalls(yield* builds.calls); + for (const call of calls) { + expect(call.dnt).toBe("1"); + expect(call.tel).toBe("1"); + } + expect( + calls + .filter((call) => ["boot", "on"].includes(call.command)) + .every((call) => call.stack === "1"), + ).toBe(true); + expect( + calls.filter((call) => call.command === "off").every((call) => call.stack === ""), + ).toBe(true); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.live("runs teardown after a launch that outlives its timeout and records the timeout", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 1, warmup: 0 }), + stackWorkload({ args: ["slow"], teardown: [["halt"]], timeoutSeconds: 1 }), + yield* Ref.make(false), + ); + + const commands = parseCalls(yield* builds.calls).map((call) => call.command); + expect(commands).toEqual(["slow", "halt", "slow", "halt"]); + expect(report.base[0]?.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + expect(report.branch[0]?.exitCode).toBe(LAUNCH_TIMEOUT_EXIT_CODE); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect( + "copies the workload's fixture into the project and links the shared artifact cache into every home", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const builds = yield* makeBuilds; + const manifestDir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-manifest-" }); + yield* fs.makeDirectory(`${manifestDir}/fixtures/seeded`, { recursive: true }); + yield* fs.writeFileString(`${manifestDir}/fixtures/seeded/seed.sql`, "select 1;"); + const cache = yield* fs.makeTempDirectoryScoped({ prefix: "bench-cache-" }); + + yield* runWorkloadBench( + baseOptions(builds.base, { + branch: builds.branch, + runs: 1, + warmup: 0, + workloadsFile: `${manifestDir}/workloads.json`, + artifactCache: cache, + }), + stackWorkload({ args: ["check"], setup: [["cache"]], fixture: "seeded" }), + yield* Ref.make(false), + ); + + const log = yield* builds.calls; + expect(log.match(/fixture present/gu)).toHaveLength(2); + expect(log.match(new RegExp(`cache linked to ${cache}`, "gu"))).toHaveLength(2); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("copies fixtureAfterSetup only once setup has finished", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const builds = yield* makeBuilds; + const manifestDir = yield* fs.makeTempDirectoryScoped({ prefix: "bench-manifest-" }); + yield* fs.makeDirectory(`${manifestDir}/fixtures/seeded`, { recursive: true }); + yield* fs.writeFileString(`${manifestDir}/fixtures/seeded/seed.sql`, "select 1;"); + + yield* runWorkloadBench( + baseOptions(builds.base, { + branch: builds.branch, + runs: 1, + warmup: 0, + workloadsFile: `${manifestDir}/workloads.json`, + }), + stackWorkload({ args: ["check"], setup: [["boot"]], fixtureAfterSetup: "seeded" }), + yield* Ref.make(false), + ); + + const log = yield* builds.calls; + expect(log).not.toContain("fixture present at setup"); + expect(log.match(/fixture present/gu)).toHaveLength(2); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect("skips the launches and records a failing exit code when setup fails", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 2, warmup: 0 }), + stackWorkload({ setup: [["fail"], ["boot"]], teardown: [["halt"]] }), + yield* Ref.make(false), + ); + + const commands = parseCalls(yield* builds.calls).map((call) => call.command); + expect(commands).not.toContain("work"); + expect(commands).not.toContain("boot"); + expect(commands.filter((command) => command === "halt")).toHaveLength(4); + for (const samples of [report.base, report.branch]) { + expect(samples.map((sample) => sample.exitCode)).toEqual([3, 3]); + } + expect(collectFlags(analyzeWorkload(report, false)).map((flag) => flag.kind)).toContain( + "non-zero-exit", + ); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); + + it.effect( + "traces only the workload's traceRuns launches per build and reports container calls from them", + () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + const { report } = yield* runWorkloadBench( + baseOptions(builds.base, { branch: builds.branch, runs: 3, warmup: 0, trace: true }), + { id: "w", args: ["work"], cwdSetup: "none", traceRuns: 1 }, + yield* Ref.make(false), + ); + + for (const samples of [report.base, report.branch]) { + expect(samples.map((sample) => sample.trace !== undefined)).toEqual([ + true, + false, + false, + ]); + } + const analysis = analyzeWorkload(report, true); + expect(analysis.base.containers).toEqual({ "container inspect": 1 }); + expect(collectFlags(analysis).map((flag) => flag.kind)).not.toContain("trace-incomplete"); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); +}); + +describe("prepareArtifacts", () => { + it.effect("installs each stack runtime once per build and skips default-mode workloads", () => + Effect.scoped( + Effect.gen(function* () { + const builds = yield* makeBuilds; + + yield* prepareArtifacts(baseOptions(builds.base, { branch: builds.branch }), [ + stackWorkload({ id: "a", runtime: "native" }), + stackWorkload({ id: "b", runtime: "native" }), + stackWorkload({ id: "c", runtime: "docker" }), + { id: "plain", args: ["start"], cwdSetup: "none", runtime: "docker" }, + ]); + + const installs = parseCalls(yield* builds.calls) + .filter((call) => call.command !== "init") + .map((call) => `${call.build}: ${call.command}`); + expect(installs).toEqual([ + "base: stack install --runtime docker", + "base: stack install --runtime native", + "branch: stack install --runtime docker", + "branch: stack install --runtime native", + ]); + }), + ).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer))), + ); }); describe("runLaunch", () => { diff --git a/apps/cli/scripts/bench-cli.ts b/apps/cli/scripts/bench-cli.ts index 5699c7aba8..f31142ad14 100644 --- a/apps/cli/scripts/bench-cli.ts +++ b/apps/cli/scripts/bench-cli.ts @@ -45,10 +45,13 @@ import { readRequestLogCount, resetReplay } from "./replay-control.ts"; import { readSpans } from "./trace-report.ts"; const USAGE = - 'usage: bun scripts/bench-cli.ts --base <path> --branch <path> [--runs N] [--warmup N] [--command "<args>"]... [--workloads <file.json>] [--api-url <url>] [--cwd-setup init] [--trace] [--json] [--out <file>] [--cpu-prof] [--update-check]\n' + + 'usage: bun scripts/bench-cli.ts --base <path> --branch <path> [--runs N] [--warmup N] [--command "<args>"]... [--workloads <file.json>] [--only <ids>] [--runtime docker|native] [--artifact-cache <dir>] [--api-url <url>] [--cwd-setup init] [--trace] [--json] [--out <file>] [--cpu-prof] [--update-check]\n' + " --command splits on whitespace; it does not support shell quoting. Use --workloads for quoted arguments.\n" + " --workloads reads a manifest of workloads and cannot be combined with --command or --cwd-setup.\n" + " --api-url points every launch at a replay server through a profile file, and counts Management API calls from the traced launches; it requires --trace.\n" + + " --only keeps the listed workload ids (comma separated); --runtime keeps workloads for that runtime, treating workloads with no runtime as docker.\n" + + " --artifact-cache links one directory in as the stack artifact cache of every launch's home, so native archives download once.\n" + + " --runs and --warmup, when given, override the per-workload values in a manifest.\n" + " --update-check keeps the GitHub release check, which otherwise adds network time to every launch."; export interface BenchOptions { @@ -56,8 +59,14 @@ export interface BenchOptions { readonly branch: string; readonly runs: number; readonly warmup: number; + /** Whether `--runs` was passed, which then overrides each workload's own `runs`. */ + readonly runsExplicit: boolean; + readonly warmupExplicit: boolean; readonly commands: ReadonlyArray<ReadonlyArray<string>>; readonly workloadsFile: string | undefined; + readonly only: ReadonlyArray<string> | undefined; + readonly runtime: "docker" | "native" | undefined; + readonly artifactCache: string | undefined; readonly cwdSetupInit: boolean; readonly apiUrl: string | undefined; readonly trace: boolean; @@ -67,6 +76,11 @@ export interface BenchOptions { readonly updateCheck: boolean; } +const DEFAULT_RUNS = 10; +const DEFAULT_WARMUP = 2; +/** The one variable a `stack: true` workload sets on its launches; it opens stack mode. */ +const STACK_FLAG = "SUPABASE_EXPERIMENTAL_STACK"; + const DEFAULT_COMMANDS: ReadonlyArray<ReadonlyArray<string>> = [["--version"], ["--help"]]; /** Dummy token for launches pointed at a replay server; it never reaches a real API. */ @@ -98,10 +112,13 @@ export function parseBenchArgs(argv: ReadonlyArray<string>): BenchOptions { options: { base: { type: "string" }, branch: { type: "string" }, - runs: { type: "string", default: "10" }, - warmup: { type: "string", default: "2" }, + runs: { type: "string" }, + warmup: { type: "string" }, command: { type: "string", multiple: true }, workloads: { type: "string" }, + only: { type: "string" }, + runtime: { type: "string" }, + "artifact-cache": { type: "string" }, "api-url": { type: "string" }, "cwd-setup": { type: "string" }, trace: { type: "boolean", default: false }, @@ -115,14 +132,17 @@ export function parseBenchArgs(argv: ReadonlyArray<string>): BenchOptions { if (values.base === undefined || values.branch === undefined) { throw new Error(USAGE); } - const runs = Number(values.runs); - const warmup = Number(values.warmup); + const runs = Number(values.runs ?? DEFAULT_RUNS); + const warmup = Number(values.warmup ?? DEFAULT_WARMUP); if (!Number.isInteger(runs) || runs < 1) { throw new Error("--runs must be a positive integer"); } if (!Number.isInteger(warmup) || warmup < 0) { throw new Error("--warmup must be a non-negative integer"); } + if (values.runtime !== undefined && values.runtime !== "docker" && values.runtime !== "native") { + throw new Error('--runtime must be "docker" or "native"'); + } if (values["cwd-setup"] !== undefined && values["cwd-setup"] !== "init") { throw new Error('--cwd-setup only supports "init"'); } @@ -147,10 +167,19 @@ export function parseBenchArgs(argv: ReadonlyArray<string>): BenchOptions { branch: hostPath.resolve(values.branch), runs, warmup, + runsExplicit: values.runs !== undefined, + warmupExplicit: values.warmup !== undefined, commands: hasCommands ? values.command!.map((raw) => raw.split(/\s+/u).filter((part) => part.length > 0)) : DEFAULT_COMMANDS, workloadsFile: values.workloads, + only: + values.only === undefined ? undefined : values.only.split(",").filter((id) => id.length > 0), + runtime: values.runtime, + artifactCache: + values["artifact-cache"] === undefined + ? undefined + : hostPath.resolve(values["artifact-cache"]), cwdSetupInit: values["cwd-setup"] === "init", apiUrl: values["api-url"], trace: values.trace ?? false, @@ -221,9 +250,20 @@ export const resolveWorkloads = Effect.fn("BenchCli.resolveWorkloads")(function* const fs = yield* FileSystem.FileSystem; return yield* parseWorkloadManifest(yield* fs.readFileString(options.workloadsFile!)); }); - const problems = validateWorkloads(workloads, { apiUrl: options.apiUrl }); + const problems = [...validateWorkloads(workloads, { apiUrl: options.apiUrl })]; + if (options.only !== undefined) { + for (const id of options.only) { + if (!workloads.some((workload) => workload.id === id)) { + problems.push(`--only names unknown workload "${id}"`); + } + } + } if (problems.length > 0) return yield* new WorkloadValidationError({ problems }); - return workloads; + return workloads.filter( + (workload) => + (options.only === undefined || options.only.includes(workload.id)) && + (options.runtime === undefined || (workload.runtime ?? "docker") === options.runtime), + ); }); interface LaunchResult { @@ -232,7 +272,8 @@ interface LaunchResult { } /** Longest a single launch may run before it is killed and counted as a failed launch. */ -export const LAUNCH_TIMEOUT = Duration.seconds(60); +export const LAUNCH_TIMEOUT_SECONDS = 60; +export const LAUNCH_TIMEOUT = Duration.seconds(LAUNCH_TIMEOUT_SECONDS); /** Exit code recorded for a launch that hit {@link LAUNCH_TIMEOUT}. */ export const LAUNCH_TIMEOUT_EXIT_CODE = 124; @@ -282,29 +323,52 @@ export function profileYaml(apiUrl: string): string { ].join("\n"); } +/** Variables set on every launch: no product telemetry, no keyring, and no update check unless asked. */ const baseLaunchEnv = ( options: BenchOptions, + workload: Workload, supabaseHome: string, profilePath: string, ): Record<string, string> => ({ SUPABASE_HOME: supabaseHome, DO_NOT_TRACK: "1", + SUPABASE_TELEMETRY_DISABLED: "1", SUPABASE_NO_KEYRING: "1", ...(options.updateCheck ? {} : { SUPABASE_NO_UPDATE_NOTIFIER: "1" }), + ...(workload.stack === true ? { [STACK_FLAG]: "1" } : {}), ...(options.apiUrl === undefined ? {} - : { - SUPABASE_PROFILE: profilePath, - SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN, - SUPABASE_TELEMETRY_DISABLED: "1", - }), + : { SUPABASE_PROFILE: profilePath, SUPABASE_ACCESS_TOKEN: BENCH_ACCESS_TOKEN }), }); +/** Longest an untimed setup, teardown, or artifact step may run. */ +export const STEP_TIMEOUT = Duration.minutes(5); + +export interface Unit { + readonly projectDir: string; + readonly env: Record<string, string>; +} + +/** Copies a fixture from the manifest's `fixtures/` directory into the project's `supabase/` directory. */ +const copyFixture = (options: BenchOptions, fixture: string, projectDir: string) => + Effect.gen(function* () { + if (options.workloadsFile === undefined) return; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // The fixtures sit outside any `supabase/` directory, which the repository ignores. + const target = path.join(projectDir, "supabase"); + yield* fs.makeDirectory(target, { recursive: true }); + yield* fs.copy(path.join(path.dirname(options.workloadsFile), "fixtures", fixture), target, { + overwrite: true, + }); + }); + /** - * A fresh project directory and `SUPABASE_HOME`, with the replay profile written and the - * workload's `cwdSetup` applied. + * A project directory and `SUPABASE_HOME` shared by one unit's setup, launches and teardown, with + * the replay profile written, the workload's `cwdSetup` applied, its fixture copied in, and the + * shared artifact cache linked into the home. */ -const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string) => +const prepareUnit = (options: BenchOptions, workload: Workload, binary: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -314,7 +378,12 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string if (options.apiUrl !== undefined) { yield* fs.writeFileString(profilePath, profileYaml(options.apiUrl)); } - const env = baseLaunchEnv(options, supabaseHome, profilePath); + if (options.artifactCache !== undefined) { + yield* fs.makeDirectory(options.artifactCache, { recursive: true }); + yield* fs.makeDirectory(path.join(supabaseHome, "cache"), { recursive: true }); + yield* fs.symlink(options.artifactCache, path.join(supabaseHome, "cache", "stack")); + } + const env = baseLaunchEnv(options, workload, supabaseHome, profilePath); if (workload.cwdSetup !== "none") { const init = yield* runLaunch({ binary, @@ -328,115 +397,341 @@ const prepareLaunch = (options: BenchOptions, workload: Workload, binary: string ); } } - return { projectDir, env }; + if (workload.fixture !== undefined) { + yield* copyFixture(options, workload.fixture, projectDir); + } + return { projectDir, env } satisfies Unit; }); -export const runWorkloadBench = ( +const runStep = Effect.fn("BenchCli.runStep")(function* ( + kind: "setup" | "teardown", + binary: string, + unit: Unit, + args: ReadonlyArray<string>, +) { + const result = yield* runLaunch({ + binary, + args, + cwd: unit.projectDir, + env: isolatedEnv(process.env, unit.env), + timeout: STEP_TIMEOUT, + }); + yield* Effect.annotateCurrentSpan({ "bench.step": kind, "bench.exit_code": result.exitCode }); + if (result.exitCode !== 0) { + yield* Console.error( + `warning: ${kind} step "${args.join(" ")}" exited ${result.exitCode}; the workload may not reflect a real stack`, + ); + } + return result.exitCode; +}); + +/** Pulls images or downloads archives for each stack runtime, so no measured launch downloads anything. */ +export const prepareArtifacts = Effect.fn("BenchCli.prepareArtifacts")(function* ( options: BenchOptions, - workload: Workload, - profiled: Ref.Ref<boolean>, -) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const plan = buildRunPlan(options.runs, options.warmup); + workloads: ReadonlyArray<Workload>, +) { + const runtimes = [ + ...new Set( + workloads.flatMap((workload) => + workload.stack === true && workload.runtime !== undefined ? [workload.runtime] : [], + ), + ), + ].sort(); + const scratch: Workload = { id: "install", args: [], cwdSetup: "init", stack: true }; + for (const binary of [options.base, options.branch]) { + for (const runtime of runtimes) { + yield* Effect.scoped( + Effect.gen(function* () { + const unit = yield* prepareUnit(options, scratch, binary); + yield* runStep("setup", binary, unit, ["stack", "install", "--runtime", runtime]); + }), + ); + } + } +}); - const samples: Record<Build, Array<RunSample>> = { base: [], branch: [] }; - let cpuProfilePath: string | undefined; +type LaunchKind = "warmup" | "timed" | "traced"; - // Resets the replay server after setup, so setup's own requests and a previous launch's - // consumed scenario never reach the launch that follows. - const resetServer = - options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); +interface LaunchOutcome { + readonly kind: LaunchKind; + readonly wallMs: number; + readonly exitCode: number; + readonly trace: RunSample["trace"]; +} - for (const planned of plan) { - const binary = planned.build === "base" ? options.base : options.branch; +/** Planned units of work for one build: a run-scoped unit holds one launch, a block-scoped one many. */ +export interface PlannedUnit { + readonly build: Build; + readonly launches: ReadonlyArray<"warmup" | "timed">; +} - yield* Effect.scoped( - Effect.gen(function* () { - const prepared = yield* prepareLaunch(options, workload, binary); - yield* resetServer; +/** Measured launches per block for `block`-scoped workloads. */ +export const BLOCK_SIZE = 5; + +/** + * Units in execution order. `run` scope reuses {@link buildRunPlan}, one launch per unit. + * `block` scope starts one unit per block of up to `blockSize` measured launches, preceded by + * `warmup` discarded launches, and alternates which build goes first in every other block. + */ +export function buildUnitPlan( + runs: number, + warmup: number, + scope: "run" | "block", + blockSize: number = BLOCK_SIZE, +): ReadonlyArray<PlannedUnit> { + if (scope === "run") { + return buildRunPlan(runs, warmup).map((planned) => ({ + build: planned.build, + launches: [planned.warmup ? "warmup" : "timed"], + })); + } + const units: Array<PlannedUnit> = []; + const blocks = Math.ceil(runs / blockSize); + for (let block = 0; block < blocks; block++) { + const measured = Math.min(blockSize, runs - block * blockSize); + const launches = [ + ...Array.from({ length: warmup }, () => "warmup" as const), + ...Array.from({ length: measured }, () => "timed" as const), + ]; + const order: ReadonlyArray<Build> = block % 2 === 0 ? ["base", "branch"] : ["branch", "base"]; + for (const build of order) units.push({ build, launches }); + } + return units; +} + +/** + * Runs one unit: setup, the launches in order, then teardown. Teardown is a finalizer registered + * before setup, so it runs after a failed step or a timed-out launch too. + */ +const runUnit = Effect.fn("BenchCli.runUnit")(function* (params: { + readonly options: BenchOptions; + readonly workload: Workload; + readonly build: Build; + readonly launches: ReadonlyArray<LaunchKind>; + readonly profiled: Ref.Ref<boolean>; + readonly onProfile: (path: string) => void; +}) { + const { options, workload, build, launches } = params; + const binary = build === "base" ? options.base : options.branch; + return yield* Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const unit = yield* prepareUnit(options, workload, binary); + yield* Effect.addFinalizer(() => + Effect.forEach(workload.teardown ?? [], (args) => runStep("teardown", binary, unit, args), { + discard: true, + }).pipe(Effect.ignore), + ); + let setupExitCode = 0; + for (const args of workload.setup ?? []) { + setupExitCode = yield* runStep("setup", binary, unit, args); + if (setupExitCode !== 0) break; + } + if (setupExitCode === 0 && workload.fixtureAfterSetup !== undefined) { + yield* copyFixture(options, workload.fixtureAfterSetup, unit.projectDir); + } + // A failed setup leaves nothing real to measure, so every launch reports the failing exit code. + if (setupExitCode !== 0) { + return launches.map((kind): LaunchOutcome => ({ + kind, + wallMs: 0, + exitCode: setupExitCode, + trace: undefined, + })); + } + + // Resets the replay server before each launch, so setup's own requests and a previous + // launch's consumed scenario never reach the launch that follows. + const resetServer = + options.apiUrl === undefined ? Effect.void : resetReplay(options.apiUrl, workload.scenario); + const timeout = Duration.seconds(workload.timeoutSeconds ?? LAUNCH_TIMEOUT_SECONDS); + const outcomes: Array<LaunchOutcome> = []; + + for (const kind of launches) { + yield* resetServer; + if (kind !== "traced") { const result = yield* runLaunch({ binary, args: workload.args, - cwd: prepared.projectDir, - env: isolatedEnv(process.env, prepared.env), + cwd: unit.projectDir, + env: isolatedEnv(process.env, unit.env), + timeout, }); - - if (planned.warmup) return; - - // Tracing and profiling slow the launch they observe, so they run in an extra, untimed - // launch in fresh directories. Both builds get traced launches; only one branch launch - // is profiled. - const traceFile = options.trace - ? yield* fs.makeTempFileScoped({ prefix: "supabase-bench-trace-", suffix: ".jsonl" }) - : undefined; - let bunOptions: string | undefined; - if (options.cpuProf && planned.build === "branch") { - const alreadyProfiled = yield* Ref.getAndSet(profiled, true); - if (!alreadyProfiled) { - const profileDir = tmpdir(); - const profileName = `supabase-bench-branch-${randomUUID()}.cpuprofile`; - cpuProfilePath = path.join(profileDir, profileName); - bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; - } + outcomes.push({ kind, ...result, trace: undefined }); + continue; + } + + // Tracing and profiling slow the launch they observe, so they run in extra, untimed launches. + const traceFile = options.trace + ? yield* fs.makeTempFileScoped({ prefix: "supabase-bench-trace-", suffix: ".jsonl" }) + : undefined; + let bunOptions: string | undefined; + if (options.cpuProf && build === "branch") { + const alreadyProfiled = yield* Ref.getAndSet(params.profiled, true); + if (!alreadyProfiled) { + const profileDir = tmpdir(); + const profileName = `supabase-bench-branch-${randomUUID()}.cpuprofile`; + params.onProfile(path.join(profileDir, profileName)); + bunOptions = `--cpu-prof --cpu-prof-dir=${profileDir} --cpu-prof-name=${profileName}`; } - - let trace: RunSample["trace"]; - let tracedExitCode: number | undefined; - if (traceFile !== undefined || bunOptions !== undefined) { - const observed = yield* prepareLaunch(options, workload, binary); - yield* resetServer; - const traced = yield* runLaunch({ - binary, - args: workload.args, - cwd: observed.projectDir, - env: isolatedEnv(process.env, { - ...observed.env, - ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), - ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), - }), - }); - tracedExitCode = traced.exitCode; - if (traceFile !== undefined) { - const requestLogCount = - options.apiUrl === undefined - ? undefined - : yield* readRequestLogCount(options.apiUrl); - // Trace data is optional; a missing or unreadable file must not discard timings. - const decoded = yield* readSpans(traceFile).pipe(Effect.option); - if (Option.isSome(decoded) && decoded.value.spans.length > 0) { - const { spans, skippedLines } = decoded.value; - const http = - options.apiUrl === undefined - ? undefined - : classifyHttpSpans(spans, options.apiUrl); - trace = { - skippedLines, - spans: summarizeSpanRun(spans), - ...(http === undefined || requestLogCount === undefined - ? {} - : { api: { ...http, requestLogCount } }), - }; - } - } + } + const traced = yield* runLaunch({ + binary, + args: workload.args, + cwd: unit.projectDir, + env: isolatedEnv(process.env, { + ...unit.env, + ...(traceFile === undefined ? {} : { SUPABASE_TRACE_FILE: traceFile }), + ...(bunOptions === undefined ? {} : { BUN_OPTIONS: bunOptions }), + }), + timeout, + }); + let trace: RunSample["trace"]; + if (traceFile !== undefined) { + const requestLogCount = + options.apiUrl === undefined ? undefined : yield* readRequestLogCount(options.apiUrl); + // Trace data is optional; a missing or unreadable file must not discard timings. + const decoded = yield* readSpans(traceFile).pipe(Effect.option); + if (Option.isSome(decoded) && decoded.value.spans.length > 0) { + const { spans, skippedLines } = decoded.value; + const http = + options.apiUrl === undefined ? undefined : classifyHttpSpans(spans, options.apiUrl); + trace = { + skippedLines, + spans: summarizeSpanRun(spans), + ...(http === undefined || requestLogCount === undefined + ? {} + : { api: { ...http, requestLogCount } }), + }; } + } + outcomes.push({ kind, ...traced, trace }); + } + return outcomes; + }), + ); +}); - samples[planned.build].push({ - wallMs: result.wallMs, - exitCode: result.exitCode, - ...(tracedExitCode === undefined ? {} : { tracedExitCode }), - ...(trace === undefined ? {} : { trace }), - }); +/** Traced launches still wanted for `build`, given how many it has had. */ +const wantsTrace = (options: BenchOptions, tracedSoFar: number, traceRuns: number) => + options.trace && tracedSoFar < traceRuns; + +export const runWorkloadBench = ( + options: BenchOptions, + workload: Workload, + profiled: Ref.Ref<boolean>, +) => + Effect.gen(function* () { + const runs = options.runsExplicit ? options.runs : (workload.runs ?? options.runs); + const warmup = options.warmupExplicit ? options.warmup : (workload.warmup ?? options.warmup); + const traceRuns = options.trace ? Math.min(workload.traceRuns ?? runs, runs) : 0; + const scope = workload.setupScope ?? "run"; + const plan = buildUnitPlan(runs, warmup, scope); + const started = yield* Clock.currentTimeNanos; + + const timed: Record<Build, Array<{ wallMs: number; exitCode: number }>> = { + base: [], + branch: [], + }; + const traced: Record<Build, Array<{ exitCode: number; trace: RunSample["trace"] }>> = { + base: [], + branch: [], + }; + let cpuProfilePath: string | undefined; + const onProfile = (path: string) => { + cpuProfilePath = path; + }; + + // A launch is traced when tracing wants more samples for the build, or a branch launch is + // still to be profiled; the profile needs no trace file. + const wantsExtra = (build: Build, tracedSoFar: number) => + Effect.map( + Ref.get(profiled), + (alreadyProfiled) => + wantsTrace(options, tracedSoFar, traceRuns) || + (options.cpuProf && build === "branch" && !alreadyProfiled), + ); + + const record = (build: Build, outcomes: ReadonlyArray<LaunchOutcome>) => { + for (const outcome of outcomes) { + if (outcome.kind === "timed") { + timed[build].push({ wallMs: outcome.wallMs, exitCode: outcome.exitCode }); + } else if (outcome.kind === "traced") { + traced[build].push({ exitCode: outcome.exitCode, trace: outcome.trace }); + } + } + }; + + for (const planned of plan) { + const { build } = planned; + if (scope === "run") { + const outcomes = yield* runUnit({ + options, + workload, + build, + launches: planned.launches, + profiled, + onProfile, + }); + record(build, outcomes); + if (planned.launches[0] === "timed" && (yield* wantsExtra(build, traced[build].length))) { + record( + build, + yield* runUnit({ + options, + workload, + build, + launches: ["traced"], + profiled, + onProfile, + }), + ); + } + continue; + } + const measured = planned.launches.filter((kind) => kind === "timed").length; + const extra: Array<"traced"> = []; + let tracedCount = traced[build].length; + for (let index = 0; index < measured; index++) { + if (!(yield* wantsExtra(build, tracedCount))) break; + extra.push("traced"); + // A profile-only launch is wanted once, not per measured launch. + if (!wantsTrace(options, tracedCount, traceRuns)) break; + tracedCount++; + } + record( + build, + yield* runUnit({ + options, + workload, + build, + launches: [...planned.launches, ...extra], + profiled, + onProfile, }), ); } + const sample = (build: Build): Array<RunSample> => + timed[build].map((launch, index) => { + const extra = traced[build][index]; + return { + ...launch, + ...(extra === undefined ? {} : { tracedExitCode: extra.exitCode }), + ...(extra?.trace === undefined ? {} : { trace: extra.trace }), + }; + }); + const finished = yield* Clock.currentTimeNanos; const report: WorkloadReport = { id: workload.id, args: workload.args, - base: samples.base, - branch: samples.branch, + ...(workload.runtime === undefined ? {} : { runtime: workload.runtime }), + ...(workload.stack === undefined ? {} : { stack: workload.stack }), + ...(options.trace ? { traceRuns } : {}), + elapsedSeconds: Math.round(Number(finished - started) / 1_000_000) / 1000, + base: sample("base"), + branch: sample("branch"), }; return { report, cpuProfilePath }; }); @@ -529,6 +824,7 @@ const main = Effect.gen(function* () { if (options === undefined) return; const workloads = yield* resolveWorkloads(options); + yield* prepareArtifacts(options, workloads); const profiled = yield* Ref.make(false); const reports: Array<WorkloadReport> = []; let cpuProfilePath: string | undefined; diff --git a/apps/cli/scripts/bench-cli.unit.test.ts b/apps/cli/scripts/bench-cli.unit.test.ts index 8906f3a380..15f8b80855 100644 --- a/apps/cli/scripts/bench-cli.unit.test.ts +++ b/apps/cli/scripts/bench-cli.unit.test.ts @@ -3,6 +3,7 @@ import { BunServices } from "@effect/platform-bun"; import { Effect } from "effect"; import { buildRunPlan, + buildUnitPlan, isolatedEnv, parseBenchArgs, profileYaml, @@ -108,6 +109,73 @@ describe("buildRunPlan", () => { }); }); +describe("buildUnitPlan", () => { + it("gives a run-scoped workload one launch per unit in the order buildRunPlan chooses", () => { + const plan = buildUnitPlan(2, 1, "run"); + + expect(plan.map((unit) => [unit.build, ...unit.launches])).toEqual([ + ["branch", "warmup"], + ["base", "warmup"], + ["base", "timed"], + ["branch", "timed"], + ["branch", "timed"], + ["base", "timed"], + ]); + }); + + it("splits a block-scoped workload into blocks of measured launches after the warmup launches, swapping the first build every other block", () => { + const plan = buildUnitPlan(7, 1, "block", 5); + + expect(plan.map((unit) => unit.build)).toEqual(["base", "branch", "branch", "base"]); + const measured = (build: "base" | "branch") => + plan + .filter((unit) => unit.build === build) + .map((unit) => unit.launches.filter((kind) => kind === "timed").length); + expect(measured("base")).toEqual([5, 2]); + expect(measured("branch")).toEqual([5, 2]); + expect(plan[0]?.launches).toEqual(["warmup", "timed", "timed", "timed", "timed", "timed"]); + }); +}); + +describe("parseBenchArgs for stack runs", () => { + it("lets explicit --runs and --warmup override the manifest, and defaults otherwise", () => { + expect(parseBenchArgs(required)).toMatchObject({ + runs: 10, + warmup: 2, + runsExplicit: false, + warmupExplicit: false, + }); + expect(parseBenchArgs([...required, "--runs", "3", "--warmup", "0"])).toMatchObject({ + runs: 3, + warmup: 0, + runsExplicit: true, + warmupExplicit: true, + }); + }); + + it("reads --only, --runtime and a resolved --artifact-cache", () => { + const options = parseBenchArgs([ + ...required, + "--only", + "a,b", + "--runtime", + "native", + "--artifact-cache", + "/tmp/cache", + ]); + + expect(options).toMatchObject({ + only: ["a", "b"], + runtime: "native", + artifactCache: "/tmp/cache", + }); + }); + + it("rejects an unknown runtime", () => { + expect(() => parseBenchArgs([...required, "--runtime", "podman"])).toThrow("--runtime"); + }); +}); + describe("profileYaml", () => { it("points the API and dashboard at the replay server and keeps project hosts local", () => { const yaml = profileYaml("http://127.0.0.1:9"); diff --git a/apps/cli/scripts/bench-fixtures.ts b/apps/cli/scripts/bench-fixtures.ts new file mode 100644 index 0000000000..b1797c7393 --- /dev/null +++ b/apps/cli/scripts/bench-fixtures.ts @@ -0,0 +1,130 @@ +/** + * Regenerates the benchmark fixtures under `apps/cli/bench/fixtures/`. The output is deterministic and committed, + * so a series of results keeps measuring the same work; changing it starts a new series. Run as + * `bun apps/cli/scripts/bench-fixtures.ts`. + */ +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Effect, FileSystem, Path } from "effect"; + +/** Tables in each schema fixture. */ +export const SCHEMA_SIZES = { small: 10, medium: 50, large: 200 } as const; +/** Migrations in the `migrations` fixture. */ +export const MIGRATION_COUNT = 20; +/** Rows inserted by the `migrations` fixture's seed. */ +export const SEED_ROWS = 1000; + +const table = (index: number) => `bench_table_${String(index).padStart(3, "0")}`; + +/** One migration creating `count` related tables, indexes, policies, views and functions. */ +export function schemaSql(count: number): string { + const blocks: Array<string> = []; + for (let index = 1; index <= count; index++) { + const name = table(index); + blocks.push( + `create table public.${name} ( + id uuid primary key default gen_random_uuid(), + owner_id uuid, + name text not null, + status text not null default 'active', + payload jsonb not null default '{}'::jsonb, + created_at timestamptz not null default now() +); +create index ${name}_owner_idx on public.${name} (owner_id); +create index ${name}_status_idx on public.${name} (status, created_at); +alter table public.${name} enable row level security; +create policy ${name}_read on public.${name} for select using (true);`, + ); + if (index % 5 === 0) { + blocks.push( + `create view public.${name}_summary as + select status, count(*) as total from public.${name} group by status;`, + ); + } + if (index % 10 === 0) { + blocks.push( + `create function public.${name}_count() returns bigint + language sql stable as $$ select count(*) from public.${name} $$;`, + ); + } + } + return `${blocks.join("\n\n")}\n`; +} + +/** Migration `index` of the `migrations` fixture; the first one owns the seeded table. */ +export function migrationSql(index: number): string { + if (index === 1) { + return `create table public.bench_items ( + id bigint generated always as identity primary key, + label text not null, + amount numeric not null, + created_at timestamptz not null default now() +); +create index bench_items_label_idx on public.bench_items (label); +`; + } + const name = table(index); + return `create table public.${name} ( + id uuid primary key default gen_random_uuid(), + item_id bigint references public.bench_items (id), + note text not null default '' +); +create index ${name}_item_idx on public.${name} (item_id); +`; +} + +/** Seed for the `migrations` fixture. */ +export const seedSql = `insert into public.bench_items (label, amount) +select 'item-' || n, (n % 97) * 1.5 from generate_series(1, ${SEED_ROWS}) as n; +`; + +const stamp = (index: number) => `2026010100${String(index).padStart(4, "0")}`; + +/** The pgTAP fixture: one table and a test file over it. */ +export const pgtapMigration = `create table public.bench_checks ( + id bigint generated always as identity primary key, + label text not null +); +`; + +export const pgtapTest = `begin; +select plan(4); +select has_table('public', 'bench_checks', 'bench_checks exists'); +select has_column('public', 'bench_checks', 'label', 'label column exists'); +select col_not_null('public', 'bench_checks', 'label', 'label is required'); +select col_is_pk('public', 'bench_checks', 'id', 'id is the primary key'); +select * from finish(); +rollback; +`; + +const main = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = path.join( + path.dirname(new URL(import.meta.url).pathname), + "..", + "bench", + "fixtures", + ); + const write = Effect.fnUntraced(function* (relative: string, content: string) { + const file = path.join(root, relative); + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(file, content); + }); + + for (const [size, count] of Object.entries(SCHEMA_SIZES)) { + yield* write(`schema-${size}/migrations/${stamp(1)}_bench_schema.sql`, schemaSql(count)); + } + for (let index = 1; index <= MIGRATION_COUNT; index++) { + yield* write( + `migrations/migrations/${stamp(index)}_bench_${String(index).padStart(2, "0")}.sql`, + migrationSql(index), + ); + } + yield* write("migrations/seed.sql", seedSql); + yield* write(`pgtap/migrations/${stamp(1)}_bench_checks.sql`, pgtapMigration); + yield* write("pgtap/tests/database/bench.test.sql", pgtapTest); +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-merge.ts b/apps/cli/scripts/bench-merge.ts new file mode 100644 index 0000000000..f0659fa0f1 --- /dev/null +++ b/apps/cli/scripts/bench-merge.ts @@ -0,0 +1,76 @@ +/** + * Merges the per-shard `bench-cli.ts --out` reports of one run into a single report, ordering + * workloads as the manifest does. Run as `bun scripts/bench-merge.ts [--workloads <file.json>] + * --out <file> <shard-report.json>...`. Binary paths, run counts and settings come from the first + * shard. + */ +import { parseArgs } from "node:util"; +import { BunRuntime, BunServices } from "@effect/platform-bun"; +import { Effect, FileSystem, Stdio } from "effect"; +import { + decodeBenchReport, + encodeBenchReport, + parseWorkloadManifest, + type BenchReport, +} from "./bench-report.ts"; + +/** Combines shard reports; a workload that appears in several shards keeps its first report. */ +export function mergeReports( + reports: ReadonlyArray<BenchReport>, + order: ReadonlyArray<string>, +): BenchReport | undefined { + const [first] = reports; + if (first === undefined) return undefined; + const byId = new Map<string, BenchReport["workloads"][number]>(); + for (const report of reports) { + for (const workload of report.workloads) { + if (!byId.has(workload.id)) byId.set(workload.id, workload); + } + } + const rank = (id: string) => { + const index = order.indexOf(id); + return index === -1 ? order.length : index; + }; + const cpuProfilePath = reports.find( + (report) => report.cpuProfilePath !== undefined, + )?.cpuProfilePath; + const apiUrl = reports.find((report) => report.apiUrl !== undefined)?.apiUrl; + return { + ...first, + ...(apiUrl === undefined ? {} : { apiUrl }), + workloads: [...byId.values()].sort((a, b) => rank(a.id) - rank(b.id)), + ...(cpuProfilePath === undefined ? {} : { cpuProfilePath }), + }; +} + +const main = Effect.gen(function* () { + const stdio = yield* Stdio.Stdio; + const argv = yield* stdio.args; + const { values, positionals } = parseArgs({ + args: [...argv], + allowPositionals: true, + options: { workloads: { type: "string" }, out: { type: "string" } }, + }); + if (values.out === undefined || positionals.length === 0) { + return yield* Effect.fail( + "usage: bun scripts/bench-merge.ts [--workloads <file.json>] --out <file> <report.json>...", + ); + } + const fs = yield* FileSystem.FileSystem; + const order = + values.workloads === undefined + ? [] + : (yield* parseWorkloadManifest(yield* fs.readFileString(values.workloads))).map( + (workload) => workload.id, + ); + const reports: Array<BenchReport> = []; + for (const file of positionals) { + reports.push(yield* decodeBenchReport(yield* fs.readFileString(file))); + } + const merged = mergeReports(reports, order); + if (merged !== undefined) yield* fs.writeFileString(values.out, yield* encodeBenchReport(merged)); +}); + +if (import.meta.main) { + BunRuntime.runMain(main.pipe(Effect.provide(BunServices.layer))); +} diff --git a/apps/cli/scripts/bench-merge.unit.test.ts b/apps/cli/scripts/bench-merge.unit.test.ts new file mode 100644 index 0000000000..2d7b899744 --- /dev/null +++ b/apps/cli/scripts/bench-merge.unit.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import type { BenchReport, RunSample } from "./bench-report.ts"; +import { mergeReports } from "./bench-merge.ts"; + +const sample = (calls: number): RunSample => ({ + wallMs: 10, + exitCode: 0, + trace: { + skippedLines: 0, + spans: {}, + api: { + apiCalls: calls, + externalCalls: 2, + errorStatuses: 0, + endpoints: {}, + requestLogCount: calls, + }, + }, +}); + +const shard = (apiUrl: string | undefined, ids: ReadonlyArray<string>, calls = 0): BenchReport => ({ + base: "/base", + branch: "/branch", + runs: 1, + warmup: 0, + trace: true, + ...(apiUrl === undefined ? {} : { apiUrl }), + workloads: ids.map((id) => ({ id, args: [id], base: [sample(calls)], branch: [sample(calls)] })), +}); + +describe("mergeReports", () => { + it("keeps each shard's API counts, including a count of zero, in manifest order", () => { + const merged = mergeReports( + [shard("http://127.0.0.1:1", ["status"], 0), shard("http://127.0.0.1:2", ["db-query"], 3)], + ["db-query", "status"], + ); + + expect(merged?.workloads.map((workload) => workload.id)).toEqual(["db-query", "status"]); + expect(merged?.workloads.map((workload) => workload.branch[0]?.trace?.api?.apiCalls)).toEqual([ + 3, 0, + ]); + }); + + it("carries an API URL even when the first shard recorded none", () => { + const merged = mergeReports([shard(undefined, ["a"]), shard("http://127.0.0.1:2", ["b"])], []); + + expect(merged?.apiUrl).toBe("http://127.0.0.1:2"); + }); +}); diff --git a/apps/cli/scripts/bench-pack.ts b/apps/cli/scripts/bench-pack.ts new file mode 100644 index 0000000000..6fab4e2435 --- /dev/null +++ b/apps/cli/scripts/bench-pack.ts @@ -0,0 +1,68 @@ +/** + * Packs benchmark workloads into shards for the full benchmark matrix. Each shard runs on one + * runner, so base and branch of a workload always share a machine and no two workloads share one + * at the same time. Free of runtime dependencies so the workflow's plan job can import it. + */ +import type { Workload } from "./bench-report.ts"; + +export type PackedWorkload = Pick<Workload, "id" | "runtime" | "estimatedSeconds">; + +export interface Shard { + readonly runtime: "docker" | "native"; + /** 1-based position within the runtime's shards. */ + readonly shard: number; + /** Shards for this runtime. */ + readonly shards: number; + readonly workloads: ReadonlyArray<string>; + readonly estimatedSeconds: number; +} + +/** Seconds assumed for a workload that has no estimate. */ +export const DEFAULT_ESTIMATE_SECONDS = 60; +/** Seconds of work each shard aims for. */ +export const DEFAULT_TARGET_SECONDS = 600; +export const DEFAULT_MAX_SHARDS = 8; + +const estimateOf = (workload: PackedWorkload) => + workload.estimatedSeconds ?? DEFAULT_ESTIMATE_SECONDS; + +/** + * Splits each runtime's workloads into `ceil(total / target)` shards (at most `maxShards`), + * assigning the longest workloads first to the lightest shard. Shards come back ordered by runtime + * then index, with each shard's workloads in manifest order. + */ +export function packShards( + workloads: ReadonlyArray<PackedWorkload>, + options: { readonly targetSeconds?: number; readonly maxShards?: number } = {}, +): ReadonlyArray<Shard> { + const target = options.targetSeconds ?? DEFAULT_TARGET_SECONDS; + const maxShards = options.maxShards ?? DEFAULT_MAX_SHARDS; + const shards: Array<Shard> = []; + for (const runtime of ["docker", "native"] as const) { + const group = workloads.filter((workload) => (workload.runtime ?? "docker") === runtime); + if (group.length === 0) continue; + const total = group.reduce((sum, workload) => sum + estimateOf(workload), 0); + const count = Math.min(maxShards, group.length, Math.max(1, Math.ceil(total / target))); + const bins = Array.from({ length: count }, () => ({ ids: new Set<string>(), seconds: 0 })); + const longestFirst = [...group].sort( + (a, b) => estimateOf(b) - estimateOf(a) || a.id.localeCompare(b.id), + ); + for (const workload of longestFirst) { + const lightest = bins.reduce((best, bin) => (bin.seconds < best.seconds ? bin : best)); + lightest.ids.add(workload.id); + lightest.seconds += estimateOf(workload); + } + bins.forEach((bin, index) => { + shards.push({ + runtime, + shard: index + 1, + shards: count, + workloads: group + .filter((workload) => bin.ids.has(workload.id)) + .map((workload) => workload.id), + estimatedSeconds: bin.seconds, + }); + }); + } + return shards; +} diff --git a/apps/cli/scripts/bench-pack.unit.test.ts b/apps/cli/scripts/bench-pack.unit.test.ts new file mode 100644 index 0000000000..7b9cff2e6d --- /dev/null +++ b/apps/cli/scripts/bench-pack.unit.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "@effect/vitest"; +import { packShards } from "./bench-pack.ts"; + +const workload = (id: string, estimatedSeconds: number, runtime?: "docker" | "native") => ({ + id, + estimatedSeconds, + ...(runtime === undefined ? {} : { runtime }), +}); + +describe("packShards", () => { + it("balances each runtime's workloads across ceil(total / target) shards and never mixes runtimes", () => { + const shards = packShards( + [ + workload("a", 400), + workload("b", 300), + workload("c", 300), + workload("d", 200), + workload("n1", 100, "native"), + ], + { targetSeconds: 600 }, + ); + + const docker = shards.filter((shard) => shard.runtime === "docker"); + expect(docker.map((shard) => shard.estimatedSeconds)).toEqual([600, 600]); + expect(docker.every((shard) => shard.shards === 2)).toBe(true); + expect(shards.filter((shard) => shard.runtime === "native")).toEqual([ + { runtime: "native", shard: 1, shards: 1, workloads: ["n1"], estimatedSeconds: 100 }, + ]); + }); + + it("puts every workload in exactly one shard and keeps manifest order within a shard", () => { + const workloads = ["w1", "w2", "w3", "w4", "w5"].map((id, index) => + workload(id, 150 + index * 10), + ); + + const shards = packShards(workloads, { targetSeconds: 300 }); + + expect(shards.flatMap((shard) => shard.workloads).sort()).toEqual([ + "w1", + "w2", + "w3", + "w4", + "w5", + ]); + for (const shard of shards) { + expect(shard.workloads).toEqual( + workloads.map((w) => w.id).filter((id) => shard.workloads.includes(id)), + ); + } + }); + + it("assumes a default estimate for a workload without one and caps the shard count", () => { + const shards = packShards( + Array.from({ length: 20 }, (_, index) => ({ id: `w${index}` })), + { targetSeconds: 10, maxShards: 3 }, + ); + + expect(shards).toHaveLength(3); + expect(shards.flatMap((shard) => shard.workloads)).toHaveLength(20); + }); + + it("returns no shards for no workloads", () => { + expect(packShards([])).toEqual([]); + }); +}); diff --git a/apps/cli/scripts/bench-report.ts b/apps/cli/scripts/bench-report.ts index 8e48662b76..fcf1097a86 100644 --- a/apps/cli/scripts/bench-report.ts +++ b/apps/cli/scripts/bench-report.ts @@ -142,11 +142,34 @@ export function exitCodesDiffer( /* Workload manifest */ +const CliArgs = Schema.Array(Schema.String); + const Workload = Schema.Struct({ id: Schema.NonEmptyString, - args: Schema.Array(Schema.String), + args: CliArgs, cwdSetup: Schema.Literals(["none", "init"]), scenario: Schema.optionalKey(Schema.String), + /** Untimed commands run with the build under test before the measured launches. */ + setup: Schema.optionalKey(Schema.Array(CliArgs)), + /** Untimed commands that always run after the measured launches, even after a failure or timeout. */ + teardown: Schema.optionalKey(Schema.Array(CliArgs)), + /** `run` repeats setup and teardown for every launch; `block` shares them across a block of launches. */ + setupScope: Schema.optionalKey(Schema.Literals(["run", "block"])), + /** Launches with `SUPABASE_EXPERIMENTAL_STACK=1`. */ + stack: Schema.optionalKey(Schema.Boolean), + /** Container runtime the workload exercises; stack workloads pass it to `stack start` themselves. */ + runtime: Schema.optionalKey(Schema.Literals(["docker", "native"])), + /** Directory under the manifest's `fixtures/` copied into the project's `supabase/` directory after `init`. */ + fixture: Schema.optionalKey(Schema.NonEmptyString), + /** Directory under `fixtures/` copied into `supabase/` after setup, so the launch finds changes setup has not applied. */ + fixtureAfterSetup: Schema.optionalKey(Schema.NonEmptyString), + runs: Schema.optionalKey(Schema.Finite), + warmup: Schema.optionalKey(Schema.Finite), + /** Traced launches per build; defaults to one per measured run. */ + traceRuns: Schema.optionalKey(Schema.Finite), + timeoutSeconds: Schema.optionalKey(Schema.Finite), + /** Expected seconds for one pass of the workload over both builds, used to pack shards. */ + estimatedSeconds: Schema.optionalKey(Schema.Finite), }); export type Workload = typeof Workload.Type; @@ -162,10 +185,44 @@ export function validateWorkloads( const problems: Array<string> = []; const seen = new Set<string>(); for (const workload of workloads) { + const name = `workload "${workload.id}"`; if (seen.has(workload.id)) problems.push(`duplicate workload id "${workload.id}"`); seen.add(workload.id); if (workload.scenario !== undefined && context.apiUrl === undefined) { - problems.push(`workload "${workload.id}" names a scenario and needs --api-url`); + problems.push(`${name} names a scenario and needs --api-url`); + } + for (const field of ["runs", "warmup", "traceRuns", "timeoutSeconds"] as const) { + const value = workload[field]; + const minimum = field === "warmup" || field === "traceRuns" ? 0 : 1; + if (value !== undefined && (!Number.isInteger(value) || value < minimum)) { + problems.push(`${name} has an invalid ${field}`); + } + } + if (workload.fixtureAfterSetup !== undefined && workload.setupScope === "block") { + problems.push( + `${name} stages fixtureAfterSetup, which a block-scoped workload applies only once`, + ); + } + if (workload.estimatedSeconds !== undefined && !(workload.estimatedSeconds > 0)) { + problems.push(`${name} has an invalid estimatedSeconds`); + } + if (workload.stack === true && workload.runtime === undefined) { + problems.push(`${name} is a stack workload and needs a runtime`); + } + if (workload.stack === true && (workload.teardown?.length ?? 0) === 0) { + problems.push(`${name} is a stack workload and needs a teardown that destroys its stack`); + } + if (workload.stack !== true && workload.runtime === "native") { + problems.push(`${name} uses the native runtime, which only stack workloads support`); + } + for (const command of [workload.args, ...(workload.setup ?? [])]) { + if (command[0] !== "stack" || command[1] !== "start" || workload.runtime === undefined) { + continue; + } + const at = command.indexOf("--runtime"); + if (at === -1 || command[at + 1] !== workload.runtime) { + problems.push(`${name} runs stack start without --runtime ${workload.runtime}`); + } } } return problems; @@ -242,6 +299,14 @@ export type RunSample = typeof RunSample.Type; const WorkloadReport = Schema.Struct({ id: Schema.String, args: Schema.Array(Schema.String), + /** Container runtime the workload exercised, when it declares one. */ + runtime: Schema.optionalKey(Schema.Literals(["docker", "native"])), + /** Whether the workload ran against the stack backend. */ + stack: Schema.optionalKey(Schema.Boolean), + /** Traced launches expected per build; absent means one per measured run. */ + traceRuns: Schema.optionalKey(Schema.Finite), + /** Wall seconds across both builds, including setup, teardown and traced launches. */ + elapsedSeconds: Schema.optionalKey(Schema.Finite), base: Schema.Array(RunSample), branch: Schema.Array(RunSample), }); @@ -312,7 +377,9 @@ export interface BuildAnalysis { readonly tracedExitCodes: ReadonlyArray<number>; readonly spans: Record<string, SpanAggregate> | undefined; readonly api: ApiSummary | undefined; - /** Measured runs without a usable trace when tracing was requested. */ + /** Median container CLI calls per run by verb, summed over `ContainerCli.spawn` and `ContainerCli.run`. */ + readonly containers: Record<string, number> | undefined; + /** Traced runs expected but without a usable trace when tracing was requested. */ readonly traceMissingRuns: number; readonly skippedTraceLines: number; } @@ -339,8 +406,28 @@ function summarizeApi(samples: ReadonlyArray<RunSample>): ApiSummary | undefined }; } -function summarizeBuild(samples: ReadonlyArray<RunSample>, traced: boolean): BuildAnalysis { +const CONTAINER_SPAN = /^ContainerCli\.(?:spawn|run)(?: \((.+)\))?$/u; + +/** Median per-run container CLI call counts keyed by verb; empty when no container call ran. */ +export function containerCounts(spans: Record<string, SpanAggregate>): Record<string, number> { + const counts: Record<string, number> = {}; + for (const [name, aggregate] of Object.entries(spans)) { + const match = CONTAINER_SPAN.exec(name); + if (match === null) continue; + const verb = match[1] ?? "(unlabelled)"; + counts[verb] = (counts[verb] ?? 0) + aggregate.medianCount; + } + return counts; +} + +function summarizeBuild( + samples: ReadonlyArray<RunSample>, + traced: boolean, + expectedTraces: number, + countContainers: boolean, +): BuildAnalysis { const withTrace = samples.flatMap((sample) => (sample.trace === undefined ? [] : [sample.trace])); + const spans = traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined; return { timing: summarizeTimings(samples.map((sample) => sample.wallMs)), samples: samples.length, @@ -348,9 +435,13 @@ function summarizeBuild(samples: ReadonlyArray<RunSample>, traced: boolean): Bui tracedExitCodes: samples.flatMap((sample) => sample.tracedExitCode === undefined ? [] : [sample.tracedExitCode], ), - spans: traced ? aggregateSpanRuns(withTrace.map((trace) => trace.spans)) : undefined, + spans, api: summarizeApi(samples), - traceMissingRuns: traced ? samples.length - withTrace.length : 0, + containers: + spans === undefined || !countContainers || withTrace.length === 0 + ? undefined + : containerCounts(spans), + traceMissingRuns: traced ? Math.max(0, expectedTraces - withTrace.length) : 0, skippedTraceLines: withTrace.reduce((sum, trace) => sum + trace.skippedLines, 0), }; } @@ -361,9 +452,17 @@ export interface EndpointChange { readonly branch: number; } +export interface ContainerChange { + readonly verb: string; + readonly base: number; + readonly branch: number; +} + export interface WorkloadAnalysis { readonly id: string; readonly args: ReadonlyArray<string>; + readonly runtime: "docker" | "native" | undefined; + readonly stack: boolean; readonly base: BuildAnalysis; readonly branch: BuildAnalysis; readonly deltaMs: number; @@ -374,6 +473,8 @@ export interface WorkloadAnalysis { /** Top phase deltas; empty unless the verdict is "slower" or "faster". */ readonly phaseRows: ReadonlyArray<SpanComparison>; readonly endpointChanges: ReadonlyArray<EndpointChange>; + /** Container verbs whose per-run call count differs between builds. */ + readonly containerChanges: ReadonlyArray<ContainerChange>; } /** Whether every launch of the workload exited 0 and every API call got a non-error response. */ @@ -402,10 +503,46 @@ function endpointChanges( .filter((change) => change.base !== change.branch); } +function containerChanges( + base: BuildAnalysis, + branch: BuildAnalysis, +): ReadonlyArray<ContainerChange> { + if (base.containers === undefined || branch.containers === undefined) return []; + const verbs = new Set([...Object.keys(base.containers), ...Object.keys(branch.containers)]); + return [...verbs] + .sort() + .map((verb) => ({ + verb, + base: base.containers?.[verb] ?? 0, + branch: branch.containers?.[verb] ?? 0, + })) + .filter((change) => change.base !== change.branch); +} + +/** Total container CLI calls per run, or undefined when container calls were not traced. */ +export function totalContainerCalls(build: BuildAnalysis): number | undefined { + return build.containers === undefined + ? undefined + : Object.values(build.containers).reduce((sum, count) => sum + count, 0); +} + /** Derives medians, comparisons and the timing verdict for one workload from its raw samples. */ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): WorkloadAnalysis { - const base = summarizeBuild(workload.base, traced); - const branch = summarizeBuild(workload.branch, traced); + const expectedTraces = (samples: number) => Math.min(workload.traceRuns ?? samples, samples); + // Stack backend container work runs in a detached owner process, so its spans never reach the CLI's trace. + const countContainers = workload.stack !== true; + const base = summarizeBuild( + workload.base, + traced, + expectedTraces(workload.base.length), + countContainers, + ); + const branch = summarizeBuild( + workload.branch, + traced, + expectedTraces(workload.branch.length), + countContainers, + ); const delta = deltaOf(base.timing.median, branch.timing.median); const differ = exitCodesDiffer(base.exitCodes, branch.exitCodes); const verdict = timingVerdict(base.timing, branch.timing, { @@ -419,6 +556,8 @@ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): Work return { id: workload.id, args: workload.args, + runtime: workload.runtime, + stack: workload.stack ?? false, base, branch, deltaMs: delta.ms, @@ -429,12 +568,16 @@ export function analyzeWorkload(workload: WorkloadReport, traced: boolean): Work phaseRows: verdict === "slower" || verdict === "faster" ? (spans ?? []).slice(0, PHASE_ROWS) : [], endpointChanges: endpointChanges(base, branch), + containerChanges: containerChanges(base, branch), }; } export type FlagKind = | "new-api-calls" | "api-calls-dropped" + | "new-container-calls" + | "container-calls-dropped" + | "container-calls-changed" | "external-calls" | "non-deterministic" | "request-log-mismatch" @@ -466,6 +609,23 @@ export function collectFlags(analysis: WorkloadAnalysis): ReadonlyArray<Flag> { if (baseCalls !== undefined && baseCalls > 0 && branchCalls === 0) { add("api-calls-dropped", `base makes ${baseCalls} API calls, branch makes 0`); } + const baseContainers = totalContainerCalls(analysis.base); + const branchContainers = totalContainerCalls(analysis.branch); + if (baseContainers !== undefined && branchContainers !== undefined) { + if (baseContainers === 0 && branchContainers > 0) { + add("new-container-calls", `base makes 0 container calls, branch makes ${branchContainers}`); + } else if (baseContainers > 0 && branchContainers === 0) { + add( + "container-calls-dropped", + `base makes ${baseContainers} container calls, branch makes 0`, + ); + } else if (analysis.containerChanges.length > 0) { + add( + "container-calls-changed", + `container calls changed from ${baseContainers} to ${branchContainers}`, + ); + } + } if (Math.min(analysis.base.samples, analysis.branch.samples) < VERDICT_MIN_SAMPLES) { add( "insufficient-samples", diff --git a/apps/cli/scripts/bench-stack.unit.test.ts b/apps/cli/scripts/bench-stack.unit.test.ts new file mode 100644 index 0000000000..d0c09b2461 --- /dev/null +++ b/apps/cli/scripts/bench-stack.unit.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { + analyzeWorkload, + collectFlags, + parseWorkloadManifest, + validateWorkloads, + type RunSample, + type WorkloadReport, +} from "./bench-report.ts"; + +const plain = (wallMs = 100): RunSample => ({ wallMs, exitCode: 0, tracedExitCode: 0 }); + +const times = (n: number, make: (i: number) => RunSample) => + Array.from({ length: n }, (_, i) => make(i)); + +describe("stack workload rules", () => { + const stackWorkload = { + id: "s", + args: ["stack", "status"], + cwdSetup: "init", + stack: true, + runtime: "native", + setup: [["stack", "start", "--runtime", "native"]], + teardown: [["stack", "destroy", "--yes"]], + } as const; + + it.effect("decodes the setup, scope, runtime, fixture and sizing fields", () => + Effect.gen(function* () { + const [parsed] = yield* parseWorkloadManifest( + `{"workloads":[{ + "id":"s","args":["stack","status"],"cwdSetup":"init","stack":true,"runtime":"native", + "setup":[["stack","start","--runtime","native"]], + "teardown":[["stack","destroy","--yes"]], + "setupScope":"block","fixture":"migrations","runs":10,"warmup":1,"traceRuns":2, + "timeoutSeconds":120,"estimatedSeconds":90 + }]}`, + ); + + expect(parsed).toMatchObject({ + setupScope: "block", + teardown: [["stack", "destroy", "--yes"]], + runtime: "native", + fixture: "migrations", + estimatedSeconds: 90, + }); + }), + ); + + it("accepts a stack workload whose stack start names its runtime", () => { + expect(validateWorkloads([stackWorkload], { apiUrl: undefined })).toEqual([]); + }); + + it.each([ + [ + "a stack workload with no runtime", + { ...stackWorkload, runtime: undefined }, + 'workload "s" is a stack workload and needs a runtime', + ], + [ + "the native runtime outside stack mode", + { ...stackWorkload, stack: undefined }, + 'workload "s" uses the native runtime, which only stack workloads support', + ], + [ + "a stack start without --runtime", + { ...stackWorkload, setup: [["stack", "start"]] }, + 'workload "s" runs stack start without --runtime native', + ], + [ + "a stack start on another runtime", + { ...stackWorkload, args: ["stack", "start", "--runtime", "docker"] }, + 'workload "s" runs stack start without --runtime native', + ], + [ + "a stack workload with no teardown", + { ...stackWorkload, teardown: undefined }, + 'workload "s" is a stack workload and needs a teardown that destroys its stack', + ], + ["a zero run count", { ...stackWorkload, runs: 0 }, 'workload "s" has an invalid runs'], + [ + "a fractional timeout", + { ...stackWorkload, timeoutSeconds: 1.5 }, + 'workload "s" has an invalid timeoutSeconds', + ], + [ + "a non-positive estimate", + { ...stackWorkload, estimatedSeconds: 0 }, + 'workload "s" has an invalid estimatedSeconds', + ], + ])("rejects %s", (_name, workload, problem) => { + expect(validateWorkloads([workload], { apiUrl: undefined })).toEqual([problem]); + }); +}); + +describe("container operations", () => { + const containerSample = (counts: Record<string, number>): RunSample => ({ + ...plain(), + trace: { + skippedLines: 0, + spans: Object.fromEntries( + Object.entries(counts).map(([name, count]) => [name, { totalMs: count * 4, count }]), + ), + }, + }); + const containerWorkload = ( + base: Record<string, number>, + branch: Record<string, number>, + extra: Partial<WorkloadReport> = {}, + ): WorkloadReport => ({ + id: "w", + args: ["status"], + base: times(5, () => containerSample(base)), + branch: times(5, () => containerSample(branch)), + ...extra, + }); + + it("sums spawn and run spans per verb and ignores other spans", () => { + const analysis = analyzeWorkload( + containerWorkload( + { + "ContainerCli.spawn (container inspect)": 2, + "ContainerCli.run (container inspect)": 1, + "cli.run": 1, + }, + { "ContainerCli.spawn (container inspect)": 3 }, + ), + true, + ); + + expect(analysis.base.containers).toEqual({ "container inspect": 3 }); + expect(analysis.branch.containers).toEqual({ "container inspect": 3 }); + expect(analysis.containerChanges).toEqual([]); + expect(collectFlags(analysis)).toEqual([]); + }); + + it("flags an extra call as a change", () => { + const analysis = analyzeWorkload( + containerWorkload( + { "ContainerCli.spawn (container inspect)": 3 }, + { "ContainerCli.spawn (container inspect)": 4 }, + ), + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).toEqual(["container-calls-changed"]); + expect(analysis.containerChanges).toEqual([{ verb: "container inspect", base: 3, branch: 4 }]); + }); + + it("flags container calls that appear from zero and ones that drop to zero", () => { + const appeared = analyzeWorkload( + containerWorkload({ "cli.run": 1 }, { "ContainerCli.spawn (ps)": 2 }), + true, + ); + const dropped = analyzeWorkload( + containerWorkload({ "ContainerCli.run (ps)": 2 }, { "cli.run": 1 }), + true, + ); + + expect(collectFlags(appeared).map((flag) => flag.kind)).toEqual(["new-container-calls"]); + expect(collectFlags(dropped).map((flag) => flag.kind)).toEqual(["container-calls-dropped"]); + }); + + it("does not flag container calls when one build has no usable trace", () => { + const analysis = analyzeWorkload( + { + ...containerWorkload({ "ContainerCli.spawn (ps)": 2 }, {}), + branch: times(5, () => plain()), + }, + true, + ); + + expect(collectFlags(analysis).map((flag) => flag.kind)).not.toContain( + "container-calls-dropped", + ); + }); + + it("leaves container calls uncounted for stack workloads and untraced runs", () => { + const stack = analyzeWorkload( + containerWorkload( + { "ContainerCli.spawn (ps)": 1 }, + { "ContainerCli.spawn (ps)": 2 }, + { stack: true }, + ), + true, + ); + const untraced = analyzeWorkload( + { + id: "w", + args: ["status"], + base: times(5, () => plain()), + branch: times(5, () => plain()), + }, + false, + ); + + expect(stack.base.containers).toBeUndefined(); + expect(collectFlags(stack)).toEqual([]); + expect(untraced.base.containers).toBeUndefined(); + }); +}); + +describe("traced run expectations", () => { + const tracedSample: RunSample = { + ...plain(), + trace: { skippedLines: 0, spans: { "cli.run": { totalMs: 50, count: 1 } } }, + }; + const withTraceRuns = (traceRuns: number): WorkloadReport => ({ + id: "w", + args: ["start"], + traceRuns, + base: [tracedSample, ...times(4, () => plain())], + branch: [tracedSample, ...times(4, () => plain())], + }); + + it("does not flag runs without a trace beyond the workload's traceRuns", () => { + expect(collectFlags(analyzeWorkload(withTraceRuns(1), true))).toEqual([]); + }); + + it("flags each build that is missing traces the workload expected", () => { + expect(collectFlags(analyzeWorkload(withTraceRuns(2), true)).map((flag) => flag.kind)).toEqual([ + "trace-incomplete", + "trace-incomplete", + ]); + }); +}); diff --git a/apps/cli/scripts/bench-summary.ts b/apps/cli/scripts/bench-summary.ts index 21b94399ee..a3f06d2ce9 100644 --- a/apps/cli/scripts/bench-summary.ts +++ b/apps/cli/scripts/bench-summary.ts @@ -1,8 +1,9 @@ /** * Renders a `bench-cli.ts --out` report as Markdown: the answer first, then a timing table, API - * calls, binary size, and collapsed details. Run as `bun scripts/bench-summary.ts --report <file>` - * with optional `--branch-build|--base-build|--replay ok|failed` step outcomes, `--head-sha`, - * `--base-ref`, `--merge-base`, and `--comment-out <file>` for the pull request + * calls, container operations (integration-heavy section), binary size, and collapsed details. Run + * as `bun scripts/bench-summary.ts --report <file>` with optional `--section + * lightweight|integration-heavy`, `--branch-build|--base-build|--replay ok|failed` step outcomes, + * `--head-sha`, `--base-ref`, `--merge-base`, and `--comment-out <file>` for the pull request * comment variant; the job summary goes to `GITHUB_STEP_SUMMARY` when set, otherwise to stdout. */ import process from "node:process"; @@ -14,18 +15,29 @@ import { collectFlags, decodeBenchReport, VERDICT_MIN_PCT, + totalContainerCalls, type BenchReport, type Flag, type WorkloadAnalysis, } from "./bench-report.ts"; const USAGE = - "usage: bun scripts/bench-summary.ts --report <file> [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha <sha>] [--base-ref <ref>] [--merge-base <sha>] [--comment-out <file>]"; + "usage: bun scripts/bench-summary.ts --report <file> [--section lightweight|integration-heavy] [--note <text>] [--branch-build ok|failed] [--base-build ok|failed] [--replay ok|failed] [--head-sha <sha>] [--base-ref <ref>] [--merge-base <sha>] [--comment-out <file>]"; type Outcome = "ok" | "failed"; +/** The benchmark a report covers; each has its own pull request comment. */ +export type Section = "lightweight" | "integration-heavy"; + +export const SECTION_NAMES: Record<Section, string> = { + lightweight: "Lightweight Commands", + "integration-heavy": "Integration Heavy Commands", +}; + export interface SummaryOptions { readonly report: string | undefined; + readonly section: Section; + readonly note: string | undefined; readonly branchBuild: Outcome; readonly baseBuild: Outcome; readonly replay: Outcome; @@ -43,6 +55,8 @@ export function parseSummaryArgs(argv: ReadonlyArray<string>): SummaryOptions { args: [...argv], options: { report: { type: "string" }, + section: { type: "string", default: "lightweight" }, + note: { type: "string" }, "branch-build": { type: "string", default: "ok" }, "base-build": { type: "string", default: "ok" }, replay: { type: "string", default: "ok" }, @@ -57,8 +71,13 @@ export function parseSummaryArgs(argv: ReadonlyArray<string>): SummaryOptions { throw new Error(`--${flag} must be "ok" or "failed"`); } } + if (values.section !== "lightweight" && values.section !== "integration-heavy") { + throw new Error('--section must be "lightweight" or "integration-heavy"'); + } const options: SummaryOptions = { report: values.report, + section: values.section, + note: values.note, branchBuild: values["branch-build"] === "failed" ? "failed" : "ok", baseBuild: values["base-build"] === "failed" ? "failed" : "ok", replay: values.replay === "failed" ? "failed" : "ok", @@ -121,7 +140,7 @@ function table(header: ReadonlyArray<string>, rows: ReadonlyArray<ReadonlyArray< const when = (build: Flag["build"]) => (build === "base" ? "before" : "after"); /** One plain-words sentence for a flag. */ -function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { +function describeFlag(flag: Flag, analysis: WorkloadAnalysis, integration: boolean): string { const name = command(analysis); const at = when(flag.build); switch (flag.kind) { @@ -137,6 +156,12 @@ function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { } case "api-calls-dropped": return `${name} no longer calls the Management API. It made ${callCount(analysis.base.api?.medianCalls)} before and now makes none.`; + case "new-container-calls": + return `${name} now runs container commands. It ran none before and now runs ${totalContainerCalls(analysis.branch) ?? "some"}.`; + case "container-calls-dropped": + return `${name} no longer runs container commands. It ran ${totalContainerCalls(analysis.base) ?? "some"} before and now runs none.`; + case "container-calls-changed": + return `${name} runs a different number of container commands: ${totalContainerCalls(analysis.base) ?? "some"} before, ${totalContainerCalls(analysis.branch) ?? "some"} now.`; case "external-calls": return `${name} made calls to hosts other than the local test server ${at}, so its timing may be skewed.`; case "non-deterministic": @@ -144,6 +169,9 @@ function describeFlag(flag: Flag, analysis: WorkloadAnalysis): string { case "request-log-mismatch": return `The API calls counted for ${name} ${at} do not match what the local test server saw, so the counts may be wrong.`; case "error-status": + if (integration) { + return `${name} called the Management API ${at} and got an error response from the local test server, which has no recorded scenario for these commands; they are expected to make no API calls.`; + } return `${name} got error responses from the local test server ${at}, so it did not follow the recorded scenario.`; case "trace-incomplete": return `Some runs of ${name} ${at} produced no usable trace, so call counts and phase timings may be incomplete.`; @@ -174,11 +202,22 @@ function changeText(analysis: WorkloadAnalysis): string { } } +/** How the workload runs: its container runtime, and `stack` when it uses the stack backend. */ +function runtimeLabel(analysis: WorkloadAnalysis): string { + const runtime = analysis.runtime ?? "docker"; + return analysis.stack ? `stack, ${runtime}` : runtime; +} + +const hasRuntimes = (analyses: ReadonlyArray<WorkloadAnalysis>) => + analyses.some((analysis) => analysis.runtime !== undefined || analysis.stack); + function timingTable(analyses: ReadonlyArray<WorkloadAnalysis>): string { + const withRuntime = hasRuntimes(analyses); return table( - ["Command", "Before", "After", "Change"], + ["Command", ...(withRuntime ? ["Runtime"] : []), "Before", "After", "Change"], analyses.map((analysis) => [ command(analysis), + ...(withRuntime ? [runtimeLabel(analysis)] : []), ms(analysis.base.timing.median), ms(analysis.branch.timing.median), changeText(analysis), @@ -186,13 +225,69 @@ function timingTable(analyses: ReadonlyArray<WorkloadAnalysis>): string { ); } +/** Commands whose container verbs are traced, with the change in each. */ +function containerOverview(analyses: ReadonlyArray<WorkloadAnalysis>): { + readonly line: string; +} { + const measured = analyses.filter( + (analysis) => + totalContainerCalls(analysis.base) !== undefined && + totalContainerCalls(analysis.branch) !== undefined, + ); + if (measured.length === 0) return { line: "Container operations were not counted." }; + const changed = measured.filter((analysis) => analysis.containerChanges.length > 0); + if (changed.length === 0) { + const any = measured.some( + (analysis) => + (totalContainerCalls(analysis.base) ?? 0) > 0 || + (totalContainerCalls(analysis.branch) ?? 0) > 0, + ); + return { + line: any + ? `Container operations: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}.` + : "Container operations: none recorded.", + }; + } + const lines = changed.map((analysis) => { + const verbs = analysis.containerChanges + .map((change) => `${code(change.verb)} ${change.base} -> ${change.branch}`) + .join(", "); + return `- ${command(analysis)} (${runtimeLabel(analysis)}): ${verbs}`; + }); + return { + line: [ + `Container operations changed for ${changed.length} ${plural(changed.length, "command", "commands")}:`, + ...lines, + ].join("\n"), + }; +} + +function containerDetailTable(analyses: ReadonlyArray<WorkloadAnalysis>): string { + const rows = analyses.flatMap((analysis) => { + const { base, branch } = analysis; + if (base.containers === undefined || branch.containers === undefined) return []; + const verbs = new Set([...Object.keys(base.containers), ...Object.keys(branch.containers)]); + return [...verbs] + .sort() + .map((verb) => [ + command(analysis), + runtimeLabel(analysis), + code(verb), + String(base.containers?.[verb] ?? 0), + String(branch.containers?.[verb] ?? 0), + ]); + }); + if (rows.length === 0) return "No container operations were recorded."; + return table(["Command", "Runtime", "Operation", "Before", "After"], rows); +} + interface ApiOverview { readonly line: string; /** Ids of the commands whose API calls changed. */ readonly changedIds: ReadonlyArray<string>; } -function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>): ApiOverview { +function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>, expectNone: boolean): ApiOverview { const measured = analyses.filter( (analysis) => analysis.base.api !== undefined && analysis.branch.api !== undefined, ); @@ -205,6 +300,18 @@ function apiOverview(analyses: ReadonlyArray<WorkloadAnalysis>): ApiOverview { const missing = analyses.length - measured.length; const suffix = missing === 0 ? "" : ` (not measured for ${missing})`; if (changed.length === 0) { + const idle = + expectNone && + measured.every( + (analysis) => + analysis.base.api?.medianCalls === 0 && analysis.branch.api?.medianCalls === 0, + ); + if (idle) { + return { + line: `API calls: none, before or after, for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, + changedIds: [], + }; + } return { line: `API calls: unchanged for ${measured.length === 1 ? "the command" : `all ${measured.length} commands`}${suffix}.`, changedIds: [], @@ -285,11 +392,10 @@ function phaseDetail(analyses: ReadonlyArray<WorkloadAnalysis>): string { return table(["Command", "Span", "Change", "Before -> after"], rows); } -export const BENCH_MARKER = "<!-- cli-bench:pr -->"; - export interface RenderContext { - /** `comment` adds the marker and the re-run footer; `summary` is the job summary. */ - readonly target: "summary" | "comment"; + /** A warning shown above the results, such as shards that did not finish. */ + readonly note?: string | undefined; + readonly section?: Section; readonly headSha?: string | undefined; readonly baseRef?: string | undefined; readonly mergeBase?: string | undefined; @@ -297,6 +403,17 @@ export interface RenderContext { const short = (sha: string) => sha.slice(0, 7); +/** "10 runs each", or the range when per-workload run counts differ. */ +function runsText(report: BenchReport): string { + const counts = report.workloads.flatMap((workload) => [ + workload.base.length, + workload.branch.length, + ]); + const low = counts.length === 0 ? report.runs : Math.min(...counts); + const high = counts.length === 0 ? report.runs : Math.max(...counts); + return low === high ? `${low} ${plural(low, "run", "runs")} each` : `${low}-${high} runs each`; +} + function contextLine(report: BenchReport | undefined, context: RenderContext): string { const parts: Array<string> = []; if (context.headSha !== undefined && context.headSha !== "") { @@ -310,7 +427,7 @@ function contextLine(report: BenchReport | undefined, context: RenderContext): s }`; parts.push(`Commit ${code(short(context.headSha))}${against}`); } - if (report !== undefined) parts.push(`${report.runs} ${plural(report.runs, "run", "runs")} each`); + if (report !== undefined) parts.push(runsText(report)); parts.push("report only, never blocks merging"); return parts.join(" · "); } @@ -321,32 +438,30 @@ function frame( context: RenderContext, report?: BenchReport, ): string { - const footer = - context.target === "comment" ? ["Comment `/bench` to run it again on the latest commit."] : []; - return ( - [ - ...(context.target === "comment" ? [BENCH_MARKER] : []), - `## CLI benchmark: ${title}`, - contextLine(report, context), - ...body, - ...footer, - ].join("\n\n") + "\n" - ); + const section = context.section ?? "lightweight"; + const heading = `## CLI benchmark (${SECTION_NAMES[section]}): ${title}`; + const note = context.note === undefined || context.note === "" ? [] : [`⚠️ ${context.note}`]; + return [heading, contextLine(report, context), ...note, ...body].join("\n\n") + "\n"; } /** The Markdown report, or `notice` in plain words when there is no report to compare. */ export function renderSummary( report: BenchReport | undefined, notice: string, - context: RenderContext = { target: "summary" }, + context: RenderContext = {}, ): string { if (report === undefined) return frame("could not run", [notice], context); const analyses = report.workloads.map((workload) => analyzeWorkload(workload, report.trace)); + const integration = (context.section ?? "lightweight") === "integration-heavy"; + // A local stack's own readiness and health traffic always leaves the test server's host. const flags = analyses.flatMap((analysis) => - collectFlags(analysis).map((flag) => ({ flag, analysis })), + collectFlags(analysis) + .filter((flag) => !(integration && flag.kind === "external-calls")) + .map((flag) => ({ flag, analysis })), ); if (analyses.length === 0) return frame("no commands were benchmarked", [], context, report); - const api = apiOverview(analyses); + const api = apiOverview(analyses, integration); + const containers = integration ? containerOverview(analyses) : undefined; const moved = analyses .filter((analysis) => analysis.verdict === "slower" || analysis.verdict === "faster") .map((analysis) => analysis.id); @@ -357,16 +472,20 @@ export function renderSummary( return frame( title, [ - ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis)}`), + ...flags.map(({ flag, analysis }) => `⚠️ ${describeFlag(flag, analysis, integration)}`), timingTable(analyses), api.line, + ...(containers === undefined ? [] : [containers.line]), sizeLine(report), [ - "<details><summary>Details: timing ranges, API calls per command, slowest phases</summary>", + `<details><summary>Details: timing ranges, API calls per command, ${integration ? "container operations, " : ""}slowest phases</summary>`, "#### Timing ranges", rangeTable(analyses), "#### API calls per command", apiDetailTable(analyses), + ...(integration + ? ["#### Container operations per command", containerDetailTable(analyses)] + : []), "#### Slowest phases", phaseDetail(analyses), "</details>", @@ -425,11 +544,13 @@ const main = Effect.gen(function* () { ? yield* loadReport(fs, options.report) : { report: undefined, notice: outcomeNotice ?? NO_REPORT }; const context = { + section: options.section, + note: options.note, headSha: options.headSha, baseRef: options.baseRef, mergeBase: options.mergeBase, }; - const markdown = renderSummary(loaded.report, loaded.notice, { target: "summary", ...context }); + const markdown = renderSummary(loaded.report, loaded.notice, context); const summaryFile = yield* Config.option(Config.String("GITHUB_STEP_SUMMARY")); if (Option.isSome(summaryFile) && summaryFile.value !== "") { yield* fs.writeFileString(summaryFile.value, markdown, { flag: "a" }); @@ -438,10 +559,7 @@ const main = Effect.gen(function* () { } if (options.commentOut !== undefined) { - yield* fs.writeFileString( - options.commentOut, - renderSummary(loaded.report, loaded.notice, { target: "comment", ...context }), - ); + yield* fs.writeFileString(options.commentOut, markdown); } }); diff --git a/apps/cli/scripts/bench-summary.unit.test.ts b/apps/cli/scripts/bench-summary.unit.test.ts index 3d5e369c6a..ba5c6719c0 100644 --- a/apps/cli/scripts/bench-summary.unit.test.ts +++ b/apps/cli/scripts/bench-summary.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { BenchReport, RunSample } from "./bench-report.ts"; -import { BENCH_MARKER, parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; +import { parseSummaryArgs, renderSummary, summaryNotice } from "./bench-summary.ts"; const run = (wallMs: number, calls: number, endpoints: Record<string, number>): RunSample => ({ wallMs, @@ -50,13 +50,12 @@ describe("renderSummary", () => { branchSizeBytes: 140269744, }; const markdown = renderSummary(sized, "unused", { - target: "summary", headSha: "abcdef1234567890", baseRef: "develop", mergeBase: "1234567890abcdef", }); - expect(markdown).toContain("## CLI benchmark: no changes found"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): no changes found"); expect(markdown).toContain( "Commit `abcdef1` against `develop` at `1234567` · 5 runs each · report only, never blocks merging", ); @@ -83,7 +82,7 @@ describe("renderSummary", () => { it("explains a new API call in plain words and counts each command once", () => { const markdown = renderSummary(report([migrationNew]), "unused"); - expect(markdown).toContain("## CLI benchmark: 1 thing to check"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): 1 thing to check"); expect(markdown).toContain( "⚠️ `supabase migration new bench` now calls the Management API. It made no calls before and now makes 1 (`GET /v1/projects/abc`).", ); @@ -99,7 +98,9 @@ describe("renderSummary", () => { it("says nothing was benchmarked instead of a clean pass when there are no commands", () => { const markdown = renderSummary(report([]), "unused"); - expect(markdown).toContain("## CLI benchmark: no commands were benchmarked"); + expect(markdown).toContain( + "## CLI benchmark (Lightweight Commands): no commands were benchmarked", + ); expect(markdown).not.toContain("no changes found"); }); @@ -119,27 +120,49 @@ describe("renderSummary", () => { }); it("names the failed step and stays a notice when there is no report", () => { - const markdown = renderSummary(undefined, "The build of this commit failed.", { - target: "summary", - }); + const markdown = renderSummary(undefined, "The build of this commit failed.", {}); - expect(markdown).toContain("## CLI benchmark: could not run"); + expect(markdown).toContain("## CLI benchmark (Lightweight Commands): could not run"); expect(markdown).toContain("The build of this commit failed."); expect(markdown).not.toContain("/bench"); }); - it("adds the marker, re-run hint only to the comment", () => { - const input = report([steady("version", ["--version"])]); - const context = { headSha: "abcdef1234" } as const; + it.each([ + { + section: "lightweight", + heading: "## CLI benchmark (Lightweight Commands): no changes found", + }, + { + section: "integration-heavy", + heading: "## CLI benchmark (Integration Heavy Commands): no changes found", + }, + ] as const)( + "titles a $section report with its name and leaves the markers and re-run note to the poster", + ({ section, heading }) => { + const input = report([steady("version", ["--version"])]); + + const comment = renderSummary(input, "unused", { section }); + + expect(comment.startsWith(`${heading}\n`)).toBe(true); + expect(comment).not.toContain("<!--"); + expect(comment).not.toContain("/bench"); + }, + ); + + it("shows a note above the results", () => { + const markdown = renderSummary(report([steady("version", ["--version"])]), "unused", { + note: "Some shards did not finish.", + }); + + expect(markdown).toContain("⚠️ Some shards did not finish."); + }); - const comment = renderSummary(input, "unused", { target: "comment", ...context }); - const summary = renderSummary(input, "unused", { target: "summary", ...context }); + it("says the range when per-command run counts differ", () => { + const fewer = { ...steady("x", ["start"]), base: runs(3, (i) => run(30 + i, 0, {})) }; - expect(BENCH_MARKER).toBe("<!-- cli-bench:pr -->"); - expect(comment.startsWith(`${BENCH_MARKER}\n`)).toBe(true); - expect(comment).toContain("Comment `/bench` to run it again on the latest commit."); - expect(summary).not.toContain(BENCH_MARKER); - expect(summary).not.toContain("/bench"); + const markdown = renderSummary(report([steady("version", ["--version"]), fewer]), "unused"); + + expect(markdown).toContain("3-5 runs each"); }); it("does not claim a missing timing verdict when commands were not comparable", () => { @@ -222,3 +245,148 @@ describe("parseSummaryArgs", () => { ).toMatchObject({ report: "r.json", replay: "ok", headSha: "abc", commentOut: "c.md" }); }); }); + +describe("renderSummary for integration-heavy commands", () => { + const withContainers = (counts: Record<string, number>): RunSample => ({ + wallMs: 1000, + exitCode: 0, + tracedExitCode: 0, + trace: { + skippedLines: 0, + spans: Object.fromEntries( + Object.entries(counts).map(([verb, count]) => [ + `ContainerCli.spawn (${verb})`, + { totalMs: 5 * count, count }, + ]), + ), + }, + }); + const heavy = ( + id: string, + base: Record<string, number>, + branch: Record<string, number>, + extra: Partial<BenchReport["workloads"][number]> = {}, + ) => ({ + id, + args: ["status"], + runtime: "docker" as const, + base: runs(5, () => withContainers(base)), + branch: runs(5, () => withContainers(branch)), + ...extra, + }); + const render = (workloads: BenchReport["workloads"]) => + renderSummary(report(workloads), "unused", { section: "integration-heavy" }); + + it("shows unchanged container operations with the runtime column", () => { + const markdown = render([ + heavy("status", { "container inspect": 3 }, { "container inspect": 3 }), + ]); + + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): no changes found"); + expect(markdown).toContain("| Command | Runtime | Before | After | Change |"); + expect(markdown).toContain("| `supabase status` | docker |"); + expect(markdown).toContain("Container operations: unchanged for the command."); + }); + + it("flags an extra container call per verb and counts the command once", () => { + const markdown = render([ + heavy("status", { "container inspect": 3 }, { "container inspect": 4 }), + ]); + + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): 1 thing to check"); + expect(markdown).toContain( + "⚠️ `supabase status` runs a different number of container commands: 3 before, 4 now.", + ); + expect(markdown).toContain("Container operations changed for 1 command:"); + expect(markdown).toContain("`container inspect` 3 -> 4"); + expect(markdown).toContain("| `supabase status` | docker | `container inspect` | 3 | 4 |"); + }); + + it("flags container commands that appear where there were none", () => { + const markdown = render([heavy("status", {}, { ps: 2 })]); + + expect(markdown).toContain( + "⚠️ `supabase status` now runs container commands. It ran none before and now runs 2.", + ); + }); + + it("does not count container operations for stack workloads, whose container work runs outside the trace", () => { + const markdown = render([ + heavy("stack-status", { ps: 1 }, { ps: 2 }, { stack: true, runtime: "native" }), + ]); + + expect(markdown).toContain("Container operations were not counted."); + expect(markdown).toContain("| `supabase status` | stack, native |"); + expect(markdown).not.toContain("container commands"); + }); + + const counted = (calls: number, extra: Partial<RunSample> = {}, external = 0): RunSample => ({ + ...run(30, calls, calls === 0 ? {} : { "GET /v1/projects": calls }), + ...extra, + trace: { + skippedLines: 0, + spans: { "cli.run": { totalMs: 15, count: 1 } }, + api: { + apiCalls: calls, + externalCalls: external, + errorStatuses: 0, + endpoints: calls === 0 ? {} : { "GET /v1/projects": calls }, + requestLogCount: calls, + }, + }, + }); + const withApi = (id: string, base: RunSample, branch: RunSample) => ({ + id, + args: ["status"], + stack: true, + runtime: "native" as const, + base: runs(5, () => base), + branch: runs(5, () => branch), + }); + + it("reports a command that made no API calls as counted, not as not counted", () => { + const markdown = render([withApi("status", counted(0), counted(0))]); + + expect(markdown).toContain("API calls: none, before or after, for the command."); + expect(markdown).not.toContain("API calls were not counted."); + expect(markdown).toContain("| `supabase status` | 0 | 0 |"); + }); + + it("says API calls were not counted when no launch was traced against the test server", () => { + const untraced = { wallMs: 30, exitCode: 0 }; + const markdown = render([withApi("status", untraced, untraced)]); + + expect(markdown).toContain("API calls were not counted."); + }); + + it("flags a Management API call that first appears on the branch", () => { + const markdown = render([withApi("status", counted(0), counted(1))]); + + expect(markdown).toContain("now calls the Management API"); + expect(markdown).toContain("## CLI benchmark (Integration Heavy Commands): 1 thing to check"); + }); + + it("does not flag a local stack's calls to other hosts", () => { + const markdown = render([withApi("status", counted(0, {}, 12), counted(0, {}, 12))]); + + expect(markdown).toContain("no changes found"); + expect(markdown).not.toContain("hosts other than"); + }); + + it("flags an error response from the empty test server in plain words", () => { + const failing = counted(1); + const errored: RunSample = { + ...failing, + trace: { ...failing.trace!, api: { ...failing.trace!.api!, errorStatuses: 1 } }, + }; + const markdown = render([withApi("status", counted(0), errored)]); + + expect(markdown).toContain("expected to make no API calls"); + }); + + it("leaves container operations out of the lightweight section", () => { + const markdown = renderSummary(report([heavy("status", { ps: 1 }, { ps: 1 })]), "unused", {}); + + expect(markdown).not.toContain("Container operations"); + }); +}); diff --git a/apps/cli/scripts/bench-workloads.unit.test.ts b/apps/cli/scripts/bench-workloads.unit.test.ts index 403ce9664f..366c4da278 100644 --- a/apps/cli/scripts/bench-workloads.unit.test.ts +++ b/apps/cli/scripts/bench-workloads.unit.test.ts @@ -4,6 +4,8 @@ import { Effect, FileSystem } from "effect"; import { parseWorkloadManifest, validateWorkloads } from "./bench-report.ts"; const manifestPath = new URL("../bench/workloads.pr.json", import.meta.url).pathname; +const fullManifestPath = new URL("../bench/workloads.full.json", import.meta.url).pathname; +const fixturesPath = new URL("../bench/fixtures/", import.meta.url).pathname; const scenariosDir = new URL("../../cli-e2e/fixtures/scenarios/", import.meta.url).pathname; const loadManifest = Effect.gen(function* () { @@ -11,6 +13,11 @@ const loadManifest = Effect.gen(function* () { return yield* parseWorkloadManifest(yield* fs.readFileString(manifestPath)); }); +const loadFullManifest = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* parseWorkloadManifest(yield* fs.readFileString(fullManifestPath)); +}); + describe("PR workload manifest", () => { it.effect("decodes and passes the structural rules when a replay server is configured", () => Effect.gen(function* () { @@ -37,3 +44,56 @@ describe("PR workload manifest", () => { }).pipe(Effect.provide(BunServices.layer)), ); }); + +describe("full workload manifest", () => { + it.effect("decodes and passes the structural rules without a replay server", () => + Effect.gen(function* () { + const workloads = yield* loadFullManifest; + + expect(workloads.length).toBeGreaterThan(0); + expect(validateWorkloads(workloads, { apiUrl: undefined })).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("names only fixtures that exist and carry a migration, seed, or test", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workloads = yield* loadFullManifest; + + for (const workload of workloads) { + if (workload.fixture === undefined) continue; + const files = yield* fs.readDirectory(`${fixturesPath}${workload.fixture}`); + expect(files.length).toBeGreaterThan(0); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("sizes every workload for sharding and leaves nothing running after it", () => + Effect.gen(function* () { + const workloads = yield* loadFullManifest; + + for (const workload of workloads) { + expect(workload.estimatedSeconds, workload.id).toBeGreaterThan(0); + expect(workload.teardown?.length ?? 0, workload.id).toBeGreaterThan(0); + const stop = workload.stack === true ? ["stack", "destroy"] : ["stop", "--no-backup"]; + expect(workload.teardown?.at(-1)?.slice(0, stop.length), workload.id).toEqual(stop); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("starts a stack before any workload that measures a command against one", () => + Effect.gen(function* () { + const workloads = yield* loadFullManifest; + + for (const workload of workloads) { + const measuresStart = + workload.args.join(" ").startsWith("stack start") || + (workload.stack !== true && workload.args[0] === "start") || + (workload.stack !== true && workload.args.join(" ") === "db start"); + if (measuresStart) continue; + const start = workload.stack === true ? ["stack", "start"] : ["start"]; + expect(workload.setup?.[0]?.slice(0, start.length), workload.id).toEqual(start); + } + }).pipe(Effect.provide(BunServices.layer)), + ); +});