From fa317d4f7e0e4a9f0be98c4caa62317565cf1797 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 28 Oct 2026 14:07:41 +0100 Subject: [PATCH 1/3] fix(cli): serialize local start and stop per project (CLI-2875) A concurrent `stop`, `stop --all`, or rollback could prune a project's Docker network between `start` creating it and attaching the first container, failing `start` with "network ... not found". `start`, `db start` (including declarative auto-start), and `stop` now hold a cross-process lifecycle lock under `$SUPABASE_HOME/locks`: a per-project lock keyed by the project's Docker label, behind a shared gate that `stop --all` takes exclusively. A blocked command prints one waiting line and gives up after 10 minutes. --- .../db-bootstrap/container-lifecycle.ts | 8 +- .../db-bootstrap/start-local-database.ts | 26 +- ...project-lifecycle-lock.integration.test.ts | 390 ++++++++++++++++++ .../project-lifecycle-lock.ts | 154 +++++++ .../cli/src/commands/db/start/SIDE_EFFECTS.md | 35 +- apps/cli/src/commands/start/SIDE_EFFECTS.md | 20 +- apps/cli/src/commands/start/start.handler.ts | 6 +- apps/cli/src/commands/stop/SIDE_EFFECTS.md | 33 +- apps/cli/src/commands/stop/stop.handler.ts | 6 +- .../telemetry/__fixtures__/error-tags.txt | 2 + apps/cli/tests/helpers/docker-model.ts | 243 +++++++++++ packages/stack/src/StackNamespace.ts | 2 +- packages/stack/src/effect.ts | 4 + packages/stack/src/namespace/Lock.ts | 40 +- 14 files changed, 914 insertions(+), 55 deletions(-) create mode 100644 apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts create mode 100644 apps/cli/src/command-internal/project-lifecycle-lock.ts create mode 100644 apps/cli/tests/helpers/docker-model.ts diff --git a/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts b/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts index ee66e005ba..ecaa78c51c 100644 --- a/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts +++ b/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts @@ -189,10 +189,10 @@ function portConflictSuggestion(hostPort: string, serviceLabel: string): string * Creates the shared docker network for this project via `docker network create --label ...`, * treating "already exists" as success. * - * Called once per `start` run rather than once per container: the network can't be deleted - * externally mid-run, so it's guaranteed to still exist for every later {@link createContainer} - * call. Returns immediately for a built-in network name (`default`, `bridge`, `host`, `none`), - * which already exists and cannot be created. + * Called once per `start` run rather than once per container: callers holding the project lifecycle + * lock keep other lifecycle commands from pruning it before every {@link createContainer} call has run. + * Returns immediately for a built-in network name (`default`, `bridge`, `host`, `none`), which + * already exists and cannot be created. */ export function ensureNetwork( spawner: Spawner, diff --git a/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts b/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts index 6997ef6b94..5919c1b82a 100644 --- a/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts +++ b/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts @@ -50,6 +50,7 @@ import { ramInBytes } from "../size-units.ts"; import { goUrlParse } from "../storage-url.ts"; import { loadLocalProjectContext } from "../local-project-context.ts"; import { cliProjectFilterValue } from "../docker-ids.ts"; +import { holdProjectLifecycleLock } from "../project-lifecycle-lock.ts"; import { buildLocalDbContainerInputs } from "./local-container-inputs.ts"; import { isLocalDbRunning } from "./local-db-running.ts"; import { rollbackStart } from "./rollback.ts"; @@ -84,15 +85,8 @@ interface StartLocalDatabaseResult { readonly status: StartLocalDatabaseStatus; } -/** - * Starts the local Postgres database, or no-ops when it is already running. See this module's - * own header for the full design rationale. - * - * `fromBackupFlag` is `db start`'s own `--from-backup` value, resolved against the caller's cwd - * before being passed in; the declarative seam's `ensureLocalDatabaseStarted` always calls with - * no argument. - */ -export const startLocalDatabase = Effect.fn("DbBootstrap.startLocalDatabase")(function* ( +/** Takes the project lifecycle lock into its scope; {@link startLocalDatabase} closes that scope after rollback. */ +const startLocalDatabaseInScope = Effect.fn("DbBootstrap.startLocalDatabase")(function* ( fromBackupFlag?: string, ) { const output = yield* Output; @@ -634,6 +628,9 @@ export const startLocalDatabase = Effect.fn("DbBootstrap.startLocalDatabase")(fu ), ); + // Held until this call returns, after any rollback has run. + yield* holdProjectLifecycleLock(context.projectId); + // If the db container is already up, tell the caller and stop here. Runs after the config // load/validation above. const running = yield* isLocalDbRunning( @@ -724,3 +721,14 @@ export const startLocalDatabase = Effect.fn("DbBootstrap.startLocalDatabase")(fu return { status: "started" } satisfies StartLocalDatabaseResult; }); + +/** + * Starts the local Postgres database, or no-ops when it is already running. See this module's + * own header for the full design rationale. + * + * `fromBackupFlag` is `db start`'s own `--from-backup` value, resolved against the caller's cwd + * before being passed in; the declarative seam's `ensureLocalDatabaseStarted` always calls with + * no argument. + */ +export const startLocalDatabase = (fromBackupFlag?: string) => + Effect.scoped(startLocalDatabaseInScope(fromBackupFlag)); diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts new file mode 100644 index 0000000000..5d1fba5bb3 --- /dev/null +++ b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts @@ -0,0 +1,390 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Deferred, Effect, Exit, Fiber, FileSystem, Layer, Option, Path } from "effect"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as TestClock from "effect/testing/TestClock"; + +import { CommandPlatformApiFactory } from "../auth/command-platform-api-factory.service.ts"; +import { start } from "../commands/start/start.handler.ts"; +import type { StartFlags } from "../commands/start/start.command.ts"; +import { stop } from "../commands/stop/stop.handler.ts"; +import type { StopFlags } from "../commands/stop/stop.command.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { CliArgs } from "../shared/cli/cli-args.service.ts"; +import { + mockCommandSettings, + mockTelemetryStateTracked, + sequentialExecBatch, +} from "../../tests/helpers/command-mocks.ts"; +import { dockerModel, type DockerModel } from "../../tests/helpers/docker-model.ts"; +import { + mockAnalytics, + mockOutput, + mockProcessControl, + mockRuntimeInfo, + mockStdin, + mockTty, +} from "../../tests/helpers/mocks.ts"; +import { unusedStackServices } from "../../tests/helpers/unused-stack.ts"; +import { DbConnection, type DbSession } from "./db-connection.service.ts"; +import { dockerRunLayer } from "./docker-run.layer.ts"; +import { DebugFlag, ExperimentalFlag, NetworkIdFlag, YesFlag } from "./global-flags.ts"; +import { + ProjectLifecycleLockBusyError, + ProjectLifecycleLockIoError, +} from "./project-lifecycle-lock.ts"; + +const startFlags: StartFlags = { exclude: [], ignoreHealthCheck: false, preview: false }; + +const stopFlags = (overrides: Partial = {}): StopFlags => ({ + projectId: Option.none(), + backup: true, + noBackup: false, + all: Option.none(), + ...overrides, +}); + +const alwaysReadyHttpClientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), +); + +/** The restart path of `start` opens a database session to converge webhooks; an empty one suffices. */ +const emptyDbSession: DbSession = { + exec: () => Effect.void, + query: () => Effect.succeed([]), + execBatch: (statements) => sequentialExecBatch(emptyDbSession)(statements), + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), +}; + +/** + * Suspends whoever calls `hold` until the test calls `Deferred.succeed(release)`. The test scope + * releases it on exit so a failed assertion cannot leave a paused rollback, which runs + * uninterruptibly, blocking cleanup. + */ +const pause = Effect.fnUntraced(function* () { + const reached = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(release, undefined)); + return { + reached, + release, + hold: Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), + }; +}); + +/** + * One CLI process: its own working directory for `project_id`, its own output, and a `waiting` + * signal that completes when the process prints the lifecycle lock's waiting line. Every process + * shares the Docker daemon and `supabaseHome`, like two terminals on one machine. + */ +const makeProcess = Effect.fnUntraced(function* ( + docker: DockerModel, + supabaseHome: string, + projectId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workdir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-lock-" }); + yield* fs.makeDirectory(path.join(workdir, "supabase"), { recursive: true }); + yield* fs.writeFileString( + path.join(workdir, "supabase", "config.toml"), + `project_id = "${projectId}"\n`, + ); + + const out = mockOutput({ format: "text" }); + const waiting = yield* Deferred.make(); + const output = Layer.effect( + Output, + Effect.gen(function* () { + const base = yield* Output; + return { + ...base, + raw: (text: string, stream?: "stdout" | "stderr") => + base + .raw(text, stream) + .pipe( + Effect.tap(() => + text.startsWith("Waiting for another Supabase CLI command") + ? Deferred.succeed(waiting, undefined) + : Effect.void, + ), + ), + }; + }), + ).pipe(Layer.provide(out.layer)); + + const layer = Layer.mergeAll( + unusedStackServices, + BunServices.layer, + output, + mockCommandSettings({ workdir, supabaseHome, projectId: Option.none() }), + mockTelemetryStateTracked().layer, + mockAnalytics().layer, + docker.layer, + alwaysReadyHttpClientLayer, + Layer.succeed(DbConnection, { connect: () => Effect.succeed(emptyDbSession) }), + dockerRunLayer.pipe(Layer.provide(docker.layer), Layer.provide(mockProcessControl().layer)), + mockProcessControl().layer, + mockRuntimeInfo({ platform: "linux" }), + Layer.succeed(CommandPlatformApiFactory, { + make: Effect.die("CommandPlatformApiFactory should not be used by a local start"), + }), + Layer.succeed(CliArgs, { args: [] }), + Layer.succeed(DebugFlag, false), + Layer.succeed(YesFlag, false), + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(NetworkIdFlag, Option.none()), + mockTty({ stdinIsTty: false }), + mockStdin(false), + ); + return { out, waiting, layer }; +}); + +type CliProcess = Effect.Success>; + +const runStart = (cli: CliProcess) => + start(startFlags).pipe(Effect.provide(cli.layer), Effect.forkChild({ startImmediately: true })); + +const runStop = (cli: CliProcess, flags: StopFlags = stopFlags()) => + stop(flags).pipe(Effect.provide(cli.layer), Effect.forkChild({ startImmediately: true })); + +/** Resolves as soon as the process either prints the waiting line or finishes. */ +const waitingOrFinished = (cli: CliProcess, fiber: Fiber.Fiber) => + Effect.raceFirst( + Deferred.await(cli.waiting).pipe(Effect.as("waiting" as const)), + Fiber.await(fiber).pipe(Effect.as("finished" as const)), + ); + +const isNetworkCreate = (args: ReadonlyArray) => + args[0] === "network" && args[1] === "create"; +const isNetworkPrune = (args: ReadonlyArray) => + args[0] === "network" && args[1] === "prune"; + +const NETWORK = "supabase_network_demo"; + +describe("project lifecycle lock", () => { + it.live( + "holds a stop of the same project until the start in flight has attached its containers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "demo"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper); + + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + expect(stopper.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with project demo...", + ); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + yield* Fiber.join(stopping); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "holds stop --all until the start in flight has attached its containers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "other"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper, stopFlags({ all: Option.some(true) })); + + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + expect(stopper.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with local Supabase projects...", + ); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + yield* Fiber.join(stopping); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "lets starts of different projects run side by side", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const alphaNetworkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => + isNetworkCreate(args) && args.at(-1) === "supabase_network_alpha" + ? alphaNetworkCreated.hold + : Effect.void, + }); + const alpha = yield* makeProcess(docker, home, "alpha"); + const beta = yield* makeProcess(docker, home, "beta"); + + const startingAlpha = yield* runStart(alpha); + yield* Deferred.await(alphaNetworkCreated.reached); + const startingBeta = yield* runStart(beta); + + expect(yield* waitingOrFinished(beta, startingBeta)).toBe("finished"); + expect(docker.containerNames()).toContain("supabase_db_beta"); + expect(docker.containerNames()).not.toContain("supabase_db_alpha"); + + yield* Deferred.succeed(alphaNetworkCreated.release, undefined); + yield* Fiber.join(startingAlpha); + yield* Fiber.join(startingBeta); + expect(docker.containerNames()).toContain("supabase_db_alpha"); + expect(docker.networkNames()).toEqual(["supabase_network_alpha", "supabase_network_beta"]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "holds a second start until the first start has finished rolling back", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const rollbackAtNetworkPrune = yield* pause(); + let firstStartFailed = false; + let rollbackPaused = false; + const docker = dockerModel({ + before: Effect.fnUntraced(function* (args: ReadonlyArray) { + if (args[0] === "create" && args.includes("supabase_auth_demo") && !firstStartFailed) { + firstStartFailed = true; + return { exitCode: 1, stderr: ["Error: no space left on device"] }; + } + if (isNetworkPrune(args) && !rollbackPaused) { + rollbackPaused = true; + yield* rollbackAtNetworkPrune.hold; + } + return undefined; + }), + }); + const first = yield* makeProcess(docker, home, "demo"); + const second = yield* makeProcess(docker, home, "demo"); + + const firstStarting = yield* runStart(first); + yield* Deferred.await(rollbackAtNetworkPrune.reached); + const secondStarting = yield* runStart(second); + + expect(yield* waitingOrFinished(second, secondStarting)).toBe("waiting"); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(rollbackAtNetworkPrune.release, undefined); + expect(Exit.isFailure(yield* Fiber.await(firstStarting))).toBe(true); + yield* Fiber.join(secondStarting); + expect(docker.networkNames()).toEqual([NETWORK]); + expect(docker.containerNames()).toContain("supabase_auth_demo"); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "leaves the start in flight untouched when a waiting stop is interrupted", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "demo"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper); + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + + yield* Fiber.interrupt(stopping); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + expect(docker.containerNames()).toContain("supabase_db_demo"); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Fiber.join(yield* runStop(stopper)); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.effect("fails a start that outwaits the lock budget without touching the owner's stack", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const owner = yield* makeProcess(docker, home, "demo"); + const waiter = yield* makeProcess(docker, home, "demo"); + + yield* runStart(owner); + yield* Deferred.await(networkCreated.reached); + const blocked = yield* start(startFlags).pipe( + Effect.provide(waiter.layer), + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + // The lock polls on the controlled clock, so step it until the waiter has polled once. + yield* Effect.raceFirst( + Effect.forever(TestClock.adjust("1 second")), + waitingOrFinished(waiter, blocked), + ); + expect(yield* waitingOrFinished(waiter, blocked)).toBe("waiting"); + yield* TestClock.adjust("10 minutes"); + + expect(yield* Fiber.join(blocked)).toBeInstanceOf(ProjectLifecycleLockBusyError); + expect(docker.networkNames()).toEqual([NETWORK]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("fails before any Docker command when the lock directory cannot be created", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const notADirectory = path.join(base, "file"); + yield* fs.writeFileString(notADirectory, ""); + const docker = dockerModel(); + const cli = yield* makeProcess(docker, path.join(notADirectory, "home"), "demo"); + + const error = yield* start(startFlags).pipe(Effect.provide(cli.layer), Effect.flip); + + expect(error).toBeInstanceOf(ProjectLifecycleLockIoError); + expect(docker.commands()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.ts b/apps/cli/src/command-internal/project-lifecycle-lock.ts new file mode 100644 index 0000000000..a770fdeb9d --- /dev/null +++ b/apps/cli/src/command-internal/project-lifecycle-lock.ts @@ -0,0 +1,154 @@ +import { + holdExclusiveLock, + holdLock, + holdSharedLock, + isLockBusy, + type NamespaceError, +} from "@supabase/stack/effect"; +import { Clock, Crypto, Data, Duration, Effect, FileSystem, Path, type Scope } from "effect"; + +import { CommandSettings } from "../config/command-settings.service.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../shared/telemetry/error-actionability.ts"; +import { cliProjectFilterValue } from "./docker-ids.ts"; + +const LOCKS_DIR = "locks"; +const GATE_FILE = "docker-lifecycle.lock"; +const LOCK_BUDGET = Duration.minutes(10); +const LOCK_SUBJECT = "the Docker lifecycle lock"; + +/** Another lifecycle command kept the lock for the whole wait budget. */ +export class ProjectLifecycleLockBusyError extends Data.TaggedError( + "ProjectLifecycleLockBusyError", +)<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return { ...actionability.invalidInput, fingerprint_suffix: "lease_held" }; + } +} + +/** The lock files could not be created or opened. */ +export class ProjectLifecycleLockIoError extends Data.TaggedError("ProjectLifecycleLockIoError")<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return { ...actionability.runtimeCrash, fingerprint_suffix: "filesystem_io" }; + } +} + +type LockError = ProjectLifecycleLockBusyError | ProjectLifecycleLockIoError; +type TakeLock = ( + file: string, + budget: Duration.Input, + subject: string, +) => Effect.Effect; + +/** + * Holds the cross-process lock that keeps lifecycle commands from pruning Docker resources another + * command is still creating, for the enclosing scope. An empty `projectId` targets every project + * (`stop --all`) and excludes all other holders; otherwise it excludes only commands for the same + * project. Take it before any Docker probe and release it after rollback. It is not reentrant, so + * never take it inside code a lock holder calls. Older binaries, raw `docker` commands, and a + * different `SUPABASE_HOME` are not covered. + */ +export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(function* ( + projectId: string, +) { + const { supabaseHome } = yield* CommandSettings; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const output = yield* Output; + const crypto = yield* Crypto.Crypto; + + const allProjects = projectId.length === 0; + const started = yield* Clock.currentTimeMillis; + let waited = false; + + const take = Effect.fnUntraced(function* ( + hold: TakeLock, + file: string, + waitingLine: string, + busyMessage: string, + ) { + const failure = (error: NamespaceError): LockError => + isLockBusy(error) + ? new ProjectLifecycleLockBusyError({ message: busyMessage }) + : new ProjectLifecycleLockIoError({ message: `failed to lock ${file}: ${error.message}` }); + yield* hold(file, Duration.zero, LOCK_SUBJECT).pipe( + Effect.catchIf(isLockBusy, () => + Effect.gen(function* () { + if (!waited) { + waited = true; + yield* output.raw(waitingLine, "stderr"); + } + const elapsed = (yield* Clock.currentTimeMillis) - started; + yield* hold( + file, + Duration.millis(Math.max(0, Duration.toMillis(LOCK_BUDGET) - elapsed)), + LOCK_SUBJECT, + ); + }), + ), + Effect.mapError(failure), + ); + }); + + yield* Effect.gen(function* () { + const dir = path.join(supabaseHome, LOCKS_DIR); + yield* fs.makeDirectory(dir, { recursive: true }).pipe( + Effect.mapError( + (error) => + new ProjectLifecycleLockIoError({ + message: `failed to create ${dir}: ${error.message}`, + }), + ), + ); + const gate = path.join(dir, GATE_FILE); + + if (allProjects) { + yield* take( + holdExclusiveLock, + gate, + "Waiting for another Supabase CLI command to finish with local Supabase projects...\n", + "Another Supabase CLI command is still starting or stopping local Supabase projects. Wait for it to finish, then retry.", + ); + return; + } + + const digest = yield* crypto.digest( + "SHA-256", + new TextEncoder().encode(cliProjectFilterValue(projectId)), + ); + const key = Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); + yield* take( + holdSharedLock, + gate, + `Waiting for another Supabase CLI command to finish with project ${projectId}...\n`, + "Another Supabase CLI command is stopping all local Supabase projects. Wait for it to finish, then retry.", + ); + yield* take( + holdLock, + path.join(dir, `docker-project-${key}.lock`), + `Waiting for another Supabase CLI command to finish with project ${projectId}...\n`, + `Another Supabase CLI command is still starting or stopping project ${projectId}. Wait for it to finish, then retry.`, + ); + }).pipe( + Effect.tap(() => Effect.annotateCurrentSpan({ "lock.outcome": "acquired" })), + Effect.tapError((error) => + Effect.annotateCurrentSpan({ + "lock.outcome": error._tag === "ProjectLifecycleLockBusyError" ? "timeout" : "error", + }), + ), + Effect.onExit(() => + Effect.annotateCurrentSpan({ + "lock.mode": allProjects ? "all" : "project", + "lock.waited": waited, + }), + ), + ); +}); diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index 7e017bd662..ce692bb5f5 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -85,13 +85,15 @@ volume was confirmed fresh this run). ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------- | -| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | -| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | -| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | -| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | -| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| Path | Format | When | +| --------------------------------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | +| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | +| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | +| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | +| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes | ## Subprocesses @@ -155,15 +157,16 @@ code is surfaced on failure. ## Exit Codes -| Code | Condition | -| ---- | --------------------------------------------------------------------------- | -| `0` | success — database started, or already running | -| `0` | `--from-backup` set and the health-check timed out (swallowed) | -| `1` | malformed `supabase/config.toml` | -| `1` | Docker daemon unreachable / inspect failure | -| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | -| `1` | a health-check timeout with `--from-backup` unset | -| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | +| Code | Condition | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success — database started, or already running | +| `0` | `--from-backup` set and the health-check timed out (swallowed) | +| `1` | malformed `supabase/config.toml` | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` to stderr first (`ProjectLifecycleLockBusyError`) | +| `1` | Docker daemon unreachable / inspect failure | +| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | +| `1` | a health-check timeout with `--from-backup` unset | +| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | ## Output diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index 76860f2153..fcddc599d4 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -116,12 +116,14 @@ command. ## Files Written -| Path | Format | When | -| ---------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | -| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | -| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | -| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | +| Path | Format | When | +| ---------------------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | +| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | +| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | +| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes | Kong's `custom_nginx.template`, Vector's `vector.yaml`, and Postgres's own bootstrap script (`postgresql.conf`-equivalent setup) are all rendered in memory and injected @@ -226,6 +228,7 @@ code is surfaced on failure. | `1` | malformed CSV in an `--exclude`/`-x` value — fails during flag parsing, before the handler and telemetry, with the exact diagnostic text on stderr; the shorthand frames it with both spellings (e.g. `invalid argument "a\"b" for "-x, --exclude" flag: parse error on line 1, column 2: bare " in non-quoted-field`; a blank-only value fails with `EOF`) — CLI-2005 | | `1` | malformed `config.toml` / `Config.Validate` failure, including an `auth.email.*.content_path` that resolves outside the project root, or that resolves in-root but is missing/unreadable (checked eagerly, before any Docker work, regardless of `auth.enabled` — see Notes) | | `1` | stopped Postgres detected but the project id sanitizes to empty — aborts before recovery removes any containers | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` to stderr first (`ProjectLifecycleLockBusyError`) | | `1` | `docker`/`podman` not spawnable, or the daemon is unreachable | | `1` | stopped-stack recovery cannot list, stop, or prune current-project containers, or prune matching networks — aborts before startup; named volumes are preserved | | `1` | image pull exhausted across every registry candidate, or the Docker daemon becomes unreachable during the pre-pull — even with `--ignore-health-check` (intentional divergence from the old Go CLI's exit-0 swallow quirk; see the CLI-1987 note under "Notes") | @@ -318,6 +321,11 @@ prose, not structured data. ## Notes +- **Lifecycle lock**: `start`, `db start`, and `stop` serialize on `$SUPABASE_HOME/locks`, so a + `docker network prune` from one never lands between another's network create and its first + container attach. The lock does not cover older CLI binaries, raw `docker` commands, a different + `SUPABASE_HOME`, projects sharing a custom `--network-id`, or Docker requests a killed process + already sent, and `stop --all` is not queued ahead of a stream of single-project commands. - **`pg_net` converges with `[experimental.webhooks]` on every non-backup start** (see `docs/go-cli-divergences.md`): a fresh volume installs `pg_net` only when webhooks are enabled or migration history contains a `create extension … pg_net`; an existing volume diff --git a/apps/cli/src/commands/start/start.handler.ts b/apps/cli/src/commands/start/start.handler.ts index cf6764dd03..8f6ef8be16 100644 --- a/apps/cli/src/commands/start/start.handler.ts +++ b/apps/cli/src/commands/start/start.handler.ts @@ -67,6 +67,7 @@ import { type ContainerIdName, } from "../../command-internal/docker-lifecycle.ts"; import { dockerRemoveAll } from "../../command-internal/docker-remove-all.ts"; +import { holdProjectLifecycleLock } from "../../command-internal/project-lifecycle-lock.ts"; import { envOverride, envOverrideApiMaxRows, @@ -594,6 +595,9 @@ export const start = Effect.fn("start")(function* (flags: StartFlags) { const dbContainerId = localDbContainerId(projectId); const filterValue = cliProjectFilterValue(projectId); + // Held until the handler scope closes, after recovery, bring-up, and any rollback have run. + yield* holdProjectLifecycleLock(projectId); + // Shared by the already-running branch (full status pipeline, re-health-checked) and the // success path (a direct pretty-print, no re-health-check). // @@ -1919,5 +1923,5 @@ export const start = Effect.fn("start")(function* (flags: StartFlags) { const { values: statusValues } = yield* buildStatusValues(statusExcluded, values); yield* output.success("", statusValues); } - }).pipe(Effect.ensuring(telemetryState.flush)); + }).pipe(Effect.scoped, Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/commands/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/stop/SIDE_EFFECTS.md index f541f79fc1..6ab0753778 100644 --- a/apps/cli/src/commands/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/stop/SIDE_EFFECTS.md @@ -25,10 +25,12 @@ model (see the CLI-1324 plan's "Critical architectural finding" for why). ## Files Written -| Path | Format | When | -| ------------------------------------------------------------------- | ------------------- | --------------------------------------------------------------------------------------- | -| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | -| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | +| Path | Format | When | +| ------------------------------------------------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | +| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project (not taken by `--all`); waits up to 10 minutes | The `start-secrets` removal is a TS-port-only hygiene step (`cleanupStartSecrets`, `command-internal/start-secrets-cleanup.ts`) — the old Go CLI never staged secrets on @@ -89,17 +91,18 @@ and ensure it is on PATH") rather than a generic "failed to ..." string. ## Exit Codes -| Code | Condition | -| ---- | -------------------------------------------------------------------------------------------------- | -| `0` | success — containers/volumes/networks pruned | -| `1` | `--project-id` and `--all` both set (`StopMutuallyExclusiveError`) | -| `1` | `config.toml` present but malformed (`StopConfigLoadError`) — an **absent** file is not an error | -| `1` | listing containers failed (`StopListError`) | -| `1` | stopping one or more containers failed (`StopContainerError`) | -| `1` | `docker container prune` failed (`StopContainerPruneError`) | -| `1` | `docker volume prune` failed, only reached when volumes are being deleted (`StopVolumePruneError`) | -| `1` | `docker network prune` failed (`StopNetworkPruneError`) | -| `1` | `docker`/`podman` both absent from `PATH` (surfaces as one of the errors above) | +| Code | Condition | +| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success — containers/volumes/networks pruned | +| `1` | `--project-id` and `--all` both set (`StopMutuallyExclusiveError`) | +| `1` | another lifecycle command still holds the lock after 10 minutes (`ProjectLifecycleLockBusyError`); `--all` waits for every other lifecycle command | +| `1` | `config.toml` present but malformed (`StopConfigLoadError`) — an **absent** file is not an error | +| `1` | listing containers failed (`StopListError`) | +| `1` | stopping one or more containers failed (`StopContainerError`) | +| `1` | `docker container prune` failed (`StopContainerPruneError`) | +| `1` | `docker volume prune` failed, only reached when volumes are being deleted (`StopVolumePruneError`) | +| `1` | `docker network prune` failed (`StopNetworkPruneError`) | +| `1` | `docker`/`podman` both absent from `PATH` (surfaces as one of the errors above) | ## Telemetry Events Fired diff --git a/apps/cli/src/commands/stop/stop.handler.ts b/apps/cli/src/commands/stop/stop.handler.ts index 23525ed499..4988aa1f2c 100644 --- a/apps/cli/src/commands/stop/stop.handler.ts +++ b/apps/cli/src/commands/stop/stop.handler.ts @@ -13,6 +13,7 @@ import { type ContainerIdName, } from "../../command-internal/docker-lifecycle.ts"; import { dockerRemoveAll } from "../../command-internal/docker-remove-all.ts"; +import { holdProjectLifecycleLock } from "../../command-internal/project-lifecycle-lock.ts"; import { cleanupStartSecrets } from "../../command-internal/start-secrets-cleanup.ts"; import { resolveLocalConfigValues } from "../../command-internal/local-config-values.ts"; import { @@ -114,6 +115,9 @@ export const stop = Effect.fn("stop")(function* (flags: StopFlags) { const deleteVolumes = flags.noBackup; const filterValue = cliProjectFilterValue(searchProjectIdFilter); + // Held through the prune and its secret cleanup; `--all` excludes every other lifecycle command. + yield* holdProjectLifecycleLock(searchProjectIdFilter); + // Printed directly (not via `output.task`'s debounced spinner) so it always appears — the // spinner's delay drops the message when Docker calls resolve faster than the threshold, as // they do against mocked/replayed Docker. @@ -192,5 +196,5 @@ export const stop = Effect.fn("stop")(function* (flags: StopFlags) { ); } } - }).pipe(Effect.ensuring(telemetryState.flush)); + }).pipe(Effect.scoped, Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index c53504bd7a..8dcf53dacc 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -423,6 +423,8 @@ PostgresConfigUpdateUnmarshalError ProfileLoadError ProfileSaveError ProjectConfigParseError +ProjectLifecycleLockBusyError +ProjectLifecycleLockIoError ProjectLinkIoError ProjectLinkMalformedError ProjectNotLinkedError diff --git a/apps/cli/tests/helpers/docker-model.ts b/apps/cli/tests/helpers/docker-model.ts new file mode 100644 index 0000000000..9c17b88394 --- /dev/null +++ b/apps/cli/tests/helpers/docker-model.ts @@ -0,0 +1,243 @@ +import { Effect, Layer, Sink, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { CLI_WORKDIR_LABEL } from "../../src/command-internal/docker-ids.ts"; + +interface DockerResult { + readonly exitCode?: number; + readonly stdout?: ReadonlyArray; + readonly stderr?: ReadonlyArray; +} + +interface DockerModelHooks { + /** Runs before the daemon handles a command; a returned result replaces the daemon's answer. */ + readonly before?: (args: ReadonlyArray) => Effect.Effect; + /** Runs after the daemon applied a command and before the CLI sees its exit. */ + readonly after?: (args: ReadonlyArray) => Effect.Effect; +} + +interface ModelContainer { + readonly name: string; + readonly id: string; + readonly labels: Readonly>; + readonly network: string | undefined; + status: "created" | "running" | "exited"; +} + +const BUILT_IN_NETWORKS = new Set(["bridge", "host", "none", "default"]); + +const STATE_JSON: Readonly> = { + created: '{"Running":false,"Status":"created"}', + running: '{"Running":true,"Status":"running","Health":{"Status":"healthy"}}', + exited: '{"Running":false,"Status":"exited"}', +}; + +function flagValues(args: ReadonlyArray, flag: string): ReadonlyArray { + const values: Array = []; + args.forEach((arg, index) => { + const value = args[index + 1]; + if (arg === flag && value !== undefined) values.push(value); + }); + return values; +} + +function parseLabels(args: ReadonlyArray): Readonly> { + return Object.fromEntries( + flagValues(args, "--label").map((label) => { + const separator = label.indexOf("="); + return separator < 0 ? [label, ""] : [label.slice(0, separator), label.slice(separator + 1)]; + }), + ); +} + +/** `--filter label=` and `--filter label==` selectors, ANDed like Docker does. */ +function matchesLabelFilters( + labels: Readonly>, + args: ReadonlyArray, +): boolean { + return flagValues(args, "--filter") + .filter((filter) => filter.startsWith("label=")) + .map((filter) => filter.slice("label=".length)) + .every((selector) => { + const separator = selector.indexOf("="); + const value = labels[separator < 0 ? selector : selector.slice(0, separator)]; + return value !== undefined && (separator < 0 || value === selector.slice(separator + 1)); + }); +} + +function containerId(name: string): string { + return [...name] + .map((char) => (char.codePointAt(0) ?? 0).toString(16).padStart(2, "0")) + .join("") + .padEnd(64, "0") + .slice(0, 64); +} + +/** + * A stateful in-memory Docker daemon behind the `docker` CLI, covering what `start`, `stop` and + * rollback ask of it. Networks and containers carry their labels, label-filtered `network prune` + * removes only networks no container is attached to, and creating a container on a network that + * does not exist fails the way the daemon does. Volumes always exist and every command it does not + * model succeeds with no output. + */ +export function dockerModel(hooks: DockerModelHooks = {}) { + const networks = new Map>>(); + const containers = new Map(); + const commands: Array> = []; + const encoder = new TextEncoder(); + + const findContainer = (ref: string | undefined) => + ref === undefined + ? undefined + : (containers.get(ref) ?? [...containers.values()].find((container) => container.id === ref)); + + const handle = (args: ReadonlyArray): DockerResult => { + const [command, subcommand] = args; + + if (command === "network" && subcommand === "inspect") { + const name = args[2] ?? ""; + return networks.has(name) + ? { exitCode: 0 } + : { exitCode: 1, stderr: [`Error response from daemon: network ${name} not found`] }; + } + if (command === "network" && subcommand === "create") { + const name = args.at(-1) ?? ""; + if (networks.has(name)) { + return { + exitCode: 1, + stderr: [`Error response from daemon: network with name ${name} already exists`], + }; + } + networks.set(name, parseLabels(args)); + return { exitCode: 0, stdout: [containerId(name)] }; + } + if (command === "network" && subcommand === "prune") { + const pruned = [...networks] + .filter( + ([name, labels]) => + matchesLabelFilters(labels, args) && + ![...containers.values()].some((container) => container.network === name), + ) + .map(([name]) => name); + for (const name of pruned) networks.delete(name); + return { exitCode: 0, stdout: ["Deleted Networks:", ...pruned] }; + } + + if (command === "create") { + const name = flagValues(args, "--name")[0] ?? `container-${commands.length}`; + const network = flagValues(args, "--network")[0]; + if (network !== undefined && !BUILT_IN_NETWORKS.has(network) && !networks.has(network)) { + return { + exitCode: 1, + stderr: [`Error response from daemon: network ${network} not found`], + }; + } + const container: ModelContainer = { + name, + id: containerId(name), + labels: parseLabels(args), + network, + status: "created", + }; + containers.set(name, container); + return { exitCode: 0, stdout: [container.id] }; + } + if (command === "start" || command === "stop") { + const container = findContainer(args[1]); + if (container === undefined) { + return { exitCode: 1, stderr: [`Error: No such container: ${args[1] ?? ""}`] }; + } + container.status = command === "start" ? "running" : "exited"; + return { exitCode: 0, stdout: [container.id] }; + } + if (command === "container" && subcommand === "inspect") { + const container = findContainer(args[2]); + return container === undefined + ? { exitCode: 1, stderr: [`Error: No such container: ${args[2] ?? ""}`] } + : { exitCode: 0, stdout: [STATE_JSON[container.status]] }; + } + if (command === "container" && subcommand === "prune") { + const pruned = [...containers.values()].filter( + (container) => + container.status !== "running" && matchesLabelFilters(container.labels, args), + ); + for (const container of pruned) containers.delete(container.name); + return { exitCode: 0, stdout: ["Deleted Containers:", ...pruned.map(({ id }) => id)] }; + } + if (command === "ps") { + const format = flagValues(args, "--format")[0] ?? ""; + const all = args.includes("--all"); + return { + exitCode: 0, + stdout: [...containers.values()] + .filter( + (container) => + (all || container.status === "running") && + matchesLabelFilters(container.labels, args), + ) + .map((container) => + format.includes("{{.Label") + ? [container.id, container.name, container.labels[CLI_WORKDIR_LABEL] ?? ""].join("\t") + : format.includes("{{.Names}}") + ? container.name + : container.id, + ), + }; + } + + if (command === "version") { + const format = flagValues(args, "--format")[0] ?? ""; + return { + exitCode: 0, + stdout: [ + format + .replaceAll("{{.Client.APIVersion}}", "1.45") + .replaceAll("{{.Server.APIVersion}}", "1.45"), + ], + }; + } + if (command === "context" && subcommand === "inspect") return { exitCode: 1 }; + return { exitCode: 0 }; + }; + + const layer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const args = ChildProcess.isStandardCommand(command) ? command.args : []; + commands.push(args); + const override = hooks.before === undefined ? undefined : yield* hooks.before(args); + const result = override ?? handle(args); + if (override === undefined && hooks.after !== undefined) yield* hooks.after(args); + + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(6000 + commands.length), + stdout: Stream.fromIterable( + (result.stdout ?? []).map((line) => encoder.encode(`${line}\n`)), + ), + stderr: Stream.fromIterable( + (result.stderr ?? []).map((line) => encoder.encode(`${line}\n`)), + ), + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.exitCode ?? 0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ); + + return { + layer, + /** Argv of every command the CLI issued, in order. */ + commands: (): ReadonlyArray> => commands, + networkNames: (): ReadonlyArray => [...networks.keys()].sort(), + containerNames: (): ReadonlyArray => [...containers.keys()].sort(), + }; +} + +export type DockerModel = ReturnType; diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index 9d94d9450f..0ea55bd1b9 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -6,7 +6,7 @@ import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; export { NamespaceError } from "./namespace/Capabilities.ts"; export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; -export { holdLock, isLockBusy } from "./namespace/Lock.ts"; +export { holdExclusiveLock, holdLock, holdSharedLock, isLockBusy } from "./namespace/Lock.ts"; export { resolveStackDataRoot, stackDataRootLabel, stackLogsRoot } from "./namespace/Paths.ts"; export { publish } from "./namespace/Publication.ts"; export { diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index a8159dcd47..3185817884 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -84,6 +84,10 @@ export { StackError } from "./Rpc.ts"; export { NamespaceError, publish } from "./StackNamespace.ts"; /** Holds an exclusive cross-process lock on a file for the enclosing scope, waiting up to a budget. */ export { holdLock } from "./StackNamespace.ts"; +/** Holds a cross-process lock shared with other shared holders, excluding `holdExclusiveLock`. */ +export { holdSharedLock } from "./StackNamespace.ts"; +/** Holds a cross-process lock that excludes every `holdSharedLock` holder. */ +export { holdExclusiveLock } from "./StackNamespace.ts"; /** Whether a `holdLock` failure means another holder kept the lock for the whole budget. */ export { isLockBusy } from "./StackNamespace.ts"; export { diff --git a/packages/stack/src/namespace/Lock.ts b/packages/stack/src/namespace/Lock.ts index db4a11d52e..7b2a0b8d44 100644 --- a/packages/stack/src/namespace/Lock.ts +++ b/packages/stack/src/namespace/Lock.ts @@ -1,7 +1,14 @@ import { Effect, Exit, Schedule, Schema, Scope } from "effect"; import type { Duration } from "effect"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; -import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; +import type { DatabaseSync } from "node:sqlite"; +import { + acquireLock, + isBusy, + takeExclusiveLock, + takeLock, + takeSharedLock, +} from "./drivers/Sqlite.ts"; class LockBusy extends Schema.TaggedError()("Namespace.LockBusy", { message: Schema.String, @@ -18,6 +25,35 @@ export const holdLock = ( file: string, budget: Duration.Input, subject: string, +): Effect.Effect => holdWith(takeLock, file, budget, subject); + +/** + * Takes a shared lock at `file` for the enclosing scope: shared holders coexist, and only a + * {@link holdExclusiveLock} holder makes them wait, up to `budget`. + */ +export const holdSharedLock = ( + file: string, + budget: Duration.Input, + subject: string, +): Effect.Effect => + holdWith(takeSharedLock, file, budget, subject); + +/** + * Takes the lock at `file` exclusively of every {@link holdSharedLock} holder for the enclosing + * scope, waiting up to `budget` for them to release. + */ +export const holdExclusiveLock = ( + file: string, + budget: Duration.Input, + subject: string, +): Effect.Effect => + holdWith(takeExclusiveLock, file, budget, subject); + +const holdWith = ( + take: (connection: DatabaseSync) => Effect.Effect, + file: string, + budget: Duration.Input, + subject: string, ): Effect.Effect => Effect.gen(function* () { const held = yield* Effect.scope; @@ -26,7 +62,7 @@ export const holdLock = ( const attempt = yield* Scope.fork(held, "sequential"); yield* acquireLock(file, "create").pipe( Scope.provide(attempt), - Effect.flatMap(takeLock), + Effect.flatMap(take), Effect.onExit((exit) => Exit.isFailure(exit) ? Scope.close(attempt, exit) : Effect.void, ), From 8e3abd3c0280386d6571962d1237310875d56033 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 28 Oct 2026 17:03:53 +0100 Subject: [PATCH 2/3] fix(cli): name stop --all in the gate waiting line and document lock I/O failures --- ...project-lifecycle-lock.integration.test.ts | 34 +++++++++++++++++++ .../project-lifecycle-lock.ts | 2 +- .../cli/src/commands/db/start/SIDE_EFFECTS.md | 21 ++++++------ apps/cli/src/commands/start/SIDE_EFFECTS.md | 3 +- apps/cli/src/commands/stop/SIDE_EFFECTS.md | 1 + packages/stack/src/effect.ts | 2 +- packages/stack/src/namespace/Lock.ts | 2 +- 7 files changed, 51 insertions(+), 14 deletions(-) diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts index 5d1fba5bb3..4aebdfa993 100644 --- a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts +++ b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts @@ -233,6 +233,40 @@ describe("project lifecycle lock", () => { 10_000, ); + it.live( + "holds a start until stop --all has finished pruning", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const pruning = yield* pause(); + const docker = dockerModel({ + before: Effect.fnUntraced(function* (args: ReadonlyArray) { + if (isNetworkPrune(args)) yield* pruning.hold; + return undefined; + }), + }); + const stopper = yield* makeProcess(docker, home, "other"); + const starter = yield* makeProcess(docker, home, "demo"); + + const stopping = yield* runStop(stopper, stopFlags({ all: Option.some(true) })); + yield* Deferred.await(pruning.reached); + const starting = yield* runStart(starter); + + expect(yield* waitingOrFinished(starter, starting)).toBe("waiting"); + expect(starter.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with local Supabase projects...", + ); + expect(docker.networkNames()).toEqual([]); + + yield* Deferred.succeed(pruning.release, undefined); + yield* Fiber.join(stopping); + yield* Fiber.join(starting); + expect(docker.networkNames()).toEqual([NETWORK]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + it.live( "lets starts of different projects run side by side", () => diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.ts b/apps/cli/src/command-internal/project-lifecycle-lock.ts index a770fdeb9d..d8a3cae578 100644 --- a/apps/cli/src/command-internal/project-lifecycle-lock.ts +++ b/apps/cli/src/command-internal/project-lifecycle-lock.ts @@ -128,7 +128,7 @@ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(f yield* take( holdSharedLock, gate, - `Waiting for another Supabase CLI command to finish with project ${projectId}...\n`, + "Waiting for another Supabase CLI command to finish with local Supabase projects...\n", "Another Supabase CLI command is stopping all local Supabase projects. Wait for it to finish, then retry.", ); yield* take( diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index ce692bb5f5..ee1191cac6 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -157,16 +157,17 @@ code is surfaced on failure. ## Exit Codes -| Code | Condition | -| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | success — database started, or already running | -| `0` | `--from-backup` set and the health-check timed out (swallowed) | -| `1` | malformed `supabase/config.toml` | -| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` to stderr first (`ProjectLifecycleLockBusyError`) | -| `1` | Docker daemon unreachable / inspect failure | -| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | -| `1` | a health-check timeout with `--from-backup` unset | -| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | +| Code | Condition | +| ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success — database started, or already running | +| `0` | `--from-backup` set and the health-check timed out (swallowed) | +| `1` | malformed `supabase/config.toml` | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | +| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | +| `1` | Docker daemon unreachable / inspect failure | +| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | +| `1` | a health-check timeout with `--from-backup` unset | +| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | ## Output diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index fcddc599d4..6fd9ec8f33 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -228,7 +228,8 @@ code is surfaced on failure. | `1` | malformed CSV in an `--exclude`/`-x` value — fails during flag parsing, before the handler and telemetry, with the exact diagnostic text on stderr; the shorthand frames it with both spellings (e.g. `invalid argument "a\"b" for "-x, --exclude" flag: parse error on line 1, column 2: bare " in non-quoted-field`; a blank-only value fails with `EOF`) — CLI-2005 | | `1` | malformed `config.toml` / `Config.Validate` failure, including an `auth.email.*.content_path` that resolves outside the project root, or that resolves in-root but is missing/unreadable (checked eagerly, before any Docker work, regardless of `auth.enabled` — see Notes) | | `1` | stopped Postgres detected but the project id sanitizes to empty — aborts before recovery removes any containers | -| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` to stderr first (`ProjectLifecycleLockBusyError`) | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | +| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | | `1` | `docker`/`podman` not spawnable, or the daemon is unreachable | | `1` | stopped-stack recovery cannot list, stop, or prune current-project containers, or prune matching networks — aborts before startup; named volumes are preserved | | `1` | image pull exhausted across every registry candidate, or the Docker daemon becomes unreachable during the pre-pull — even with `--ignore-health-check` (intentional divergence from the old Go CLI's exit-0 swallow quirk; see the CLI-1987 note under "Notes") | diff --git a/apps/cli/src/commands/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/stop/SIDE_EFFECTS.md index 6ab0753778..bcc38c28d8 100644 --- a/apps/cli/src/commands/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/stop/SIDE_EFFECTS.md @@ -96,6 +96,7 @@ and ensure it is on PATH") rather than a generic "failed to ..." string. | `0` | success — containers/volumes/networks pruned | | `1` | `--project-id` and `--all` both set (`StopMutuallyExclusiveError`) | | `1` | another lifecycle command still holds the lock after 10 minutes (`ProjectLifecycleLockBusyError`); `--all` waits for every other lifecycle command | +| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | | `1` | `config.toml` present but malformed (`StopConfigLoadError`) — an **absent** file is not an error | | `1` | listing containers failed (`StopListError`) | | `1` | stopping one or more containers failed (`StopContainerError`) | diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 3185817884..49e2bbc628 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -88,7 +88,7 @@ export { holdLock } from "./StackNamespace.ts"; export { holdSharedLock } from "./StackNamespace.ts"; /** Holds a cross-process lock that excludes every `holdSharedLock` holder. */ export { holdExclusiveLock } from "./StackNamespace.ts"; -/** Whether a `holdLock` failure means another holder kept the lock for the whole budget. */ +/** Whether a lock acquisition failure means another holder kept the lock for the whole budget. */ export { isLockBusy } from "./StackNamespace.ts"; export { Lock, diff --git a/packages/stack/src/namespace/Lock.ts b/packages/stack/src/namespace/Lock.ts index 7b2a0b8d44..946e48bb38 100644 --- a/packages/stack/src/namespace/Lock.ts +++ b/packages/stack/src/namespace/Lock.ts @@ -14,7 +14,7 @@ class LockBusy extends Schema.TaggedError()("Namespace.LockBusy", { message: Schema.String, }) {} -/** Whether `error` is a {@link holdLock} failure because another holder kept the lock for the whole budget. */ +/** Whether `error` is a lock acquisition failure because another holder kept the lock for the whole budget. */ export const isLockBusy = (error: NamespaceError): boolean => Schema.is(LockBusy)(error.cause); /** From d99df44c5f9bcb8188f705e6ada8b57fee16f40b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Thu, 29 Oct 2026 18:18:48 +0100 Subject: [PATCH 3/3] fix(cli): continue without the lifecycle lock when SUPABASE_HOME is unwritable --- ...project-lifecycle-lock.integration.test.ts | 46 +++++++++---- .../project-lifecycle-lock.ts | 66 ++++++++++--------- .../cli/src/commands/db/start/SIDE_EFFECTS.md | 19 +++--- apps/cli/src/commands/start/SIDE_EFFECTS.md | 33 +++++----- apps/cli/src/commands/stop/SIDE_EFFECTS.md | 13 ++-- .../telemetry/__fixtures__/error-tags.txt | 1 - 6 files changed, 98 insertions(+), 80 deletions(-) diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts index 1212e6f205..e8afa86829 100644 --- a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts +++ b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts @@ -30,10 +30,7 @@ import { unusedStackServices } from "../../tests/helpers/unused-stack.ts"; import { DbConnection, type DbSession } from "./db-connection.service.ts"; import { dockerRunLayer } from "./docker-run.layer.ts"; import { DebugFlag, ExperimentalFlag, NetworkIdFlag, YesFlag } from "./global-flags.ts"; -import { - ProjectLifecycleLockBusyError, - ProjectLifecycleLockIoError, -} from "./project-lifecycle-lock.ts"; +import { ProjectLifecycleLockBusyError } from "./project-lifecycle-lock.ts"; const startFlags: StartFlags = { exclude: [], ignoreHealthCheck: false, preview: false }; @@ -433,20 +430,43 @@ describe("project lifecycle lock", () => { ); } - it.effect("fails before any Docker command when the lock directory cannot be created", () => - Effect.gen(function* () { + describe("when the lock directory cannot be created", () => { + const unwritableHome = Effect.fnUntraced(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const base = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); const notADirectory = path.join(base, "file"); yield* fs.writeFileString(notADirectory, ""); - const docker = dockerModel(); - const cli = yield* makeProcess(docker, path.join(notADirectory, "home"), "demo"); + return path.join(notADirectory, "home"); + }); + + it.effect("starts the stack without the lock", () => + Effect.gen(function* () { + const docker = dockerModel(); + const cli = yield* makeProcess(docker, yield* unwritableHome(), "demo"); - const error = yield* start(startFlags).pipe(Effect.provide(cli.layer), Effect.flip); + yield* start(startFlags).pipe(Effect.provide(cli.layer)); - expect(error).toBeInstanceOf(ProjectLifecycleLockIoError); - expect(docker.commands()).toEqual([]); - }).pipe(Effect.provide(BunServices.layer)), - ); + expect(docker.containerNames()).toContain("supabase_db_demo"); + expect(cli.out.stderrText).not.toContain("Waiting for another Supabase CLI command"); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("stops the stack without the lock", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const docker = dockerModel(); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, yield* unwritableHome(), "demo"); + yield* start(startFlags).pipe(Effect.provide(starter.layer)); + expect(docker.containerNames()).toContain("supabase_db_demo"); + + yield* stop(stopFlags()).pipe(Effect.provide(stopper.layer)); + + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + }); }); diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.ts b/apps/cli/src/command-internal/project-lifecycle-lock.ts index 57bf5a013f..1826720cda 100644 --- a/apps/cli/src/command-internal/project-lifecycle-lock.ts +++ b/apps/cli/src/command-internal/project-lifecycle-lock.ts @@ -5,7 +5,17 @@ import { isLockBusy, type NamespaceError, } from "@supabase/stack/effect"; -import { Clock, Crypto, Data, Duration, Effect, FileSystem, Path, type Scope } from "effect"; +import { + Clock, + Crypto, + Data, + Duration, + Effect, + FileSystem, + Option, + Path, + type Scope, +} from "effect"; import { CommandSettings } from "../config/command-settings.service.ts"; import { Output } from "../shared/output/output.service.ts"; @@ -14,6 +24,7 @@ import { type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../shared/telemetry/error-actionability.ts"; +import { DebugLogger } from "./debug-logger.service.ts"; import { cliProjectFilterValue } from "./docker-ids.ts"; const LOCKS_DIR = "locks"; @@ -32,16 +43,6 @@ export class ProjectLifecycleLockBusyError extends Data.TaggedError( } } -/** The lock files could not be created or opened. */ -export class ProjectLifecycleLockIoError extends Data.TaggedError("ProjectLifecycleLockIoError")<{ - readonly message: string; -}> { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return { ...actionability.runtimeCrash, fingerprint_suffix: "filesystem_io" }; - } -} - -type LockError = ProjectLifecycleLockBusyError | ProjectLifecycleLockIoError; type TakeLock = ( file: string, budget: Duration.Input, @@ -53,8 +54,9 @@ type TakeLock = ( * command is still creating, for the enclosing scope. An empty `projectId` targets every project * (`stop --all`) and excludes all other holders; otherwise it excludes only commands for the same * project. Take it before any Docker probe and release it after rollback. It is not reentrant, so - * never take it inside code a lock holder calls. Older binaries, raw `docker` commands, and a - * different `SUPABASE_HOME` are not covered. + * never take it inside code a lock holder calls. A lock that cannot be created or opened is + * skipped with a debug line. Older binaries, raw `docker` commands, and a different `SUPABASE_HOME` + * are not covered. */ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(function* ( projectId: string, @@ -64,6 +66,7 @@ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(f const path = yield* Path.Path; const output = yield* Output; const crypto = yield* Crypto.Crypto; + const debugLogger = yield* Effect.serviceOption(DebugLogger); const allProjects = projectId.length === 0; const started = yield* Clock.currentTimeMillis; @@ -75,10 +78,6 @@ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(f waitingLine: string, busyMessage: string, ) { - const failure = (error: NamespaceError): LockError => - isLockBusy(error) - ? new ProjectLifecycleLockBusyError({ message: busyMessage }) - : new ProjectLifecycleLockIoError({ message: `failed to lock ${file}: ${error.message}` }); yield* hold(file, Duration.zero, LOCK_SUBJECT).pipe( Effect.catchIf(isLockBusy, () => Effect.gen(function* () { @@ -94,20 +93,15 @@ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(f ); }), ), - Effect.mapError(failure), + Effect.mapError((error) => + isLockBusy(error) ? new ProjectLifecycleLockBusyError({ message: busyMessage }) : error, + ), ); }); yield* Effect.gen(function* () { const dir = path.join(supabaseHome, LOCKS_DIR); - yield* fs.makeDirectory(dir, { recursive: true }).pipe( - Effect.mapError( - (error) => - new ProjectLifecycleLockIoError({ - message: `failed to create ${dir}: ${error.message}`, - }), - ), - ); + yield* fs.makeDirectory(dir, { recursive: true }); const gate = path.join(dir, GATE_FILE); if (allProjects) { @@ -138,12 +132,20 @@ export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(f `Another Supabase CLI command is still starting or stopping project ${projectId}. Wait for it to finish, then retry.`, ); }).pipe( - Effect.tap(() => Effect.annotateCurrentSpan({ "lock.outcome": "acquired" })), - Effect.tapError((error) => - Effect.annotateCurrentSpan({ - "lock.outcome": error._tag === "ProjectLifecycleLockBusyError" ? "timeout" : "error", - }), - ), + Effect.matchEffect({ + onSuccess: () => Effect.annotateCurrentSpan({ "lock.outcome": "acquired" }), + onFailure: (error) => + error instanceof ProjectLifecycleLockBusyError + ? Effect.annotateCurrentSpan({ "lock.outcome": "timeout" }).pipe( + Effect.andThen(Effect.fail(error)), + ) + : (Option.isSome(debugLogger) + ? debugLogger.value.debug( + `Continuing without the Docker lifecycle lock: ${error.message}`, + ) + : Effect.void + ).pipe(Effect.andThen(Effect.annotateCurrentSpan({ "lock.outcome": "unavailable" }))), + }), Effect.onExit(() => Effect.annotateCurrentSpan({ "lock.mode": allProjects ? "all" : "project", diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index d60eb20eb3..7ba533b7f2 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -110,15 +110,15 @@ failure is logged to stderr and never replaces the original error. ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | -| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | -| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | -| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | -| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | -| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | -| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes | +| Path | Format | When | +| --------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | +| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | +| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | +| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | +| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | ## Subprocesses @@ -190,7 +190,6 @@ code is surfaced on failure. | `0` | `--from-backup` set and the health-check timed out (swallowed) | | `1` | malformed `supabase/config.toml` | | `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | -| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | | `1` | Docker daemon unreachable / inspect failure | | `1` | db container exists but is not running, in Bitbucket Pipelines (`supabase stop` suggestion) | | `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index bc758ba51e..ebf29df5fe 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -192,14 +192,14 @@ command. ## Files Written -| Path | Format | When | -| ---------------------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | -| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | -| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | -| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | -| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | -| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes | +| Path | Format | When | +| ---------------------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | +| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | +| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | +| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | Kong's `custom_nginx.template`, Vector's `vector.yaml`, and Postgres's own bootstrap script (`postgresql.conf`-equivalent setup) are all rendered in memory and injected @@ -306,7 +306,6 @@ code is surfaced on failure. | `1` | stopped Postgres detected but the project id sanitizes to empty — aborts before recovery removes any containers | | `1` | Postgres container exists but is not running and is not recovered (Bitbucket Pipelines, or Docker reported no state) — ` container is not running: ` (`unknown` for an empty status) with a `supabase stop` then `supabase start` suggestion; the "already running" banner is never printed | | `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | -| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | | `1` | `docker`/`podman` not spawnable, or the daemon is unreachable | | `1` | stopped-stack recovery cannot list, stop, or prune current-project containers, or prune matching networks — aborts before startup; named volumes are preserved | | `1` | image pull exhausted across every registry candidate, or the Docker daemon becomes unreachable during the pre-pull — even with `--ignore-health-check` (intentional divergence from the old Go CLI's exit-0 swallow quirk; see the CLI-1987 note under "Notes") | @@ -404,14 +403,14 @@ prose, not structured data. - **Lifecycle lock**: `start`, `db start`, and `stop` serialize on `$SUPABASE_HOME/locks`, so a `docker network prune` from one never lands between another's network create and its first - container attach. The lock (SQLite `BEGIN IMMEDIATE` locks the OS releases when the process dies) - is taken once the project id is resolved, before inspecting any container, and held until the - command ends, after any rollback. A command that has to wait prints `Waiting for another Supabase -CLI command to finish with project ...` to stderr (text format only), waits up to 10 minutes, - then fails with `ProjectLifecycleLockBusyError`. The lock does not cover older CLI binaries, raw - `docker` commands, a different `SUPABASE_HOME`, projects sharing a custom `--network-id`, or - Docker requests a killed process already sent, and `stop --all` is not queued ahead of a stream - of single-project commands. + container attach. The lock (SQLite `BEGIN IMMEDIATE` locks the OS releases when the process + dies) is taken once the project id is resolved, before inspecting any container, and held until + the command ends, after any rollback. A command that has to wait prints a waiting line to stderr + (text format only), waits up to 10 minutes, then fails with `ProjectLifecycleLockBusyError`. A lock location that cannot be + created or opened never blocks the command: it continues unlocked and logs the reason under + `--debug`. The lock does not cover older CLI binaries, raw `docker` commands, a different + `SUPABASE_HOME`, projects sharing a custom `--network-id`, or Docker requests a killed process + already sent, and `stop --all` is not queued ahead of a stream of single-project commands. - **`pg_net` converges with `[experimental.webhooks]` on every non-backup start** (see `docs/go-cli-divergences.md`): a fresh volume installs `pg_net` only when webhooks are enabled or migration history contains a `create extension … pg_net`; an existing volume diff --git a/apps/cli/src/commands/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/stop/SIDE_EFFECTS.md index bcc38c28d8..2ec6a785d6 100644 --- a/apps/cli/src/commands/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/stop/SIDE_EFFECTS.md @@ -25,12 +25,12 @@ model (see the CLI-1324 plan's "Critical architectural finding" for why). ## Files Written -| Path | Format | When | -| ------------------------------------------------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | -| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | -| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command | -| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project (not taken by `--all`); waits up to 10 minutes | +| Path | Format | When | +| ------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | +| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project (not taken by `--all`); waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | The `start-secrets` removal is a TS-port-only hygiene step (`cleanupStartSecrets`, `command-internal/start-secrets-cleanup.ts`) — the old Go CLI never staged secrets on @@ -96,7 +96,6 @@ and ensure it is on PATH") rather than a generic "failed to ..." string. | `0` | success — containers/volumes/networks pruned | | `1` | `--project-id` and `--all` both set (`StopMutuallyExclusiveError`) | | `1` | another lifecycle command still holds the lock after 10 minutes (`ProjectLifecycleLockBusyError`); `--all` waits for every other lifecycle command | -| `1` | `$SUPABASE_HOME/locks` cannot be created or a lock file there cannot be opened — fails before any Docker command (`ProjectLifecycleLockIoError`) | | `1` | `config.toml` present but malformed (`StopConfigLoadError`) — an **absent** file is not an error | | `1` | listing containers failed (`StopListError`) | | `1` | stopping one or more containers failed (`StopContainerError`) | diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index 142335c8b3..3aba4e8029 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -424,7 +424,6 @@ ProfileLoadError ProfileSaveError ProjectConfigParseError ProjectLifecycleLockBusyError -ProjectLifecycleLockIoError ProjectLinkIoError ProjectLinkMalformedError ProjectNotLinkedError