diff --git a/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts b/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts index 7b56d105cd..85457f9137 100644 --- a/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts +++ b/apps/cli/src/command-internal/db-bootstrap/container-lifecycle.ts @@ -191,10 +191,10 @@ function portConflictSuggestion(hostPort: string, serviceLabel: string): string * Creates the shared docker network for this project via `docker network create --label ...`, * treating "already exists" as success. * - * Called once per `start` run rather than once per container: the network can't be deleted - * externally mid-run, so it's guaranteed to still exist for every later {@link createContainer} - * call. Returns immediately for a built-in network name (`default`, `bridge`, `host`, `none`), - * which already exists and cannot be created. + * Called once per `start` run rather than once per container: callers holding the project lifecycle + * lock keep other lifecycle commands from pruning it before every {@link createContainer} call has run. + * Returns immediately for a built-in network name (`default`, `bridge`, `host`, `none`), which + * already exists and cannot be created. */ export function ensureNetwork( spawner: Spawner, diff --git a/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts b/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts index 68177d6256..37ac986ec0 100644 --- a/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts +++ b/apps/cli/src/command-internal/db-bootstrap/start-local-database.ts @@ -28,7 +28,7 @@ import { parseUrl } from "../storage-url.ts"; import { loadLocalProjectContext } from "../local-project-context.ts"; import { cliProjectFilterValue, localDbContainerId } from "../docker-ids.ts"; import { isBitbucketPipeline } from "../bitbucket-pipeline.ts"; -import { holdLocalStackStartLock } from "../local-stack-start-lock.ts"; +import { holdProjectLifecycleLock } from "../project-lifecycle-lock.ts"; import { dbNotRunningError } from "../status-errors.ts"; import { buildLocalDbContainerInputs } from "./local-container-inputs.ts"; import { removeContainer } from "./container-lifecycle.ts"; @@ -61,6 +61,7 @@ interface StartLocalDatabaseResult { readonly status: StartLocalDatabaseStatus; } +/** Takes the project lifecycle lock into its scope; {@link startLocalDatabase} closes that scope after rollback. */ const startLocalDatabaseInScope = Effect.fn("DbBootstrap.startLocalDatabase")(function* ( fromBackupFlag?: string, ) { @@ -173,8 +174,8 @@ const startLocalDatabaseInScope = Effect.fn("DbBootstrap.startLocalDatabase")(fu document, ).pipe(Effect.mapError((cause) => new DbConfigLoadError({ message: cause.message }))); - // Held through bring-up and rollback so a concurrent start cannot prune this run's containers. - yield* holdLocalStackStartLock(context.projectId); + // Held until this call returns, after any rollback has run. + yield* holdProjectLifecycleLock(context.projectId); // If the db container is already up, tell the caller and stop here. Runs after the config // load/validation above. diff --git a/apps/cli/src/command-internal/local-stack-start-lock.ts b/apps/cli/src/command-internal/local-stack-start-lock.ts deleted file mode 100644 index b6dd7a785b..0000000000 --- a/apps/cli/src/command-internal/local-stack-start-lock.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { holdLock, isLockBusy } from "@supabase/stack/effect"; -import { Clock, Data, Effect, FileSystem, Option, Path } from "effect"; - -import { CommandSettings } from "../config/command-settings.service.ts"; -import { Output } from "../shared/output/output.service.ts"; -import { - actionability, - type CliErrorActionabilityDeclaration, - ErrorActionabilityId, -} from "../shared/telemetry/error-actionability.ts"; -import { DebugLogger } from "../shared/output/debug-logger.service.ts"; -import { aqua } from "./colors.ts"; - -const SUBJECT = "the start lock"; -const NO_WAIT = "0 millis"; -const WAIT_BUDGET = "10 minutes"; -const WAITING_MESSAGE = "Waiting for another supabase start for this project to finish...\n"; - -/** Another `supabase start` for this project held the start lock for the whole wait budget. */ -export class LocalStackStartBusyError extends Data.TaggedError("LocalStackStartBusyError")<{ - readonly message: string; - readonly suggestion?: string; -}> { - get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { - return { ...actionability.startStack, fingerprint_suffix: "lease_held" }; - } -} - -/** - * Serialises `supabase start` and `supabase db start` per project for the enclosing scope. An - * unwritable lock location never blocks a start; it continues unlocked. - */ -export const holdLocalStackStartLock = Effect.fn("LocalStack.startLock")(function* ( - projectId: string, -) { - if (projectId.length === 0) return; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const output = yield* Output; - const { supabaseHome } = yield* CommandSettings; - const debugLogger = yield* Effect.serviceOption(DebugLogger); - const locksDir = path.join(supabaseHome, "locks"); - const lockFile = path.join(locksDir, `start-${projectId}.lock`); - const started = yield* Clock.currentTimeMillis; - - const record = (outcome: "acquired" | "busy" | "unavailable") => - Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => - Effect.annotateCurrentSpan({ "lock.outcome": outcome, "lock.wait_ms": now - started }), - ), - ); - const continueUnlocked = (reason: string) => - (Option.isSome(debugLogger) - ? debugLogger.value.debug(`Continuing without the start lock: ${reason}`) - : Effect.void - ).pipe(Effect.andThen(record("unavailable"))); - const waitForHolder = Effect.gen(function* () { - if (output.format === "text") { - yield* output.raw(WAITING_MESSAGE, "stderr"); - } - yield* holdLock(lockFile, WAIT_BUDGET, SUBJECT); - }); - - yield* fs.makeDirectory(locksDir, { recursive: true }).pipe( - Effect.matchEffect({ - onFailure: (error) => continueUnlocked(error.message), - onSuccess: () => - holdLock(lockFile, NO_WAIT, SUBJECT).pipe( - Effect.catchIf(isLockBusy, () => waitForHolder), - Effect.matchEffect({ - onSuccess: () => record("acquired"), - onFailure: (error) => - isLockBusy(error) - ? record("busy").pipe( - Effect.andThen( - Effect.fail( - new LocalStackStartBusyError({ - message: "another supabase start is still running for this project", - suggestion: `Wait for it to finish or cancel it, then run ${aqua("supabase start")} again.`, - }), - ), - ), - ) - : continueUnlocked(error.message), - }), - ), - }), - ); -}); diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts new file mode 100644 index 0000000000..e38174ca5c --- /dev/null +++ b/apps/cli/src/command-internal/project-lifecycle-lock.integration.test.ts @@ -0,0 +1,474 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Deferred, Effect, Exit, Fiber, FileSystem, Layer, Option, Path } from "effect"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as TestClock from "effect/testing/TestClock"; + +import { CommandPlatformApiFactory } from "../auth/command-platform-api-factory.service.ts"; +import { start } from "../commands/start/start.handler.ts"; +import type { StartFlags } from "../commands/start/start.command.ts"; +import { stop } from "../commands/stop/stop.handler.ts"; +import type { StopFlags } from "../commands/stop/stop.command.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { CliArgs } from "../shared/cli/cli-args.service.ts"; +import { + mockCommandSettings, + mockTelemetryStateTracked, + sequentialExecBatch, +} from "../../tests/helpers/command-mocks.ts"; +import { cliConfigValuesTestLayer } from "../../tests/helpers/config-values-layer.ts"; +import { dockerModel, type DockerModel } from "../../tests/helpers/docker-model.ts"; +import { + mockAnalytics, + mockOutput, + mockProcessControl, + mockRuntimeInfo, + mockStdin, + mockTty, +} from "../../tests/helpers/mocks.ts"; +import { unusedStackServices } from "../../tests/helpers/unused-stack.ts"; +import { DbConnection, type DbSession } from "./db-connection.service.ts"; +import { dockerRunLayer } from "./docker-run.layer.ts"; +import { DebugFlag, ExperimentalFlag, NetworkIdFlag, YesFlag } from "./global-flags.ts"; +import { ProjectLifecycleLockBusyError } from "./project-lifecycle-lock.ts"; + +const startFlags: StartFlags = { exclude: [], ignoreHealthCheck: false, preview: false }; + +const stopFlags = (overrides: Partial = {}): StopFlags => ({ + projectId: Option.none(), + backup: true, + noBackup: false, + all: Option.none(), + ...overrides, +}); + +const alwaysReadyHttpClientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), +); + +/** The restart path of `start` opens a database session to converge webhooks; an empty one suffices. */ +const emptyDbSession: DbSession = { + exec: () => Effect.void, + query: () => Effect.succeed([]), + execBatch: (statements) => sequentialExecBatch(emptyDbSession)(statements), + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), +}; + +/** + * Suspends whoever calls `hold` until the test calls `Deferred.succeed(release)`. The test scope + * releases it on exit so a failed assertion cannot leave a paused rollback, which runs + * uninterruptibly, blocking cleanup. + */ +const pause = Effect.fnUntraced(function* () { + const reached = yield* Deferred.make(); + const release = yield* Deferred.make(); + yield* Effect.addFinalizer(() => Deferred.succeed(release, undefined)); + return { + reached, + release, + hold: Deferred.succeed(reached, undefined).pipe(Effect.andThen(Deferred.await(release))), + }; +}); + +/** + * One CLI process: its own working directory for `project_id`, its own output, and a `waiting` + * signal that completes when the process prints the lifecycle lock's waiting line. Every process + * shares the Docker daemon and `supabaseHome`, like two terminals on one machine. + */ +const makeProcess = Effect.fnUntraced(function* ( + docker: DockerModel, + supabaseHome: string, + projectId: string, + format: "text" | "json" | "stream-json" = "text", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const workdir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-lock-" }); + yield* fs.makeDirectory(path.join(workdir, "supabase"), { recursive: true }); + yield* fs.writeFileString( + path.join(workdir, "supabase", "config.toml"), + `project_id = "${projectId}"\n`, + ); + + const out = mockOutput({ format }); + const waiting = yield* Deferred.make(); + const output = Layer.effect( + Output, + Effect.gen(function* () { + const base = yield* Output; + return { + ...base, + raw: (text: string, stream?: "stdout" | "stderr") => + base + .raw(text, stream) + .pipe( + Effect.tap(() => + text.startsWith("Waiting for another Supabase CLI command") + ? Deferred.succeed(waiting, undefined) + : Effect.void, + ), + ), + }; + }), + ).pipe(Layer.provide(out.layer)); + + const layer = Layer.mergeAll( + unusedStackServices, + BunServices.layer, + cliConfigValuesTestLayer, + output, + mockCommandSettings({ workdir, supabaseHome, projectId: Option.none() }), + mockTelemetryStateTracked().layer, + mockAnalytics().layer, + docker.layer, + alwaysReadyHttpClientLayer, + Layer.succeed(DbConnection, { connect: () => Effect.succeed(emptyDbSession) }), + dockerRunLayer.pipe(Layer.provide(docker.layer), Layer.provide(mockProcessControl().layer)), + mockProcessControl().layer, + mockRuntimeInfo({ platform: "linux" }), + Layer.succeed(CommandPlatformApiFactory, { + make: Effect.die("CommandPlatformApiFactory should not be used by a local start"), + }), + Layer.succeed(CliArgs, { args: [] }), + Layer.succeed(DebugFlag, false), + Layer.succeed(YesFlag, false), + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(NetworkIdFlag, Option.none()), + mockTty({ stdinIsTty: false }), + mockStdin(false), + ); + return { out, waiting, layer }; +}); + +type CliProcess = Effect.Success>; + +const runStart = (cli: CliProcess) => + start(startFlags).pipe(Effect.provide(cli.layer), Effect.forkChild({ startImmediately: true })); + +const runStop = (cli: CliProcess, flags: StopFlags = stopFlags()) => + stop(flags).pipe(Effect.provide(cli.layer), Effect.forkChild({ startImmediately: true })); + +/** Resolves as soon as the process either prints the waiting line or finishes. */ +const waitingOrFinished = (cli: CliProcess, fiber: Fiber.Fiber) => + Effect.raceFirst( + Deferred.await(cli.waiting).pipe(Effect.as("waiting" as const)), + Fiber.await(fiber).pipe(Effect.as("finished" as const)), + ); + +const isNetworkCreate = (args: ReadonlyArray) => + args[0] === "network" && args[1] === "create"; +const isNetworkPrune = (args: ReadonlyArray) => + args[0] === "network" && args[1] === "prune"; + +const NETWORK = "supabase_network_demo"; + +describe("project lifecycle lock", () => { + it.live( + "holds a stop of the same project until the start in flight has attached its containers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "demo"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper); + + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + expect(stopper.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with project demo...", + ); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + yield* Fiber.join(stopping); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "holds stop --all until the start in flight has attached its containers", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "other"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper, stopFlags({ all: Option.some(true) })); + + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + expect(stopper.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with local Supabase projects...", + ); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + yield* Fiber.join(stopping); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "holds a start until stop --all has finished pruning", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const pruning = yield* pause(); + const docker = dockerModel({ + before: Effect.fnUntraced(function* (args: ReadonlyArray) { + if (isNetworkPrune(args)) yield* pruning.hold; + return undefined; + }), + }); + const stopper = yield* makeProcess(docker, home, "other"); + const starter = yield* makeProcess(docker, home, "demo"); + + const stopping = yield* runStop(stopper, stopFlags({ all: Option.some(true) })); + yield* Deferred.await(pruning.reached); + const starting = yield* runStart(starter); + + expect(yield* waitingOrFinished(starter, starting)).toBe("waiting"); + expect(starter.out.stderrText).toContain( + "Waiting for another Supabase CLI command to finish with local Supabase projects...", + ); + expect(docker.networkNames()).toEqual([]); + + yield* Deferred.succeed(pruning.release, undefined); + yield* Fiber.join(stopping); + yield* Fiber.join(starting); + expect(docker.networkNames()).toEqual([NETWORK]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "lets starts of different projects run side by side", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const alphaNetworkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => + isNetworkCreate(args) && args.at(-1) === "supabase_network_alpha" + ? alphaNetworkCreated.hold + : Effect.void, + }); + const alpha = yield* makeProcess(docker, home, "alpha"); + const beta = yield* makeProcess(docker, home, "beta"); + + const startingAlpha = yield* runStart(alpha); + yield* Deferred.await(alphaNetworkCreated.reached); + const startingBeta = yield* runStart(beta); + + expect(yield* waitingOrFinished(beta, startingBeta)).toBe("finished"); + expect(docker.containerNames()).toContain("supabase_db_beta"); + expect(docker.containerNames()).not.toContain("supabase_db_alpha"); + + yield* Deferred.succeed(alphaNetworkCreated.release, undefined); + yield* Fiber.join(startingAlpha); + yield* Fiber.join(startingBeta); + expect(docker.containerNames()).toContain("supabase_db_alpha"); + expect(docker.networkNames()).toEqual(["supabase_network_alpha", "supabase_network_beta"]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "holds a second start until the first start has finished rolling back", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const rollbackAtNetworkPrune = yield* pause(); + let firstStartFailed = false; + let rollbackPaused = false; + const docker = dockerModel({ + before: Effect.fnUntraced(function* (args: ReadonlyArray) { + if (args[0] === "create" && args.includes("supabase_auth_demo") && !firstStartFailed) { + firstStartFailed = true; + return { exitCode: 1, stderr: ["Error: no space left on device"] }; + } + if (isNetworkPrune(args) && !rollbackPaused) { + rollbackPaused = true; + yield* rollbackAtNetworkPrune.hold; + } + return undefined; + }), + }); + const first = yield* makeProcess(docker, home, "demo"); + const second = yield* makeProcess(docker, home, "demo"); + + const firstStarting = yield* runStart(first); + yield* Deferred.await(rollbackAtNetworkPrune.reached); + const secondStarting = yield* runStart(second); + + expect(yield* waitingOrFinished(second, secondStarting)).toBe("waiting"); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(rollbackAtNetworkPrune.release, undefined); + expect(Exit.isFailure(yield* Fiber.await(firstStarting))).toBe(true); + yield* Fiber.join(secondStarting); + expect(docker.networkNames()).toEqual([NETWORK]); + expect(docker.containerNames()).toContain("supabase_auth_demo"); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.live( + "leaves the start in flight untouched when a waiting stop is interrupted", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, home, "demo"); + + const starting = yield* runStart(starter); + yield* Deferred.await(networkCreated.reached); + const stopping = yield* runStop(stopper); + expect(yield* waitingOrFinished(stopper, stopping)).toBe("waiting"); + + yield* Fiber.interrupt(stopping); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Deferred.succeed(networkCreated.release, undefined); + yield* Fiber.join(starting); + expect(docker.containerNames()).toContain("supabase_db_demo"); + expect(docker.networkNames()).toEqual([NETWORK]); + + yield* Fiber.join(yield* runStop(stopper)); + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + 10_000, + ); + + it.effect("fails a start that outwaits the lock budget without touching the owner's stack", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const owner = yield* makeProcess(docker, home, "demo"); + const waiter = yield* makeProcess(docker, home, "demo"); + + yield* runStart(owner); + yield* Deferred.await(networkCreated.reached); + const blocked = yield* start(startFlags).pipe( + Effect.provide(waiter.layer), + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + // The lock polls on the controlled clock, so step it until the waiter has polled once. + yield* Effect.raceFirst( + Effect.forever(TestClock.adjust("1 second")), + waitingOrFinished(waiter, blocked), + ); + expect(yield* waitingOrFinished(waiter, blocked)).toBe("waiting"); + yield* TestClock.adjust("10 minutes"); + + expect(yield* Fiber.join(blocked)).toBeInstanceOf(ProjectLifecycleLockBusyError); + expect(docker.networkNames()).toEqual([NETWORK]); + }).pipe(Effect.provide(BunServices.layer)), + ); + + for (const format of ["json", "stream-json"] as const) { + it.effect(`keeps the waiting line out of ${format} output while waiting for the lock`, () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const networkCreated = yield* pause(); + const docker = dockerModel({ + after: (args) => (isNetworkCreate(args) ? networkCreated.hold : Effect.void), + }); + const owner = yield* makeProcess(docker, home, "demo"); + const waiter = yield* makeProcess(docker, home, "demo", format); + + yield* runStart(owner); + yield* Deferred.await(networkCreated.reached); + const blocked = yield* start(startFlags).pipe( + Effect.provide(waiter.layer), + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + yield* Effect.raceFirst(Effect.forever(TestClock.adjust("1 minute")), Fiber.await(blocked)); + + expect(yield* Fiber.join(blocked)).toBeInstanceOf(ProjectLifecycleLockBusyError); + expect(waiter.out.rawChunks).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + } + + describe("when the lock directory cannot be created", () => { + const unwritableHome = Effect.fnUntraced(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const notADirectory = path.join(base, "file"); + yield* fs.writeFileString(notADirectory, ""); + return path.join(notADirectory, "home"); + }); + + it.effect("starts the stack without the lock", () => + Effect.gen(function* () { + const docker = dockerModel(); + const cli = yield* makeProcess(docker, yield* unwritableHome(), "demo"); + + yield* start(startFlags).pipe(Effect.provide(cli.layer)); + + expect(docker.containerNames()).toContain("supabase_db_demo"); + expect(cli.out.stderrText).not.toContain("Waiting for another Supabase CLI command"); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("stops the stack without the lock", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-home-" }); + const docker = dockerModel(); + const starter = yield* makeProcess(docker, home, "demo"); + const stopper = yield* makeProcess(docker, yield* unwritableHome(), "demo"); + yield* start(startFlags).pipe(Effect.provide(starter.layer)); + expect(docker.containerNames()).toContain("supabase_db_demo"); + + yield* stop(stopFlags()).pipe(Effect.provide(stopper.layer)); + + expect(docker.containerNames()).toEqual([]); + expect(docker.networkNames()).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)), + ); + }); +}); diff --git a/apps/cli/src/command-internal/project-lifecycle-lock.ts b/apps/cli/src/command-internal/project-lifecycle-lock.ts new file mode 100644 index 0000000000..609f176ca7 --- /dev/null +++ b/apps/cli/src/command-internal/project-lifecycle-lock.ts @@ -0,0 +1,156 @@ +import { + holdExclusiveLock, + holdLock, + holdSharedLock, + isLockBusy, + type NamespaceError, +} from "@supabase/stack/effect"; +import { + Clock, + Crypto, + Data, + Duration, + Effect, + FileSystem, + Option, + Path, + type Scope, +} from "effect"; + +import { CommandSettings } from "../config/command-settings.service.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../shared/telemetry/error-actionability.ts"; +import { DebugLogger } from "../shared/output/debug-logger.service.ts"; +import { cliProjectFilterValue } from "./docker-ids.ts"; + +const LOCKS_DIR = "locks"; +const GATE_FILE = "docker-lifecycle.lock"; +const LOCK_BUDGET = Duration.minutes(10); +const LOCK_SUBJECT = "the Docker lifecycle lock"; + +/** Another lifecycle command kept the lock for the whole wait budget. */ +export class ProjectLifecycleLockBusyError extends Data.TaggedError( + "ProjectLifecycleLockBusyError", +)<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return { ...actionability.invalidInput, fingerprint_suffix: "lease_held" }; + } +} + +type TakeLock = ( + file: string, + budget: Duration.Input, + subject: string, +) => Effect.Effect; + +/** + * Holds the cross-process lock that keeps lifecycle commands from pruning Docker resources another + * command is still creating, for the enclosing scope. An empty `projectId` targets every project + * (`stop --all`) and excludes all other holders; otherwise it excludes only commands for the same + * project. Take it before any Docker probe and release it after rollback. It is not reentrant, so + * never take it inside code a lock holder calls. A lock that cannot be created or opened is + * skipped with a debug line. Older binaries, raw `docker` commands, and a different `SUPABASE_HOME` + * are not covered. + */ +export const holdProjectLifecycleLock = Effect.fn("ProjectLifecycleLock.hold")(function* ( + projectId: string, +) { + const { supabaseHome } = yield* CommandSettings; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const output = yield* Output; + const crypto = yield* Crypto.Crypto; + const debugLogger = yield* Effect.serviceOption(DebugLogger); + + const allProjects = projectId.length === 0; + const started = yield* Clock.currentTimeMillis; + let waited = false; + + const take = Effect.fnUntraced(function* ( + hold: TakeLock, + file: string, + waitingLine: string, + busyMessage: string, + ) { + yield* hold(file, Duration.zero, LOCK_SUBJECT).pipe( + Effect.catchIf(isLockBusy, () => + Effect.gen(function* () { + if (!waited) { + waited = true; + if (output.format === "text") yield* output.raw(waitingLine, "stderr"); + } + const elapsed = (yield* Clock.currentTimeMillis) - started; + yield* hold( + file, + Duration.millis(Math.max(0, Duration.toMillis(LOCK_BUDGET) - elapsed)), + LOCK_SUBJECT, + ); + }), + ), + Effect.mapError((error) => + isLockBusy(error) ? new ProjectLifecycleLockBusyError({ message: busyMessage }) : error, + ), + ); + }); + + yield* Effect.gen(function* () { + const dir = path.join(supabaseHome, LOCKS_DIR); + yield* fs.makeDirectory(dir, { recursive: true }); + const gate = path.join(dir, GATE_FILE); + + if (allProjects) { + yield* take( + holdExclusiveLock, + gate, + "Waiting for another Supabase CLI command to finish with local Supabase projects...\n", + "Another Supabase CLI command is still starting or stopping local Supabase projects. Wait for it to finish, then retry.", + ); + return; + } + + const digest = yield* crypto.digest( + "SHA-256", + new TextEncoder().encode(cliProjectFilterValue(projectId)), + ); + const key = Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); + yield* take( + holdSharedLock, + gate, + "Waiting for another Supabase CLI command to finish with local Supabase projects...\n", + "Another Supabase CLI command is stopping all local Supabase projects. Wait for it to finish, then retry.", + ); + yield* take( + holdLock, + path.join(dir, `docker-project-${key}.lock`), + `Waiting for another Supabase CLI command to finish with project ${projectId}...\n`, + `Another Supabase CLI command is still starting or stopping project ${projectId}. Wait for it to finish, then retry.`, + ); + }).pipe( + Effect.matchEffect({ + onSuccess: () => Effect.annotateCurrentSpan({ "lock.outcome": "acquired" }), + onFailure: (error) => + error instanceof ProjectLifecycleLockBusyError + ? Effect.annotateCurrentSpan({ "lock.outcome": "timeout" }).pipe( + Effect.andThen(Effect.fail(error)), + ) + : (Option.isSome(debugLogger) + ? debugLogger.value.debug( + `Continuing without the Docker lifecycle lock: ${error.message}`, + ) + : Effect.void + ).pipe(Effect.andThen(Effect.annotateCurrentSpan({ "lock.outcome": "unavailable" }))), + }), + Effect.onExit(() => + Effect.annotateCurrentSpan({ + "lock.mode": allProjects ? "all" : "project", + "lock.waited": waited, + }), + ), + ); +}); diff --git a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md index b029913b55..1fdb8e1b07 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md @@ -32,12 +32,13 @@ formatting without disabling safe compaction. ## Files Written -| Path | Format | When | -| ----------------------------------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/schemas/**/*.sql` (default declarative dir, or invocation-local `--output`) | SQL | selected destination is wiped + rewritten after confirmation | -| `/.pgdelta-export.json` | JSON | export metadata | -| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | with `PGDELTA_DEBUG` | -| `$SUPABASE_HOME/locks/start-.lock` | SQLite | when the local database is started — the per-project start lock shared with `supabase start`; a concurrent start prints `Waiting for another supabase start for this project to finish...` on stderr (see `db start`) | +| Path | Format | When | +| ----------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/schemas/**/*.sql` (default declarative dir, or invocation-local `--output`) | SQL | selected destination is wiped + rewritten after confirmation | +| `/.pgdelta-export.json` | JSON | export metadata | +| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | with `PGDELTA_DEBUG` | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite | when the local database is started — the shared Docker lifecycle lock held by `supabase start`, `db start` and `stop` (see `db start`) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite | when the local database is started — serializes lifecycle commands for the same project; a waiting command prints `Waiting for another Supabase CLI command to finish with project ...` on stderr | ## Subprocesses / Containers diff --git a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md index 93e9868014..86606fca80 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md @@ -43,7 +43,8 @@ disabling safe compaction. | `/supabase/migrations/_[_].sql` | SQL | changes; bundled engine may emit ordered segments | | `/supabase/schemas/extension.sql` | SQL | accepted legacy-extension repair | | `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | -| `$SUPABASE_HOME/locks/start-.lock` | SQLite | when the local database is started — the per-project start lock shared with `supabase start`; a concurrent start prints `Waiting for another supabase start for this project to finish...` on stderr (see `db start`) | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite | when the local database is started — the shared Docker lifecycle lock held by `supabase start`, `db start` and `stop` (see `db start`) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite | when the local database is started — serializes lifecycle commands for the same project; a waiting command prints `Waiting for another Supabase CLI command to finish with project ...` on stderr | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot — migrations/declarative shadows (`--no-cache` bypasses the snapshot cache entirely — neither read nor written); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index a7a957d3e8..62bdc1dc41 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -4,11 +4,8 @@ Fully native — see `db reset --local`'s own `SIDE_EFFECTS.md`. This is `db sta **not** the top-level `supabase start`: no status table, no `cli_stack_started` event, no `Finished` line, no `--exclude`, no `--ignore-health-check`. -The handler validates config, takes the per-project start lock shared with `supabase start` -(`$SUPABASE_HOME/locks/start-.lock`, held until the command ends: a second start for -the same project prints `Waiting for another supabase start for this project to finish...` to -stderr in text format, waits up to 10 minutes, then fails with `LocalStackStartBusyError`; a lock -location that cannot be opened never blocks the command), inspects the local Postgres +The handler validates config, takes the project lifecycle lock (see Files Written; held until the +command ends), inspects the local Postgres container (`localDbState` in `command-internal/db-bootstrap/local-db-running.ts` — a direct Docker Engine API `GET /containers//json` over the active context's unix socket / named pipe, falling back to a spawned `docker container inspect` whenever the Engine gives no @@ -113,14 +110,15 @@ failure is logged to stderr and never replaces the original error. ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | -| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | -| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | -| `$SUPABASE_HOME/locks/start-.lock` | SQLite | the per-project start lock shared with `supabase start`; never deleted, released when the command ends (by the OS if the process dies) | -| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | -| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| Path | Format | When | +| --------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | +| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | +| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | +| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | +| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | ## Subprocesses @@ -185,17 +183,17 @@ code is surfaced on failure. ## Exit Codes -| Code | Condition | -| ---- | ----------------------------------------------------------------------------------------------------------- | -| `0` | success — database started, or already running | -| `0` | `--from-backup` set and the health-check timed out (swallowed) | -| `1` | malformed `supabase/config.toml` | -| `1` | Docker daemon unreachable / inspect failure | -| `1` | db container exists but is not running, in Bitbucket Pipelines (`supabase stop` suggestion) | -| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | -| `1` | a health-check timeout with `--from-backup` unset | -| `1` | another `supabase start`/`db start` still held the start lock after 10 minutes (`LocalStackStartBusyError`) | -| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | +| Code | Condition | +| ---- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success — database started, or already running | +| `0` | `--from-backup` set and the health-check timed out (swallowed) | +| `1` | malformed `supabase/config.toml` | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | +| `1` | Docker daemon unreachable / inspect failure | +| `1` | db container exists but is not running, in Bitbucket Pipelines (`supabase stop` suggestion) | +| `1` | `backup volume already exists` (`--from-backup` against a non-fresh volume) | +| `1` | a health-check timeout with `--from-backup` unset | +| `1` | any other container-bootstrap failure (network/volume/create/start/setup) | ## Output diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index b987bdc478..f60124ae61 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -2,7 +2,6 @@ import { BunServices } from "@effect/platform-bun"; import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest"; import { Cause, - Clock, Deferred, Effect, Exit, @@ -28,7 +27,6 @@ import { mockRuntimeInfo, } from "../../../../tests/helpers/mocks.ts"; import { EXEC_FAILED_STATE } from "../../../../tests/helpers/docker-health.ts"; -import { observeMakeDirectory, startLockState } from "../../../../tests/helpers/start-lock.ts"; import { assertPermissionDrift } from "../../../../tests/helpers/permission-drift.ts"; import { configValuesLayer } from "../../../../tests/helpers/config-values-layer.ts"; import { unusedGateway } from "../../../../tests/helpers/unused-stack.ts"; @@ -42,7 +40,6 @@ import { withEmptyConfigEnv, } from "../../../../tests/helpers/command-mocks.ts"; import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; -import { classifyCliErrorActionability } from "../../../shared/telemetry/error-actionability.ts"; import { DebugFlag, ExperimentalFlag, @@ -55,7 +52,6 @@ import { HealthCheckTimeoutError } from "../../../command-internal/db-bootstrap/ import { DbConnectError } from "../../../command-internal/db-connection.errors.ts"; import { DbConnection, type DbSession } from "../../../command-internal/db-connection.service.ts"; import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; -import { LocalStackStartBusyError } from "../../../command-internal/local-stack-start-lock.ts"; import { dbStart } from "./start.handler.ts"; import type { DbStartFlags } from "./start.command.ts"; import { stackLocalDatabaseConn } from "../../../command-internal/stack-local-database.ts"; @@ -68,7 +64,8 @@ import { } from "../../../command-internal/stack-catalog-setup.ts"; import { StackError, - holdLock, + holdExclusiveLock, + isLockBusy, type DatabaseInstance, type ServiceCreation, type ServiceCreationInput, @@ -1220,76 +1217,34 @@ describe("db start", () => { } }); - describe("per-project start lock", () => { - const WAITING_LINE = "Waiting for another supabase start for this project to finish...\n"; - - const startLockFile = Effect.fnUntraced(function* (workdir: string) { + describe("project lifecycle lock", () => { + // `db start` holds the shared gate, so an exclusive probe reads `busy` while it is in flight. + const lifecycleGateFile = Effect.fnUntraced(function* (workdir: string) { const path = yield* Path.Path; - return path.join(workdir, ".supabase", "locks", "start-test.lock"); + return path.join(workdir, ".supabase", "locks", "docker-lifecycle.lock"); }); - - it.effect( - "waits the 10 minute budget for a concurrent start of the same project, then fails without touching Docker", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const lockFile = yield* startLockFile(tempRoot.current); - yield* fs.makeDirectory(path.dirname(lockFile), { recursive: true }); - yield* holdLock(lockFile, "0 millis", "concurrent start"); - const lockDirReady = yield* Deferred.make(); - const { layer, out, child } = setup(); - - const forkedAt = yield* Clock.currentTimeMillis; - const fiber = yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide( - Layer.mergeAll( - layer, - observeMakeDirectory((made) => - made === path.dirname(lockFile) - ? Deferred.succeed(lockDirReady, undefined).pipe(Effect.asVoid) - : Effect.void, - ), - ), - ), - Effect.flip, - Effect.forkChild({ startImmediately: true }), - ); - yield* Effect.raceFirst(Deferred.await(lockDirReady), Fiber.await(fiber)); - for (let minute = 0; minute < 60 && fiber.pollUnsafe() === undefined; minute++) { - yield* TestClock.adjust("1 minute"); - } - const error = yield* Fiber.join(fiber); - const waitedMillis = (yield* Clock.currentTimeMillis) - forkedAt; - - expect(error).toBeInstanceOf(LocalStackStartBusyError); - expect(error).toMatchObject({ - _tag: "LocalStackStartBusyError", - message: "another supabase start is still running for this project", - suggestion: expect.stringContaining("supabase start"), - }); - expect(classifyCliErrorActionability(error)).toMatchObject({ - error_fingerprint: "tag:LocalStackStartBusyError:lease_held", - }); - expect(out.stderrText).toContain(WAITING_LINE); - expect(waitedMillis).toBeGreaterThanOrEqual(10 * 60_000); - expect(waitedMillis).toBeLessThanOrEqual(11 * 60_000); - expect(child.spawned).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - 60_000, - ); + const gateState = (gate: string) => + TestClock.withLive( + holdExclusiveLock(gate, "0 millis", "probe").pipe( + Effect.scoped, + Effect.match({ + onSuccess: () => "free" as const, + onFailure: (error) => (isLockBusy(error) ? ("busy" as const) : ("error" as const)), + }), + ), + ); for (const outcome of ["succeeds", "fails"] as const) { it.effect( - `holds the lock while the database comes up and releases it when the start ${outcome}`, + `holds the lifecycle lock while the database comes up and releases it when the start ${outcome}`, () => { const { layer, dbSession } = setup({ route: freshVolumeRoute(defaultRoute()) }); return Effect.gen(function* () { - const lockFile = yield* startLockFile(tempRoot.current); + const gate = yield* lifecycleGateFile(tempRoot.current); const stateAtConnect: Array<"free" | "busy" | "error"> = []; const probingConnection = Layer.succeed(DbConnection, { connect: () => - startLockState(lockFile).pipe( + gateState(gate).pipe( Effect.flatMap((state) => { stateAtConnect.push(state); return outcome === "fails" && stateAtConnect.length > 1 @@ -1307,7 +1262,7 @@ describe("db start", () => { expect(Exit.isSuccess(exit)).toBe(outcome === "succeeds"); expect(stateAtConnect.length).toBeGreaterThan(1); expect(stateAtConnect.every((state) => state === "busy")).toBe(true); - expect(yield* startLockState(lockFile)).toBe("free"); + expect(yield* gateState(gate)).toBe("free"); }).pipe(Effect.provide(BunServices.layer)); }, ); diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index fbfdd12eca..f0f247f9e9 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -193,13 +193,14 @@ command. ## Files Written -| Path | Format | When | -| ---------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | -| `$SUPABASE_HOME/locks/start-.lock` | SQLite | on every start with a non-empty project id — the per-project start lock (see "Concurrent starts"); never deleted, released when the command ends (by the OS if the process dies) | -| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | -| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | -| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | +| Path | Format | When | +| ---------------------------------------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/.branches/_current_branch` | text | on every start, only if absent — writes `"main"` | +| `/supabase/snippets/` | — | when Studio is enabled, only if absent — created before any container and kept on rollback; `podman create -v`, unlike Docker, rejects a missing bind source | +| `/supabase/.temp/start-secrets//{env,multiline-env}/` | varies | Edge Runtime's own JWT/service-role-key/secret env artifacts — see below | +| auth email `content_path` files (SELinux security label only, no content change) | xattr | when the container runtime applies SELinux labels (enforcing or permissive): the `rw,z` Kong bind relabels each mounted template to the shared container context; persists after `supabase stop` | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project; waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | Kong's `custom_nginx.template`, Vector's `vector.yaml`, and Postgres's own bootstrap script (`postgresql.conf`-equivalent setup) are all rendered in memory and injected @@ -306,7 +307,7 @@ code is surfaced on failure. | `1` | an unparsable `SUPABASE_*` config override (e.g. a non-boolean `SUPABASE_AUTH_ENABLED`) | | `1` | stopped Postgres detected but the project id sanitizes to empty — aborts before recovery removes any containers | | `1` | Postgres container exists but is not running and is not recovered (Bitbucket Pipelines, or Docker reported no state) — ` container is not running: ` (`unknown` for an empty status) with a `supabase stop` then `supabase start` suggestion; the "already running" banner is never printed | -| `1` | another `supabase start`/`supabase db start` for this project still held the start lock after the 10-minute wait (`LocalStackStartBusyError`) — fails before any container work, with a suggestion to wait for or cancel the other start | +| `1` | another lifecycle command still holds this project's lock after 10 minutes — prints `Waiting for another Supabase CLI command to finish with project ...` (or `...with local Supabase projects...` behind `stop --all`) to stderr first (`ProjectLifecycleLockBusyError`) | | `1` | `docker`/`podman` not spawnable, or the daemon is unreachable | | `1` | stopped-stack recovery cannot list, stop, or prune current-project containers, or prune matching networks — aborts before startup; named volumes are preserved | | `1` | image pull exhausted across every registry candidate, or the Docker daemon becomes unreachable during the pre-pull — even with `--ignore-health-check` (never swallowed into exit 0; see the `--ignore-health-check` note under "Notes") | @@ -401,6 +402,16 @@ prose, not structured data. ## Notes +- **Lifecycle lock**: `start`, `db start`, and `stop` serialize on `$SUPABASE_HOME/locks`, so a + `docker network prune` from one never lands between another's network create and its first + container attach. The lock (SQLite `BEGIN IMMEDIATE` locks the OS releases when the process + dies) is taken once the project id is resolved, before inspecting any container, and held until + the command ends, after any rollback. A command that has to wait prints a waiting line to stderr + (text format only), waits up to 10 minutes, then fails with `ProjectLifecycleLockBusyError`. A lock location that cannot be + created or opened never blocks the command: it continues unlocked and logs the reason under + `--debug`. The lock does not cover older CLI binaries, raw `docker` commands, a different + `SUPABASE_HOME`, projects sharing a custom `--network-id`, or Docker requests a killed process + already sent, and `stop --all` is not queued ahead of a stream of single-project commands. - **`pg_net` converges with `[experimental.webhooks]` on every non-backup start**: a fresh volume installs `pg_net` only when webhooks are enabled or migration history contains a `create extension … pg_net`; an existing volume @@ -463,14 +474,6 @@ prose, not structured data. create` and `docker start`) is recovered like any other stopped stack. The named volume is preserved, and the data-directory probe (see "Fresh-volume DB setup") decides whether the next bring-up still runs the setup pipeline and bucket seeding. -- **Concurrent starts:** `start` and `db start` take a per-project - lock at `$SUPABASE_HOME/locks/start-.lock` (a SQLite `BEGIN IMMEDIATE` lock the OS - releases when the process dies) once the project id is resolved and before inspecting any - container, and hold it until the command ends. A second run prints `Waiting for another supabase -start for this project to finish...` to stderr (text format only) and waits up to 10 - minutes, then fails with `LocalStackStartBusyError`. A lock location that cannot be created or - opened never blocks a start: it continues unlocked and logs the reason under `--debug`. `stop` - does not take the lock, so it still runs while another start is in progress. - **Spec-strict import-map key matching:** Edge Runtime bind mounts are computed by the same functions import scanner as `functions deploy`/`functions serve` (`walkImportPaths`/`substituteImportMapValue`, shared code), which matches import-map keys diff --git a/apps/cli/src/commands/start/start.handler.ts b/apps/cli/src/commands/start/start.handler.ts index b902ac74b3..94c11f5c73 100644 --- a/apps/cli/src/commands/start/start.handler.ts +++ b/apps/cli/src/commands/start/start.handler.ts @@ -64,6 +64,7 @@ import { type ContainerIdName, } from "../../command-internal/docker-lifecycle.ts"; import { dockerRemoveAll } from "../../command-internal/docker-remove-all.ts"; +import { holdProjectLifecycleLock } from "../../command-internal/project-lifecycle-lock.ts"; import { resolveAuthCaptcha, resolveAuthEmail, @@ -94,7 +95,6 @@ import { import { resolveConfigSubtree } from "../../config/cli-config-subtree.ts"; import { CliConfigValueError } from "../../config/cli-config.errors.ts"; import { seedBucketsRun } from "../../command-internal/seed-buckets.ts"; -import { holdLocalStackStartLock } from "../../command-internal/local-stack-start-lock.ts"; import { cleanupStartSecrets } from "../../command-internal/start-secrets-cleanup.ts"; import { dbNotRunningError, @@ -542,6 +542,9 @@ export const start = Effect.fn("start")(function* (flags: StartFlags) { const filterValue = cliProjectFilterValue(projectId); const dbNotRunning = (status: string) => dbNotRunningError(dbContainerId, status); + // Held until the handler scope closes, after recovery, bring-up, and any rollback have run. + yield* holdProjectLifecycleLock(projectId); + // Shared by the already-running branch (full status pipeline, re-health-checked) and the // success path (a direct pretty-print, no re-health-check). // @@ -570,9 +573,6 @@ export const start = Effect.fn("start")(function* (flags: StartFlags) { Effect.mapError((error) => new StartConfigLoadError({ message: error.message })), ); - // A concurrent start's rollback would otherwise prune this run's containers and network. - yield* holdLocalStackStartLock(projectId); - // 3. A missing container proceeds to startup; other inspect failures propagate. Stopped // stacks are recovered unless Bitbucket's lack of named volumes makes removal destructive. const inspectDbState = inspectContainerState(spawner, dbContainerId).pipe( diff --git a/apps/cli/src/commands/start/start.integration.test.ts b/apps/cli/src/commands/start/start.integration.test.ts index 9839226be0..29e3121e6c 100644 --- a/apps/cli/src/commands/start/start.integration.test.ts +++ b/apps/cli/src/commands/start/start.integration.test.ts @@ -2,7 +2,6 @@ import { generateKeyPairSync } from "node:crypto"; import { BunServices } from "@effect/platform-bun"; import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest"; -import { holdLock } from "@supabase/stack/effect"; import { Cause, Clock, @@ -50,7 +49,6 @@ import { inspectedState, } from "../../../tests/helpers/docker-health.ts"; import { assertPermissionDrift } from "../../../tests/helpers/permission-drift.ts"; -import { observeMakeDirectory, startLockState } from "../../../tests/helpers/start-lock.ts"; import { unusedStackServices } from "../../../tests/helpers/unused-stack.ts"; import { CliArgs } from "../../shared/cli/cli-args.service.ts"; import { @@ -71,7 +69,6 @@ import { HealthCheckTimeoutSeconds } from "../../command-internal/db-bootstrap/h import { DbConnectError } from "../../command-internal/db-connection.errors.ts"; import { DbConnection, type DbSession } from "../../command-internal/db-connection.service.ts"; import { dockerRunLayer } from "../../command-internal/docker-run.layer.ts"; -import { LocalStackStartBusyError } from "../../command-internal/local-stack-start-lock.ts"; import { START_EXCLUDABLE_KEYS } from "./start.exclude.ts"; import type { StartFlags } from "./start.command.ts"; import { start } from "./start.handler.ts"; @@ -646,8 +643,6 @@ interface SetupOpts { readonly experimental?: boolean; /** Replaces the `DbConnection.connect` behind the Postgres readiness probe. Defaults to succeeding. */ readonly dbConnect?: (typeof DbConnection)["Service"]["connect"]; - /** Overrides `$SUPABASE_HOME`, which the start lock lives under. Defaults to `/.supabase`. */ - readonly supabaseHome?: string; } const setup = Effect.fnUntraced(function* (opts: SetupOpts = {}) { @@ -661,7 +656,7 @@ const setup = Effect.fnUntraced(function* (opts: SetupOpts = {}) { const out = mockOutput({ format: opts.format ?? "text" }); const telemetry = mockTelemetryStateTracked(); const analytics = mockAnalytics(); - const cliSettings = mockCommandSettings({ workdir, supabaseHome: opts.supabaseHome }); + const cliSettings = mockCommandSettings({ workdir }); const child = mockStartContainerCliSpawner(opts.route ?? defaultRoute(), { failSpawn: opts.failSpawn, onSecretCopy: opts.onSecretCopy, @@ -1546,148 +1541,6 @@ describe("start integration", () => { ); }); - describe("per-project start lock", () => { - const WAITING_LINE = "Waiting for another supabase start for this project to finish...\n"; - - const startLockFile = Effect.fnUntraced(function* (workdir: string) { - const path = yield* Path.Path; - return path.join(workdir, ".supabase", "locks", "start-demo.lock"); - }); - - const startAgainstHeldLock = (format: "text" | "json" | "stream-json") => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const lockFile = yield* startLockFile(tempRoot.current); - yield* fs.makeDirectory(path.dirname(lockFile), { recursive: true }); - yield* holdLock(lockFile, "0 millis", "concurrent start"); - const lockDirReady = yield* Deferred.make(); - const { layer, out, child } = yield* setup({ format }); - - const forkedAt = yield* Clock.currentTimeMillis; - const fiber = yield* start(flags()).pipe( - Effect.provide( - Layer.mergeAll( - layer, - observeMakeDirectory((made) => - made === path.dirname(lockFile) - ? Deferred.succeed(lockDirReady, undefined).pipe(Effect.asVoid) - : Effect.void, - ), - ), - ), - Effect.flip, - Effect.forkChild({ startImmediately: true }), - ); - yield* Effect.raceFirst(Deferred.await(lockDirReady), Fiber.await(fiber)); - for (let minute = 0; minute < 60 && fiber.pollUnsafe() === undefined; minute++) { - yield* TestClock.adjust("1 minute"); - } - const error = yield* Fiber.join(fiber); - const waitedMillis = (yield* Clock.currentTimeMillis) - forkedAt; - return { error, out, child, waitedMillis }; - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)); - - it.effect( - "waits the 10 minute budget for a concurrent start of the same project, then fails without touching Docker", - () => - Effect.gen(function* () { - const { error, out, child, waitedMillis } = yield* startAgainstHeldLock("text"); - - expect(error).toBeInstanceOf(LocalStackStartBusyError); - expect(error).toMatchObject({ - _tag: "LocalStackStartBusyError", - message: "another supabase start is still running for this project", - suggestion: expect.stringContaining("supabase start"), - }); - expect(classifyCliErrorActionability(error)).toMatchObject({ - error_kind: "user_actionable", - error_fingerprint: "tag:LocalStackStartBusyError:lease_held", - }); - expect(out.stderrText).toContain(WAITING_LINE); - expect(waitedMillis).toBeGreaterThanOrEqual(10 * 60_000); - expect(waitedMillis).toBeLessThanOrEqual(11 * 60_000); - expect(child.spawned).toEqual([]); - }), - 60_000, - ); - - for (const format of ["json", "stream-json"] as const) { - it.effect( - `keeps the waiting line out of ${format} output while waiting for a concurrent start`, - () => - Effect.gen(function* () { - const { error, out } = yield* startAgainstHeldLock(format); - - expect(error).toMatchObject({ _tag: "LocalStackStartBusyError" }); - expect(out.rawChunks).toEqual([]); - }), - 60_000, - ); - } - - it.effect("holds the lock from before the first Docker inspect through bring-up", () => - Effect.scoped( - Effect.gen(function* () { - const lockFile = yield* startLockFile(tempRoot.current); - const base = defaultRoute(); - const stateAtInspect: Array<"free" | "busy" | "error"> = []; - const { layer } = yield* setup({ - route: (args) => - args[0] === "container" && args[1] === "inspect" && args[2] === "supabase_db_demo" - ? startLockState(lockFile).pipe( - Effect.map((state) => { - stateAtInspect.push(state); - return base(args); - }), - ) - : base(args), - }); - - yield* start(flags()).pipe(Effect.provide(layer)); - - expect(stateAtInspect.length).toBeGreaterThan(1); - expect(stateAtInspect.every((state) => state === "busy")).toBe(true); - expect(yield* startLockState(lockFile)).toBe("free"); - }), - ).pipe(Effect.provide(BunServices.layer)), - ); - - it.effect("releases the lock when start fails", () => - Effect.scoped( - Effect.gen(function* () { - const lockFile = yield* startLockFile(tempRoot.current); - const { layer } = yield* setup({ - route: (args) => - args[0] === "container" && args[1] === "inspect" ? { stdout: [""] } : { exitCode: 0 }, - }); - - const error = yield* start(flags()).pipe(Effect.provide(layer), Effect.flip); - - expect(error).toMatchObject({ _tag: "StatusDbNotRunningError" }); - expect(yield* startLockState(lockFile)).toBe("free"); - }), - ).pipe(Effect.provide(BunServices.layer)), - ); - - it.effect("starts without a lock when the lock location cannot be created", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const blocker = path.join(tempRoot.current, "not-a-directory"); - yield* fs.writeFileString(blocker, ""); - const { layer, out, child } = yield* setup({ - supabaseHome: path.join(blocker, "home"), - }); - - yield* start(flags()).pipe(Effect.provide(layer)); - - expect(createdContainerNames(child.spawned)).toContain("supabase_db_demo"); - expect(out.stderrText).not.toContain(WAITING_LINE); - }).pipe(Effect.provide(BunServices.layer)), - ); - }); - describe("config load / validation failures", () => { it.effect("fails when --workdir/SUPABASE_WORKDIR points at a missing path", () => Effect.gen(function* () { diff --git a/apps/cli/src/commands/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/stop/SIDE_EFFECTS.md index 09df6da0f0..84fa021d7e 100644 --- a/apps/cli/src/commands/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/stop/SIDE_EFFECTS.md @@ -25,10 +25,12 @@ model (see the CLI-1324 plan's "Critical architectural finding" for why). ## Files Written -| Path | Format | When | -| ------------------------------------------------------------------- | ------------------- | --------------------------------------------------------------------------------------- | -| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | -| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | +| Path | Format | When | +| ------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `~/.supabase/telemetry.json` | JSON | always (in `Effect.ensuring`) at end of command | +| `/supabase/.temp/start-secrets/` (removed) | plaintext, per-file | after teardown succeeds, for every container name torn down that had a staged directory | +| `$SUPABASE_HOME/locks/docker-lifecycle.lock` | SQLite lock file | held for the whole run: shared for one project, exclusive for `stop --all`; waits up to 10 minutes for a conflicting command (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | +| `$SUPABASE_HOME/locks/docker-project-.lock` | SQLite lock file | held for the whole run: serializes lifecycle commands for the same project (not taken by `--all`); waits up to 10 minutes (skipped, with a debug line, when `$SUPABASE_HOME/locks` cannot be created or opened) | The `start-secrets` removal is a hygiene step (`cleanupStartSecrets`, `command-internal/start-secrets-cleanup.ts`). Only Edge Runtime's own @@ -92,6 +94,7 @@ and ensure it is on PATH") rather than a generic "failed to ..." string. | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `0` | success — containers/volumes/networks pruned | | `1` | `--project-id` and `--all` both set (`StopMutuallyExclusiveError`) | +| `1` | another lifecycle command still holds the lock after 10 minutes (`ProjectLifecycleLockBusyError`); `--all` waits for every other lifecycle command | | `1` | `config.toml` present but malformed (`StopConfigLoadError`), or a config value is invalid (`CliConfigValueError`, naming its source) — an **absent** file is not an error | | `1` | listing containers failed (`StopListError`) | | `1` | stopping one or more containers failed (`StopContainerError`) | diff --git a/apps/cli/src/commands/stop/stop.handler.ts b/apps/cli/src/commands/stop/stop.handler.ts index ff16d14962..77daf71ec1 100644 --- a/apps/cli/src/commands/stop/stop.handler.ts +++ b/apps/cli/src/commands/stop/stop.handler.ts @@ -13,6 +13,7 @@ import { type ContainerIdName, } from "../../command-internal/docker-lifecycle.ts"; import { dockerRemoveAll } from "../../command-internal/docker-remove-all.ts"; +import { holdProjectLifecycleLock } from "../../command-internal/project-lifecycle-lock.ts"; import { cleanupStartSecrets } from "../../command-internal/start-secrets-cleanup.ts"; import { CliConfigValueError } from "../../config/cli-config.errors.ts"; import { loadResolvedConfigSurfacingValueErrors } from "../../command-internal/config-value-passthrough.ts"; @@ -118,6 +119,9 @@ export const stop = Effect.fn("stop")(function* (flags: StopFlags) { const deleteVolumes = flags.noBackup; const filterValue = cliProjectFilterValue(searchProjectIdFilter); + // Held through the prune and its secret cleanup; `--all` excludes every other lifecycle command. + yield* holdProjectLifecycleLock(searchProjectIdFilter); + // Printed directly (not via `output.task`'s debounced spinner) so it always appears — the // spinner's delay drops the message when Docker calls resolve faster than the threshold, as // they do against mocked/replayed Docker. @@ -196,5 +200,5 @@ export const stop = Effect.fn("stop")(function* (flags: StopFlags) { ); } } - }).pipe(Effect.ensuring(telemetryState.flush)); + }).pipe(Effect.scoped, Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index 0c943d1917..a6c3fdeac1 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -313,7 +313,6 @@ LinkProjectStatusNetworkError LinkRefArgConflictError LinkedDbNetworkBannedError LocalDbRunningError -LocalStackStartBusyError LockResolutionError LoginCryptoError LoginDecryptError @@ -397,6 +396,7 @@ PostgresConfigUpdateUnmarshalError ProfileLoadError ProfileSaveError ProjectConfigParseError +ProjectLifecycleLockBusyError ProjectLinkIoError ProjectLinkMalformedError ProjectNotLinkedError diff --git a/apps/cli/tests/helpers/docker-model.ts b/apps/cli/tests/helpers/docker-model.ts new file mode 100644 index 0000000000..9c17b88394 --- /dev/null +++ b/apps/cli/tests/helpers/docker-model.ts @@ -0,0 +1,243 @@ +import { Effect, Layer, Sink, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +import { CLI_WORKDIR_LABEL } from "../../src/command-internal/docker-ids.ts"; + +interface DockerResult { + readonly exitCode?: number; + readonly stdout?: ReadonlyArray; + readonly stderr?: ReadonlyArray; +} + +interface DockerModelHooks { + /** Runs before the daemon handles a command; a returned result replaces the daemon's answer. */ + readonly before?: (args: ReadonlyArray) => Effect.Effect; + /** Runs after the daemon applied a command and before the CLI sees its exit. */ + readonly after?: (args: ReadonlyArray) => Effect.Effect; +} + +interface ModelContainer { + readonly name: string; + readonly id: string; + readonly labels: Readonly>; + readonly network: string | undefined; + status: "created" | "running" | "exited"; +} + +const BUILT_IN_NETWORKS = new Set(["bridge", "host", "none", "default"]); + +const STATE_JSON: Readonly> = { + created: '{"Running":false,"Status":"created"}', + running: '{"Running":true,"Status":"running","Health":{"Status":"healthy"}}', + exited: '{"Running":false,"Status":"exited"}', +}; + +function flagValues(args: ReadonlyArray, flag: string): ReadonlyArray { + const values: Array = []; + args.forEach((arg, index) => { + const value = args[index + 1]; + if (arg === flag && value !== undefined) values.push(value); + }); + return values; +} + +function parseLabels(args: ReadonlyArray): Readonly> { + return Object.fromEntries( + flagValues(args, "--label").map((label) => { + const separator = label.indexOf("="); + return separator < 0 ? [label, ""] : [label.slice(0, separator), label.slice(separator + 1)]; + }), + ); +} + +/** `--filter label=` and `--filter label==` selectors, ANDed like Docker does. */ +function matchesLabelFilters( + labels: Readonly>, + args: ReadonlyArray, +): boolean { + return flagValues(args, "--filter") + .filter((filter) => filter.startsWith("label=")) + .map((filter) => filter.slice("label=".length)) + .every((selector) => { + const separator = selector.indexOf("="); + const value = labels[separator < 0 ? selector : selector.slice(0, separator)]; + return value !== undefined && (separator < 0 || value === selector.slice(separator + 1)); + }); +} + +function containerId(name: string): string { + return [...name] + .map((char) => (char.codePointAt(0) ?? 0).toString(16).padStart(2, "0")) + .join("") + .padEnd(64, "0") + .slice(0, 64); +} + +/** + * A stateful in-memory Docker daemon behind the `docker` CLI, covering what `start`, `stop` and + * rollback ask of it. Networks and containers carry their labels, label-filtered `network prune` + * removes only networks no container is attached to, and creating a container on a network that + * does not exist fails the way the daemon does. Volumes always exist and every command it does not + * model succeeds with no output. + */ +export function dockerModel(hooks: DockerModelHooks = {}) { + const networks = new Map>>(); + const containers = new Map(); + const commands: Array> = []; + const encoder = new TextEncoder(); + + const findContainer = (ref: string | undefined) => + ref === undefined + ? undefined + : (containers.get(ref) ?? [...containers.values()].find((container) => container.id === ref)); + + const handle = (args: ReadonlyArray): DockerResult => { + const [command, subcommand] = args; + + if (command === "network" && subcommand === "inspect") { + const name = args[2] ?? ""; + return networks.has(name) + ? { exitCode: 0 } + : { exitCode: 1, stderr: [`Error response from daemon: network ${name} not found`] }; + } + if (command === "network" && subcommand === "create") { + const name = args.at(-1) ?? ""; + if (networks.has(name)) { + return { + exitCode: 1, + stderr: [`Error response from daemon: network with name ${name} already exists`], + }; + } + networks.set(name, parseLabels(args)); + return { exitCode: 0, stdout: [containerId(name)] }; + } + if (command === "network" && subcommand === "prune") { + const pruned = [...networks] + .filter( + ([name, labels]) => + matchesLabelFilters(labels, args) && + ![...containers.values()].some((container) => container.network === name), + ) + .map(([name]) => name); + for (const name of pruned) networks.delete(name); + return { exitCode: 0, stdout: ["Deleted Networks:", ...pruned] }; + } + + if (command === "create") { + const name = flagValues(args, "--name")[0] ?? `container-${commands.length}`; + const network = flagValues(args, "--network")[0]; + if (network !== undefined && !BUILT_IN_NETWORKS.has(network) && !networks.has(network)) { + return { + exitCode: 1, + stderr: [`Error response from daemon: network ${network} not found`], + }; + } + const container: ModelContainer = { + name, + id: containerId(name), + labels: parseLabels(args), + network, + status: "created", + }; + containers.set(name, container); + return { exitCode: 0, stdout: [container.id] }; + } + if (command === "start" || command === "stop") { + const container = findContainer(args[1]); + if (container === undefined) { + return { exitCode: 1, stderr: [`Error: No such container: ${args[1] ?? ""}`] }; + } + container.status = command === "start" ? "running" : "exited"; + return { exitCode: 0, stdout: [container.id] }; + } + if (command === "container" && subcommand === "inspect") { + const container = findContainer(args[2]); + return container === undefined + ? { exitCode: 1, stderr: [`Error: No such container: ${args[2] ?? ""}`] } + : { exitCode: 0, stdout: [STATE_JSON[container.status]] }; + } + if (command === "container" && subcommand === "prune") { + const pruned = [...containers.values()].filter( + (container) => + container.status !== "running" && matchesLabelFilters(container.labels, args), + ); + for (const container of pruned) containers.delete(container.name); + return { exitCode: 0, stdout: ["Deleted Containers:", ...pruned.map(({ id }) => id)] }; + } + if (command === "ps") { + const format = flagValues(args, "--format")[0] ?? ""; + const all = args.includes("--all"); + return { + exitCode: 0, + stdout: [...containers.values()] + .filter( + (container) => + (all || container.status === "running") && + matchesLabelFilters(container.labels, args), + ) + .map((container) => + format.includes("{{.Label") + ? [container.id, container.name, container.labels[CLI_WORKDIR_LABEL] ?? ""].join("\t") + : format.includes("{{.Names}}") + ? container.name + : container.id, + ), + }; + } + + if (command === "version") { + const format = flagValues(args, "--format")[0] ?? ""; + return { + exitCode: 0, + stdout: [ + format + .replaceAll("{{.Client.APIVersion}}", "1.45") + .replaceAll("{{.Server.APIVersion}}", "1.45"), + ], + }; + } + if (command === "context" && subcommand === "inspect") return { exitCode: 1 }; + return { exitCode: 0 }; + }; + + const layer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const args = ChildProcess.isStandardCommand(command) ? command.args : []; + commands.push(args); + const override = hooks.before === undefined ? undefined : yield* hooks.before(args); + const result = override ?? handle(args); + if (override === undefined && hooks.after !== undefined) yield* hooks.after(args); + + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(6000 + commands.length), + stdout: Stream.fromIterable( + (result.stdout ?? []).map((line) => encoder.encode(`${line}\n`)), + ), + stderr: Stream.fromIterable( + (result.stderr ?? []).map((line) => encoder.encode(`${line}\n`)), + ), + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result.exitCode ?? 0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ); + + return { + layer, + /** Argv of every command the CLI issued, in order. */ + commands: (): ReadonlyArray> => commands, + networkNames: (): ReadonlyArray => [...networks.keys()].sort(), + containerNames: (): ReadonlyArray => [...containers.keys()].sort(), + }; +} + +export type DockerModel = ReturnType; diff --git a/apps/cli/tests/helpers/start-lock.ts b/apps/cli/tests/helpers/start-lock.ts deleted file mode 100644 index ffff322750..0000000000 --- a/apps/cli/tests/helpers/start-lock.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { BunServices } from "@effect/platform-bun"; -import { holdLock, isLockBusy } from "@supabase/stack/effect"; -import { Effect, FileSystem, Layer } from "effect"; -import * as TestClock from "effect/testing/TestClock"; - -/** Probes a start lock on the real clock, so a held lock reads `busy` without waiting on the TestClock. */ -export const startLockState = (lockFile: string) => - TestClock.withLive( - holdLock(lockFile, "0 millis", "probe").pipe( - Effect.scoped, - Effect.match({ - onSuccess: () => "free" as const, - onFailure: (error) => (isLockBusy(error) ? ("busy" as const) : ("error" as const)), - }), - ), - ); - -/** A `FileSystem` that reports each successful `makeDirectory`, the last real I/O before the first lock attempt. */ -export const observeMakeDirectory = (onMade: (path: string) => Effect.Effect) => - Layer.effect( - FileSystem.FileSystem, - Effect.map(FileSystem.FileSystem, (real) => - FileSystem.FileSystem.of({ - ...real, - makeDirectory: (path, options) => - real.makeDirectory(path, options).pipe(Effect.tap(() => onMade(path))), - }), - ), - ).pipe(Layer.provide(BunServices.layer)); diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts index 9d94d9450f..0ea55bd1b9 100644 --- a/packages/stack/src/StackNamespace.ts +++ b/packages/stack/src/StackNamespace.ts @@ -6,7 +6,7 @@ import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; export { NamespaceError } from "./namespace/Capabilities.ts"; export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; -export { holdLock, isLockBusy } from "./namespace/Lock.ts"; +export { holdExclusiveLock, holdLock, holdSharedLock, isLockBusy } from "./namespace/Lock.ts"; export { resolveStackDataRoot, stackDataRootLabel, stackLogsRoot } from "./namespace/Paths.ts"; export { publish } from "./namespace/Publication.ts"; export { diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 2ac54a3964..72bcfe45f3 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -84,7 +84,11 @@ export { StackError } from "./Rpc.ts"; export { NamespaceError, publish } from "./StackNamespace.ts"; /** Holds an exclusive cross-process lock on a file for the enclosing scope, waiting up to a budget. */ export { holdLock } from "./StackNamespace.ts"; -/** Whether a `holdLock` failure means another holder kept the lock for the whole budget. */ +/** Holds a cross-process lock shared with other shared holders, excluding `holdExclusiveLock`. */ +export { holdSharedLock } from "./StackNamespace.ts"; +/** Holds a cross-process lock that excludes every `holdSharedLock` holder. */ +export { holdExclusiveLock } from "./StackNamespace.ts"; +/** Whether a lock acquisition failure means another holder kept the lock for the whole budget. */ export { isLockBusy } from "./StackNamespace.ts"; export { HOSTED_LOCK_SCHEMA_URL, diff --git a/packages/stack/src/namespace/Lock.ts b/packages/stack/src/namespace/Lock.ts index db4a11d52e..946e48bb38 100644 --- a/packages/stack/src/namespace/Lock.ts +++ b/packages/stack/src/namespace/Lock.ts @@ -1,13 +1,20 @@ import { Effect, Exit, Schedule, Schema, Scope } from "effect"; import type { Duration } from "effect"; import { namespaceError, type NamespaceError } from "./Capabilities.ts"; -import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; +import type { DatabaseSync } from "node:sqlite"; +import { + acquireLock, + isBusy, + takeExclusiveLock, + takeLock, + takeSharedLock, +} from "./drivers/Sqlite.ts"; class LockBusy extends Schema.TaggedError()("Namespace.LockBusy", { message: Schema.String, }) {} -/** Whether `error` is a {@link holdLock} failure because another holder kept the lock for the whole budget. */ +/** Whether `error` is a lock acquisition failure because another holder kept the lock for the whole budget. */ export const isLockBusy = (error: NamespaceError): boolean => Schema.is(LockBusy)(error.cause); /** @@ -18,6 +25,35 @@ export const holdLock = ( file: string, budget: Duration.Input, subject: string, +): Effect.Effect => holdWith(takeLock, file, budget, subject); + +/** + * Takes a shared lock at `file` for the enclosing scope: shared holders coexist, and only a + * {@link holdExclusiveLock} holder makes them wait, up to `budget`. + */ +export const holdSharedLock = ( + file: string, + budget: Duration.Input, + subject: string, +): Effect.Effect => + holdWith(takeSharedLock, file, budget, subject); + +/** + * Takes the lock at `file` exclusively of every {@link holdSharedLock} holder for the enclosing + * scope, waiting up to `budget` for them to release. + */ +export const holdExclusiveLock = ( + file: string, + budget: Duration.Input, + subject: string, +): Effect.Effect => + holdWith(takeExclusiveLock, file, budget, subject); + +const holdWith = ( + take: (connection: DatabaseSync) => Effect.Effect, + file: string, + budget: Duration.Input, + subject: string, ): Effect.Effect => Effect.gen(function* () { const held = yield* Effect.scope; @@ -26,7 +62,7 @@ export const holdLock = ( const attempt = yield* Scope.fork(held, "sequential"); yield* acquireLock(file, "create").pipe( Scope.provide(attempt), - Effect.flatMap(takeLock), + Effect.flatMap(take), Effect.onExit((exit) => Exit.isFailure(exit) ? Scope.close(attempt, exit) : Effect.void, ),