From 90f0df5180c4313287280fe047e22ccb8df33feb Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 22:57:32 -0700 Subject: [PATCH 1/5] fix(server): supervise background service updates --- apps/server/src/cli/service.ts | 39 ++- apps/server/src/cloud/selfUpdate.test.ts | 126 ++++--- apps/server/src/cloud/selfUpdate.ts | 171 +++++----- .../src/cloud/systemdSelfUpdate.test.ts | 165 ++++++++++ apps/server/src/cloud/systemdSelfUpdate.ts | 310 ++++++++++++++++++ .../server/src/persistence/Migrations.test.ts | 53 +++ apps/server/src/persistence/Migrations.ts | 56 ++++ docs/internals/server-updates.md | 39 ++- docs/user/updating.md | 4 + .../client-runtime/src/state/server.test.ts | 23 ++ packages/client-runtime/src/state/server.ts | 97 +++++- 11 files changed, 895 insertions(+), 188 deletions(-) create mode 100644 apps/server/src/cloud/systemdSelfUpdate.test.ts create mode 100644 apps/server/src/cloud/systemdSelfUpdate.ts create mode 100644 apps/server/src/persistence/Migrations.test.ts diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index bd846eeee34..003f3c272f3 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -2,11 +2,14 @@ import * as Console from "effect/Console"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Terminal from "effect/Terminal"; -import { Command, GlobalFlag, Prompt } from "effect/unstable/cli"; +import { Command, Flag, GlobalFlag, Prompt } from "effect/unstable/cli"; +import { FetchHttpClient } from "effect/unstable/http"; import packageJson from "../../package.json" with { type: "json" }; import * as BootService from "../cloud/bootService.ts"; +import { applySystemdSelfUpdatePlan } from "../cloud/systemdSelfUpdate.ts"; import type * as ServerConfig from "../config.ts"; +import { validateMigrationIdentities } from "../persistence/Migrations.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; @@ -142,6 +145,38 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe( ), ); +/** Internal entrypoint launched by systemd-run outside t3code.service. */ +const serviceApplyUpdateCommand = Command.make("_apply-update", { + plan: Flag.string("plan"), +}).pipe( + Command.withHidden, + Command.withHandler(({ plan }) => + applySystemdSelfUpdatePlan(plan).pipe( + Effect.provide(Layer.mergeAll(ProcessRunner.layer, FetchHttpClient.layer)), + ), + ), +); + +/** Target-artifact preflight used before the running server is stopped. */ +const serviceValidateUpdateCommand = Command.make("_validate-update", { + database: Flag.string("database"), +}).pipe( + Command.withHidden, + Command.withHandler(({ database }) => + Effect.gen(function* () { + // Keep node:sqlite behind the same lazy bundle boundary as normal server + // startup; most service commands never open the database. + const NodeSqliteClient = yield* Effect.promise( + () => import("../persistence/NodeSqliteClient.ts"), + ); + yield* validateMigrationIdentities().pipe( + Effect.scoped, + Effect.provide(NodeSqliteClient.layer({ filename: database, readonly: true })), + ); + }), + ), +); + export const offerServiceDuringOnboarding = Effect.gen(function* () { const service = yield* BootService.BootService; const { supported, installed, current } = yield* service.status; @@ -195,5 +230,7 @@ export const serviceCommand = Command.make("service").pipe( serviceUninstallCommand, serviceUpdateCommand, serviceStatusCommand, + serviceApplyUpdateCommand, + serviceValidateUpdateCommand, ]), ); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index bfac916a59d..ff47d490dc6 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; @@ -23,36 +24,15 @@ import { renderBootServiceUnit, } from "./bootService.ts"; import * as SelfUpdate from "./selfUpdate.ts"; +import { + SYSTEMD_SELF_UPDATE_DIRECTORY, + SYSTEMD_SELF_UPDATE_PLAN_FILE, + SYSTEMD_SELF_UPDATE_UNIT, + SystemdSelfUpdatePlan, +} from "./systemdSelfUpdate.ts"; const NODE_PATH = "/usr/local/bin/node"; -const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* ( - filePath: string, - expected: string, -) { - const fs = yield* FileSystem.FileSystem; - for (let iteration = 0; iteration < 1_000; iteration += 1) { - const contents = yield* fs.readFileString(filePath); - if (contents === expected) { - return; - } - // The rollback performs real filesystem I/O on a detached fiber, which - // advancing TestClock does not await. - yield* Effect.yieldNow; - } - return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`)); -}); - -const eventuallyTrue = Effect.fn("test.eventuallyTrue")(function* (predicate: () => boolean) { - for (let iteration = 0; iteration < 1_000; iteration += 1) { - if (predicate()) { - return; - } - yield* Effect.yieldNow; - } - return yield* Effect.die(new Error("Expected condition was not observed.")); -}); - interface RecordedCommand { readonly command: string; readonly args: ReadonlyArray; @@ -445,7 +425,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" }); assert.deepEqual( context.commands.map((entry) => entry.command), - [NODE_PATH, "npm", NODE_PATH], + [NODE_PATH, "npm", NODE_PATH, NODE_PATH], ); }).pipe(Effect.provide(TestClock.layer())), ); @@ -506,6 +486,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { [ `npm install --prefix ${context.path.join(context.baseDir, "runtime/versions/0.0.29")} --no-fund --no-audit t3@0.0.29`, `${NODE_PATH} ${pinnedEntry} --version`, + `${NODE_PATH} ${pinnedEntry} service _validate-update --database ${context.path.join(context.baseDir, "userdata/state.sqlite")}`, ], ); @@ -521,9 +502,17 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("rewrites the systemd unit and restarts the boot service", () => + it.effect("stages the systemd unit and hands activation to a transient service", () => Effect.gen(function* () { const context = yield* makeContext({ bootService: true }); + const unitPath = context.path.join( + context.home, + ".config", + "systemd", + "user", + BOOT_SERVICE_UNIT_FILE, + ); + const previousUnit = yield* context.fs.readFileString(unitPath); const result = yield* context.service.update({ targetVersion: "0.0.29" }); assert.deepEqual(result, { targetVersion: "0.0.29", method: "boot-service" }); @@ -531,46 +520,55 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { context.baseDir, "runtime/versions/0.0.29/node_modules/t3/dist/bin.mjs", ); - const unit = yield* context.fs.readFileString( - context.path.join(context.home, ".config", "systemd", "user", "t3code.service"), + // The RPC process does not rewrite the unit it is currently running + // under. The independent transient service applies the staged unit. + assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); + const planPath = context.path.join( + context.baseDir, + "userdata", + SYSTEMD_SELF_UPDATE_DIRECTORY, + SYSTEMD_SELF_UPDATE_PLAN_FILE, + ); + const plan = yield* context.fs + .readFileString(planPath) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan))), + ); + assert.equal(plan.previousUnit, previousUnit); + assert.include(plan.nextUnit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); + assert.equal( + plan.runtimeStatePath, + context.path.join(context.baseDir, "userdata/server-runtime.json"), ); - assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); assert.deepEqual( context.commands.map((entry) => entry.command), - ["npm", NODE_PATH, "systemctl"], + ["npm", NODE_PATH, NODE_PATH, "systemd-run"], ); - assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]); - - // Restart waits until after the update acknowledgement can flush. - yield* TestClock.adjust(Duration.seconds(10)); - assert.deepEqual(context.commands[3], { - command: "systemctl", - args: ["--user", "restart", "--no-block", "t3code.service"], - }); + assert.include(context.commands[3]?.args ?? [], `--unit=${SYSTEMD_SELF_UPDATE_UNIT}`); + assert.include(context.commands[3]?.args ?? [], planPath); assert.lengthOf(context.spawns, 0); - // systemd replaces the process; the server must not exit itself. assert.equal(context.exitCount(), 0); - // The queued restart returns while this process is still shutting - // down; the lock must stay held so a second update cannot rewrite the - // unit mid-teardown. + // The lock stays held after a successful handoff because the transient + // service is now responsible for replacing this process. const concurrentError = yield* context.service .update({ targetVersion: "0.0.30" }) .pipe(Effect.flip); assert.include(concurrentError.reason, "already in progress"); - }).pipe(Effect.provide(TestClock.layer())), + }), ); - it.effect("restores the previous unit and permits a retry when systemd restart fails", () => + it.effect("leaves the live unit untouched and permits a retry when the handoff fails", () => Effect.gen(function* () { - let failRestart = true; + let failHandoff = true; const context = yield* makeContext({ bootService: true, failWhen: (command, args) => { - if (command !== "systemctl" || args[1] !== "restart" || !failRestart) { + if (command !== "systemd-run" || !failHandoff) { return false; } - failRestart = false; + assert.include(args, `--unit=${SYSTEMD_SELF_UPDATE_UNIT}`); + failHandoff = false; return true; }, }); @@ -583,29 +581,20 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { ); const previousUnit = yield* context.fs.readFileString(unitPath); - const first = yield* context.service.update({ targetVersion: "0.0.29" }); - assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); - yield* TestClock.adjust(Duration.seconds(10)); - yield* eventuallyFileString(unitPath, previousUnit); - yield* eventuallyTrue(() => context.commands.at(-1)?.args[1] === "daemon-reload"); - assert.deepEqual( - context.commands.slice(-2).map((entry) => entry.args), - [ - ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], - ["--user", "daemon-reload"], - ], - ); + const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); + assert.include(first.reason, "Systemd rejected the update handoff"); + assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); const retry = yield* context.service.update({ targetVersion: "0.0.30" }); assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" }); - }).pipe(Effect.provide(TestClock.layer())), + }), ); - it.effect("restores the previous systemd unit when daemon-reload fails", () => + it.effect("rejects a target that cannot validate the current database", () => Effect.gen(function* () { const context = yield* makeContext({ bootService: true, - failWhen: (command) => command === "systemctl", + failWhen: (command, args) => command === NODE_PATH && args.includes("_validate-update"), }); const unitPath = context.path.join( context.home, @@ -617,16 +606,15 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { const previousUnit = yield* context.fs.readFileString(unitPath); const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(error.reason, "Reloading systemd units failed"); + assert.include(error.reason, "not compatible with this server's database"); assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); assert.deepEqual( context.commands.map((entry) => entry.command), - ["npm", NODE_PATH, "systemctl", "systemctl"], + ["npm", NODE_PATH, NODE_PATH], ); - yield* TestClock.adjust(Duration.seconds(10)); assert.lengthOf(context.spawns, 0); assert.equal(context.exitCount(), 0); - }).pipe(Effect.provide(TestClock.layer())), + }), ); }); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 9dcb713e1a1..b318cb73b3b 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -24,8 +24,8 @@ import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; +import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; -import { writeFileStringAtomically } from "../atomicWrite.ts"; import * as ProcessRunner from "../processRunner.ts"; import { BOOT_SERVICE_UNIT_ENV, @@ -34,6 +34,14 @@ import { renderBootServiceUnit, } from "./bootService.ts"; import { ensurePinnedRuntimeInstalled, removePinnedRuntimeInstallation } from "./pinnedRuntime.ts"; +import { + SYSTEMD_SELF_UPDATE_DIRECTORY, + SYSTEMD_SELF_UPDATE_PLAN_FILE, + SYSTEMD_SELF_UPDATE_RECEIPT_FILE, + SYSTEMD_SELF_UPDATE_UNIT, + SystemdSelfUpdatePlan, + writeSystemdSelfUpdatePlan, +} from "./systemdSelfUpdate.ts"; /** * Lets a connected client replace this server with another published `t3` @@ -221,11 +229,6 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option Effect.andThen(restart), Effect.forkDetach({ startImmediately: true }), ); - const writeUnitAtomically = (filePath: string, contents: string) => - writeFileStringAtomically({ filePath, contents }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - ); const update: ServerSelfUpdate["Service"]["update"] = Effect.fn( "cloud.server_self_update.update", @@ -301,6 +304,33 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option ); } + // Run this command from the target artifact while the known-good server + // is still alive. Older artifacts without the compatibility check, or a + // build whose migration identities disagree with this database, cannot + // be activated remotely. + const databasePreflight = yield* runner + .run({ + command: host.execPath, + args: [ + runtimePaths.entryPath, + "service", + "_validate-update", + "--database", + serverConfig.dbPath, + ], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError((cause) => + failWith(`Could not verify t3@${targetVersion} against this database.`, cause), + ), + ); + if (databasePreflight.code !== 0) { + return yield* failWith( + `The installed t3@${targetVersion} is not compatible with this server's database (exit code ${String(databasePreflight.code)}).`, + ); + } + if (activeMethod === "boot-service") { const homeDir = env.HOME ?? ""; const unitPath = path.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE); @@ -311,98 +341,63 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option ); // Same shape bootService.install writes, so host lifecycle commands // still recognize the unit as current. - const unit = renderBootServiceUnit({ + const nextUnit = renderBootServiceUnit({ nodePath: host.execPath, t3EntryPath: runtimePaths.entryPath, baseDir: serverConfig.baseDir, logPath: path.join(serverConfig.logsDir, "boot-service.log"), unitPath, }); - yield* writeUnitAtomically(unitPath, unit).pipe( - Effect.mapError((cause) => failWith("Could not update the systemd unit.", cause)), - ); - - const reloadSystemd = Effect.fn("cloud.server_self_update.reload_systemd")(function* () { - const reload = yield* runner - .run({ command: "systemctl", args: ["--user", "daemon-reload"] }) - .pipe(Effect.mapError((cause) => failWith("Could not reload systemd units.", cause))); - if (reload.code !== 0) { - return yield* failWith( - `Reloading systemd units failed (exit code ${String(reload.code)}).`, - ); - } + const updateDirectory = path.join(serverConfig.stateDir, SYSTEMD_SELF_UPDATE_DIRECTORY); + const planPath = path.join(updateDirectory, SYSTEMD_SELF_UPDATE_PLAN_FILE); + const plan = new SystemdSelfUpdatePlan({ + version: 1, + fromVersion: packageJson.version, + targetVersion, + currentPid: process.pid, + unitPath, + previousUnit, + nextUnit, + runtimeStatePath: serverConfig.serverRuntimeStatePath, + receiptPath: path.join(updateDirectory, SYSTEMD_SELF_UPDATE_RECEIPT_FILE), }); - - yield* reloadSystemd().pipe( - Effect.catch((reloadError) => - writeUnitAtomically(unitPath, previousUnit).pipe( - Effect.mapError((rollbackCause) => - failWith("Could not restore the previous systemd unit.", { - reloadError, - rollbackCause, - }), - ), - // Systemd should still have the old unit in memory after the - // failed reload, but retry after restoring in case it applied a - // partial update before returning an error. - Effect.andThen(reloadSystemd().pipe(Effect.ignore)), - Effect.andThen(Effect.fail(reloadError)), - ), - ), + yield* writeSystemdSelfUpdatePlan(planPath, plan).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.mapError((cause) => failWith("Could not stage the systemd update plan.", cause)), ); - yield* Effect.logInfo("Server self-update installed; restarting boot service.", { + + // The transient unit has its own cgroup, so it survives the restart of + // t3code.service and remains responsible for readiness and rollback. + const handoff = yield* runner + .run({ + command: "systemd-run", + args: [ + "--user", + "--collect", + "--service-type=exec", + `--unit=${SYSTEMD_SELF_UPDATE_UNIT}`, + host.execPath, + host.cliEntryPath, + "service", + "_apply-update", + "--plan", + planPath, + ], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError((cause) => failWith("Could not hand the update to systemd.", cause)), + ); + if (handoff.code !== 0) { + return yield* failWith( + `Systemd rejected the update handoff (exit code ${String(handoff.code)}).`, + ); + } + yield* Effect.logInfo("Server self-update staged; systemd will activate it.", { targetVersion, + planPath, }); - // Restart after the acknowledgement has had time to cross any relay - // hop. --no-block queues the restart job and exits before systemd - // stops this unit: a blocking restart's SIGTERM reaches the systemctl - // child (it shares this service's cgroup), which read as a restart - // failure and rolled the new unit back while the old server finished - // shutting down. With the handoff race gone, a non-zero exit or spawn - // error means systemd genuinely rejected the job while this process is - // still alive, so restoring the previous unit below stays correct. - yield* scheduleRestart( - Effect.gen(function* () { - const restart = yield* runner - .run({ - command: "systemctl", - args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], - }) - .pipe( - Effect.mapError((cause) => - failWith("Could not restart the systemd boot service.", cause), - ), - ); - if (restart.code !== 0) { - return yield* failWith( - `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`, - ); - } - }).pipe( - Effect.catch((restartError) => - writeUnitAtomically(unitPath, previousUnit).pipe( - Effect.andThen(reloadSystemd()), - Effect.mapError((rollbackError) => - failWith("Could not restore the previous systemd unit.", { - restartError, - rollbackError, - }), - ), - Effect.andThen(Effect.fail(restartError)), - ), - ), - Effect.catch((error) => - Effect.logError("Server self-update could not restart the boot service.").pipe( - Effect.annotateLogs({ targetVersion, error: error.reason }), - // Permit a retry only after the failed handoff was rolled - // back. A queued restart returns while this process is still - // shutting down; releasing the lock then would let a second - // update rewrite the unit mid-teardown. - Effect.andThen(Ref.set(inFlight, false)), - ), - ), - ), - ); } else { // Spawn the shim before acknowledging the RPC so ENOENT/EACCES and // other launch failures leave this server alive and return a useful diff --git a/apps/server/src/cloud/systemdSelfUpdate.test.ts b/apps/server/src/cloud/systemdSelfUpdate.test.ts new file mode 100644 index 00000000000..07f4e1eff2d --- /dev/null +++ b/apps/server/src/cloud/systemdSelfUpdate.test.ts @@ -0,0 +1,165 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; + +import * as ProcessRunner from "../processRunner.ts"; +import { + SystemdSelfUpdateActivationError, + SystemdSelfUpdatePlan, + SystemdSelfUpdateReceipt, + activateSystemdSelfUpdatePlan, +} from "./systemdSelfUpdate.ts"; + +interface RecordedCommand { + readonly command: string; + readonly args: ReadonlyArray; +} + +const makeRunnerLayer = (commands: Array) => + Layer.succeed( + ProcessRunner.ProcessRunner, + ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.sync(() => { + commands.push({ command: input.command, args: input.args }); + return { + stdout: "", + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + }), + ); + +it.layer(NodeServices.layer)("systemd self-update activation", (it) => { + const makeFixture = Effect.fn("test.makeSystemdUpdateFixture")(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-systemd-update-test-" }); + const unitPath = path.join(directory, "t3code.service"); + const receiptPath = path.join(directory, "receipt.json"); + const previousUnit = "old unit\n"; + yield* fs.writeFileString(unitPath, previousUnit); + return { + fs, + unitPath, + receiptPath, + previousUnit, + plan: new SystemdSelfUpdatePlan({ + version: 1, + fromVersion: "0.0.28", + targetVersion: "0.0.29", + currentPid: 123, + unitPath, + previousUnit, + nextUnit: "new unit\n", + runtimeStatePath: path.join(directory, "server-runtime.json"), + receiptPath, + }), + }; + }); + + const readReceipt = Effect.fn("test.readSystemdUpdateReceipt")(function* (receiptPath: string) { + const fs = yield* FileSystem.FileSystem; + return yield* fs + .readFileString(receiptPath) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdateReceipt))), + ); + }); + + it.effect("activates and verifies the new service from the helper", () => + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const commands: Array = []; + const readinessVersions: Array = []; + + yield* activateSystemdSelfUpdatePlan( + fixture.plan, + (input) => Effect.sync(() => readinessVersions.push(input.expectedVersion)), + { restartDelay: Duration.zero }, + ).pipe(Effect.provide(makeRunnerLayer(commands))); + + assert.equal(yield* fixture.fs.readFileString(fixture.unitPath), "new unit\n"); + assert.deepEqual(readinessVersions, ["0.0.29"]); + assert.deepEqual( + commands.map(({ args }) => args), + [ + ["--user", "daemon-reload"], + ["--user", "restart", "t3code.service"], + ], + ); + assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "healthy"); + }), + ); + + it.effect("restores and verifies the previous service when the target never becomes ready", () => + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const commands: Array = []; + const readinessVersions: Array = []; + + yield* activateSystemdSelfUpdatePlan( + fixture.plan, + (input) => + Effect.sync(() => readinessVersions.push(input.expectedVersion)).pipe( + Effect.andThen( + input.expectedVersion === fixture.plan.targetVersion + ? Effect.fail( + new SystemdSelfUpdateActivationError({ + reason: "target did not become ready", + }), + ) + : Effect.void, + ), + ), + { restartDelay: Duration.zero }, + ).pipe(Effect.provide(makeRunnerLayer(commands))); + + assert.equal(yield* fixture.fs.readFileString(fixture.unitPath), fixture.previousUnit); + assert.deepEqual(readinessVersions, ["0.0.29", "0.0.28"]); + assert.deepEqual( + commands.map(({ args }) => args), + [ + ["--user", "daemon-reload"], + ["--user", "restart", "t3code.service"], + ["--user", "daemon-reload"], + ["--user", "reset-failed", "t3code.service"], + ["--user", "restart", "t3code.service"], + ], + ); + const receipt = yield* readReceipt(fixture.receiptPath); + assert.equal(receipt.phase, "rolled-back"); + assert.equal(receipt.detail, "target did not become ready"); + }), + ); + + it.effect("records a recovery failure instead of claiming the update completed", () => + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const commands: Array = []; + const error = yield* activateSystemdSelfUpdatePlan( + fixture.plan, + (input) => + Effect.fail( + new SystemdSelfUpdateActivationError({ + reason: `${input.expectedVersion} did not become ready`, + }), + ), + { restartDelay: Duration.zero }, + ).pipe(Effect.provide(makeRunnerLayer(commands)), Effect.flip); + + assert.include(error.reason, "previous server could not be restored"); + assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed"); + }), + ); +}); diff --git a/apps/server/src/cloud/systemdSelfUpdate.ts b/apps/server/src/cloud/systemdSelfUpdate.ts new file mode 100644 index 00000000000..4deeee5ec39 --- /dev/null +++ b/apps/server/src/cloud/systemdSelfUpdate.ts @@ -0,0 +1,310 @@ +import { ExecutionEnvironmentDescriptor } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; + +import { writeFileStringAtomically } from "../atomicWrite.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; +import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts"; + +export const SYSTEMD_SELF_UPDATE_UNIT = "t3code-self-update.service"; +export const SYSTEMD_SELF_UPDATE_DIRECTORY = "self-update"; +export const SYSTEMD_SELF_UPDATE_PLAN_FILE = "plan.json"; +export const SYSTEMD_SELF_UPDATE_RECEIPT_FILE = "receipt.json"; + +const DEFAULT_RESTART_DELAY = Duration.seconds(2); +const DEFAULT_READINESS_TIMEOUT = Duration.seconds(60); +const DEFAULT_READINESS_INTERVAL = Duration.millis(500); +const READINESS_REQUEST_TIMEOUT = Duration.seconds(2); +const WELL_KNOWN_ENVIRONMENT_PATH = "/.well-known/t3/environment"; + +export class SystemdSelfUpdatePlan extends Schema.Class( + "SystemdSelfUpdatePlan", +)({ + version: Schema.Literal(1), + fromVersion: Schema.String, + targetVersion: Schema.String, + currentPid: Schema.Int, + unitPath: Schema.String, + previousUnit: Schema.String, + nextUnit: Schema.String, + runtimeStatePath: Schema.String, + receiptPath: Schema.String, +}) {} + +export const SystemdSelfUpdateReceipt = Schema.Struct({ + version: Schema.Literal(1), + targetVersion: Schema.String, + phase: Schema.Literals(["prepared", "activating", "healthy", "rolled-back", "recovery-failed"]), + updatedAt: Schema.String, + detail: Schema.optional(Schema.String), +}); +export type SystemdSelfUpdateReceipt = typeof SystemdSelfUpdateReceipt.Type; + +export class SystemdSelfUpdateActivationError extends Schema.TaggedErrorClass()( + "SystemdSelfUpdateActivationError", + { + reason: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.reason; + } +} + +const decodePlan = Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan)); +const encodePlan = Schema.encodeEffect(Schema.fromJsonString(SystemdSelfUpdatePlan)); +const encodeReceipt = Schema.encodeEffect(Schema.fromJsonString(SystemdSelfUpdateReceipt)); + +const provideFileServices = ( + effect: Effect.Effect, + fs: FileSystem.FileSystem, + path: Path.Path, +) => + effect.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + +const writeAtomically = ( + filePath: string, + contents: string, + fs: FileSystem.FileSystem, + path: Path.Path, +) => provideFileServices(writeFileStringAtomically({ filePath, contents }), fs, path); + +export const writeSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.writePlan")(function* ( + planPath: string, + plan: SystemdSelfUpdatePlan, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const encoded = yield* encodePlan(plan); + yield* writeAtomically(planPath, `${encoded}\n`, fs, path); + yield* writeReceipt(plan, "prepared"); +}); + +const writeReceipt = Effect.fn("systemdSelfUpdate.writeReceipt")(function* ( + plan: SystemdSelfUpdatePlan, + phase: SystemdSelfUpdateReceipt["phase"], + detail?: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const now = yield* DateTime.now; + const receipt = { + version: 1, + targetVersion: plan.targetVersion, + phase, + updatedAt: DateTime.formatIso(now), + ...(detail === undefined ? {} : { detail }), + } satisfies SystemdSelfUpdateReceipt; + const encoded = yield* encodeReceipt(receipt); + yield* writeAtomically(plan.receiptPath, `${encoded}\n`, fs, path); +}); + +const runSystemctl = Effect.fn("systemdSelfUpdate.runSystemctl")(function* ( + args: ReadonlyArray, + failureDescription: string, +) { + const runner = yield* ProcessRunner.ProcessRunner; + const result = yield* runner.run({ command: "systemctl", args }).pipe( + Effect.mapError( + (cause) => + new SystemdSelfUpdateActivationError({ + reason: failureDescription, + cause, + }), + ), + ); + if (result.code !== 0) { + return yield* new SystemdSelfUpdateActivationError({ + reason: `${failureDescription} (exit code ${String(result.code)}).`, + }); + } +}); + +const installUnit = Effect.fn("systemdSelfUpdate.installUnit")(function* ( + plan: SystemdSelfUpdatePlan, + contents: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* writeAtomically(plan.unitPath, contents, fs, path).pipe( + Effect.mapError( + (cause) => + new SystemdSelfUpdateActivationError({ + reason: "Could not write the systemd boot service unit.", + cause, + }), + ), + ); + yield* runSystemctl(["--user", "daemon-reload"], "Could not reload systemd units"); +}); + +const restartBootService = runSystemctl( + ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + "Could not restart the systemd boot service", +); + +export interface SystemdSelfUpdateReadinessInput { + readonly expectedVersion: string; + readonly previousPid: number; + readonly runtimeStatePath: string; +} + +export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.waitForReadiness")( + function* ( + input: SystemdSelfUpdateReadinessInput, + options: { + readonly timeout?: Duration.Input; + readonly interval?: Duration.Input; + } = {}, + ) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const client = yield* HttpClient.HttpClient; + const attempt = Effect.gen(function* () { + const state = yield* provideFileServices( + readPersistedServerRuntimeState(input.runtimeStatePath), + fs, + path, + ).pipe( + Effect.mapError( + (cause) => + new SystemdSelfUpdateActivationError({ + reason: "Could not read the restarted server's runtime state.", + cause, + }), + ), + ); + if (Option.isNone(state) || state.value.pid === input.previousPid) { + return yield* new SystemdSelfUpdateActivationError({ + reason: "The restarted server has not published fresh runtime state yet.", + }); + } + + const request = HttpClientRequest.get( + new URL(WELL_KNOWN_ENVIRONMENT_PATH, state.value.origin).toString(), + ); + const descriptor = yield* client.execute(request).pipe( + Effect.timeout(READINESS_REQUEST_TIMEOUT), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)), + Effect.mapError( + (cause) => + new SystemdSelfUpdateActivationError({ + reason: "The restarted server is not ready yet.", + cause, + }), + ), + ); + if (descriptor.serverVersion !== input.expectedVersion) { + return yield* new SystemdSelfUpdateActivationError({ + reason: `The restarted server reported t3@${descriptor.serverVersion}, expected t3@${input.expectedVersion}.`, + }); + } + }); + + const ready = yield* attempt.pipe( + Effect.retry(Schedule.spaced(options.interval ?? DEFAULT_READINESS_INTERVAL)), + Effect.timeoutOption(options.timeout ?? DEFAULT_READINESS_TIMEOUT), + ); + if (Option.isNone(ready)) { + return yield* new SystemdSelfUpdateActivationError({ + reason: `The restarted server did not become ready as t3@${input.expectedVersion}.`, + }); + } + }, +); + +export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activatePlan")(function* < + R, +>( + plan: SystemdSelfUpdatePlan, + waitForReadiness: ( + input: SystemdSelfUpdateReadinessInput, + ) => Effect.Effect, + options: { readonly restartDelay?: Duration.Input } = {}, +) { + const activate = Effect.gen(function* () { + yield* writeReceipt(plan, "activating").pipe(Effect.ignore); + yield* Effect.sleep(options.restartDelay ?? DEFAULT_RESTART_DELAY); + yield* installUnit(plan, plan.nextUnit); + yield* restartBootService; + yield* waitForReadiness({ + expectedVersion: plan.targetVersion, + previousPid: plan.currentPid, + runtimeStatePath: plan.runtimeStatePath, + }); + yield* writeReceipt(plan, "healthy").pipe(Effect.ignore); + }); + + yield* activate.pipe( + Effect.catch((activationError) => + Effect.gen(function* () { + yield* Effect.logError("Systemd self-update failed; restoring the previous version.").pipe( + Effect.annotateLogs({ + targetVersion: plan.targetVersion, + error: activationError.reason, + }), + ); + const recovery = Effect.gen(function* () { + yield* installUnit(plan, plan.previousUnit); + yield* runSystemctl( + ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], + "Could not reset the failed systemd boot service", + ); + yield* restartBootService; + yield* waitForReadiness({ + expectedVersion: plan.fromVersion, + previousPid: plan.currentPid, + runtimeStatePath: plan.runtimeStatePath, + }); + yield* writeReceipt(plan, "rolled-back", activationError.reason).pipe(Effect.ignore); + }); + + yield* recovery.pipe( + Effect.catch((recoveryError) => + writeReceipt(plan, "recovery-failed", recoveryError.reason).pipe( + Effect.ignore, + Effect.andThen( + Effect.fail( + new SystemdSelfUpdateActivationError({ + reason: "The update failed and the previous server could not be restored.", + cause: { activationError, recoveryError }, + }), + ), + ), + ), + ), + ); + }), + ), + ); +}); + +export const applySystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.applyPlan")(function* ( + planPath: string, +) { + const fs = yield* FileSystem.FileSystem; + const plan = yield* fs.readFileString(planPath).pipe( + Effect.flatMap(decodePlan), + Effect.mapError( + (cause) => + new SystemdSelfUpdateActivationError({ + reason: `Could not read the staged update plan at ${planPath}.`, + cause, + }), + ), + ); + yield* activateSystemdSelfUpdatePlan(plan, waitForSystemdSelfUpdateReadiness); +}); diff --git a/apps/server/src/persistence/Migrations.test.ts b/apps/server/src/persistence/Migrations.test.ts new file mode 100644 index 00000000000..c65b28f7090 --- /dev/null +++ b/apps/server/src/persistence/Migrations.test.ts @@ -0,0 +1,53 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { MigrationIdentityMismatchError, validateMigrationIdentities } from "./Migrations.ts"; +import * as NodeSqliteClient from "./NodeSqliteClient.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("migration identity validation", (it) => { + const createTrackingTable = Effect.fn("test.createMigrationTrackingTable")(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql` + CREATE TABLE IF NOT EXISTS effect_sql_migrations ( + migration_id integer PRIMARY KEY NOT NULL, + name varchar(255) NOT NULL, + created_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `; + yield* sql`DELETE FROM effect_sql_migrations`; + }); + + it.effect("rejects a reused migration ID before migrations run", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* createTrackingTable(); + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES (35, 'DifferentMigration') + `; + + const error = yield* validateMigrationIdentities().pipe(Effect.flip); + assert.instanceOf(error, MigrationIdentityMismatchError); + assert.equal(error.migrationId, 35); + assert.equal(error.expectedName, "ProjectionThreadTitleRegeneration"); + assert.equal(error.actualName, "DifferentMigration"); + }), + ); + + it.effect("allows migration IDs this build does not know about", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* createTrackingTable(); + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES (999, 'FutureMigration') + `; + + yield* validateMigrationIdentities(); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 95cb6b17f84..c9799d0f98f 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -11,6 +11,8 @@ import * as Migrator from "effect/unstable/sql/Migrator"; import * as Layer from "effect/Layer"; import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; // Import all migrations statically import Migration0001 from "./Migrations/001_OrchestrationEvents.ts"; @@ -112,6 +114,59 @@ export const makeMigrationLoader = (throughId?: number) => */ const run = Migrator.make({}); +export class MigrationIdentityMismatchError extends Schema.TaggedErrorClass()( + "MigrationIdentityMismatchError", + { + migrationId: Schema.Int, + expectedName: Schema.String, + actualName: Schema.String, + }, +) { + override get message(): string { + return `Migration ${String(this.migrationId)} is recorded as '${this.actualName}', but this t3 build expects '${this.expectedName}'.`; + } +} + +/** + * Refuse to run a build whose migration history disagrees with the database. + * Unknown newer IDs are intentionally allowed so a compatible older build can + * inspect a database created by a newer release without rewriting its history. + */ +export const validateMigrationIdentities = Effect.fn("validateMigrationIdentities")(function* () { + const sql = yield* SqlClient.SqlClient; + const tables = yield* sql<{ readonly name: string }>` + SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'effect_sql_migrations' + `; + if (tables.length === 0) { + return; + } + + const expectedNames = new Map( + migrationEntries.map(([migrationId, name]) => [migrationId, name]), + ); + const persisted = yield* sql<{ + readonly migration_id: number; + readonly name: string; + }>` + SELECT migration_id, name + FROM effect_sql_migrations + ORDER BY migration_id + `; + + for (const migration of persisted) { + const expectedName = expectedNames.get(migration.migration_id); + if (expectedName !== undefined && expectedName !== migration.name) { + return yield* new MigrationIdentityMismatchError({ + migrationId: migration.migration_id, + expectedName, + actualName: migration.name, + }); + } + } +}); + export interface RunMigrationsOptions { readonly toMigrationInclusive?: number | undefined; } @@ -129,6 +184,7 @@ export interface RunMigrationsOptions { export const runMigrations = Effect.fn("runMigrations")(function* ({ toMigrationInclusive, }: RunMigrationsOptions = {}) { + yield* validateMigrationIdentities(); const executedMigrations = yield* run({ loader: makeMigrationLoader(toMigrationInclusive) }); const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`); yield* migrations.length === 0 diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1e017737148..9e2317ea28d 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -66,10 +66,15 @@ flowchart TD H --> I[Run version preflight] I -->|bad code or version| J[Remove candidate runtime and keep current server] I -->|cannot run preflight| J2[Keep candidate and current server] - I -->|passes| K{Handoff method} - K -->|boot-service| L[Rewrite and restart T3 systemd unit] + I -->|passes| I2[Run target database compatibility check] + I2 -->|fails| J2 + I2 -->|passes| K{Handoff method} + K -->|boot-service| L[Stage plan and start transient systemd helper] K -->|respawn| M[Start delayed replacement and exit current process] - L --> N[Reconnect with fresh backoff] + L --> L2[Helper activates unit and verifies target] + L2 -->|target fails| L3[Restore and verify previous unit] + L2 -->|target ready| N[Reconnect with fresh backoff] + L3 --> N M --> N N --> O[Replacement publishes ready at target version] ``` @@ -84,8 +89,10 @@ The update service permits one update at a time. It installs `t3@` unde successfully. Boot-service setup and self-update share the same process-wide installation lock, so they cannot mutate a pinned runtime concurrently. -Before any restart, the current Node executable runs the replacement with `--version`. A failed -install, failed preflight, or wrong reported version leaves the current server running. +Before any restart, the current Node executable runs the replacement with `--version`, then asks +the target artifact to validate its migration identities against the live database in read-only +mode. A failed install, version preflight, or compatibility check leaves the current server running. +Targets old enough to lack the compatibility command are not eligible for remote activation. Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when the preflight process actually completes and reports a bad exit code or the wrong version: that is @@ -107,10 +114,14 @@ authorization; it does not uninstall the host service. ## Process Handoff -For `boot-service`, the server atomically rewrites the T3-managed user unit to point at the verified -runtime and reloads systemd. It acknowledges the handoff, then restarts the unit after the same -short grace period used by foreground respawn. A rejected deferred restart restores the previous -unit and is logged by the still-running process. +For `boot-service`, the request process writes a typed plan but leaves its live unit untouched. It +starts a fixed-name `t3code-self-update.service` transient unit with `systemd-run --user`; the +transient unit has a separate cgroup and acts as the cross-process update lock. After a short grace +period it atomically installs the candidate unit, reloads systemd, performs a blocking restart, and +probes the public environment descriptor for the exact target version. If activation or readiness +fails, the helper restores the previous unit, clears systemd's failed state, restarts it, and proves +the previous version is ready. The plan and latest receipt live under the environment state +directory's `self-update` folder for diagnosis. For `respawn`, the server starts a detached, delayed replacement that replays the original CLI arguments. It then acknowledges the request and schedules the current process to exit. The delays @@ -118,10 +129,10 @@ give the acknowledgement time to cross direct or relayed connections before the Progress-capable servers emit `downloading` before installing the pinned runtime and `installing` before preflight and handoff. A terminal stream event acknowledges that restart is scheduled. The -client then enters `resuming`, waits for the replacement lifecycle stream to publish `ready` with -the target version, and only then completes the operation. It watches for the intentional -disconnect's first backoff state and requests one fresh retry, which clears historical backoff debt -without adding a separate reconnect loop. +client then enters `resuming`, waits for the intentional disconnect before accepting a new lifecycle +event, and requests one fresh retry to clear historical backoff debt. The first new `ready` event +must report the target version. A verified rollback therefore becomes an immediate wrong-version +error instead of looking like a two-minute timeout. ## Release Invariant @@ -134,7 +145,9 @@ the hosted web deployment depends on that release. See [Release Checklist](../op - Capability contract: `packages/contracts/src/environment.ts` - Update RPC contract: `packages/contracts/src/server.ts` and `packages/contracts/src/rpc.ts` - Capability detection and handoff: `apps/server/src/cloud/selfUpdate.ts` +- Systemd activation, health check, and rollback: `apps/server/src/cloud/systemdSelfUpdate.ts` - Host service commands: `apps/server/src/cli/service.ts` - Pinned runtime installation: `apps/server/src/cloud/pinnedRuntime.ts` +- Migration identity validation: `apps/server/src/persistence/Migrations.ts` - Client version comparison: `apps/web/src/versionSkew.ts` - Shared update action: `apps/web/src/components/ServerUpdateAction.tsx` diff --git a/docs/user/updating.md b/docs/user/updating.md index 8e1fac81854..a5bc497aaec 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -59,6 +59,10 @@ Keep the web or desktop app open while the server restarts. The update completes replacement server reports the requested version and is ready to accept commands. The warning and progress rail then disappear. +When a background-service replacement cannot become ready, T3 Code restores and verifies the +previous server version automatically. The update remains failed so you can retry after reading the +reported reason, but the remote server should remain reachable. + If a step fails: 1. Retry the offered action once. diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index 12925a99867..768ba0bc346 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -34,6 +34,8 @@ import { resolveServerUpdateProgressResult, serverUpdateStateForProgressEvent, serverUpdateStateForServerVersion, + ServerUpdateWrongVersionError, + validateResumedServerVersion, } from "./server.ts"; const CONFIG = { @@ -152,6 +154,27 @@ describe("server state projection", () => { }); }); + it.effect("reports the rollback version as soon as the server resumes", () => + Effect.gen(function* () { + const error = yield* validateResumedServerVersion({ + environmentId: TARGET.environmentId, + targetVersion: "0.0.31", + actualVersion: "0.0.30", + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(ServerUpdateWrongVersionError); + expect(error.message).toBe("The server resumed on t3@0.0.30 instead of t3@0.0.31."); + }), + ); + + it.effect("accepts the requested version after reconnect", () => + validateResumedServerVersion({ + environmentId: TARGET.environmentId, + targetVersion: "0.0.31", + actualVersion: "0.0.31", + }), + ); + it("keeps active update state and hides stale failures after a version change", () => { const running = { status: "running" as const, diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index edd1893f739..f374be4696f 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -86,6 +86,43 @@ export class ServerUpdateResumeTimeoutError extends Schema.TaggedErrorClass()( + "ServerUpdateRestartTimeoutError", + { + environmentId: Schema.String, + targetVersion: Schema.String, + }, +) { + override get message(): string { + return `The server accepted t3@${this.targetVersion} but did not restart.`; + } +} + +export class ServerUpdateWrongVersionError extends Schema.TaggedErrorClass()( + "ServerUpdateWrongVersionError", + { + environmentId: Schema.String, + targetVersion: Schema.String, + actualVersion: Schema.String, + }, +) { + override get message(): string { + return `The server resumed on t3@${this.actualVersion} instead of t3@${this.targetVersion}.`; + } +} + +export const validateResumedServerVersion = Effect.fn("server.validateResumedUpdateVersion")( + function* (input: { + readonly environmentId: EnvironmentId; + readonly targetVersion: string; + readonly actualVersion: string; + }) { + if (input.actualVersion !== input.targetVersion) { + return yield* new ServerUpdateWrongVersionError(input); + } + }, +); + export class ServerUpdateProgressIncompleteError extends Schema.TaggedErrorClass()( "ServerUpdateProgressIncompleteError", { @@ -446,6 +483,7 @@ export function createServerEnvironmentAtoms( atomRegistry.get(configValueAtom(target.environmentId))?.environment.serverVersion ?? targetVersion; let currentStage: ServerUpdateStage = "downloading"; + let transportAlreadyDisconnected = false; atomRegistry.set(stateAtom, { status: "running", stage: currentStage, @@ -500,9 +538,14 @@ export function createServerEnvironmentAtoms( ), Effect.exit, ); + const terminalResult = yield* Ref.get(terminal); + transportAlreadyDisconnected = + Option.isSome(terminalResult) && + Exit.isFailure(streamExit) && + isLegacyUpdateHandoffLoss(streamExit.cause); return yield* resolveServerUpdateProgressResult( targetVersion, - yield* Ref.get(terminal), + terminalResult, streamExit, ); }) @@ -521,6 +564,7 @@ export function createServerEnvironmentAtoms( // Older servers can tear down the transport before their // unary acknowledgement arrives. Treat only that transport // loss as a handoff, then prove it by waiting for target ready. + transportAlreadyDisconnected = true; return { targetVersion, method: selfUpdateMethod }; } return yield* Effect.failCause(exit.cause); @@ -537,26 +581,40 @@ export function createServerEnvironmentAtoms( }), ); - // The update restart is intentional. As soon as the supervisor sees - // that first failed connection, discard any prior backoff debt and - // retry immediately instead of carrying an old 16-second delay. - yield* environmentRegistry.stateChanges(target.environmentId).pipe( - Stream.filter((state) => state.phase === "backoff"), - Stream.take(1), - Stream.runDrain, - Effect.andThen(environmentRegistry.retryNow(target.environmentId)), - Effect.timeoutOption(Duration.seconds(30)), - Effect.ignore, - Effect.forkChild, - ); + // Subscribe before following lifecycle again so a cached ready event + // from the still-running old process cannot satisfy the update. The + // helper waits before restarting, giving this listener time to attach. + if (!transportAlreadyDisconnected) { + const disconnected = yield* environmentRegistry.stateChanges(target.environmentId).pipe( + Stream.filter((state) => state.phase === "backoff"), + Stream.take(1), + Stream.runHead, + Effect.timeoutOption(Duration.seconds(30)), + Effect.map(Option.flatten), + ); + if (Option.isNone(disconnected)) { + return yield* new ServerUpdateRestartTimeoutError({ + environmentId: target.environmentId, + targetVersion, + }); + } + // The update restart is intentional. Discard any prior backoff debt + // instead of carrying an old 16-second delay into recovery. + yield* environmentRegistry.retryNow(target.environmentId); + } else { + // Legacy servers may close the RPC before this listener can attach. + // If the supervisor is still waiting, clear that delay; if it has + // already reconnected, do not interrupt the healthy replacement. + const state = yield* environmentRegistry.state(target.environmentId); + if (state.phase === "backoff") { + yield* environmentRegistry.retryNow(target.environmentId); + } + } const resumed = yield* environmentRegistry .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {})) .pipe( - Stream.filter( - (event) => - event.type === "ready" && event.payload.environment.serverVersion === targetVersion, - ), + Stream.filter((event) => event.type === "ready"), Stream.runHead, Effect.timeoutOption(Duration.seconds(120)), Effect.map(Option.flatten), @@ -567,6 +625,11 @@ export function createServerEnvironmentAtoms( targetVersion, }); } + yield* validateResumedServerVersion({ + environmentId: target.environmentId, + targetVersion, + actualVersion: resumed.value.payload.environment.serverVersion, + }); atomRegistry.set(stateAtom, IDLE_SERVER_UPDATE_STATE); return result; From 70c24206f17e6e32b338a471391247e54aaa1975 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 23:01:47 -0700 Subject: [PATCH 2/5] fix(server): structure update activation errors --- .../src/cloud/systemdSelfUpdate.test.ts | 10 ++- apps/server/src/cloud/systemdSelfUpdate.ts | 87 +++++++++++++------ 2 files changed, 68 insertions(+), 29 deletions(-) diff --git a/apps/server/src/cloud/systemdSelfUpdate.test.ts b/apps/server/src/cloud/systemdSelfUpdate.test.ts index 07f4e1eff2d..559bcb75725 100644 --- a/apps/server/src/cloud/systemdSelfUpdate.test.ts +++ b/apps/server/src/cloud/systemdSelfUpdate.test.ts @@ -116,7 +116,8 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { input.expectedVersion === fixture.plan.targetVersion ? Effect.fail( new SystemdSelfUpdateActivationError({ - reason: "target did not become ready", + operation: "await-readiness", + expectedVersion: input.expectedVersion, }), ) : Effect.void, @@ -139,7 +140,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { ); const receipt = yield* readReceipt(fixture.receiptPath); assert.equal(receipt.phase, "rolled-back"); - assert.equal(receipt.detail, "target did not become ready"); + assert.equal(receipt.detail, "The restarted server did not become ready as t3@0.0.29."); }), ); @@ -152,13 +153,14 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { (input) => Effect.fail( new SystemdSelfUpdateActivationError({ - reason: `${input.expectedVersion} did not become ready`, + operation: "await-readiness", + expectedVersion: input.expectedVersion, }), ), { restartDelay: Duration.zero }, ).pipe(Effect.provide(makeRunnerLayer(commands)), Effect.flip); - assert.include(error.reason, "previous server could not be restored"); + assert.include(error.message, "previous server could not be restored"); assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed"); }), ); diff --git a/apps/server/src/cloud/systemdSelfUpdate.ts b/apps/server/src/cloud/systemdSelfUpdate.ts index 4deeee5ec39..fe557436bbc 100644 --- a/apps/server/src/cloud/systemdSelfUpdate.ts +++ b/apps/server/src/cloud/systemdSelfUpdate.ts @@ -48,15 +48,48 @@ export const SystemdSelfUpdateReceipt = Schema.Struct({ }); export type SystemdSelfUpdateReceipt = typeof SystemdSelfUpdateReceipt.Type; +export const SystemdSelfUpdateOperation = Schema.Literals([ + "read-plan", + "write-unit", + "daemon-reload", + "restart", + "reset-failed", + "await-readiness", + "recover", +]); +export type SystemdSelfUpdateOperation = typeof SystemdSelfUpdateOperation.Type; + export class SystemdSelfUpdateActivationError extends Schema.TaggedErrorClass()( "SystemdSelfUpdateActivationError", { - reason: Schema.String, + operation: SystemdSelfUpdateOperation, + path: Schema.optional(Schema.String), + expectedVersion: Schema.optional(Schema.String), + actualVersion: Schema.optional(Schema.String), + exitCode: Schema.optional(Schema.Number), cause: Schema.optional(Schema.Defect()), }, ) { override get message(): string { - return this.reason; + const exitCode = this.exitCode === undefined ? "" : ` (exit code ${String(this.exitCode)})`; + switch (this.operation) { + case "read-plan": + return `Could not read the staged update plan${this.path === undefined ? "" : ` at ${this.path}`}.`; + case "write-unit": + return "Could not write the systemd boot service unit."; + case "daemon-reload": + return `Could not reload systemd units${exitCode}.`; + case "restart": + return `Could not restart the systemd boot service${exitCode}.`; + case "reset-failed": + return `Could not reset the failed systemd boot service${exitCode}.`; + case "await-readiness": + return this.actualVersion === undefined + ? `The restarted server did not become ready${this.expectedVersion === undefined ? "" : ` as t3@${this.expectedVersion}`}.` + : `The restarted server reported t3@${this.actualVersion}, expected t3@${this.expectedVersion ?? "unknown"}.`; + case "recover": + return "The update failed and the previous server could not be restored."; + } } } @@ -113,21 +146,22 @@ const writeReceipt = Effect.fn("systemdSelfUpdate.writeReceipt")(function* ( const runSystemctl = Effect.fn("systemdSelfUpdate.runSystemctl")(function* ( args: ReadonlyArray, - failureDescription: string, + operation: Extract, ) { const runner = yield* ProcessRunner.ProcessRunner; const result = yield* runner.run({ command: "systemctl", args }).pipe( Effect.mapError( (cause) => new SystemdSelfUpdateActivationError({ - reason: failureDescription, + operation, cause, }), ), ); if (result.code !== 0) { return yield* new SystemdSelfUpdateActivationError({ - reason: `${failureDescription} (exit code ${String(result.code)}).`, + operation, + exitCode: Number(result.code), }); } }); @@ -142,18 +176,16 @@ const installUnit = Effect.fn("systemdSelfUpdate.installUnit")(function* ( Effect.mapError( (cause) => new SystemdSelfUpdateActivationError({ - reason: "Could not write the systemd boot service unit.", + operation: "write-unit", + path: plan.unitPath, cause, }), ), ); - yield* runSystemctl(["--user", "daemon-reload"], "Could not reload systemd units"); + yield* runSystemctl(["--user", "daemon-reload"], "daemon-reload"); }); -const restartBootService = runSystemctl( - ["--user", "restart", BOOT_SERVICE_UNIT_FILE], - "Could not restart the systemd boot service", -); +const restartBootService = runSystemctl(["--user", "restart", BOOT_SERVICE_UNIT_FILE], "restart"); export interface SystemdSelfUpdateReadinessInput { readonly expectedVersion: string; @@ -181,14 +213,16 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa Effect.mapError( (cause) => new SystemdSelfUpdateActivationError({ - reason: "Could not read the restarted server's runtime state.", + operation: "await-readiness", + expectedVersion: input.expectedVersion, cause, }), ), ); if (Option.isNone(state) || state.value.pid === input.previousPid) { return yield* new SystemdSelfUpdateActivationError({ - reason: "The restarted server has not published fresh runtime state yet.", + operation: "await-readiness", + expectedVersion: input.expectedVersion, }); } @@ -202,14 +236,17 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa Effect.mapError( (cause) => new SystemdSelfUpdateActivationError({ - reason: "The restarted server is not ready yet.", + operation: "await-readiness", + expectedVersion: input.expectedVersion, cause, }), ), ); if (descriptor.serverVersion !== input.expectedVersion) { return yield* new SystemdSelfUpdateActivationError({ - reason: `The restarted server reported t3@${descriptor.serverVersion}, expected t3@${input.expectedVersion}.`, + operation: "await-readiness", + expectedVersion: input.expectedVersion, + actualVersion: descriptor.serverVersion, }); } }); @@ -220,7 +257,8 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa ); if (Option.isNone(ready)) { return yield* new SystemdSelfUpdateActivationError({ - reason: `The restarted server did not become ready as t3@${input.expectedVersion}.`, + operation: "await-readiness", + expectedVersion: input.expectedVersion, }); } }, @@ -254,32 +292,30 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa yield* Effect.logError("Systemd self-update failed; restoring the previous version.").pipe( Effect.annotateLogs({ targetVersion: plan.targetVersion, - error: activationError.reason, + operation: activationError.operation, + error: activationError.message, }), ); const recovery = Effect.gen(function* () { yield* installUnit(plan, plan.previousUnit); - yield* runSystemctl( - ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], - "Could not reset the failed systemd boot service", - ); + yield* runSystemctl(["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], "reset-failed"); yield* restartBootService; yield* waitForReadiness({ expectedVersion: plan.fromVersion, previousPid: plan.currentPid, runtimeStatePath: plan.runtimeStatePath, }); - yield* writeReceipt(plan, "rolled-back", activationError.reason).pipe(Effect.ignore); + yield* writeReceipt(plan, "rolled-back", activationError.message).pipe(Effect.ignore); }); yield* recovery.pipe( Effect.catch((recoveryError) => - writeReceipt(plan, "recovery-failed", recoveryError.reason).pipe( + writeReceipt(plan, "recovery-failed", recoveryError.message).pipe( Effect.ignore, Effect.andThen( Effect.fail( new SystemdSelfUpdateActivationError({ - reason: "The update failed and the previous server could not be restored.", + operation: "recover", cause: { activationError, recoveryError }, }), ), @@ -301,7 +337,8 @@ export const applySystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.applyPlan Effect.mapError( (cause) => new SystemdSelfUpdateActivationError({ - reason: `Could not read the staged update plan at ${planPath}.`, + operation: "read-plan", + path: planPath, cause, }), ), From c072888cff068a70c50cbfcdb598ed2abbeb2516 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 23:09:34 -0700 Subject: [PATCH 3/5] fix(server): release failed update handoffs --- apps/server/src/cloud/selfUpdate.test.ts | 26 +++++++++++++++ apps/server/src/cloud/selfUpdate.ts | 40 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index ff47d490dc6..5e33a0ce896 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -590,6 +590,32 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { }), ); + it.effect("permits a retry when the transient helper stops before replacing this process", () => + Effect.gen(function* () { + let helperActive = true; + const context = yield* makeContext({ + bootService: true, + failWhen: (command, args) => + command === "systemctl" && args.includes("is-active") && !helperActive, + }); + + const first = yield* context.service.update({ targetVersion: "0.0.29" }); + assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); + + helperActive = false; + yield* TestClock.adjust(Duration.seconds(2)); + yield* Effect.yieldNow; + + const retry = yield* context.service.update({ targetVersion: "0.0.30" }); + assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" }); + assert.isTrue( + context.commands.some( + ({ command, args }) => command === "systemctl" && args.includes("is-active"), + ), + ); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("rejects a target that cannot validate the current database", () => Effect.gen(function* () { const context = yield* makeContext({ diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index b318cb73b3b..eef988f76d5 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -19,6 +19,7 @@ import * as NodeChildProcess from "node:child_process"; import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -230,6 +231,44 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option Effect.forkDetach({ startImmediately: true }), ); + /** + * A successful systemd-run handoff normally ends by replacing this process, + * so its in-memory lock disappears with it. If the transient helper exits + * before that happens, release the lock so the still-healthy old server can + * accept a retry. The fixed transient unit remains the cross-process lock. + */ + const releaseLockIfSystemdHelperStops = Effect.gen(function* () { + let consecutiveProbeFailures = 0; + for (;;) { + yield* Effect.sleep(Duration.seconds(1)); + const probe = yield* runner + .run({ + command: "systemctl", + args: ["--user", "is-active", "--quiet", SYSTEMD_SELF_UPDATE_UNIT], + timeout: Duration.seconds(5), + }) + .pipe(Effect.exit); + if (Exit.isFailure(probe)) { + consecutiveProbeFailures += 1; + if (consecutiveProbeFailures < 3) { + continue; + } + } else { + consecutiveProbeFailures = 0; + if (probe.value.code === 0) { + continue; + } + } + + yield* Effect.logWarning( + "Systemd self-update helper stopped before replacing this server; updates are retryable.", + { targetUnit: SYSTEMD_SELF_UPDATE_UNIT }, + ); + yield* Ref.set(inFlight, false); + return; + } + }); + const update: ServerSelfUpdate["Service"]["update"] = Effect.fn( "cloud.server_self_update.update", )(function* (input, reportProgress = () => Effect.void) { @@ -394,6 +433,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option `Systemd rejected the update handoff (exit code ${String(handoff.code)}).`, ); } + yield* releaseLockIfSystemdHelperStops.pipe(Effect.forkDetach({ startImmediately: true })); yield* Effect.logInfo("Server self-update staged; systemd will activate it.", { targetVersion, planPath, From cd25a3de9a3a8a2efb3da050577381621d0e543e Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sat, 1 Aug 2026 00:07:09 -0700 Subject: [PATCH 4/5] fix(server): validate update rollback boundaries --- apps/server/src/cli/service.ts | 4 +- apps/server/src/cloud/selfUpdate.test.ts | 1 + apps/server/src/cloud/selfUpdate.ts | 1 + .../systemdSelfUpdate.acceptance.test.ts | 237 ++++++++++++++++++ .../src/cloud/systemdSelfUpdate.test.ts | 60 ++++- apps/server/src/cloud/systemdSelfUpdate.ts | 15 +- apps/server/src/http.ts | 25 +- .../server/src/persistence/Migrations.test.ts | 52 +++- apps/server/src/persistence/Migrations.ts | 76 ++++-- apps/server/src/server.test.ts | 25 ++ apps/server/src/serverRuntimeStartup.ts | 5 + docs/internals/server-updates.md | 13 +- docs/operations/release.md | 17 ++ packages/contracts/src/environmentHttp.ts | 16 +- 14 files changed, 499 insertions(+), 48 deletions(-) create mode 100644 apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index 003f3c272f3..ff297ac3682 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -9,7 +9,7 @@ import packageJson from "../../package.json" with { type: "json" }; import * as BootService from "../cloud/bootService.ts"; import { applySystemdSelfUpdatePlan } from "../cloud/systemdSelfUpdate.ts"; import type * as ServerConfig from "../config.ts"; -import { validateMigrationIdentities } from "../persistence/Migrations.ts"; +import { validateAutomaticUpdateMigrationFrontier } from "../persistence/Migrations.ts"; import * as ProcessRunner from "../processRunner.ts"; import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts"; @@ -169,7 +169,7 @@ const serviceValidateUpdateCommand = Command.make("_validate-update", { const NodeSqliteClient = yield* Effect.promise( () => import("../persistence/NodeSqliteClient.ts"), ); - yield* validateMigrationIdentities().pipe( + yield* validateAutomaticUpdateMigrationFrontier().pipe( Effect.scoped, Effect.provide(NodeSqliteClient.layer({ filename: database, readonly: true })), ); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 5e33a0ce896..1704eb453ab 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -535,6 +535,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan))), ); assert.equal(plan.previousUnit, previousUnit); + assert.equal(plan.serviceUnit, BOOT_SERVICE_UNIT_FILE); assert.include(plan.nextUnit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); assert.equal( plan.runtimeStatePath, diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index eef988f76d5..0fa32e00fa7 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -394,6 +394,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option fromVersion: packageJson.version, targetVersion, currentPid: process.pid, + serviceUnit: BOOT_SERVICE_UNIT_FILE, unitPath, previousUnit, nextUnit, diff --git a/apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts b/apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts new file mode 100644 index 00000000000..1780d2d8fe3 --- /dev/null +++ b/apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts @@ -0,0 +1,237 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * Opt-in acceptance coverage for the real systemd handoff and rollback path. + * Every unit and file is uniquely named; this must never target t3code.service. + * + * Run on Linux with a user manager: + * T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \ + * apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { FetchHttpClient } from "effect/unstable/http"; +import { describe } from "vite-plus/test"; + +import * as ProcessRunner from "../processRunner.ts"; +import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; +import { quoteSystemdValue } from "./bootService.ts"; +import { + SystemdSelfUpdatePlan, + SystemdSelfUpdateReceipt, + waitForSystemdSelfUpdateReadiness, + writeSystemdSelfUpdatePlan, +} from "./systemdSelfUpdate.ts"; + +const OLD_VERSION = "0.0.28"; +const TARGET_VERSION = "0.0.29"; +const decodeLaunch = Schema.decodeUnknownSync( + Schema.fromJsonString( + Schema.Struct({ + pid: Schema.Int, + version: Schema.String, + }), + ), +); + +const fixtureServerSource = ` +import * as fs from "node:fs"; +import * as http from "node:http"; + +const [runtimeStatePath, launchesPath, version, readiness] = process.argv.slice(2); +const descriptor = { + environmentId: "systemd-self-update-acceptance", + label: "Systemd self-update acceptance", + platform: { os: "linux", arch: "x64" }, + serverVersion: version, + capabilities: { repositoryIdentity: true }, +}; +const server = http.createServer((request, response) => { + if (request.url === "/.well-known/t3/ready" && readiness === "ready") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(descriptor)); + return; + } + response.writeHead(503); + response.end(); +}); +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") process.exit(1); + fs.appendFileSync(launchesPath, JSON.stringify({ pid: process.pid, version }) + "\\n"); + fs.writeFileSync(runtimeStatePath, JSON.stringify({ + version: 1, + pid: process.pid, + port: address.port, + origin: "http://127.0.0.1:" + address.port, + startedAt: new Date().toISOString(), + }) + "\\n"); +}); +`; + +const runAcceptanceScenario = Effect.fn("test.runSystemdSelfUpdateAcceptance")(function* ( + scenario: "healthy" | "rollback", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runner = yield* ProcessRunner.ProcessRunner; + const home = process.env.HOME; + if (!home) { + return yield* Effect.die(new Error("HOME is required for a systemd user-manager test.")); + } + + const directory = yield* fs.makeTempDirectoryScoped({ prefix: `t3-systemd-${scenario}-` }); + const runtimeStatePath = path.join(directory, "server-runtime.json"); + const launchesPath = path.join(directory, "launches.jsonl"); + const receiptPath = path.join(directory, "receipt.json"); + const planPath = path.join(directory, "plan.json"); + const fixtureServerPath = path.join(directory, "fixture-server.mjs"); + yield* fs.writeFileString(fixtureServerPath, fixtureServerSource); + + const suffix = `${String(process.pid)}-${String(yield* Clock.currentTimeMillis)}-${scenario}`; + const serviceUnit = `t3code-self-update-acceptance-${suffix}.service`; + const helperUnit = `t3code-self-update-acceptance-helper-${suffix}.service`; + const userUnitDirectory = path.join(home, ".config", "systemd", "user"); + const unitPath = path.join(userUnitDirectory, serviceUnit); + yield* fs.makeDirectory(userUnitDirectory, { recursive: true }); + + const renderUnit = (version: string, readiness: "ready" | "blocked") => + [ + "[Unit]", + "Description=T3 self-update acceptance fixture", + "", + "[Service]", + "Type=simple", + `ExecStart=${quoteSystemdValue(process.execPath)} ${quoteSystemdValue(fixtureServerPath)} ${quoteSystemdValue(runtimeStatePath)} ${quoteSystemdValue(launchesPath)} ${version} ${readiness}`, + "Restart=no", + "TimeoutStopSec=5", + "", + ].join("\n"); + const previousUnit = renderUnit(OLD_VERSION, "ready"); + const nextUnit = renderUnit(TARGET_VERSION, scenario === "healthy" ? "ready" : "blocked"); + + const systemctl = (args: ReadonlyArray) => + runner.run({ command: "systemctl", args: ["--user", ...args] }); + const cleanup = Effect.gen(function* () { + yield* systemctl(["stop", helperUnit, serviceUnit]).pipe(Effect.ignore); + yield* systemctl(["reset-failed", helperUnit, serviceUnit]).pipe(Effect.ignore); + yield* fs.remove(unitPath, { force: true }).pipe(Effect.ignore); + yield* systemctl(["daemon-reload"]).pipe(Effect.ignore); + }); + yield* Effect.addFinalizer(() => cleanup); + + yield* fs.writeFileString(unitPath, previousUnit); + const reload = yield* systemctl(["daemon-reload"]); + assert.equal(Number(reload.code), 0); + const start = yield* systemctl(["start", serviceUnit]); + assert.equal(Number(start.code), 0); + yield* waitForSystemdSelfUpdateReadiness( + { + expectedVersion: OLD_VERSION, + previousPid: 0, + runtimeStatePath, + }, + { timeout: Duration.seconds(10), interval: Duration.millis(100) }, + ); + const initialState = yield* readPersistedServerRuntimeState(runtimeStatePath); + if (Option.isNone(initialState)) { + return yield* Effect.die( + new Error("The disposable systemd service did not publish runtime state."), + ); + } + + const plan = new SystemdSelfUpdatePlan({ + version: 1, + fromVersion: OLD_VERSION, + targetVersion: TARGET_VERSION, + currentPid: initialState.value.pid, + serviceUnit, + unitPath, + previousUnit, + nextUnit, + runtimeStatePath, + receiptPath, + }); + yield* writeSystemdSelfUpdatePlan(planPath, plan); + + const repositoryRoot = path.resolve(import.meta.dirname, "../../../.."); + const cliEntryPath = path.join(repositoryRoot, "apps", "server", "src", "bin.ts"); + const handoff = yield* runner.run({ + command: "systemd-run", + args: [ + "--user", + "--wait", + "--collect", + "--service-type=exec", + `--unit=${helperUnit}`, + process.execPath, + "--experimental-strip-types", + cliEntryPath, + "service", + "_apply-update", + "--plan", + planPath, + ], + timeout: Duration.seconds(90), + }); + assert.equal(Number(handoff.code), 0, handoff.stderr); + + const receipt = yield* fs + .readFileString(receiptPath) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdateReceipt))), + ); + assert.equal(receipt.phase, scenario === "healthy" ? "healthy" : "rolled-back"); + assert.equal( + yield* fs.readFileString(unitPath), + scenario === "healthy" ? nextUnit : previousUnit, + ); + + yield* waitForSystemdSelfUpdateReadiness( + { + expectedVersion: scenario === "healthy" ? TARGET_VERSION : OLD_VERSION, + previousPid: initialState.value.pid, + runtimeStatePath, + }, + { timeout: Duration.seconds(10), interval: Duration.millis(100) }, + ); + const launches = (yield* fs.readFileString(launchesPath)) + .trim() + .split("\n") + .map((line) => decodeLaunch(line)); + assert.deepEqual( + launches.map(({ version }) => version), + scenario === "healthy" + ? [OLD_VERSION, TARGET_VERSION] + : [OLD_VERSION, TARGET_VERSION, OLD_VERSION], + ); +}); + +describe.runIf(process.env.T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE === "1")( + "systemd self-update acceptance", + () => { + const layer = ProcessRunner.layer.pipe( + Layer.provideMerge(NodeServices.layer), + Layer.merge(FetchHttpClient.layer), + ); + + it.live( + "keeps the helper alive while replacing the host service", + () => runAcceptanceScenario("healthy").pipe(Effect.provide(layer)), + 30_000, + ); + + it.live( + "rolls back a server that binds HTTP without becoming runtime-ready", + () => runAcceptanceScenario("rollback").pipe(Effect.provide(layer)), + 120_000, + ); + }, +); diff --git a/apps/server/src/cloud/systemdSelfUpdate.test.ts b/apps/server/src/cloud/systemdSelfUpdate.test.ts index 559bcb75725..5c910767583 100644 --- a/apps/server/src/cloud/systemdSelfUpdate.test.ts +++ b/apps/server/src/cloud/systemdSelfUpdate.test.ts @@ -6,14 +6,17 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import { HttpClient, HttpClientResponse } from "effect/unstable/http"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import * as ProcessRunner from "../processRunner.ts"; +import { PersistedServerRuntimeState } from "../serverRuntimeState.ts"; import { SystemdSelfUpdateActivationError, SystemdSelfUpdatePlan, SystemdSelfUpdateReceipt, activateSystemdSelfUpdatePlan, + waitForSystemdSelfUpdateReadiness, } from "./systemdSelfUpdate.ts"; interface RecordedCommand { @@ -59,6 +62,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { fromVersion: "0.0.28", targetVersion: "0.0.29", currentPid: 123, + serviceUnit: "t3code-self-update-test.service", unitPath, previousUnit, nextUnit: "new unit\n", @@ -95,7 +99,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { commands.map(({ args }) => args), [ ["--user", "daemon-reload"], - ["--user", "restart", "t3code.service"], + ["--user", "restart", "t3code-self-update-test.service"], ], ); assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "healthy"); @@ -132,10 +136,10 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { commands.map(({ args }) => args), [ ["--user", "daemon-reload"], - ["--user", "restart", "t3code.service"], + ["--user", "restart", "t3code-self-update-test.service"], ["--user", "daemon-reload"], - ["--user", "reset-failed", "t3code.service"], - ["--user", "restart", "t3code.service"], + ["--user", "reset-failed", "t3code-self-update-test.service"], + ["--user", "restart", "t3code-self-update-test.service"], ], ); const receipt = yield* readReceipt(fixture.receiptPath); @@ -164,4 +168,52 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed"); }), ); + + it.effect("probes the runtime-readiness endpoint on the replacement process", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-systemd-ready-test-" }); + const runtimeStatePath = path.join(directory, "server-runtime.json"); + const encodeRuntimeState = Schema.encodeEffect( + Schema.fromJsonString(PersistedServerRuntimeState), + ); + const encodedRuntimeState = yield* encodeRuntimeState({ + version: 1, + pid: 456, + port: 3210, + origin: "http://127.0.0.1:3210", + startedAt: "2026-01-01T00:00:00.000Z", + }); + yield* fs.writeFileString(runtimeStatePath, `${encodedRuntimeState}\n`); + + const requests: Array = []; + const httpClientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => { + requests.push(request.url); + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ + environmentId: "environment-test", + label: "Test environment", + platform: { os: "linux", arch: "x64" }, + serverVersion: "0.0.29", + capabilities: { repositoryIdentity: true }, + }), + ), + ); + }), + ); + + yield* waitForSystemdSelfUpdateReadiness({ + expectedVersion: "0.0.29", + previousPid: 123, + runtimeStatePath, + }).pipe(Effect.provide(httpClientLayer)); + + assert.deepEqual(requests, ["http://127.0.0.1:3210/.well-known/t3/ready"]); + }), + ); }); diff --git a/apps/server/src/cloud/systemdSelfUpdate.ts b/apps/server/src/cloud/systemdSelfUpdate.ts index fe557436bbc..29a1bfee31d 100644 --- a/apps/server/src/cloud/systemdSelfUpdate.ts +++ b/apps/server/src/cloud/systemdSelfUpdate.ts @@ -12,7 +12,6 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab import { writeFileStringAtomically } from "../atomicWrite.ts"; import * as ProcessRunner from "../processRunner.ts"; import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts"; -import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts"; export const SYSTEMD_SELF_UPDATE_UNIT = "t3code-self-update.service"; export const SYSTEMD_SELF_UPDATE_DIRECTORY = "self-update"; @@ -23,7 +22,7 @@ const DEFAULT_RESTART_DELAY = Duration.seconds(2); const DEFAULT_READINESS_TIMEOUT = Duration.seconds(60); const DEFAULT_READINESS_INTERVAL = Duration.millis(500); const READINESS_REQUEST_TIMEOUT = Duration.seconds(2); -const WELL_KNOWN_ENVIRONMENT_PATH = "/.well-known/t3/environment"; +const WELL_KNOWN_READY_PATH = "/.well-known/t3/ready"; export class SystemdSelfUpdatePlan extends Schema.Class( "SystemdSelfUpdatePlan", @@ -32,6 +31,7 @@ export class SystemdSelfUpdatePlan extends Schema.Class( fromVersion: Schema.String, targetVersion: Schema.String, currentPid: Schema.Int, + serviceUnit: Schema.String, unitPath: Schema.String, previousUnit: Schema.String, nextUnit: Schema.String, @@ -185,7 +185,8 @@ const installUnit = Effect.fn("systemdSelfUpdate.installUnit")(function* ( yield* runSystemctl(["--user", "daemon-reload"], "daemon-reload"); }); -const restartBootService = runSystemctl(["--user", "restart", BOOT_SERVICE_UNIT_FILE], "restart"); +const restartBootService = (serviceUnit: string) => + runSystemctl(["--user", "restart", serviceUnit], "restart"); export interface SystemdSelfUpdateReadinessInput { readonly expectedVersion: string; @@ -227,7 +228,7 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa } const request = HttpClientRequest.get( - new URL(WELL_KNOWN_ENVIRONMENT_PATH, state.value.origin).toString(), + new URL(WELL_KNOWN_READY_PATH, state.value.origin).toString(), ); const descriptor = yield* client.execute(request).pipe( Effect.timeout(READINESS_REQUEST_TIMEOUT), @@ -277,7 +278,7 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa yield* writeReceipt(plan, "activating").pipe(Effect.ignore); yield* Effect.sleep(options.restartDelay ?? DEFAULT_RESTART_DELAY); yield* installUnit(plan, plan.nextUnit); - yield* restartBootService; + yield* restartBootService(plan.serviceUnit); yield* waitForReadiness({ expectedVersion: plan.targetVersion, previousPid: plan.currentPid, @@ -298,8 +299,8 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa ); const recovery = Effect.gen(function* () { yield* installUnit(plan, plan.previousUnit); - yield* runSystemctl(["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], "reset-failed"); - yield* restartBootService; + yield* runSystemctl(["--user", "reset-failed", plan.serviceUnit], "reset-failed"); + yield* restartBootService(plan.serviceUnit); yield* waitForReadiness({ expectedVersion: plan.fromVersion, previousPid: plan.currentPid, diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 5a380be8fe2..d98acc0d5e5 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -40,6 +40,7 @@ import { } from "./auth/http.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; +import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); @@ -177,13 +178,23 @@ export const serverEnvironmentHttpApiLayer = HttpApiBuilder.group( "metadata", Effect.fnUntraced(function* (handlers) { const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; - return handlers.handle( - "descriptor", - Effect.fn("environment.metadata.descriptor")(function* (args) { - yield* annotateEnvironmentRequest(args.endpoint.name); - return yield* serverEnvironment.getDescriptor; - }, traceRelayRequest), - ); + const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup; + return handlers + .handle( + "descriptor", + Effect.fn("environment.metadata.descriptor")(function* (args) { + yield* annotateEnvironmentRequest(args.endpoint.name); + return yield* serverEnvironment.getDescriptor; + }, traceRelayRequest), + ) + .handle( + "ready", + Effect.fn("environment.metadata.ready")(function* (args) { + yield* annotateEnvironmentRequest(args.endpoint.name); + yield* startup.awaitReady.pipe(Effect.orDie); + return yield* serverEnvironment.getDescriptor; + }, traceRelayRequest), + ); }), ); diff --git a/apps/server/src/persistence/Migrations.test.ts b/apps/server/src/persistence/Migrations.test.ts index c65b28f7090..26fe57f1152 100644 --- a/apps/server/src/persistence/Migrations.test.ts +++ b/apps/server/src/persistence/Migrations.test.ts @@ -3,7 +3,12 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as SqlClient from "effect/unstable/sql/SqlClient"; -import { MigrationIdentityMismatchError, validateMigrationIdentities } from "./Migrations.ts"; +import { + AutomaticUpdateMigrationFrontierError, + MigrationIdentityMismatchError, + validateAutomaticUpdateMigrationFrontier, + validateMigrationIdentities, +} from "./Migrations.ts"; import * as NodeSqliteClient from "./NodeSqliteClient.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); @@ -50,4 +55,49 @@ layer("migration identity validation", (it) => { yield* validateMigrationIdentities(); }), ); + + it.effect("rejects automatic downgrade when the database has a newer migration", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* createTrackingTable(); + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES (999, 'FutureMigration') + `; + + const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip); + assert.instanceOf(error, AutomaticUpdateMigrationFrontierError); + assert.equal(error.databaseMigrationId, 999); + assert.equal(error.targetMigrationId, 35); + }), + ); + + it.effect("rejects automatic updates that would run migrations", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* createTrackingTable(); + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES (34, 'ProjectionThreadsSnoozed') + `; + + const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip); + assert.instanceOf(error, AutomaticUpdateMigrationFrontierError); + assert.equal(error.databaseMigrationId, 34); + assert.equal(error.targetMigrationId, 35); + }), + ); + + it.effect("allows automatic updates on the same migration frontier", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* createTrackingTable(); + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES (35, 'ProjectionThreadTitleRegeneration') + `; + + yield* validateAutomaticUpdateMigrationFrontier(); + }), + ); }); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index c9799d0f98f..b66a6996aa5 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -127,33 +127,53 @@ export class MigrationIdentityMismatchError extends Schema.TaggedErrorClass()( + "AutomaticUpdateMigrationFrontierError", + { + databaseMigrationId: Schema.Int, + targetMigrationId: Schema.Int, + }, +) { + override get message(): string { + return this.databaseMigrationId > this.targetMigrationId + ? `This database is newer than the requested t3 build (migration ${String(this.databaseMigrationId)} vs ${String(this.targetMigrationId)}).` + : `The requested t3 build would migrate this database (migration ${String(this.databaseMigrationId)} to ${String(this.targetMigrationId)}), which is not safe during an automatic update.`; + } +} + +const readPersistedMigrationIdentities = Effect.fn("readPersistedMigrationIdentities")( + function* () { + const sql = yield* SqlClient.SqlClient; + const tables = yield* sql<{ readonly name: string }>` + SELECT name + FROM sqlite_master + WHERE type = 'table' AND name = 'effect_sql_migrations' + `; + if (tables.length === 0) { + return []; + } + + return yield* sql<{ + readonly migration_id: number; + readonly name: string; + }>` + SELECT migration_id, name + FROM effect_sql_migrations + ORDER BY migration_id + `; + }, +); + /** * Refuse to run a build whose migration history disagrees with the database. * Unknown newer IDs are intentionally allowed so a compatible older build can * inspect a database created by a newer release without rewriting its history. */ export const validateMigrationIdentities = Effect.fn("validateMigrationIdentities")(function* () { - const sql = yield* SqlClient.SqlClient; - const tables = yield* sql<{ readonly name: string }>` - SELECT name - FROM sqlite_master - WHERE type = 'table' AND name = 'effect_sql_migrations' - `; - if (tables.length === 0) { - return; - } - const expectedNames = new Map( migrationEntries.map(([migrationId, name]) => [migrationId, name]), ); - const persisted = yield* sql<{ - readonly migration_id: number; - readonly name: string; - }>` - SELECT migration_id, name - FROM effect_sql_migrations - ORDER BY migration_id - `; + const persisted = yield* readPersistedMigrationIdentities(); for (const migration of persisted) { const expectedName = expectedNames.get(migration.migration_id); @@ -167,6 +187,26 @@ export const validateMigrationIdentities = Effect.fn("validateMigrationIdentitie } }); +/** + * Automatic activation is safe only when the target and database are already + * on the same schema. A binary rollback cannot undo a migration, and an older + * target cannot safely interpret migration IDs that it does not know about. + */ +export const validateAutomaticUpdateMigrationFrontier = Effect.fn( + "validateAutomaticUpdateMigrationFrontier", +)(function* () { + yield* validateMigrationIdentities(); + const persisted = yield* readPersistedMigrationIdentities(); + const databaseMigrationId = persisted.at(-1)?.migration_id ?? 0; + const targetMigrationId = migrationEntries.at(-1)?.[0] ?? 0; + if (databaseMigrationId !== targetMigrationId) { + return yield* new AutomaticUpdateMigrationFrontierError({ + databaseMigrationId, + targetMigrationId, + }); + } +}); + export interface RunMigrationsOptions { readonly toMigrationInclusive?: number | undefined; } diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 569e8a51c37..1070a13261d 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -782,6 +782,7 @@ const buildAppUnderTest = (options?: { Layer.provide( Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({ awaitCommandReady: Effect.void, + awaitReady: Effect.void, markHttpListening: Effect.void, enqueueCommand: (effect) => effect, ...options?.layers?.serverRuntimeStartup, @@ -1361,6 +1362,30 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("does not report update readiness before the runtime publishes ready", () => + Effect.gen(function* () { + const runtimeReady = yield* Deferred.make(); + yield* buildAppUnderTest({ + layers: { + serverRuntimeStartup: { + awaitReady: Deferred.await(runtimeReady), + }, + }, + }); + + const url = yield* getHttpServerUrl("/.well-known/t3/ready"); + const responseFiber = yield* fetchEffect(url).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + assert.isUndefined(responseFiber.pollUnsafe()); + + yield* Deferred.succeed(runtimeReady, undefined); + const response = yield* Fiber.join(responseFiber); + const body = yield* responseJsonEffect(response); + assert.equal(response.status, 200); + assert.deepEqual(body, testEnvironmentDescriptor); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("compresses large JSON responses through the composed routes", () => Effect.gen(function* () { const descriptor = { diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index b52b577c5b5..06feba57b2a 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -59,6 +59,7 @@ export class ServerRuntimeStartup extends Context.Service< ServerRuntimeStartup, { readonly awaitCommandReady: Effect.Effect; + readonly awaitReady: Effect.Effect; readonly markHttpListening: Effect.Effect; readonly enqueueCommand: ( effect: Effect.Effect, @@ -300,6 +301,7 @@ export const make = Effect.gen(function* () { const commandGate = yield* makeCommandGate; const httpListening = yield* Deferred.make(); + const runtimeReady = yield* Deferred.make(); const reactorScope = yield* Scope.make("sequential"); yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void)); @@ -425,6 +427,7 @@ export const make = Effect.gen(function* () { }); yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause }); yield* commandGate.failCommandReady(error); + yield* Deferred.fail(runtimeReady, error).pipe(Effect.orDie); return; } @@ -444,6 +447,7 @@ export const make = Effect.gen(function* () { }, }), ); + yield* Deferred.succeed(runtimeReady, undefined).pipe(Effect.orDie); yield* Effect.logDebug("startup phase: recording startup heartbeat"); yield* launchStartupHeartbeat; @@ -470,6 +474,7 @@ export const make = Effect.gen(function* () { return { awaitCommandReady: commandGate.awaitCommandReady, + awaitReady: Deferred.await(runtimeReady), markHttpListening: Deferred.succeed(httpListening, undefined), enqueueCommand: commandGate.enqueueCommand, } satisfies ServerRuntimeStartup["Service"]; diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 9e2317ea28d..54b856b7d03 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -90,9 +90,12 @@ successfully. Boot-service setup and self-update share the same process-wide ins they cannot mutate a pinned runtime concurrently. Before any restart, the current Node executable runs the replacement with `--version`, then asks -the target artifact to validate its migration identities against the live database in read-only -mode. A failed install, version preflight, or compatibility check leaves the current server running. -Targets old enough to lack the compatibility command are not eligible for remote activation. +the target artifact to validate the live database in read-only mode. Automatic activation requires +the target and database to be on the exact same migration frontier. This rejects both downgrades and +updates that would migrate the database because a binary rollback cannot undo a schema change. Use +the manual host service update path for those releases. A failed install, version preflight, or +compatibility check leaves the current server running. Targets old enough to lack the compatibility +command are not eligible for remote activation. Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when the preflight process actually completes and reports a bad exit code or the wrong version: that is @@ -118,7 +121,9 @@ For `boot-service`, the request process writes a typed plan but leaves its live starts a fixed-name `t3code-self-update.service` transient unit with `systemd-run --user`; the transient unit has a separate cgroup and acts as the cross-process update lock. After a short grace period it atomically installs the candidate unit, reloads systemd, performs a blocking restart, and -probes the public environment descriptor for the exact target version. If activation or readiness +probes the public runtime-readiness endpoint for the exact target version. The endpoint does not +respond until the server's command runtime is usable and its lifecycle `ready` event has been +published, so an HTTP listener alone cannot make an activation healthy. If activation or readiness fails, the helper restores the previous unit, clears systemd's failed state, restarts it, and proves the previous version is ready. The plan and latest receipt live under the environment state directory's `self-update` folder for diagnosis. diff --git a/docs/operations/release.md b/docs/operations/release.md index 9b33e94cc60..a18285d382f 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -190,6 +190,23 @@ connect the new client to a server on the previous version and verify that the u reconnects to the matching server. Test one automatic path and the manual or desktop-managed guidance when those environments are available. +The automatic boot-service path also requires a real systemd user-manager acceptance run. Use a +disposable T3 home and uniquely named host and helper units, never the developer's daily +`t3code.service`. Exercise both outcomes: + +1. A same-migration-frontier candidate becomes runtime-ready, the helper survives the host-unit + restart, and the receipt reaches `healthy` with a new runtime PID and the exact target version. +2. A candidate that binds HTTP but never becomes runtime-ready times out, the previous unit is + restored, and the receipt reaches `rolled-back` with the old version command-ready again. + +Treat the systemd run as a release blocker when the update lifecycle changes. CI jobs without a +user manager may skip it, but a Linux release host must record the run before publishing. + +```bash +T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \ + apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts +``` + ## Desktop auto-update notes - Updater runtime: `apps/desktop/src/updates/DesktopUpdates.ts`. diff --git a/packages/contracts/src/environmentHttp.ts b/packages/contracts/src/environmentHttp.ts index 2d40dad60cc..2e014e42337 100644 --- a/packages/contracts/src/environmentHttp.ts +++ b/packages/contracts/src/environmentHttp.ts @@ -371,11 +371,17 @@ export const AuthOtherClientSessionsRevokeResult = Schema.Struct({ }); export type AuthOtherClientSessionsRevokeResult = typeof AuthOtherClientSessionsRevokeResult.Type; -export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata").add( - HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", { - success: ExecutionEnvironmentDescriptor, - }), -) {} +export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata") + .add( + HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", { + success: ExecutionEnvironmentDescriptor, + }), + ) + .add( + HttpApiEndpoint.get("ready", "/.well-known/t3/ready", { + success: ExecutionEnvironmentDescriptor, + }), + ) {} export class EnvironmentAuthHttpApi extends HttpApiGroup.make("auth") .add( From da8ebd3fcd65beaf0ecb64d5d6bf8189ae6b8a71 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Sat, 1 Aug 2026 00:11:26 -0700 Subject: [PATCH 5/5] test(web): handle update readiness metadata --- apps/web/test/environmentHttpTest.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/web/test/environmentHttpTest.ts b/apps/web/test/environmentHttpTest.ts index ce43faacc40..27c3df55e8e 100644 --- a/apps/web/test/environmentHttpTest.ts +++ b/apps/web/test/environmentHttpTest.ts @@ -73,13 +73,15 @@ export async function installEnvironmentHttpTest(scenario: EnvironmentHttpTestSc Effect.provide([ NodeHttpServer.layerHttpServices, HttpApiBuilder.group(EnvironmentHttpApi, "metadata", (handlers) => - handlers.handle( - "descriptor", - Effect.fn("test.environment.metadata.descriptor")(function* () { - calls.descriptor += 1; - return yield* scenario.descriptor?.() ?? unexpectedEndpoint("metadata.descriptor"); - }), - ), + handlers + .handle( + "descriptor", + Effect.fn("test.environment.metadata.descriptor")(function* () { + calls.descriptor += 1; + return yield* scenario.descriptor?.() ?? unexpectedEndpoint("metadata.descriptor"); + }), + ) + .handle("ready", () => unexpectedEndpoint("metadata.ready")), ), HttpApiBuilder.group(EnvironmentHttpApi, "auth", (handlers) => handlers