(handle: (params: P) => Effect.Effect) =>
+export const actsAsCaller = (
+ handle: (params: P, caller: Caller) => Effect.Effect,
+) =>
declare(
(params: P) =>
requireThreadCaller.pipe(
Effect.andThen(writingCaller),
- Effect.flatMap(() => handle(params)),
+ Effect.flatMap((caller) => handle(params, caller)),
),
undefined,
);
diff --git a/apps/server/src/mcp/toolkits/pluginTools/handlers.test.ts b/apps/server/src/mcp/toolkits/pluginTools/handlers.test.ts
new file mode 100644
index 0000000000000..42d5b60e9cf92
--- /dev/null
+++ b/apps/server/src/mcp/toolkits/pluginTools/handlers.test.ts
@@ -0,0 +1,187 @@
+import { expect, it } from "@effect/vitest";
+import {
+ EnvironmentId,
+ PluginInstallationId,
+ PluginToolError,
+ ProviderInstanceId,
+ ThreadId,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import { McpSchema, McpServer } from "effect/ai";
+
+import * as PluginTools from "../../../plugins/PluginTools.ts";
+import * as McpHttpServer from "../../McpHttpServer.ts";
+import * as McpInvocationContext from "../../McpInvocationContext.ts";
+import { liveThreadShell } from "../../McpToolAccess.testkit.ts";
+import * as ThreadManagement from "../../../orchestration-v2/ThreadManagementService.ts";
+
+const grants = [{ installationId: PluginInstallationId.make("installation-1"), generation: 3 }];
+const invocation = {
+ environmentId: EnvironmentId.make("environment-plugin-tools"),
+ capabilities: new Set(),
+ issuedAt: 1,
+ requestNamespace: "thread:thread-plugin-tools",
+ thread: {
+ threadId: ThreadId.make("thread-plugin-tools"),
+ providerSessionId: "provider-session-plugin-tools",
+ providerInstanceId: ProviderInstanceId.make("codex"),
+ },
+ client: undefined,
+ pluginToolGrants: grants,
+} satisfies McpInvocationContext.McpInvocationScope;
+// An MCP OAuth client signed in from outside T3 Code: no thread and no grants.
+const outsideClient = {
+ environmentId: invocation.environmentId,
+ capabilities: new Set(),
+ issuedAt: 1,
+ requestNamespace: "client:session-outside",
+ thread: undefined,
+ client: { sessionId: "session-outside", label: "outside", access: "full-access" },
+} satisfies McpInvocationContext.McpInvocationScope;
+const client = McpSchema.McpServerClient.of({
+ clientId: 1,
+ clientCapabilities: {},
+ clientInfo: { name: "mcp-test", version: "1.0.0" },
+ protocolVersion: "2025-06-18",
+ initializePayload: {
+ protocolVersion: "2025-06-18",
+ capabilities: {},
+ clientInfo: { name: "mcp-test", version: "1.0.0" },
+ },
+ getClient: Effect.die("unused"),
+});
+
+it.effect(
+ "takes grants and thread from the credential and marks the call tool conservatively",
+ () => {
+ const seen: Array = [];
+ let shell = liveThreadShell(invocation.thread.threadId);
+ const threads = Layer.mock(ThreadManagement.ThreadManagementService)({
+ getThreadShell: () => Effect.sync(() => shell),
+ });
+ const tools = Layer.mock(PluginTools.PluginTools)({
+ list: (granted, options) =>
+ Effect.sync(() => {
+ seen.push({ granted, options });
+ return { tools: [], notInThisSession: [] };
+ }),
+ call: (granted, request) =>
+ Effect.suspend(() => {
+ seen.push({ granted, request });
+ return request.tool === "acme.search/lookup"
+ ? Effect.succeed({ hits: 1 })
+ : Effect.fail(
+ new PluginToolError({ reason: "unknown-tool", message: "No such tool." }),
+ );
+ }),
+ });
+ return Effect.gen(function* () {
+ const server = yield* McpServer.McpServer;
+ const registered = Object.fromEntries(server.tools.map(({ tool }) => [tool.name, tool]));
+ expect(registered.plugin_tools_list?.annotations).toMatchObject({
+ readOnlyHint: true,
+ destructiveHint: false,
+ });
+ expect(registered.plugin_tool_call?.annotations).toMatchObject({
+ readOnlyHint: false,
+ destructiveHint: true,
+ openWorldHint: true,
+ });
+ const callTool = (
+ name: string,
+ args: Record,
+ scope: McpInvocationContext.McpInvocationScope = invocation,
+ ) =>
+ server
+ .callTool({ name, arguments: args })
+ .pipe(
+ Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
+ Effect.provideService(McpSchema.McpServerClient, client),
+ );
+
+ const listed = yield* callTool("plugin_tools_list", { plugin: "acme.search", cursor: "a" });
+ expect(listed.isError).toBe(false);
+ // A context the agent passes is only input; the plugin gets the credential's.
+ const called = yield* callTool("plugin_tool_call", {
+ tool: "acme.search/lookup",
+ input: { q: "x", context: { threadId: "spoofed" } },
+ });
+ expect(called).toMatchObject({ isError: false, structuredContent: { result: { hits: 1 } } });
+ const failed = yield* callTool("plugin_tool_call", { tool: "acme.search/nope" });
+ expect(failed.isError).toBe(true);
+ expect(failed.content).toEqual([{ type: "text", text: "No such tool." }]);
+
+ expect(seen).toEqual([
+ { granted: grants, options: { plugin: "acme.search", cursor: "a" } },
+ {
+ granted: grants,
+ request: {
+ tool: "acme.search/lookup",
+ input: { q: "x", context: { threadId: "spoofed" } },
+ context: {
+ environmentId: invocation.environmentId,
+ threadId: invocation.thread.threadId,
+ },
+ },
+ },
+ {
+ granted: grants,
+ request: {
+ tool: "acme.search/nope",
+ input: {},
+ context: {
+ environmentId: invocation.environmentId,
+ threadId: invocation.thread.threadId,
+ },
+ },
+ },
+ ]);
+
+ // Outside clients see no plugin tools and cannot call one; the plugin is never reached.
+ seen.length = 0;
+ const outsideList = yield* callTool("plugin_tools_list", {}, outsideClient);
+ expect(outsideList.isError).toBe(false);
+ const outsideCall = yield* callTool(
+ "plugin_tool_call",
+ { tool: "acme.search/lookup" },
+ outsideClient,
+ );
+ expect(outsideCall.isError).toBe(true);
+ expect(outsideCall.content).toEqual([
+ {
+ type: "text",
+ text: "This tool acts as the calling T3 thread, so it needs an agent running inside T3 Code. This MCP client signed in from outside a thread.",
+ },
+ ]);
+ expect(outsideList.structuredContent).toEqual({ tools: [], notInThisSession: [] });
+ expect(seen).toEqual([]);
+ for (const modes of [
+ { runtimeMode: "approval-required", interactionMode: "default" },
+ { runtimeMode: "auto-accept-edits", interactionMode: "default" },
+ { runtimeMode: "auto", interactionMode: "default" },
+ { runtimeMode: "full-access", interactionMode: "plan" },
+ ] as const) {
+ shell = liveThreadShell(invocation.thread.threadId, modes);
+ const refused = yield* callTool("plugin_tool_call", { tool: "acme.search/lookup" });
+ expect(refused.isError).toBe(true);
+ expect(refused.content).toEqual([
+ {
+ type: "text",
+ text: "Calling a trusted plugin needs a full-access thread in default mode.",
+ },
+ ]);
+ expect(seen).toEqual([]);
+ }
+ }).pipe(
+ Effect.scoped,
+ Effect.provide(
+ McpHttpServer.layerPluginToolsToolkit.pipe(
+ Layer.provideMerge(McpServer.McpServer.layer),
+ Layer.provide(tools),
+ Layer.provide(threads),
+ ),
+ ),
+ );
+ },
+);
diff --git a/apps/server/src/mcp/toolkits/pluginTools/handlers.ts b/apps/server/src/mcp/toolkits/pluginTools/handlers.ts
new file mode 100644
index 0000000000000..c512ef8802770
--- /dev/null
+++ b/apps/server/src/mcp/toolkits/pluginTools/handlers.ts
@@ -0,0 +1,50 @@
+import * as Effect from "effect/Effect";
+
+import * as PluginTools from "../../../plugins/PluginTools.ts";
+import * as McpInvocationContext from "../../McpInvocationContext.ts";
+import * as McpToolAccess from "../../McpToolAccess.ts";
+import { PluginToolsToolkit } from "./tools.ts";
+import { assertFullAccess } from "../../threadAccess.ts";
+
+const make = Effect.gen(function* () {
+ const tools = yield* PluginTools.PluginTools;
+ return {
+ // Scope and grants come from the session's credential, never from the agent.
+ // A caller without a thread can call no plugin tool, so it learns of no plugins either.
+ plugin_tools_list: McpToolAccess.reads((input) =>
+ McpInvocationContext.McpInvocationContext.pipe(
+ Effect.flatMap((scope) =>
+ scope.thread === undefined
+ ? Effect.succeed({ tools: [], notInThisSession: [] })
+ : tools.list(scope.pluginToolGrants ?? [], {
+ ...(input.plugin === undefined ? {} : { plugin: input.plugin }),
+ ...(input.cursor === undefined ? {} : { cursor: input.cursor }),
+ }),
+ ),
+ ),
+ ),
+ // A plugin tool runs on behalf of the calling thread while its run is live, so a client
+ // signed in from outside T3 Code (no thread, no grants) cannot call one.
+ plugin_tool_call: McpToolAccess.actsAsCaller((input, caller) =>
+ assertFullAccess(
+ caller,
+ "Calling a trusted plugin needs a full-access thread in default mode.",
+ ).pipe(
+ Effect.andThen(McpInvocationContext.McpInvocationContext),
+ Effect.flatMap((scope) =>
+ McpInvocationContext.requireThreadScope(scope, "plugin_tool_call"),
+ ),
+ Effect.flatMap((scope) =>
+ tools.call(scope.pluginToolGrants ?? [], {
+ tool: input.tool,
+ input: input.input ?? {},
+ context: { environmentId: scope.environmentId, threadId: scope.thread.threadId },
+ }),
+ ),
+ Effect.map((result) => ({ result })),
+ ),
+ ),
+ } satisfies McpToolAccess.Handlers;
+});
+
+export const layer = McpToolAccess.toLayer(PluginToolsToolkit, make);
diff --git a/apps/server/src/mcp/toolkits/pluginTools/tools.ts b/apps/server/src/mcp/toolkits/pluginTools/tools.ts
new file mode 100644
index 0000000000000..8cfbdc395c09c
--- /dev/null
+++ b/apps/server/src/mcp/toolkits/pluginTools/tools.ts
@@ -0,0 +1,64 @@
+import { OrchestratorMcpFailure, PluginToolError, PluginToolsListResult } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+import * as Tool from "effect/ai/Tool";
+import * as Toolkit from "effect/ai/Toolkit";
+
+import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts";
+import * as McpInvocationContext from "../../McpInvocationContext.ts";
+
+const dependencies = [McpInvocationContext.McpInvocationContext];
+
+const PluginToolsListTool = Tool.make("plugin_tools_list", {
+ description:
+ "List the tools offered by the user's trusted local T3 Code plugins that this session may use. Each entry has the tool name to pass to plugin_tool_call, its inputSchema, and its sideEffect (read, write or destructive). Listing starts no plugin. Results come in pages ordered by plugin id: when nextCursor is present, pass it as cursor for more. Pass plugin to list one plugin's tools. Plugins enabled after this session started appear under notInThisSession and need a new session.",
+ parameters: Schema.Struct({
+ plugin: Schema.optionalKey(
+ Schema.String.check(Schema.isMaxLength(128)).annotate({
+ description: "A plugin id, such as acme.search, to list only that plugin's tools.",
+ }),
+ ),
+ cursor: Schema.optionalKey(
+ Schema.String.check(Schema.isMaxLength(128)).annotate({
+ description: "The nextCursor of the previous page.",
+ }),
+ ),
+ }),
+ success: PluginToolsListResult,
+ failure: PluginToolError,
+ dependencies,
+})
+ .annotate(Tool.Title, "List plugin tools")
+ .annotate(Tool.Readonly, true)
+ .annotate(Tool.Destructive, false)
+ .annotate(Tool.Idempotent, true)
+ .annotate(Tool.OpenWorld, false);
+
+/**
+ * One fixed tool calls every plugin tool, so its hints describe the most a
+ * plugin tool may do. Each tool's own sideEffect is in plugin_tools_list.
+ */
+const PluginToolCallTool = Tool.make("plugin_tool_call", {
+ description:
+ "Call a tool from plugin_tools_list. Pass its tool name and an input object that matches its inputSchema. Check the tool's sideEffect first: write and destructive tools change things, so follow the user's instructions about such changes.",
+ parameters: Schema.Struct({
+ tool: Schema.String.check(Schema.isMaxLength(200)).annotate({
+ description: "The tool value from plugin_tools_list, such as acme.search/lookup.",
+ }),
+ input: Schema.optionalKey(
+ Schema.Record(Schema.String, Schema.Unknown).annotate({
+ description: "Arguments matching the tool's inputSchema. Defaults to {}.",
+ }),
+ ),
+ }),
+ success: Schema.Struct({ result: Schema.Unknown }),
+ // The caller check refuses an outside client or a thread whose run has ended.
+ failure: Schema.Union([PluginToolError, OrchestratorMcpFailure]),
+ dependencies: [...dependencies, ThreadManagementService.ThreadManagementService],
+})
+ .annotate(Tool.Title, "Call plugin tool")
+ .annotate(Tool.Readonly, false)
+ .annotate(Tool.Destructive, true)
+ .annotate(Tool.Idempotent, false)
+ .annotate(Tool.OpenWorld, true);
+
+export const PluginToolsToolkit = Toolkit.make(PluginToolsListTool, PluginToolCallTool);
diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts
index 1e3388e4136ad..223078de04242 100644
--- a/apps/server/src/observability/RpcInstrumentation.ts
+++ b/apps/server/src/observability/RpcInstrumentation.ts
@@ -92,6 +92,28 @@ const RPC_AGGREGATES = {
[WS_METHODS.scheduledTasksSetEnabled]: "scheduledTasks",
[WS_METHODS.scheduledTasksDelete]: "scheduledTasks",
[WS_METHODS.scheduledTasksRunNow]: "scheduledTasks",
+ [WS_METHODS.pluginsList]: "plugins",
+ [WS_METHODS.pluginsSubscribe]: "plugins",
+ [WS_METHODS.pluginsAdd]: "plugins",
+ [WS_METHODS.pluginsRefresh]: "plugins",
+ [WS_METHODS.pluginsConsent]: "plugins",
+ [WS_METHODS.pluginsEnable]: "plugins",
+ [WS_METHODS.pluginsDisable]: "plugins",
+ [WS_METHODS.pluginsRemove]: "plugins",
+ [WS_METHODS.pluginsResume]: "plugins",
+ [WS_METHODS.pluginsSettingsSubscribe]: "plugins",
+ [WS_METHODS.pluginsSettingsUpdate]: "plugins",
+ [WS_METHODS.pluginActionsSubscribe]: "plugins",
+ [WS_METHODS.pluginActionsInvoke]: "plugins",
+ [WS_METHODS.pluginsNpmList]: "plugins",
+ [WS_METHODS.pluginsNpmAdd]: "plugins",
+ [WS_METHODS.pluginsNpmStageUpdate]: "plugins",
+ [WS_METHODS.pluginsNpmApplyUpdate]: "plugins",
+ [WS_METHODS.pluginsNpmDiscardUpdate]: "plugins",
+ [WS_METHODS.pluginsNotificationsSubscribe]: "plugins",
+ [WS_METHODS.pluginViewsSubscribe]: "pluginViews",
+ [WS_METHODS.pluginViewsReadBundle]: "pluginViews",
+ [WS_METHODS.pluginViewsCall]: "pluginViews",
[WS_METHODS.scheduledTasksRotateWebhookToken]: "scheduledTasks",
[WS_METHODS.scheduledTasksListWebhookDeliveries]: "scheduledTasks",
[WS_METHODS.scheduledTasksGetWebhookDelivery]: "scheduledTasks",
@@ -172,6 +194,7 @@ const RPC_AGGREGATES = {
[WS_METHODS.subscribeWorktreeSetup]: "vcs",
[WS_METHODS.worktreeSetupCancel]: "vcs",
[WS_METHODS.subscribeResourceTelemetry]: "server",
+ [WS_METHODS.subscribeContributionStatus]: "server",
[WS_METHODS.vcsRefreshStatus]: "vcs",
[WS_METHODS.vcsPull]: "git",
[WS_METHODS.gitRunStackedAction]: "vcs",
diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts
index 92e1e01ad43f2..f353c7ccf8730 100644
--- a/apps/server/src/orchestration-v2/EffectOutbox.ts
+++ b/apps/server/src/orchestration-v2/EffectOutbox.ts
@@ -207,6 +207,10 @@ export interface EffectOutboxV2Shape {
readonly listByCommandId: (
commandId: CommandId,
) => Effect.Effect, EffectOutboxError>;
+ /**
+ * Cancelling a checkpoint capture abandons its run's finalization. Do that
+ * through `EventSink.commitCommand`, which records the failure.
+ */
readonly cancelUnsettled: (input: {
readonly threadId: ThreadId;
readonly effectTypes: ReadonlyArray;
diff --git a/apps/server/src/orchestration-v2/EffectWorker.test.ts b/apps/server/src/orchestration-v2/EffectWorker.test.ts
index 2329f2aa20cf0..7ed64568f0e06 100644
--- a/apps/server/src/orchestration-v2/EffectWorker.test.ts
+++ b/apps/server/src/orchestration-v2/EffectWorker.test.ts
@@ -135,7 +135,10 @@ function layerExecutorFor(input: {
),
Layer.succeed(
RunFinalizationService.RunFinalizationService,
- RunFinalizationService.RunFinalizationService.of({ finalize: () => Effect.void }),
+ RunFinalizationService.RunFinalizationService.of({
+ finalize: () => Effect.void,
+ abandon: () => Effect.succeed(true),
+ }),
),
Layer.succeed(
CheckpointRollbackService.CheckpointRollbackServiceV2,
diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts
index 534043aeb4eb8..2cb554d15e04c 100644
--- a/apps/server/src/orchestration-v2/EffectWorker.ts
+++ b/apps/server/src/orchestration-v2/EffectWorker.ts
@@ -61,6 +61,18 @@ export interface OrchestrationEffectExecutorV2Shape {
effect: EffectOutbox.OrchestrationEffectV2,
options?: { readonly willRetry: boolean },
) => Effect.Effect;
+ /**
+ * Settles an effect whose last attempt failed, in place of `outbox.fail`, so
+ * its failure record commits with the terminal status. Returns undefined
+ * for effects that keep no record, and false when the worker no longer
+ * owns the lease.
+ */
+ readonly fail?: (input: {
+ readonly effect: EffectOutbox.OrchestrationEffectV2;
+ readonly workerId: string;
+ readonly error: string;
+ readonly cause: Cause.Cause;
+ }) => Effect.Effect | undefined;
}
export class OrchestrationEffectExecutorV2 extends Context.Service<
@@ -573,6 +585,36 @@ export const layerExecutor: Layer.Layer<
);
}
}),
+ fail: ({ effect, workerId, error, cause }) => {
+ if (effect.request.type !== "checkpoint.capture") return undefined;
+ const failure = Cause.findErrorOption(cause).pipe(
+ Option.map((executionError) => executionError.cause),
+ Option.filter(RunFinalizationService.isRunFinalizationError),
+ );
+ return runFinalization
+ .abandon({
+ threadId: effect.threadId,
+ runId: effect.request.runId,
+ scopeId: effect.request.scopeId,
+ operation: Option.match(failure, {
+ onNone: () => "capture-checkpoint" as const,
+ onSome: (finalizationError) => finalizationError.operation,
+ }),
+ effectId: effect.id,
+ workerId,
+ error,
+ })
+ .pipe(
+ Effect.mapError(
+ (abandonCause) =>
+ new OrchestrationEffectExecutionError({
+ effectId: effect.id,
+ effectType: effect.request.type,
+ cause: abandonCause,
+ }),
+ ),
+ );
+ },
});
}),
);
@@ -655,9 +697,12 @@ export const layerWithOptions = (
}),
),
);
+ const retryDelayMs = (effect: EffectOutbox.OrchestrationEffectV2) =>
+ Math.min(30_000, 100 * 2 ** Math.max(0, effect.attemptCount - 1));
const requeueClaim = (
effect: EffectOutbox.OrchestrationEffectV2,
cause: Cause.Cause,
+ delayMs = 0,
) =>
Cause.hasInterruptsOnly(cause)
? Effect.void
@@ -666,7 +711,7 @@ export const layerWithOptions = (
effectId: effect.id,
workerId,
error: `Worker failed before settling the claimed effect: ${Cause.pretty(cause)}`,
- delayMs: 0,
+ delayMs,
})
.pipe(
Effect.flatMap((requeued) =>
@@ -814,22 +859,32 @@ export const layerWithOptions = (
nonRetryable,
error,
});
+ const retry = outbox
+ .retry({
+ effectId: effect.id,
+ workerId,
+ error,
+ delayMs: retryDelayMs(effect),
+ })
+ .pipe(Effect.onError((cause) => requeueClaim(effect, cause)));
+ const recordedFailure = executor.fail?.({ effect, workerId, error, cause: exit.cause });
+ // Terminal interrupt races succeed; recorded failures remain recoverable until their write commits.
+ // Backoff avoids repeating a failed recording immediately; interruption replays the effect.
const updated = nonRetryable
? yield* outbox
.succeed({ effectId: effect.id, workerId })
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
- : effect.attemptCount >= maxAttempts
- ? yield* outbox
- .fail({ effectId: effect.id, workerId, error })
- .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
- : yield* outbox
- .retry({
- effectId: effect.id,
- workerId,
- error,
- delayMs: Math.min(30_000, 100 * 2 ** Math.max(0, effect.attemptCount - 1)),
- })
- .pipe(Effect.onError((cause) => requeueClaim(effect, cause)));
+ : effect.attemptCount < maxAttempts
+ ? yield* retry
+ : recordedFailure === undefined
+ ? yield* outbox
+ .fail({ effectId: effect.id, workerId, error })
+ .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
+ : Cause.hasInterruptsOnly(exit.cause)
+ ? yield* retry
+ : yield* recordedFailure.pipe(
+ Effect.onError((cause) => requeueClaim(effect, cause, retryDelayMs(effect))),
+ );
if (!updated) {
if (yield* wasCancelled(effect.id)) return true;
return yield* new OrchestrationEffectWorkerError({
diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts
index b210d3c9840ae..603c79f8620bd 100644
--- a/apps/server/src/orchestration-v2/EventSink.ts
+++ b/apps/server/src/orchestration-v2/EventSink.ts
@@ -33,6 +33,7 @@ import * as EffectOutbox from "./EffectOutbox.ts";
import * as EventStore from "./EventStore.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProjectStore from "./ProjectStore.ts";
+import * as RunFinalized from "./RunFinalized.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
export class EventSinkWriteError extends Schema.TaggedError()(
@@ -157,6 +158,25 @@ export interface EventSinkV2Shape {
},
EventSinkV2Error
>;
+ /**
+ * Fails a claimed effect and records `events` in one transaction. Commits
+ * nothing when `workerId` no longer holds the effect's lease.
+ */
+ readonly failEffect: (input: {
+ readonly commandId: CommandId;
+ readonly effectId: string;
+ readonly workerId: string;
+ readonly error: string;
+ readonly events: ReadonlyArray;
+ }) => Effect.Effect<
+ {
+ readonly committed: boolean;
+ readonly storedEvents: ReadonlyArray;
+ },
+ EventSinkV2Error
+ >;
+ /** Whether the run recorded `run.finalized` or `run.finalization-failed`. */
+ readonly hasRunFinalization: (runId: RunId) => Effect.Effect;
readonly commitRejectedCommand: (input: {
readonly commandId: CommandId;
readonly threadId: ThreadId;
@@ -329,7 +349,105 @@ const layerBase: Layer.Layer<
});
});
- const normalizeEvents = (events: ReadonlyArray) => {
+ const isRunFinalizationRecorded = (runId: RunId) =>
+ sql<{ readonly found: number }>`
+ SELECT 1 AS found
+ FROM orchestration_events
+ WHERE event_id = ${RunFinalized.runFinalizedEventId(runId)}
+ LIMIT 1
+ `.pipe(Effect.map((rows) => rows.length > 0));
+
+ // Capture runs finalize through RunFinalizationService; uncaptured or abandoned runs finalize with their terminal write.
+ // Stored state prevents duplicate milestones; the enclosing EventSink operation owns tracing.
+ const withRunFinalizedEvents = Effect.fnUntraced(function* (
+ events: ReadonlyArray,
+ effects: ReadonlyArray,
+ ) {
+ const isFinalizationRecord = (event: OrchestrationV2DomainEvent) =>
+ event.type === "run.finalized" || event.type === "run.finalization-failed";
+ if (!events.some((event) => event.type === "run.updated" || isFinalizationRecord(event))) {
+ return events;
+ }
+ const captureStatus = (runId: RunId) =>
+ effects.some(
+ (effect) =>
+ effect.request.type === "checkpoint.capture" && effect.request.runId === runId,
+ )
+ ? Effect.succeedSome("pending")
+ : effectOutbox
+ .get(RunFinalized.checkpointCaptureEffectId(runId))
+ .pipe(Effect.map(Option.map((effect) => effect.status)));
+ const statuses = new Map();
+ const previousStatus = (runId: RunId) =>
+ statuses.has(runId)
+ ? Effect.succeed(statuses.get(runId))
+ : sql<{ readonly status: string }>`
+ SELECT status
+ FROM orchestration_v2_projection_runs
+ WHERE run_id = ${runId}
+ LIMIT 1
+ `.pipe(Effect.map((rows) => rows[0]?.status));
+ const finalized = new Set();
+ const result: Array = [];
+ // Appended last so the milestone follows every write in its commit.
+ const milestones: Array = [];
+ for (const event of events) {
+ if (event.type === "run.finalized" || event.type === "run.finalization-failed") {
+ const runId = event.payload.runId;
+ if (!finalized.has(runId) && !(yield* isRunFinalizationRecorded(runId))) {
+ finalized.add(runId);
+ result.push(event);
+ }
+ continue;
+ }
+ result.push(event);
+ if (event.type === "run.created") {
+ statuses.set(event.payload.id, event.payload.status);
+ continue;
+ }
+ if (event.type !== "run.updated") continue;
+ const run = event.payload;
+ const previous = yield* previousStatus(run.id);
+ statuses.set(run.id, run.status);
+ const outcome = RunFinalized.runFinalizedOutcome(run.status);
+ // Only the transition into a final status finalizes, so later
+ // updates to old runs never produce a late milestone.
+ if (
+ outcome === null ||
+ finalized.has(run.id) ||
+ (previous !== undefined && RunFinalized.isSettledRunStatus(previous)) ||
+ (yield* isRunFinalizationRecorded(run.id))
+ ) {
+ continue;
+ }
+ const capture = yield* captureStatus(run.id);
+ const abandoned =
+ Option.isSome(capture) && (capture.value === "failed" || capture.value === "cancelled");
+ if (Option.isSome(capture) && !abandoned) continue;
+ finalized.add(run.id);
+ milestones.push(
+ abandoned
+ ? RunFinalized.makeRunFinalizationFailedEvent({
+ run,
+ operation: RunFinalized.abandonedOperation(run),
+ occurredAt: event.occurredAt,
+ })
+ : RunFinalized.makeRunFinalizedEvent({ run, outcome, occurredAt: event.occurredAt }),
+ );
+ }
+ return [...result, ...milestones];
+ });
+
+ // The enclosing EventSink operation already traces this persistence work.
+ const normalizeEvents = Effect.fnUntraced(function* (
+ input: ReadonlyArray,
+ effects: ReadonlyArray = [],
+ ) {
+ const events = yield* withRunFinalizedEvents(input, effects);
+ return yield* normalizeTurnItemPositions(events);
+ });
+
+ const normalizeTurnItemPositions = (events: ReadonlyArray) => {
const runOrdinals = new Map(
events.flatMap((event) =>
event.type === "run.created" || event.type === "run.updated"
@@ -441,6 +559,7 @@ const layerBase: Layer.Layer<
input.guardPendingUserInputCancellations === true
? yield* guardUserInputCancellations(events)
: events,
+ input.effects,
);
const committed = yield* eventStore.append({
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
@@ -496,10 +615,13 @@ const layerBase: Layer.Layer<
};
}
+ // The checkpoint capture enqueued below decides how a run that
+ // settles here finalizes, so it must be visible to normalization.
const normalized = yield* normalizeEvents(
input.guardPendingUserInputCancellations === true
? yield* guardUserInputCancellations(input.events)
: input.events,
+ input.effects,
);
const storedEvents = yield* eventStore.append({
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
@@ -632,6 +754,79 @@ const layerBase: Layer.Layer<
return { receipt: existing.value, storedEvents };
});
+ // Cancelling a checkpoint capture abandons its run's finalization, so the
+ // cancelling commit records `run.finalization-failed` for that run.
+ // The enclosing EventSink operation already traces this persistence work.
+ const recordAbandonedCaptures = Effect.fnUntraced(function* (
+ commandId: CommandId,
+ cancelledEffectIds: ReadonlyArray,
+ occurredAt: DateTime.Utc,
+ ) {
+ const events: Array = [];
+ for (const effectId of cancelledEffectIds) {
+ const effect = yield* effectOutbox.get(effectId);
+ if (Option.isNone(effect) || effect.value.request.type !== "checkpoint.capture") continue;
+ const { run } = yield* projectionStore.getCheckpointCaptureContext(
+ effect.value.threadId,
+ effect.value.request,
+ );
+ if (run === undefined || run.status === "rolled_back") continue;
+ events.push(
+ RunFinalized.makeRunFinalizationFailedEvent({
+ run,
+ operation: RunFinalized.abandonedOperation(run),
+ occurredAt,
+ }),
+ );
+ }
+ if (events.length === 0) return [];
+ const storedEvents = yield* eventStore.append({
+ commandId,
+ events: yield* normalizeEvents(events),
+ });
+ yield* applyStoredEvents(storedEvents);
+ return storedEvents;
+ });
+
+ const failEffectEffect = Effect.fn("orchestrationV2.EventSink.failEffect")(function* (
+ input: Parameters[0],
+ ) {
+ yield* Effect.annotateCurrentSpan({
+ "orchestration_v2.command_id": input.commandId,
+ "orchestration_v2.effect_id": input.effectId,
+ "orchestration_v2.event_count": input.events.length,
+ });
+
+ return yield* commitThenPublish(
+ Effect.gen(function* () {
+ const failed = yield* effectOutbox.fail({
+ effectId: input.effectId,
+ workerId: input.workerId,
+ error: input.error,
+ });
+ if (!failed) {
+ return {
+ committed: false as const,
+ storedEvents: [] as ReadonlyArray,
+ };
+ }
+ const normalized = yield* normalizeEvents(input.events);
+ const storedEvents =
+ normalized.length === 0
+ ? []
+ : yield* eventStore.append({ commandId: input.commandId, events: normalized });
+ const outcomes = yield* applyStoredEvents(storedEvents);
+ return { committed: true as const, storedEvents, outcomes };
+ }),
+ (result) =>
+ result.committed
+ ? effectOutbox
+ .notifyAvailable()
+ .pipe(Effect.andThen(publishStoredEvents(result.storedEvents, result.outcomes)))
+ : Effect.void,
+ );
+ });
+
const commitCommandEffect = Effect.fn("orchestrationV2.EventSink.commitCommand")(function* (
input: Parameters[0],
) {
@@ -656,14 +851,15 @@ const layerBase: Layer.Layer<
runId === undefined
? input.events
: yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId),
+ input.effects,
);
- const storedEvents = yield* eventStore.append({
+ const appended = yield* eventStore.append({
commandId: input.commandId,
events: normalized,
});
// A Stop that changes nothing can still queue the stop of its delegated tasks.
const sequence =
- storedEvents.at(-1)?.sequence ??
+ appended.at(-1)?.sequence ??
(input.effects.length === 0
? undefined
: yield* eventStore.latestSequence({ threadId: input.threadId }));
@@ -672,25 +868,37 @@ const layerBase: Layer.Layer<
new Error(`Command ${input.commandId} produced no orchestration events.`),
);
}
- const outcomes = yield* applyStoredEvents(storedEvents);
+ const outcomes = yield* applyStoredEvents(appended);
yield* effectOutbox.enqueue(input.effects);
+ const cancelledEffectIds =
+ input.cancelUnsettledEffects === undefined
+ ? []
+ : yield* effectOutbox.cancelUnsettled({
+ threadId: input.threadId,
+ ...input.cancelUnsettledEffects,
+ });
+ const storedEvents = input.cancelUnsettledEffects?.effectTypes.includes(
+ "checkpoint.capture",
+ )
+ ? [
+ ...appended,
+ ...(yield* recordAbandonedCaptures(
+ input.commandId,
+ cancelledEffectIds,
+ input.acceptedAt,
+ )),
+ ]
+ : appended;
const receipt: CommandReceiptStore.CommandReceiptV2 = {
commandId: input.commandId,
threadId: input.threadId,
commandType: input.commandType,
acceptedAt: input.acceptedAt,
- resultSequence: sequence,
+ resultSequence: storedEvents.at(-1)?.sequence ?? sequence,
status: "accepted",
error: null,
};
yield* commandReceipts.upsert(receipt);
- const cancelledEffectIds =
- input.cancelUnsettledEffects === undefined
- ? []
- : yield* effectOutbox.cancelUnsettled({
- threadId: input.threadId,
- ...input.cancelUnsettledEffects,
- });
return { receipt, storedEvents, outcomes, committed: true as const, cancelledEffectIds };
}),
(result) =>
@@ -939,6 +1147,21 @@ const layerBase: Layer.Layer<
}),
),
),
+ failEffect: (input) =>
+ failEffectEffect(input).pipe(
+ Effect.mapError(
+ (cause) =>
+ new EventSinkWriteError({
+ commandId: input.commandId,
+ eventCount: input.events.length,
+ cause,
+ }),
+ ),
+ ),
+ hasRunFinalization: (runId) =>
+ isRunFinalizationRecorded(runId).pipe(
+ Effect.mapError((cause) => new EventSinkStreamError({ cause })),
+ ),
commitRejectedCommand: (input) =>
commitRejectedCommandEffect(input).pipe(
Effect.mapError(
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts
index 95c0f7a4bae53..55ceb83c029f3 100644
--- a/apps/server/src/orchestration-v2/ProjectionStore.ts
+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts
@@ -832,6 +832,10 @@ export function applyToProjection(
),
),
});
+ // Milestones over state the run rows already hold.
+ case "run.finalized":
+ case "run.finalization-failed":
+ return projection;
case "run.background-work-cancelled":
return {
...base,
@@ -2001,6 +2005,9 @@ export const layer: Layer.Layer =
`;
break;
}
+ case "run.finalized":
+ case "run.finalization-failed":
+ break;
case "run.background-work-cancelled": {
// Only this field changes, so a concurrent lifecycle write is never regressed.
const workJson = yield* encodeRestartCancelledBackgroundWork(
@@ -2717,6 +2724,8 @@ export const layer: Layer.Layer =
event.type !== "thread.interaction-mode-updated" &&
event.type !== "thread.model-selection-updated" &&
event.type !== "thread.provider-switched" &&
+ event.type !== "run.finalized" &&
+ event.type !== "run.finalization-failed" &&
event.type !== "thread.guest-share-updated"
) {
const updatedAt = yield* encodeThreadUpdatedAt(event.occurredAt);
diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
index 5bbb3926e2b66..fdd1beed5b2b2 100644
--- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
@@ -10,6 +10,7 @@ import {
type OrchestrationV2ProviderCapabilities,
type OrchestrationV2ProviderSession,
type OrchestrationV2ProviderThread,
+ PluginInstallationId,
type Project,
ProjectId,
ProviderDriverKind,
@@ -34,6 +35,7 @@ import { HttpServer } from "effect/http";
import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
+import * as PluginTools from "../plugins/PluginTools.ts";
import * as ProjectService from "../project/ProjectService.ts";
import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
@@ -476,11 +478,9 @@ function layerTest(input: {
readonly failReleaseEventWrites?: boolean;
readonly flakyReleaseWrites?: FlakyReleaseWrites;
readonly pauseAttachWrite?: Parameters[0];
+ /** Once armed, the next call of that registry step hangs until interrupted or crashes. */
+ readonly pauseMcpRegistry?: PauseMcpRegistry;
readonly beforeUnchangedWrite?: Ref.Ref>;
- readonly pauseResolve?: {
- readonly armed: Ref.Ref;
- readonly paused: Deferred.Deferred;
- };
readonly hasPendingBackgroundWork?: Effect.Effect;
readonly hasPendingBackgroundWorkForThread?: Effect.Effect;
readonly backgroundTasksEndWithProcess?: boolean;
@@ -492,6 +492,7 @@ function layerTest(input: {
readonly scopeCloseReached?: Deferred.Deferred;
readonly serverSettingsLayer?: ReturnType;
readonly projectServiceLayer?: Layer.Layer;
+ readonly pluginToolsLayer?: Layer.Layer;
}) {
const layerConfiguredEventSink =
input.flakyReleaseWrites !== undefined
@@ -531,9 +532,9 @@ function layerTest(input: {
}).pipe(Effect.map(ProviderAdapterRegistry.layerSingle)),
);
const layerConfiguredMcpRegistry =
- input.pauseResolve === undefined
+ input.pauseMcpRegistry === undefined
? layerTestMcpRegistry
- : layerPausingMcpRegistry(input.pauseResolve);
+ : layerPausingMcpRegistry(input.pauseMcpRegistry);
const layerProviderEventIngestorTest = ProviderEventIngestor.layer.pipe(
Layer.provide(
Layer.mergeAll(
@@ -574,6 +575,7 @@ function layerTest(input: {
layerTestStores,
...(input.serverSettingsLayer === undefined ? [] : [input.serverSettingsLayer]),
...(input.projectServiceLayer === undefined ? [] : [input.projectServiceLayer]),
+ ...(input.pluginToolsLayer === undefined ? [] : [input.pluginToolsLayer]),
),
),
),
@@ -599,23 +601,41 @@ const layerTestMcpRegistry = Layer.effect(
Layer.provide(NodeServices.layer),
);
-const layerPausingMcpRegistry = (pause: {
+interface PauseMcpRegistry {
+ readonly step: "resolve" | "setPluginToolGrants";
+ readonly outcome: "hang" | "crash";
readonly armed: Ref.Ref;
readonly paused: Deferred.Deferred;
-}) =>
+}
+
+const layerPausingMcpRegistry = (pause: PauseMcpRegistry) =>
Layer.effect(
McpSessionRegistry.McpSessionRegistry,
Effect.gen(function* () {
const delegate = yield* McpSessionRegistry.McpSessionRegistry;
+ const holdIfArmed = (step: PauseMcpRegistry["step"], run: Effect.Effect) =>
+ step !== pause.step
+ ? run
+ : Ref.getAndSet(pause.armed, false).pipe(
+ Effect.flatMap((armed) =>
+ armed
+ ? Deferred.succeed(pause.paused, undefined).pipe(
+ Effect.andThen(
+ pause.outcome === "hang"
+ ? Effect.never
+ : Effect.die(new Error(`${step} crashed`)),
+ ),
+ )
+ : run,
+ ),
+ );
return McpSessionRegistry.McpSessionRegistry.of({
...delegate,
- resolve: (rawToken) =>
- Ref.getAndSet(pause.armed, false).pipe(
- Effect.flatMap((armed) =>
- armed
- ? Deferred.succeed(pause.paused, undefined).pipe(Effect.andThen(Effect.never))
- : delegate.resolve(rawToken),
- ),
+ resolve: (rawToken) => holdIfArmed("resolve", delegate.resolve(rawToken)),
+ setPluginToolGrants: (providerSessionId, grants) =>
+ holdIfArmed(
+ "setPluginToolGrants",
+ delegate.setPluginToolGrants(providerSessionId, grants),
),
});
}),
@@ -1845,6 +1865,49 @@ it.effect(
}),
);
+it.effect("ProviderSessionManagerV2 snapshots the enabled tool plugins into the credential", () =>
+ Effect.gen(function* () {
+ const state = yield* Ref.make(emptyState);
+ const mcpConfigs = yield* Ref.make<
+ ReadonlyArray
+ >([]);
+ const enabled = yield* Ref.make([
+ { installationId: PluginInstallationId.make("installation-1"), generation: 1 },
+ ]);
+ const pluginToolsLayer = Layer.mock(PluginTools.PluginTools)({ grants: Ref.get(enabled) });
+ yield* Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
+ const registry = yield* McpSessionRegistry.McpSessionRegistry;
+ const now = yield* DateTime.now;
+ const threadId = ThreadId.make("thread-provider-session-manager-plugin-tools");
+ yield* eventSink.write({
+ events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
+ });
+ const openAndResolve = Effect.gen(function* () {
+ const providerSessionId = yield* idAllocator.allocate.providerSession({
+ providerInstanceId: modelSelection.instanceId,
+ threadId,
+ });
+ yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
+ const config = (yield* Ref.get(mcpConfigs)).at(-1);
+ const token = config!.authorizationHeader.replace(/^Bearer\s+/, "");
+ const resolved = yield* registry.resolve(token);
+ yield* manager.close(providerSessionId);
+ return resolved?.pluginToolGrants;
+ });
+
+ assert.deepEqual(yield* openAndResolve, yield* Ref.get(enabled));
+ // A plugin enabled later reaches the next session, not the one already prepared.
+ yield* Ref.set(enabled, []);
+ assert.deepEqual(yield* openAndResolve, []);
+ }).pipe(
+ Effect.provide(layerTest({ state, idleTimeoutMs: 1_000, mcpConfigs, pluginToolsLayer })),
+ );
+ }),
+);
+
it.effect("ProviderSessionManagerV2 honors a project browser-access opt-out", () =>
Effect.gen(function* () {
const captured = yield* runBrowserAccessScenario({
@@ -2075,10 +2138,14 @@ it.effect(
}),
);
-// regression: #15571 an interrupted session start did not clean up
-it.effect(
- "ProviderSessionManagerV2 revokes a reused credential after a resume stopped while checking it",
- () =>
+it.effect.each([
+ ["stopped while checking it", "resolve", "hang"],
+ ["stopped while updating its plugin tool grants", "setPluginToolGrants", "hang"],
+ ["crashed while checking it", "resolve", "crash"],
+ ["crashed while updating its plugin tool grants", "setPluginToolGrants", "crash"],
+] as const)(
+ "ProviderSessionManagerV2 revokes a reused credential after a resume %s",
+ ([, step, outcome]) =>
Effect.gen(function* () {
const state = yield* Ref.make(emptyState);
const armed = yield* Ref.make(false);
@@ -2090,8 +2157,8 @@ it.effect(
const registry = yield* McpSessionRegistry.McpSessionRegistry;
const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
const now = yield* DateTime.now;
- const owner = ThreadId.make("thread-provider-session-manager-resolve-stop-owner");
- const threadId = ThreadId.make("thread-provider-session-manager-resolve-stop");
+ const owner = ThreadId.make(`thread-provider-session-manager-${step}-${outcome}-owner`);
+ const threadId = ThreadId.make(`thread-provider-session-manager-${step}-${outcome}`);
const providerSessionId = idAllocator.derive.providerSession({
providerInstanceId: modelSelection.instanceId,
});
@@ -2114,7 +2181,7 @@ it.effect(
threadId,
providerSessionId,
now,
- nativeThreadId: "native-resolve-stop",
+ nativeThreadId: `native-${step}-${outcome}`,
}),
});
yield* resume;
@@ -2123,16 +2190,22 @@ it.effect(
assert.isDefined(config);
const token = config!.authorizationHeader.replace(/^Bearer\s+/, "");
+ // A re-attach is stopped, or crashes, while it reuses that credential.
yield* Ref.set(armed, true);
const stopped = yield* resume.pipe(Effect.forkChild({ startImmediately: true }));
yield* Deferred.await(paused);
- yield* Fiber.interrupt(stopped);
+ // A crashed attach is logged and rolled back rather than failing the resume.
+ yield* outcome === "hang" ? Fiber.interrupt(stopped) : Fiber.await(stopped);
yield* manager.release({ providerSessionId, reason: "manual_shutdown" });
assert.isUndefined(yield* registry.resolve(token));
}).pipe(
Effect.provide(
- layerTest({ state, idleTimeoutMs: 60_000, pauseResolve: { armed, paused } }),
+ layerTest({
+ state,
+ idleTimeoutMs: 60_000,
+ pauseMcpRegistry: { step, outcome, armed, paused },
+ }),
),
);
}),
diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts
index 03a7278774a64..d029d7df2079a 100644
--- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts
+++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts
@@ -45,6 +45,7 @@ import * as ProjectService from "../project/ProjectService.ts";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
import * as ServerSettings from "../serverSettings.ts";
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
+import * as PluginTools from "../plugins/PluginTools.ts";
import * as EventSink from "./EventSink.ts";
import * as CommandOutputHub from "./CommandOutputHub.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
@@ -340,6 +341,8 @@ export const layerWithOptions = (
/** Unreadable settings must preserve explicit browser-access opt-outs. */
const serverSettings = yield* Effect.serviceOption(ServerSettings.ServerSettingsService);
const projectService = yield* Effect.serviceOption(ProjectService.ProjectService);
+ // Optional for the same reason; without it a session gets no plugin tools.
+ const pluginTools = yield* Effect.serviceOption(PluginTools.PluginTools);
const eventSink = yield* EventSink.EventSinkV2;
const commandOutput = yield* CommandOutputHub.CommandOutputHub;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
@@ -457,6 +460,10 @@ export const layerWithOptions = (
>(["orchestration", "worktree", "pull-requests"]);
if (browserToolsAvailable) capabilities.add("preview");
if (deviceToolsAvailable) capabilities.add("device");
+ // Taken at every preparation; each call still checks the plugin is enabled now.
+ const pluginToolGrants = Option.isSome(pluginTools)
+ ? yield* pluginTools.value.grants
+ : [];
const subagent = yield* projectionStore.getThread(threadId).pipe(
Effect.map((thread) => thread.lineage.relationshipToParent === "subagent"),
Effect.orElseSucceed(() => false),
@@ -466,25 +473,37 @@ export const layerWithOptions = (
// Reserve before async validation so release cannot revoke between validation and reservation.
reserveMcpCredential(threadId, existing.providerSessionId);
const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, "");
- // Interrupted resolution must release the reservation before the caller learns of it.
- const resolved = yield* mcpSessionRegistry
- .resolve(rawToken)
- .pipe(
- Effect.onInterrupt(() =>
- Effect.sync(() =>
- dropMcpCredentialReservation(threadId, existing.providerSessionId),
- ),
- ),
+ // The caller only learns of the reservation once this returns,
+ // so a stop or crash while resolving or updating grants must drop it here.
+ const reused = yield* Effect.gen(function* () {
+ const resolved = yield* mcpSessionRegistry.resolve(rawToken);
+ if (
+ resolved === undefined ||
+ resolved.thread.threadId !== threadId ||
+ resolved.thread.providerInstanceId !== providerInstanceId ||
+ // A flipped browser-access setting must not survive through
+ // credential reuse: rotate so the new scope reflects it.
+ resolved.capabilities.has("preview") !== browserToolsAvailable ||
+ resolved.capabilities.has("device") !== deviceToolsAvailable ||
+ (resolved.thread.subagent === true) !== subagent
+ ) {
+ return false;
+ }
+ // The provider keeps this credential, and the plugin tools are fixed meta-tools,
+ // so new grants apply to it without rotating the token.
+ yield* mcpSessionRegistry.setPluginToolGrants(
+ existing.providerSessionId,
+ pluginToolGrants,
);
- if (
- resolved !== undefined &&
- resolved.thread.threadId === threadId &&
- resolved.thread.providerInstanceId === providerInstanceId &&
- resolved.capabilities.has("preview") === browserToolsAvailable &&
- resolved.capabilities.has("device") === deviceToolsAvailable &&
- // A credential issued while the lineage read failed is replaced once it works.
- (resolved.thread.subagent === true) === subagent
- ) {
+ return true;
+ }).pipe(
+ Effect.onError(() =>
+ Effect.sync(() =>
+ dropMcpCredentialReservation(threadId, existing.providerSessionId),
+ ),
+ ),
+ );
+ if (reused) {
return { mcpCredentialId: existing.providerSessionId, issued: false };
}
dropMcpCredentialReservation(threadId, existing.providerSessionId);
@@ -495,6 +514,7 @@ export const layerWithOptions = (
providerInstanceId,
browserToolsAvailable,
capabilities,
+ pluginToolGrants,
subagent,
});
yield* mcpSessions.set(credential.config);
diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts
index b9b2b09bd227e..f690bfea04678 100644
--- a/apps/server/src/orchestration-v2/RunExecutionService.ts
+++ b/apps/server/src/orchestration-v2/RunExecutionService.ts
@@ -53,6 +53,7 @@ import {
makeProviderFailureTurnItem,
} from "@t3tools/provider-core/server/failure";
import * as RunFinalizationService from "./RunFinalizationService.ts";
+import { checkpointCaptureEffectId } from "./RunFinalized.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
import { userFacingDatabaseErrorMessage } from "./UserFacingErrors.ts";
@@ -742,7 +743,7 @@ export const layer: Layer.Layer<
input.terminal.status === "cancelled"
? [
{
- id: `effect:checkpoint.capture:${input.run.id}`,
+ id: checkpointCaptureEffectId(input.run.id),
commandId: checkpointCaptureCommandId,
threadId: input.run.threadId,
request: {
diff --git a/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts b/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts
index c2b9e0aabcd5b..fa1589c9c52a3 100644
--- a/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts
+++ b/apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts
@@ -11,6 +11,7 @@ import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts";
import * as CheckpointCapture from "./CheckpointCaptureService.ts";
import * as CheckpointRollback from "./CheckpointRollbackService.ts";
+import * as EventSink from "./EventSink.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as EffectWorker from "./EffectWorker.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -37,6 +38,15 @@ it.effect.each([
cause: "checkpoint storage unavailable",
});
const projections = Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getCheckpointCaptureContext: () =>
+ Effect.succeed({
+ run: undefined,
+ rootNode: undefined,
+ scope: undefined,
+ providerThread: undefined,
+ readyCheckpointOrdinals: [],
+ sideChat: false,
+ }),
getCheckpointContext: () =>
Effect.succeed({
runs: [],
@@ -70,6 +80,9 @@ it.effect.each([
Layer.mergeAll(
projections,
observer,
+ Layer.mock(EventSink.EventSinkV2)({
+ hasRunFinalization: () => Effect.succeed(false),
+ }),
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
execute: () =>
Effect.gen(function* () {
diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
index 38a18192531a5..286b09ae26f81 100644
--- a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
+++ b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
@@ -1,8 +1,10 @@
-import { assert, it } from "@effect/vitest";
+import { assert, it, vi } from "@effect/vitest";
import {
CheckpointScopeId,
+ ProviderInstanceId,
RunId,
ThreadId,
+ type OrchestrationV2Run,
type OrchestrationV2ThreadShell,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
@@ -12,9 +14,68 @@ import * as PullRequestService from "../pullRequest/PullRequestService.ts";
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts";
import * as CheckpointCapture from "./CheckpointCaptureService.ts";
+import * as EventSink from "./EventSink.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as RunFinalization from "./RunFinalizationService.ts";
+it.effect("refreshes workspace after checkpoint capture, then records finalization", () => {
+ const threadId = ThreadId.make("thread_finalize");
+ const runId = RunId.make("run_finalize");
+ const scopeId = CheckpointScopeId.make("scope_finalize");
+ const steps: Array = [];
+ const capture = vi.fn(() => Effect.sync(() => steps.push("capture")));
+ const refresh = vi.fn(() => Effect.sync(() => steps.push("refresh")));
+ const write = vi.fn(() => Effect.sync(() => (steps.push("record"), [])));
+ const checkpointContext = {
+ runs: [],
+ checkpointScopes: [{ id: scopeId, runId, kind: "root_run" as const, cwd: "/repo" }],
+ checkpoints: [],
+ };
+ const layer = RunFinalization.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({ execute: capture }),
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getThreadProjection: () =>
+ Effect.die("workspace refresh must not load transcript history"),
+ getCheckpointContext: () => Effect.succeed(checkpointContext),
+ getCheckpointCaptureContext: () =>
+ Effect.succeed({
+ run: {
+ id: runId,
+ threadId,
+ rootNodeId: null,
+ providerInstanceId: ProviderInstanceId.make("codex"),
+ status: "completed",
+ checkpointId: null,
+ } as OrchestrationV2Run,
+ rootNode: undefined,
+ scope: undefined,
+ providerThread: undefined,
+ readyCheckpointOrdinals: [],
+ sideChat: false,
+ }),
+ }),
+ Layer.mock(EventSink.EventSinkV2)({
+ write,
+ hasRunFinalization: () => Effect.succeed(false),
+ }),
+ Layer.succeed(RunFinalization.RunFinalizationObserver, {
+ refresh,
+ refreshAfterTurn: () => Effect.void,
+ }),
+ ),
+ ),
+ );
+ return Effect.gen(function* () {
+ const service = yield* RunFinalization.RunFinalizationService;
+ yield* service.finalize({ threadId, runId, scopeId });
+ assert.equal(capture.mock.calls.length, 1);
+ assert.deepEqual(refresh.mock.calls[0], [{ cwd: "/repo", threadId, runId }]);
+ assert.deepEqual(steps, ["capture", "refresh", "record"]);
+ }).pipe(Effect.provide(layer));
+});
+
it.effect("completes finalization when the post-checkpoint workspace refresh fails", () => {
const threadId = ThreadId.make("thread_refresh_failure");
const runId = RunId.make("run_refresh_failure");
@@ -23,6 +84,10 @@ it.effect("completes finalization when the post-checkpoint workspace refresh fai
const layer = RunFinalization.layer.pipe(
Layer.provide(
Layer.mergeAll(
+ Layer.mock(EventSink.EventSinkV2)({
+ hasRunFinalization: () => Effect.succeed(false),
+ write: () => Effect.sync(() => (captured.push("finalized"), [])),
+ }),
Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
execute: () =>
Effect.sync(() => {
@@ -36,6 +101,22 @@ it.effect("completes finalization when the post-checkpoint workspace refresh fai
checkpointScopes: [{ id: scopeId, runId, kind: "root_run" as const, cwd: "/repo" }],
checkpoints: [],
}),
+ getCheckpointCaptureContext: () =>
+ Effect.succeed({
+ run: {
+ id: runId,
+ threadId,
+ rootNodeId: null,
+ providerInstanceId: ProviderInstanceId.make("codex"),
+ status: "completed",
+ checkpointId: null,
+ } as OrchestrationV2Run,
+ rootNode: undefined,
+ scope: undefined,
+ providerThread: undefined,
+ readyCheckpointOrdinals: [],
+ sideChat: false,
+ }),
}),
Layer.succeed(RunFinalization.RunFinalizationObserver, {
refresh: () =>
@@ -54,7 +135,81 @@ it.effect("completes finalization when the post-checkpoint workspace refresh fai
return Effect.gen(function* () {
const service = yield* RunFinalization.RunFinalizationService;
yield* service.finalize({ threadId, runId, scopeId });
- assert.deepEqual(captured, ["checkpoint", "refresh"]);
+ assert.deepEqual(captured, ["checkpoint", "refresh", "finalized"]);
+ }).pipe(Effect.provide(layer));
+});
+
+it.effect("preserves checkpoint failures without refreshing the workspace", () => {
+ const threadId = ThreadId.make("thread_capture_failure");
+ const runId = RunId.make("run_capture_failure");
+ const scopeId = CheckpointScopeId.make("scope_capture_failure");
+ const captureError = new CheckpointCapture.CheckpointCaptureExecutionError({
+ threadId,
+ runId,
+ scopeId,
+ cause: new Error("Checkpoint capture failed"),
+ });
+ const refresh = vi.fn(() => Effect.void);
+ const layer = RunFinalization.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(EventSink.EventSinkV2)({
+ hasRunFinalization: () => Effect.succeed(false),
+ }),
+ Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({
+ execute: () => Effect.fail(captureError),
+ }),
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getCheckpointContext: () => Effect.die("capture failed before context lookup"),
+ }),
+ Layer.succeed(RunFinalization.RunFinalizationObserver, {
+ refresh,
+ refreshAfterTurn: () => Effect.void,
+ }),
+ ),
+ ),
+ );
+ return Effect.gen(function* () {
+ const service = yield* RunFinalization.RunFinalizationService;
+ const error = yield* service.finalize({ threadId, runId, scopeId }).pipe(Effect.flip);
+ assert.equal(error.operation, "capture-checkpoint");
+ assert.strictEqual(error.cause, captureError);
+ assert.equal(refresh.mock.calls.length, 0);
+ }).pipe(Effect.provide(layer));
+});
+
+it.effect("preserves checkpoint context read failures without refreshing the workspace", () => {
+ const threadId = ThreadId.make("thread_context_failure");
+ const runId = RunId.make("run_context_failure");
+ const scopeId = CheckpointScopeId.make("scope_context_failure");
+ const contextError = new ProjectionStore.ProjectionStoreReadError({
+ threadId,
+ cause: new Error("Checkpoint context read failed"),
+ });
+ const refresh = vi.fn(() => Effect.void);
+ const layer = RunFinalization.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(EventSink.EventSinkV2)({
+ hasRunFinalization: () => Effect.succeed(false),
+ }),
+ Layer.mock(CheckpointCapture.CheckpointCaptureServiceV2)({ execute: () => Effect.void }),
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getCheckpointContext: () => Effect.fail(contextError),
+ }),
+ Layer.succeed(RunFinalization.RunFinalizationObserver, {
+ refresh,
+ refreshAfterTurn: () => Effect.void,
+ }),
+ ),
+ ),
+ );
+ return Effect.gen(function* () {
+ const service = yield* RunFinalization.RunFinalizationService;
+ const error = yield* service.finalize({ threadId, runId, scopeId }).pipe(Effect.flip);
+ assert.equal(error.operation, "refresh-workspace");
+ assert.strictEqual(error.cause, contextError);
+ assert.equal(refresh.mock.calls.length, 0);
}).pipe(Effect.provide(layer));
});
diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.ts b/apps/server/src/orchestration-v2/RunFinalizationService.ts
index 40882033ba527..34cde32358965 100644
--- a/apps/server/src/orchestration-v2/RunFinalizationService.ts
+++ b/apps/server/src/orchestration-v2/RunFinalizationService.ts
@@ -1,5 +1,14 @@
-import { CheckpointScopeId, ProjectId, RunId, ThreadId } from "@t3tools/contracts";
+import {
+ CheckpointScopeId,
+ CommandId,
+ OrchestrationV2RunFinalizationOperation,
+ ProjectId,
+ RunId,
+ ThreadId,
+} from "@t3tools/contracts";
+import * as Cause from "effect/Cause";
import * as Context from "effect/Context";
+import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Schema from "effect/Schema";
@@ -8,7 +17,9 @@ import * as PullRequestService from "../pullRequest/PullRequestService.ts";
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts";
import * as CheckpointCapture from "./CheckpointCaptureService.ts";
+import * as EventSink from "./EventSink.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
+import * as RunFinalized from "./RunFinalized.ts";
export class RunFinalizationError extends Schema.TaggedError()(
"RunFinalizationError",
@@ -16,11 +27,13 @@ export class RunFinalizationError extends Schema.TaggedError()(
"RunFinalizationRefreshError",
{ cwd: Schema.String, cause: Schema.Defect() },
@@ -40,51 +53,134 @@ export class RunFinalizationObserver extends Context.Reference<{
export class RunFinalizationService extends Context.Service<
RunFinalizationService,
{
+ /**
+ * Captures the run's checkpoint, refreshes its workspace, then records
+ * `run.finalized`. At-least-once: every step is safe to repeat, and a run
+ * that already recorded its finalization is left as recorded. Every
+ * failure outside the best-effort display refresh names the step that failed.
+ */
readonly finalize: (input: {
readonly threadId: ThreadId;
readonly runId: RunId;
readonly scopeId: CheckpointScopeId;
}) => Effect.Effect;
+ /**
+ * Gives up on a run's finalization after the worker's last attempt failed
+ * at `operation`: fails the claimed capture effect and records
+ * `run.finalization-failed` in one transaction. Returns false, recording
+ * nothing, when `workerId` no longer holds the effect's lease.
+ */
+ readonly abandon: (input: {
+ readonly threadId: ThreadId;
+ readonly runId: RunId;
+ readonly scopeId: CheckpointScopeId;
+ readonly operation: OrchestrationV2RunFinalizationOperation;
+ readonly effectId: string;
+ readonly workerId: string;
+ readonly error: string;
+ }) => Effect.Effect;
}
>()("t3/orchestration-v2/RunFinalizationService") {}
const make = Effect.gen(function* () {
const checkpointCapture = yield* CheckpointCapture.CheckpointCaptureServiceV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const eventSink = yield* EventSink.EventSinkV2;
const observer = yield* RunFinalizationObserver;
- const finalize: RunFinalizationService["Service"]["finalize"] = Effect.fn(
- "RunFinalizationService.finalize",
- )(function* (input) {
- yield* checkpointCapture
- .execute(input)
- .pipe(
- Effect.mapError(
- (cause) => new RunFinalizationError({ ...input, operation: "capture-checkpoint", cause }),
- ),
- );
- const projection = yield* projections
- .getCheckpointContext(input.threadId)
- .pipe(
+ const finalize = Effect.fn("RunFinalizationService.finalize")(function* (input: {
+ readonly threadId: ThreadId;
+ readonly runId: RunId;
+ readonly scopeId: CheckpointScopeId;
+ }) {
+ // Unexpected defects fail the step too; an interruption is replayed.
+ const failStep =
+ (operation: OrchestrationV2RunFinalizationOperation) => (cause: Cause.Cause) =>
+ Cause.hasInterruptsOnly(cause)
+ ? Effect.interrupt
+ : Effect.fail(
+ new RunFinalizationError({ ...input, operation, cause: Cause.squash(cause) }),
+ );
+
+ // A replay after a crash honours the disposition already recorded.
+ if (
+ yield* eventSink
+ .hasRunFinalization(input.runId)
+ .pipe(Effect.catchCause(failStep("capture-checkpoint")))
+ )
+ return;
+ yield* checkpointCapture.execute(input).pipe(Effect.catchCause(failStep("capture-checkpoint")));
+ yield* Effect.gen(function* () {
+ const projection = yield* projections.getCheckpointContext(input.threadId);
+ const cwd = projection.checkpointScopes.find((scope) => scope.id === input.scopeId)?.cwd;
+ if (cwd !== undefined) {
+ // A display refresh failure must not retry an already durable checkpoint.
+ yield* observer.refresh({ cwd, threadId: input.threadId, runId: input.runId }).pipe(
+ Effect.catchTags({
+ RunFinalizationRefreshError: (cause) =>
+ Effect.logWarning("Failed to refresh workspace after checkpoint capture", {
+ ...input,
+ cause,
+ }),
+ }),
+ );
+ }
+ }).pipe(Effect.catchCause(failStep("refresh-workspace")));
+ yield* Effect.gen(function* () {
+ const { run } = yield* projections.getCheckpointCaptureContext(input.threadId, input);
+ // A rolled-back run was discarded before its capture ran.
+ const outcome = run === undefined ? null : RunFinalized.runFinalizedOutcome(run.status);
+ if (run === undefined || outcome === null) return;
+ // EventSink drops the event when this run already recorded its finalization.
+ yield* eventSink.write({
+ commandId: CommandId.make(`command:effect:run.finalized:${run.id}`),
+ events: [
+ RunFinalized.makeRunFinalizedEvent({ run, outcome, occurredAt: yield* DateTime.now }),
+ ],
+ });
+ }).pipe(Effect.catchCause(failStep("record-finalized")));
+ });
+
+ const abandon: RunFinalizationService["Service"]["abandon"] = Effect.fn(
+ "RunFinalizationService.abandon",
+ )(
+ function* (input) {
+ const { run } = yield* projections.getCheckpointCaptureContext(input.threadId, input);
+ const { committed } = yield* eventSink.failEffect({
+ commandId: CommandId.make(`command:effect:run.finalization-failed:${input.runId}`),
+ effectId: input.effectId,
+ workerId: input.workerId,
+ error: input.error,
+ // A rolled-back run was discarded; there is nothing to report.
+ events:
+ run === undefined || run.status === "rolled_back"
+ ? []
+ : [
+ RunFinalized.makeRunFinalizationFailedEvent({
+ run,
+ operation: input.operation,
+ occurredAt: yield* DateTime.now,
+ }),
+ ],
+ });
+ return committed;
+ },
+ (effect, input) =>
+ effect.pipe(
Effect.mapError(
- (cause) => new RunFinalizationError({ ...input, operation: "refresh-workspace", cause }),
- ),
- );
- const cwd = projection.checkpointScopes.find((scope) => scope.id === input.scopeId)?.cwd;
- if (cwd !== undefined) {
- // Retrying a display refresh after the checkpoint is durable blocks the next turn.
- yield* observer.refresh({ cwd, threadId: input.threadId, runId: input.runId }).pipe(
- Effect.catchTags({
- RunFinalizationRefreshError: (cause) =>
- Effect.logWarning("Failed to refresh workspace after checkpoint capture", {
- ...input,
+ (cause) =>
+ new RunFinalizationError({
+ threadId: input.threadId,
+ runId: input.runId,
+ scopeId: input.scopeId,
+ operation: input.operation,
cause,
}),
- }),
- );
- }
- });
- return RunFinalizationService.of({ finalize });
+ ),
+ ),
+ );
+
+ return RunFinalizationService.of({ finalize, abandon });
});
export const layer = Layer.effect(RunFinalizationService, make);
diff --git a/apps/server/src/orchestration-v2/RunFinalized.test.ts b/apps/server/src/orchestration-v2/RunFinalized.test.ts
new file mode 100644
index 0000000000000..40b352ff625a5
--- /dev/null
+++ b/apps/server/src/orchestration-v2/RunFinalized.test.ts
@@ -0,0 +1,924 @@
+import { assert, it } from "@effect/vitest";
+import {
+ CheckpointId,
+ CheckpointScopeId,
+ CommandId,
+ EventId,
+ MessageId,
+ NodeId,
+ type OrchestrationV2DomainEvent,
+ type OrchestrationV2Run,
+ type OrchestrationV2StoredEvent,
+ ProjectId,
+ ProviderInstanceId,
+ ProviderThreadId,
+ RunAttemptId,
+ RunId,
+ ThreadId,
+} from "@t3tools/contracts";
+import * as DateTime from "effect/DateTime";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Fiber from "effect/Fiber";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
+
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import * as CheckpointCapture from "./CheckpointCaptureService.ts";
+import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
+import * as EffectOutbox from "./EffectOutbox.ts";
+import * as EffectWorker from "./EffectWorker.ts";
+import * as EventSink from "./EventSink.ts";
+import * as EventStore from "./EventStore.ts";
+import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
+import * as ProjectionStore from "./ProjectionStore.ts";
+import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
+import * as ProviderSessionManager from "./ProviderSessionManager.ts";
+import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
+import * as ProviderTurnStartService from "./ProviderTurnStartService.ts";
+import * as RunFinalization from "./RunFinalizationService.ts";
+import { checkpointCaptureEffectId, makeRunFinalizationFailedEvent } from "./RunFinalized.ts";
+import * as RuntimeRequestService from "./RuntimeRequestService.ts";
+import * as ThreadManagementService from "./ThreadManagementService.ts";
+import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
+
+const threadId = ThreadId.make("thread:run-finalized");
+const runId = RunId.make("run:run-finalized");
+const scopeId = CheckpointScopeId.make("scope:run-finalized");
+const rootNodeId = NodeId.make("node:run-finalized-root");
+const providerThreadId = ProviderThreadId.make("provider-thread:run-finalized");
+const providerInstanceId = ProviderInstanceId.make("codex");
+const checkpointId = CheckpointId.make("checkpoint:run-finalized");
+const attemptId = RunAttemptId.make("attempt:run-finalized");
+
+const maxAttempts = 2;
+/** The worker's backoff after a capture's first failed attempt. */
+const firstRetryDelay = "100 millis";
+/** The worker's backoff after a capture's last attempt, when its failure record cannot commit. */
+const lastRetryDelay = "200 millis";
+
+/**
+ * Real stores, outbox, worker, finalization and restart recovery. Checkpoint
+ * capture and workspace refresh are stubs. `finalizationSink` can inject
+ * faults into the event sink the finalization service writes through.
+ */
+const makeLayer = (
+ capture: Effect.Effect<
+ void,
+ CheckpointCapture.CheckpointCaptureExecutionError,
+ EventSink.EventSinkV2
+ >,
+ refresh: () => Effect.Effect = () =>
+ Effect.void,
+ finalizationSink: (sink: EventSink.EventSinkV2Shape) => EventSink.EventSinkV2Shape = (sink) =>
+ sink,
+) => {
+ const stores = Layer.mergeAll(
+ SqlitePersistence.layerMemory,
+ EventStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
+ ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
+ );
+ const eventSink = EventSink.layer.pipe(Layer.provide(stores));
+ const outbox = EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory));
+ const finalization = RunFinalization.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ stores,
+ Layer.effect(
+ EventSink.EventSinkV2,
+ EventSink.EventSinkV2.pipe(Effect.map(finalizationSink)),
+ ).pipe(Layer.provide(eventSink)),
+ Layer.effect(
+ CheckpointCapture.CheckpointCaptureServiceV2,
+ Effect.gen(function* () {
+ const sink = yield* EventSink.EventSinkV2;
+ return { execute: () => Effect.provideService(capture, EventSink.EventSinkV2, sink) };
+ }),
+ ).pipe(Layer.provide(eventSink)),
+ Layer.succeed(RunFinalization.RunFinalizationObserver, {
+ refresh,
+ refreshAfterTurn: () => Effect.void,
+ }),
+ ),
+ ),
+ );
+ const executor = EffectWorker.layerExecutor.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ finalization,
+ Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}),
+ Layer.mock(CheckpointRollbackService.CheckpointRollbackServiceV2)({}),
+ Layer.mock(ProviderTurnControlService.ProviderTurnControlServiceV2)({}),
+ Layer.mock(ProviderTurnStartService.ProviderTurnStartServiceV2)({}),
+ Layer.mock(RuntimeRequestService.RuntimeRequestServiceV2)({}),
+ Layer.mock(ThreadTitleRegenerationService.ThreadTitleRegenerationService)({}),
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
+ ServerSettings.layerTest(),
+ ),
+ ),
+ );
+ const worker = EffectWorker.layerWithOptions({
+ workerId: "worker:run-finalized",
+ maxAttempts,
+ }).pipe(Layer.provide(Layer.merge(outbox, executor)));
+ const recovery = ProviderRuntimeRecovery.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ stores,
+ eventSink,
+ outbox,
+ worker,
+ IdAllocator.layer,
+ ServerSettings.layerTest(),
+ ),
+ ),
+ );
+ return Layer.mergeAll(stores, eventSink, outbox, finalization, worker, recovery);
+};
+
+const makeRun = (
+ now: DateTime.Utc,
+ status: OrchestrationV2Run["status"],
+ overrides: Partial = {},
+): OrchestrationV2Run => ({
+ id: runId,
+ threadId,
+ ordinal: 1,
+ providerInstanceId,
+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
+ providerThreadId,
+ userMessageId: MessageId.make("message:run-finalized"),
+ rootNodeId,
+ activeAttemptId: null,
+ status,
+ requestedAt: now,
+ startedAt: now,
+ completedAt: null,
+ checkpointId: null,
+ contextHandoffId: null,
+ ...overrides,
+});
+
+let eventCounter = 0;
+const runEvent = (
+ type: "run.created" | "run.updated",
+ run: OrchestrationV2Run,
+ occurredAt: DateTime.Utc,
+): OrchestrationV2DomainEvent => ({
+ id: EventId.make(`event:run-finalized-test:${(eventCounter += 1)}`),
+ type,
+ threadId,
+ runId: run.id,
+ providerInstanceId,
+ occurredAt,
+ payload: run,
+});
+
+const captureEffect = {
+ id: checkpointCaptureEffectId(runId),
+ commandId: CommandId.make(`command:effect:checkpoint.capture:${runId}`),
+ threadId,
+ request: { type: "checkpoint.capture" as const, runId, scopeId },
+};
+
+const seedThread = Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ yield* eventSink.write({
+ events: [
+ {
+ id: EventId.make("event:run-finalized-test:thread"),
+ type: "thread.created",
+ threadId,
+ providerInstanceId,
+ occurredAt: now,
+ payload: {
+ createdBy: "user",
+ creationSource: "web",
+ id: threadId,
+ projectId: ProjectId.make("project:run-finalized"),
+ title: "Run finalized",
+ providerInstanceId,
+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ activeProviderThreadId: null,
+ lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
+ forkedFrom: null,
+ createdAt: now,
+ updatedAt: now,
+ archivedAt: null,
+ settledOverride: null,
+ settledAt: null,
+ lastVisitedAt: null,
+ deletedAt: null,
+ },
+ },
+ {
+ id: EventId.make("event:run-finalized-test:scope"),
+ type: "checkpoint-scope.created",
+ threadId,
+ occurredAt: now,
+ payload: {
+ id: scopeId,
+ threadId,
+ runId,
+ nodeId: rootNodeId,
+ parentScopeId: null,
+ providerThreadId,
+ kind: "root_run",
+ ordinalWithinParent: 0,
+ advancesAppRunCount: true,
+ cwd: "/repo",
+ createdAt: now,
+ },
+ },
+ runEvent("run.created", makeRun(now, "running"), now),
+ ],
+ });
+});
+
+const storedEvents = Effect.gen(function* () {
+ const eventStore = yield* EventStore.EventStoreV2;
+ return Array.from(yield* eventStore.read({ threadId }).pipe(Stream.runCollect));
+});
+
+const finalizedEvents = (events: ReadonlyArray) =>
+ events.flatMap((stored) => (stored.event.type === "run.finalized" ? [stored] : []));
+
+/** Every finalization record for the thread, success or failure. */
+const finalizationRecords = storedEvents.pipe(
+ Effect.map((events) =>
+ events.flatMap((stored) =>
+ stored.event.type === "run.finalized" || stored.event.type === "run.finalization-failed"
+ ? [{ type: stored.event.type, payload: stored.event.payload }]
+ : [],
+ ),
+ ),
+);
+
+/** Runs every claimable effect on the real worker. */
+const drainWorker = EffectWorker.OrchestrationEffectWorkerV2.pipe(
+ Effect.flatMap((worker) => worker.drain()),
+);
+
+const captureStatus = EffectOutbox.EffectOutboxV2.pipe(
+ Effect.flatMap((outbox) => outbox.get(captureEffect.id)),
+ Effect.map(Option.map((effect) => effect.status)),
+);
+
+const runStatus = ProjectionStore.ProjectionStoreV2.pipe(
+ Effect.flatMap((projections) =>
+ projections.getCheckpointCaptureContext(threadId, { runId, scopeId }),
+ ),
+ Effect.map(({ run }) => run?.status),
+);
+
+/** The startup recovery a restarted server runs before its worker resumes. */
+const restartServer = ProviderRuntimeRecovery.ProviderRuntimeRecoveryService.pipe(
+ Effect.flatMap((recovery) => recovery.recover),
+);
+
+/** Stands in for CheckpointCaptureService: commits the checkpoint once, like the real one. */
+const commitCapture = (status: "completed" | "interrupted" | "cancelled") =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ yield* eventSink.commitCommand({
+ commandId: captureEffect.commandId,
+ threadId,
+ commandType: "checkpoint.capture",
+ acceptedAt: now,
+ effects: [],
+ events: [
+ runEvent("run.updated", makeRun(now, status, { checkpointId, completedAt: now }), now),
+ ],
+ });
+ }).pipe(Effect.orDie);
+
+const failCapture = Effect.fail(
+ new CheckpointCapture.CheckpointCaptureExecutionError({
+ threadId,
+ runId,
+ scopeId,
+ cause: "simulated capture failure",
+ }),
+);
+
+const failRefresh = () =>
+ Effect.fail(
+ new RunFinalization.RunFinalizationRefreshError({
+ cwd: "/repo",
+ cause: "simulated refresh failure",
+ }),
+ );
+
+/** Ends the run waiting on its capture, as RunExecutionService does. */
+const finishProviderTurn = (status: "waiting" | "interrupted" | "cancelled") =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ yield* eventSink.writeWithEffects({
+ events: [
+ runEvent(
+ "run.updated",
+ makeRun(now, status, status === "waiting" ? {} : { completedAt: now }),
+ now,
+ ),
+ ],
+ effects: [captureEffect],
+ });
+ });
+
+it.effect("finalizes a completed run once, after its checkpoint and workspace refresh", () => {
+ const refreshedAtSequence: Array = [];
+ let probe: Effect.Effect = Effect.void;
+ return Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ probe = eventSink.latestSequence({ threadId }).pipe(
+ Effect.map((sequence) => {
+ refreshedAtSequence.push(sequence);
+ }),
+ Effect.orDie,
+ );
+ yield* drainWorker;
+
+ const events = yield* storedEvents;
+ const finalized = finalizedEvents(events);
+ assert.lengthOf(finalized, 1);
+ const [stored] = finalized;
+ assert.deepEqual(stored?.event.payload, { runId, outcome: "completed", checkpointId });
+ assert.equal(stored?.event.id, EventId.make(`event:run-finalized:${runId}`));
+ const completedAt = events.find(
+ (event) => event.event.type === "run.updated" && event.event.payload.status === "completed",
+ )?.sequence;
+ assert.isDefined(completedAt);
+ assert.deepEqual(refreshedAtSequence, [completedAt!]);
+ assert.isTrue(stored!.sequence > completedAt!);
+ assert.deepEqual(yield* captureStatus, Option.some("succeeded"));
+ }).pipe(Effect.provide(makeLayer(commitCapture("completed"), () => probe)));
+});
+
+it.effect("recording the milestone leaves thread activity where the run left it", () =>
+ Effect.gen(function* () {
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ yield* drainWorker;
+
+ const events = yield* storedEvents;
+ const completed = events.find(
+ (event) => event.event.type === "run.updated" && event.event.payload.status === "completed",
+ );
+ const [milestone] = finalizedEvents(events);
+ assert.isDefined(completed);
+ assert.isDefined(milestone);
+ // The slow refresh put the milestone a minute after the run's last write.
+ assert.isTrue(DateTime.isGreaterThan(milestone.event.occurredAt, completed.event.occurredAt));
+ const shell = yield* projections.getThreadShell(threadId);
+ assert.deepEqual(shell?.updatedAt, completed.event.occurredAt);
+ }).pipe(
+ Effect.provide(makeLayer(commitCapture("completed"), () => TestClock.adjust("60 seconds"))),
+ ),
+);
+
+it.effect("a restart after finalizing but before settling records one milestone", () => {
+ let captures = 0;
+ return Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
+ const finalization = yield* RunFinalization.RunFinalizationService;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ // A worker finalized, then its process died before settling the effect.
+ yield* outbox.claimNext({ workerId: "worker:crashed" });
+ yield* finalization.finalize({ threadId, runId, scopeId });
+ assert.lengthOf(finalizedEvents(yield* storedEvents), 1);
+
+ // Restart requeues the capture; the worker settles it without capturing again.
+ const summary = yield* restartServer;
+ assert.equal(summary.requeuedEffects, 1);
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("succeeded"));
+ assert.equal(captures, 1);
+ // A later write of the finished run adds nothing either.
+ const now = yield* DateTime.now;
+ yield* eventSink.write({
+ events: [
+ runEvent("run.updated", makeRun(now, "completed", { checkpointId, completedAt: now }), now),
+ ],
+ });
+ assert.deepEqual(yield* finalizationRecords, [
+ { type: "run.finalized", payload: { runId, outcome: "completed", checkpointId } },
+ ]);
+ }).pipe(
+ Effect.provide(
+ makeLayer(
+ Effect.suspend(() => {
+ captures += 1;
+ return commitCapture("completed");
+ }),
+ ),
+ ),
+ );
+});
+
+it.effect.each(["interrupted", "cancelled"] as const)(
+ "a stopped run with a capture finalizes as %s after the capture",
+ (status) =>
+ Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn(status);
+ assert.lengthOf(yield* finalizationRecords, 0);
+ yield* drainWorker;
+ assert.deepEqual(yield* finalizationRecords, [
+ { type: "run.finalized", payload: { runId, outcome: status, checkpointId } },
+ ]);
+ }).pipe(Effect.provide(makeLayer(commitCapture(status)))),
+);
+
+/** Ends a running attempt through the ownership guard, as RunExecutionService does. */
+const finishGuardedTurn = (
+ status: "interrupted" | "cancelled",
+ guard: { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: "running" | "waiting" },
+) =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ yield* eventSink.write({
+ events: [
+ runEvent("run.updated", makeRun(now, "running", { activeAttemptId: attemptId }), now),
+ ],
+ });
+ return yield* eventSink.writeIfRunCurrent({
+ threadId,
+ runId,
+ ...guard,
+ events: [
+ runEvent(
+ "run.updated",
+ makeRun(now, status, { activeAttemptId: attemptId, completedAt: now }),
+ now,
+ ),
+ ],
+ effects: [captureEffect],
+ });
+ });
+
+it.effect.each(["interrupted", "cancelled"] as const)(
+ "a guarded stop that queues a capture finalizes as %s after the capture",
+ (status) =>
+ Effect.gen(function* () {
+ yield* seedThread;
+ const result = yield* finishGuardedTurn(status, {
+ activeAttemptId: attemptId,
+ expectedStatus: "running",
+ });
+ assert.isTrue(result.committed);
+ assert.lengthOf(yield* finalizationRecords, 0);
+ yield* drainWorker;
+ assert.deepEqual(yield* finalizationRecords, [
+ { type: "run.finalized", payload: { runId, outcome: status, checkpointId } },
+ ]);
+ }).pipe(Effect.provide(makeLayer(commitCapture(status)))),
+);
+
+it.effect("a guarded stop that no longer owns the run commits and queues nothing", () =>
+ Effect.gen(function* () {
+ yield* seedThread;
+ const result = yield* finishGuardedTurn("interrupted", {
+ activeAttemptId: RunAttemptId.make("attempt:run-finalized-stale"),
+ expectedStatus: "running",
+ });
+ assert.isFalse(result.committed);
+ assert.deepEqual(yield* captureStatus, Option.none());
+ assert.equal(yield* drainWorker, 0);
+ assert.equal(yield* runStatus, "running");
+ assert.lengthOf(yield* finalizationRecords, 0);
+ }).pipe(Effect.provide(makeLayer(commitCapture("interrupted")))),
+);
+
+it.effect("a run that ends without a capture finalizes in its terminal commit", () =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ yield* seedThread;
+ const now = yield* DateTime.now;
+ const failed = makeRun(now, "failed", { completedAt: now });
+ const [terminal, milestone] = yield* eventSink.write({
+ events: [runEvent("run.updated", failed, now)],
+ });
+ assert.equal(terminal?.event.type, "run.updated");
+ assert.equal(milestone?.event.type, "run.finalized");
+ assert.equal(milestone?.commandId, terminal?.commandId);
+ // A repeated terminal write and a later update to the finished run add nothing.
+ yield* eventSink.write({ events: [runEvent("run.updated", failed, now)] });
+ yield* eventSink.write({
+ events: [runEvent("run.updated", { ...failed, status: "cancelled" }, now)],
+ });
+ assert.deepEqual(
+ finalizedEvents(yield* storedEvents).map((stored) => stored.event.payload),
+ [{ runId, outcome: "failed", checkpointId: null }],
+ );
+ }).pipe(Effect.provide(makeLayer(Effect.void))),
+);
+
+it.effect("unfinished, discarded, and previously finished runs never finalize", () =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const finalization = yield* RunFinalization.RunFinalizationService;
+ yield* seedThread;
+ const now = yield* DateTime.now;
+ // Waiting on a capture that never ran.
+ yield* finishProviderTurn("waiting");
+ assert.lengthOf(yield* finalizationRecords, 0);
+ // Rolled back before the capture ran: discarded, and the capture skips it.
+ yield* eventSink.write({
+ events: [runEvent("run.updated", makeRun(now, "rolled_back", { completedAt: now }), now)],
+ });
+ yield* finalization.finalize({ threadId, runId, scopeId });
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ // A run that finished before this milestone existed has no event. A later
+ // update to it must not invent one.
+ const historical = makeRun(now, "completed", {
+ id: RunId.make("run:run-finalized-historical"),
+ ordinal: 2,
+ completedAt: now,
+ });
+ yield* eventSink.write({ events: [runEvent("run.created", historical, now)] });
+ yield* eventSink.write({ events: [runEvent("run.updated", historical, now)] });
+ assert.lengthOf(yield* finalizationRecords, 0);
+ }).pipe(Effect.provide(makeLayer(Effect.void))),
+);
+
+it.effect.each(["waiting", "interrupted"] as const)(
+ "a capture that gives up after a %s turn records the failure, never run.finalized",
+ (status) =>
+ Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn(status);
+ // The first failure will be retried, so nothing is recorded yet.
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("pending"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ yield* TestClock.adjust(firstRetryDelay);
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("failed"));
+ const failure = [
+ {
+ type: "run.finalization-failed" as const,
+ payload: { runId, operation: "capture-checkpoint" as const },
+ },
+ ];
+ assert.deepEqual(yield* finalizationRecords, failure);
+
+ // A restart cancels a run still waiting on the failed capture. That is
+ // not a finalization either.
+ yield* restartServer;
+ assert.equal(yield* runStatus, status === "waiting" ? "cancelled" : "interrupted");
+ assert.deepEqual(yield* finalizationRecords, failure);
+ }).pipe(Effect.provide(makeLayer(failCapture))),
+);
+
+it.effect("a failed display refresh finalizes the durable checkpoint across restart", () =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ yield* drainWorker;
+ yield* TestClock.adjust(firstRetryDelay);
+ yield* drainWorker;
+
+ assert.deepEqual(yield* captureStatus, Option.some("succeeded"));
+ assert.equal(yield* runStatus, "completed");
+ const finalized = [
+ {
+ type: "run.finalized" as const,
+ payload: { runId, outcome: "completed" as const, checkpointId },
+ },
+ ];
+ assert.deepEqual(yield* finalizationRecords, finalized);
+ // Neither a restart nor a later write duplicates the recorded finalization.
+ yield* restartServer;
+ const now = yield* DateTime.now;
+ yield* eventSink.write({
+ events: [
+ runEvent("run.updated", makeRun(now, "completed", { checkpointId, completedAt: now }), now),
+ ],
+ });
+ assert.deepEqual(yield* finalizationRecords, finalized);
+ }).pipe(Effect.provide(makeLayer(commitCapture("completed"), failRefresh))),
+);
+
+it.effect("a failed display refresh is not retried after run.finalized", () => {
+ let refreshes = 0;
+ return Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ yield* drainWorker;
+ yield* TestClock.adjust(firstRetryDelay);
+ yield* drainWorker;
+ assert.equal(refreshes, 1);
+ assert.deepEqual(yield* finalizationRecords, [
+ { type: "run.finalized", payload: { runId, outcome: "completed", checkpointId } },
+ ]);
+ }).pipe(
+ Effect.provide(
+ makeLayer(commitCapture("completed"), () =>
+ Effect.suspend(() => ((refreshes += 1) === 1 ? failRefresh() : Effect.void)),
+ ),
+ ),
+ );
+});
+
+it.effect(
+ "a capture cancelled outside a command records the failure when restart ends the run",
+ () =>
+ Effect.gen(function* () {
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ yield* outbox.cancelUnsettled({
+ threadId,
+ effectTypes: ["checkpoint.capture"],
+ reason: "test",
+ });
+ assert.deepEqual(yield* captureStatus, Option.some("cancelled"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+ yield* restartServer;
+ assert.equal(yield* runStatus, "cancelled");
+ assert.deepEqual(yield* finalizationRecords, [
+ {
+ type: "run.finalization-failed",
+ payload: { runId, operation: "capture-checkpoint" },
+ },
+ ]);
+ }).pipe(Effect.provide(makeLayer(Effect.void))),
+);
+
+it.effect("a command that cancels a capture records the failure in its commit", () =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ yield* seedThread;
+ yield* finishProviderTurn("interrupted");
+ const now = yield* DateTime.now;
+ const commandId = CommandId.make("command:run-finalized-test:cancel-capture");
+ const { storedEvents, receipt } = yield* eventSink.commitCommand({
+ commandId,
+ threadId,
+ commandType: "test.cancel-capture",
+ acceptedAt: now,
+ events: [runEvent("run.updated", makeRun(now, "interrupted", { completedAt: now }), now)],
+ effects: [],
+ cancelUnsettledEffects: { effectTypes: ["checkpoint.capture"], reason: "test" },
+ });
+ const failure = [
+ {
+ type: "run.finalization-failed" as const,
+ payload: { runId, operation: "capture-checkpoint" as const },
+ },
+ ];
+ assert.deepEqual(yield* captureStatus, Option.some("cancelled"));
+ assert.deepEqual(yield* finalizationRecords, failure);
+ const recorded = storedEvents.at(-1);
+ assert.equal(recorded?.event.type, "run.finalization-failed");
+ assert.equal(recorded?.commandId, commandId);
+ assert.equal(receipt.resultSequence, recorded?.sequence);
+
+ // Nothing runs the cancelled capture, and a restart adds nothing.
+ assert.equal(yield* drainWorker, 0);
+ yield* restartServer;
+ assert.deepEqual(yield* finalizationRecords, failure);
+ }).pipe(Effect.provide(makeLayer(Effect.die("a cancelled capture must not run")))),
+);
+
+const captureDefect = Effect.die("simulated unexpected checkpoint defect");
+const refreshDefect = () => Effect.die("simulated unexpected refresh defect");
+
+it.effect.each([
+ {
+ label: "a capture defect after a waiting turn",
+ turn: "waiting" as const,
+ capture: captureDefect,
+ refresh: undefined,
+ operation: "capture-checkpoint" as const,
+ runAfterRestart: "cancelled",
+ },
+ {
+ label: "a capture defect after an interrupted turn",
+ turn: "interrupted" as const,
+ capture: captureDefect,
+ refresh: undefined,
+ operation: "capture-checkpoint" as const,
+ runAfterRestart: "interrupted",
+ },
+ {
+ label: "a refresh defect after a completed run's checkpoint",
+ turn: "waiting" as const,
+ capture: commitCapture("completed"),
+ refresh: refreshDefect,
+ operation: "refresh-workspace" as const,
+ runAfterRestart: "completed",
+ },
+ {
+ label: "a refresh defect after an interrupted run's checkpoint",
+ turn: "interrupted" as const,
+ capture: commitCapture("interrupted"),
+ refresh: refreshDefect,
+ operation: "refresh-workspace" as const,
+ runAfterRestart: "interrupted",
+ },
+])("$label records the failure when retries run out", (testCase) =>
+ Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn(testCase.turn);
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("pending"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ yield* TestClock.adjust(firstRetryDelay);
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("failed"));
+ const failure = [
+ {
+ type: "run.finalization-failed" as const,
+ payload: { runId, operation: testCase.operation },
+ },
+ ];
+ assert.deepEqual(yield* finalizationRecords, failure);
+
+ yield* restartServer;
+ yield* drainWorker;
+ assert.equal(yield* runStatus, testCase.runAfterRestart);
+ assert.deepEqual(yield* captureStatus, Option.some("failed"));
+ assert.deepEqual(yield* finalizationRecords, failure);
+ }).pipe(Effect.provide(makeLayer(testCase.capture, testCase.refresh))),
+);
+
+it.effect("a failure record that cannot commit keeps the capture recoverable", () => {
+ let faults = 1;
+ let captures = 0;
+ return Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ yield* drainWorker;
+ yield* TestClock.adjust(firstRetryDelay);
+ // The last attempt fails and its failure record cannot commit.
+ const exit = yield* Effect.exit(drainWorker);
+ assert.isTrue(Exit.isFailure(exit));
+ assert.equal(faults, 0);
+ // The rolled-back commit neither failed the capture nor recorded anything.
+ assert.deepEqual(yield* captureStatus, Option.some("pending"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+ // The capture waits out the backoff instead of running again at once.
+ yield* drainWorker;
+ assert.equal(captures, 2);
+ assert.deepEqual(yield* captureStatus, Option.some("pending"));
+
+ // A restart keeps the work; the next attempt gives up and records it.
+ yield* restartServer;
+ yield* TestClock.adjust(lastRetryDelay);
+ yield* drainWorker;
+ assert.deepEqual(yield* captureStatus, Option.some("failed"));
+ assert.deepEqual(yield* finalizationRecords, [
+ {
+ type: "run.finalization-failed",
+ payload: { runId, operation: "capture-checkpoint" },
+ },
+ ]);
+ }).pipe(
+ Effect.provide(
+ makeLayer(
+ Effect.suspend(() => {
+ captures += 1;
+ return failCapture;
+ }),
+ undefined,
+ (sink) => ({
+ ...sink,
+ // Also append an event whose id is taken, so the commit fails after
+ // the capture's terminal status was written inside it.
+ failEffect: (input) =>
+ Effect.suspend(() => {
+ if (faults === 0) return sink.failEffect(input);
+ faults -= 1;
+ const at = input.events[0]!.occurredAt;
+ return sink.failEffect({
+ ...input,
+ events: [
+ ...input.events,
+ {
+ ...runEvent("run.updated", makeRun(at, "waiting"), at),
+ id: EventId.make("event:run-finalized-test:thread"),
+ },
+ ],
+ });
+ }),
+ }),
+ ),
+ ),
+ );
+});
+
+it.effect("a restart after a failure record but before settling never finalizes again", () => {
+ let captures = 0;
+ return Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ // A worker recorded the failure, then its process died before settling
+ // the capture (the state a non-atomic writer could leave).
+ yield* outbox.claimNext({ workerId: "worker:crashed" });
+ const context = yield* ProjectionStore.ProjectionStoreV2.pipe(
+ Effect.flatMap((projections) =>
+ projections.getCheckpointCaptureContext(threadId, { runId, scopeId }),
+ ),
+ );
+ yield* eventSink.write({
+ events: [
+ makeRunFinalizationFailedEvent({
+ run: context.run!,
+ operation: "capture-checkpoint",
+ occurredAt: yield* DateTime.now,
+ }),
+ ],
+ });
+
+ const summary = yield* restartServer;
+ assert.equal(summary.requeuedEffects, 1);
+ yield* drainWorker;
+ assert.equal(captures, 0);
+ assert.deepEqual(yield* captureStatus, Option.some("succeeded"));
+ assert.deepEqual(yield* finalizationRecords, [
+ {
+ type: "run.finalization-failed",
+ payload: { runId, operation: "capture-checkpoint" },
+ },
+ ]);
+ }).pipe(
+ Effect.provide(
+ makeLayer(
+ Effect.suspend(() => {
+ captures += 1;
+ return commitCapture("completed");
+ }),
+ ),
+ ),
+ );
+});
+
+it.effect("an interrupted capture is replayed, never recorded as a failure", () => {
+ let captures = 0;
+ const started = Deferred.makeUnsafe();
+ return Effect.gen(function* () {
+ yield* seedThread;
+ yield* finishProviderTurn("waiting");
+ // Interrupted on every attempt in a live worker: retried, not given up.
+ yield* drainWorker;
+ yield* TestClock.adjust(firstRetryDelay);
+ yield* drainWorker;
+ yield* TestClock.adjust("30 seconds");
+ assert.deepEqual(yield* captureStatus, Option.some("pending"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ // A shutdown interrupts the worker mid-capture and leaves the claim running.
+ const shuttingDown = yield* Effect.forkChild(drainWorker);
+ yield* Deferred.await(started);
+ yield* Fiber.interrupt(shuttingDown);
+ assert.deepEqual(yield* captureStatus, Option.some("running"));
+ assert.lengthOf(yield* finalizationRecords, 0);
+
+ // Restart replays it and the capture now succeeds.
+ yield* restartServer;
+ yield* drainWorker;
+ assert.equal(captures, 4);
+ assert.deepEqual(yield* finalizationRecords, [
+ { type: "run.finalized", payload: { runId, outcome: "completed", checkpointId } },
+ ]);
+ }).pipe(
+ Effect.provide(
+ makeLayer(
+ Effect.suspend(() => {
+ captures += 1;
+ return captures <= 2
+ ? Effect.interrupt
+ : captures === 3
+ ? Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never))
+ : commitCapture("completed");
+ }),
+ ),
+ ),
+ );
+});
diff --git a/apps/server/src/orchestration-v2/RunFinalized.ts b/apps/server/src/orchestration-v2/RunFinalized.ts
new file mode 100644
index 0000000000000..c63292032030b
--- /dev/null
+++ b/apps/server/src/orchestration-v2/RunFinalized.ts
@@ -0,0 +1,88 @@
+import {
+ EventId,
+ type OrchestrationV2DomainEvent,
+ type OrchestrationV2Run,
+ type OrchestrationV2RunFinalizationOperation,
+ type OrchestrationV2RunFinalizedOutcome,
+ type RunId,
+} from "@t3tools/contracts";
+import type * as DateTime from "effect/DateTime";
+
+/**
+ * The id of a run's one finalization record, `run.finalized` or
+ * `run.finalization-failed`. The event log's unique event id keeps a run to
+ * one of them, and consumers can deduplicate deliveries by it.
+ */
+export const runFinalizedEventId = (runId: RunId) => EventId.make(`event:run-finalized:${runId}`);
+
+/**
+ * The checkpoint capture a finished run enqueues. RunFinalizationService
+ * records `run.finalized` once capture and refresh succeed, or
+ * `run.finalization-failed` in the same commit that gives up on the capture.
+ * Cancelling the capture records that failure too.
+ */
+export const checkpointCaptureEffectId = (runId: RunId) => `effect:checkpoint.capture:${runId}`;
+
+const SETTLED_RUN_STATUSES: ReadonlySet = new Set([
+ "completed",
+ "failed",
+ "interrupted",
+ "cancelled",
+ "rolled_back",
+]);
+
+/** Whether a persisted run status is final, including a discarded (rolled-back) run. */
+export const isSettledRunStatus = (status: string) => SETTLED_RUN_STATUSES.has(status);
+
+/** The finalized outcome for a run status, or null when the run is not finished or was discarded. */
+export const runFinalizedOutcome = (
+ status: OrchestrationV2Run["status"],
+): OrchestrationV2RunFinalizedOutcome | null =>
+ status === "completed" ||
+ status === "failed" ||
+ status === "interrupted" ||
+ status === "cancelled"
+ ? status
+ : null;
+
+/**
+ * The step a run's finalization stopped at when its capture was abandoned
+ * without reporting one: before the checkpoint commit or after it.
+ */
+export const abandonedOperation = (
+ run: OrchestrationV2Run,
+): OrchestrationV2RunFinalizationOperation =>
+ run.checkpointId === null ? "capture-checkpoint" : "refresh-workspace";
+
+const recordEnvelope = (run: OrchestrationV2Run, occurredAt: DateTime.Utc) => ({
+ id: runFinalizedEventId(run.id),
+ threadId: run.threadId,
+ runId: run.id,
+ ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
+ providerInstanceId: run.providerInstanceId,
+ occurredAt,
+});
+
+export const makeRunFinalizedEvent = (input: {
+ readonly run: OrchestrationV2Run;
+ readonly outcome: OrchestrationV2RunFinalizedOutcome;
+ readonly occurredAt: DateTime.Utc;
+}): Extract => ({
+ ...recordEnvelope(input.run, input.occurredAt),
+ type: "run.finalized",
+ payload: {
+ runId: input.run.id,
+ outcome: input.outcome,
+ checkpointId: input.run.checkpointId,
+ },
+});
+
+export const makeRunFinalizationFailedEvent = (input: {
+ readonly run: OrchestrationV2Run;
+ readonly operation: OrchestrationV2RunFinalizationOperation;
+ readonly occurredAt: DateTime.Utc;
+}): Extract => ({
+ ...recordEnvelope(input.run, input.occurredAt),
+ type: "run.finalization-failed",
+ payload: { runId: input.run.id, operation: input.operation },
+});
diff --git a/apps/server/src/orchestration-v2/RunMetrics.test.ts b/apps/server/src/orchestration-v2/RunMetrics.test.ts
index fead1c29a969e..aa07165f7c5bf 100644
--- a/apps/server/src/orchestration-v2/RunMetrics.test.ts
+++ b/apps/server/src/orchestration-v2/RunMetrics.test.ts
@@ -271,8 +271,20 @@ describe("EventSink run outcomes", () => {
yield* Fiber.join(terminal),
yield* Fiber.join(unrelated),
];
- assert.strictEqual(terminalEvents.length, 1);
- assert.strictEqual(unrelatedEvents.length, 1);
+ assert.deepEqual(
+ terminalEvents.map((stored) => stored.event.type),
+ ["run.updated", "run.finalized"],
+ );
+ assert.deepEqual(terminalEvents[0]?.event, runEvent("run.updated", "completed"));
+ assert.deepEqual(terminalEvents[1]?.event.payload, {
+ runId,
+ outcome: "completed",
+ checkpointId: null,
+ });
+ assert.deepEqual(
+ unrelatedEvents.map((stored) => stored.event.type),
+ ["thread.created"],
+ );
const counter = (yield* runSnapshots("p-sink")).find(
(snapshot) => snapshot.id === "t3_runs_total",
);
diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
index 8e1ca75fcac98..da5ebe62ff3d9 100644
--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts
+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
@@ -20,6 +20,7 @@ import {
type OrchestrationV2Run,
type OrchestrationV2RuntimeRequest,
ProjectId,
+ PluginInstallationId,
type PullRequestDetail,
type PullRequestComment,
PullRequestOperationError,
@@ -40,6 +41,8 @@ import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
+import * as Ref from "effect/Ref";
+import * as Exit from "effect/Exit";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import * as SqlClient from "effect/sql/SqlClient";
@@ -53,6 +56,8 @@ import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.t
import * as ProjectService from "../project/ProjectService.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts";
+import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
+import * as PluginTools from "../plugins/PluginTools.ts";
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
@@ -273,25 +278,30 @@ const moveProject = (projectId: ProjectId, workspaceRoot: string, updatedAt: str
}),
);
-const layerTest = Layer.mergeAll(
- RuntimeLayer.layer,
- RuntimeLayer.layerEventSink,
- ProjectStore.layer,
- ProjectionStore.layer,
- EffectOutbox.layer,
- ThreadCommandExecutor.layer,
-).pipe(
- Layer.provide(McpSessionRegistryTestkit.layer),
- Layer.provide(McpProviderSessions.layer),
- Layer.provide(SqlitePersistence.layerMemory),
- Layer.provide(layerCheckpointStoreTest),
- Layer.provide(layerServerConfig),
- Layer.provide(ServerSettings.layerTest()),
- Layer.provide(layerTestProviderInstanceRegistry),
- Layer.provide(layerGitWorkflowTest),
- Layer.provide(layerProjectServiceTest),
- Layer.provide(layerPlatformTest),
-);
+const layerTestWithMcpRegistry = (
+ registryLayer: Layer.Layer,
+) =>
+ Layer.mergeAll(
+ RuntimeLayer.layer,
+ RuntimeLayer.layerEventSink,
+ ProjectStore.layer,
+ ProjectionStore.layer,
+ EffectOutbox.layer,
+ ThreadCommandExecutor.layer,
+ ).pipe(
+ Layer.provide(registryLayer),
+ Layer.provide(McpProviderSessions.layer),
+ Layer.provide(SqlitePersistence.layerMemory),
+ Layer.provide(layerCheckpointStoreTest),
+ Layer.provide(layerServerConfig),
+ Layer.provide(ServerSettings.layerTest()),
+ Layer.provide(layerTestProviderInstanceRegistry),
+ Layer.provide(layerGitWorkflowTest),
+ Layer.provide(layerProjectServiceTest),
+ Layer.provide(layerPlatformTest),
+ );
+
+const layerTest = layerTestWithMcpRegistry(McpSessionRegistryTestkit.layer);
const layerLegacyImportTest = RuntimeLayer.layer.pipe(
Layer.provide(McpSessionRegistryTestkit.layer),
@@ -8501,3 +8511,70 @@ it.layer(layerQueueResumeDisabledInstanceTest)(
);
},
);
+
+it.effect("runtime layers snapshot outer plugin tool grants into session credentials", () =>
+ Effect.gen(function* () {
+ const requests = yield* Ref.make>([]);
+ const grants = [
+ { installationId: PluginInstallationId.make("runtime-tool-plugin"), generation: 3 },
+ ];
+ const registryLayer = Layer.effect(
+ McpSessionRegistry.McpSessionRegistry,
+ Effect.gen(function* () {
+ const delegate = yield* McpSessionRegistry.McpSessionRegistry;
+ return McpSessionRegistry.McpSessionRegistry.of({
+ ...delegate,
+ issue: (request) =>
+ delegate
+ .issue(request)
+ .pipe(Effect.tap(() => Ref.update(requests, (current) => [...current, request]))),
+ });
+ }),
+ ).pipe(Layer.provide(McpSessionRegistryTestkit.layer));
+ yield* Effect.gen(function* () {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
+ const threadId = ThreadId.make("runtime-plugin-grants-thread");
+ yield* orchestrator.dispatch({
+ type: "thread.create",
+ createdBy: "user",
+ creationSource: "web",
+ commandId: CommandId.make("runtime-plugin-grants-create"),
+ threadId,
+ projectId: ProjectId.make("runtime-plugin-grants-project"),
+ title: "Plugin grants",
+ modelSelection,
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: process.cwd(),
+ });
+ // The lifecycle adapter deliberately refuses to open; credentials are issued before that boundary.
+ const opened = yield* manager
+ .open({
+ threadId,
+ providerSessionId: ProviderSessionId.make("runtime-plugin-grants-session"),
+ modelSelection,
+ runtimePolicy: {
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ cwd: process.cwd(),
+ },
+ })
+ .pipe(Effect.exit);
+ assert.isTrue(Exit.isFailure(opened));
+ assert.deepEqual(
+ (yield* Ref.get(requests)).map((request) => request.pluginToolGrants),
+ [grants],
+ );
+ }).pipe(
+ Effect.provide(
+ layerTestWithMcpRegistry(registryLayer).pipe(
+ Layer.provideMerge(
+ Layer.mock(PluginTools.PluginTools)({ grants: Effect.succeed(grants) }),
+ ),
+ ),
+ ),
+ );
+ }),
+);
diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts
index 1b54627595372..803d8b13aa674 100644
--- a/apps/server/src/orchestration-v2/runtimeLayer.ts
+++ b/apps/server/src/orchestration-v2/runtimeLayer.ts
@@ -207,7 +207,9 @@ const layerCheckpointCaptureServiceProvided = CheckpointCaptureService.layer.pip
),
);
const layerRunFinalizationServiceProvided = RunFinalizationService.layer.pipe(
- Layer.provide(Layer.merge(layerCheckpointCaptureServiceProvided, ProjectionStore.layer)),
+ Layer.provide(
+ Layer.mergeAll(layerCheckpointCaptureServiceProvided, layerEventSink, ProjectionStore.layer),
+ ),
);
const layerOrchestratorProvided = Orchestrator.layer.pipe(
diff --git a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts
index 77e8bae8f085f..8a3b6a7f08df4 100644
--- a/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts
+++ b/apps/server/src/orchestration-v2/testkit/OrchestratorScenario.ts
@@ -210,8 +210,18 @@ function scenarioCommands(scenario: OrchestratorV2Scenario): ReadonlyArray
- projection.runtimeRequests.find((request) => request.status === "pending");
+ projection.runtimeRequests.find(
+ (request) =>
+ request.status === "pending" &&
+ projection.turnItems.some(
+ (item) =>
+ (item.type === "approval_request" || item.type === "user_input_request") &&
+ item.requestId === request.id,
+ ),
+ );
const hasActiveRun = (projection: OrchestrationV2ThreadProjection) =>
projection.runs.some((run) =>
diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
index afd11a9ab7ecb..6402c10d4a983 100644
--- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
@@ -444,7 +444,9 @@ export function layerWithRegistry(
),
);
const layerRunFinalizationServiceProvided = RunFinalizationService.layer.pipe(
- Layer.provide(Layer.merge(layerCheckpointCaptureServiceProvided, layerStores)),
+ Layer.provide(
+ Layer.mergeAll(layerCheckpointCaptureServiceProvided, layerEventSinkProvided, layerStores),
+ ),
);
const layerThreadTitleRegenerationTest = Layer.succeed(
ThreadTitleRegenerationService.ThreadTitleRegenerationService,
diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts
index e13c2b449d9e7..a16b2973b8fa3 100644
--- a/apps/server/src/persistence/Migrations.ts
+++ b/apps/server/src/persistence/Migrations.ts
@@ -1,6 +1,7 @@
import * as Migrator from "effect/sql/Migrator";
import * as Effect from "effect/Effect";
import * as SqlClient from "effect/sql/SqlClient";
+import { reconcilePluginPreviewMigrations } from "./reconcilePluginPreviewMigrations.ts";
import { reconcileV2PreviewMigration } from "./reconcileV2PreviewMigration.ts";
import Migration0001 from "./Migrations/001_OrchestrationEvents.ts";
@@ -79,6 +80,9 @@ import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts";
import Migration0074 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts";
import Migration0075 from "./Migrations/075_SubagentDeliveryState.ts";
import Migration0076 from "./Migrations/076_ProjectionProjectsPinArchive.ts";
+import Migration0077 from "./Migrations/077_PluginInstallations.ts";
+import Migration0078 from "./Migrations/078_PluginEventCursors.ts";
+import Migration0079 from "./Migrations/079_PluginSettings.ts";
const migrationEntries = [
[1, "OrchestrationEvents", Migration0001],
@@ -158,6 +162,9 @@ const migrationEntries = [
[74, "ExcludeAttachmentPlaceholdersFromSearch", Migration0074],
[75, "SubagentDeliveryState", Migration0075],
[76, "ProjectionProjectsPinArchive", Migration0076],
+ [77, "PluginInstallations", Migration0077],
+ [78, "PluginEventCursors", Migration0078],
+ [79, "PluginSettings", Migration0079],
] as const;
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
@@ -218,11 +225,15 @@ export const runMigrations = Effect.fn("runMigrations")(function* ({
const includesV2 =
toMigrationInclusive === undefined || toMigrationInclusive >= FIRST_V2_MIGRATION_ID;
const previewMigrations = includesV2 ? yield* reconcileV2PreviewMigration() : [];
+ const pluginMigrations = includesV2
+ ? yield* reconcilePluginPreviewMigrations(migrationManifest, toMigrationInclusive)
+ : [];
if (includesV2) {
yield* assertV2MigrationsCanRun(toMigrationInclusive);
}
const executedMigrations = [
...previewMigrations,
+ ...pluginMigrations,
...(yield* run({ loader: makeMigrationLoader(toMigrationInclusive) })),
];
const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`);
diff --git a/apps/server/src/persistence/Migrations/077_PluginInstallations.ts b/apps/server/src/persistence/Migrations/077_PluginInstallations.ts
new file mode 100644
index 0000000000000..fc8e31c2cd87e
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/077_PluginInstallations.ts
@@ -0,0 +1,16 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // One row per plugin directory added to this environment. `record_json` holds the last
+ // inspection and the consent; the directory is unique so one folder is one installation.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS plugin_installations (
+ installation_id TEXT PRIMARY KEY,
+ directory TEXT NOT NULL UNIQUE,
+ record_json TEXT NOT NULL
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/Migrations/078_PluginEventCursors.ts b/apps/server/src/persistence/Migrations/078_PluginEventCursors.ts
new file mode 100644
index 0000000000000..cf24e4b054559
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/078_PluginEventCursors.ts
@@ -0,0 +1,16 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // One row per plugin installation that receives events: the event log sequence it has
+ // acknowledged through. Removing the installation removes its row.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS plugin_event_cursors (
+ installation_id TEXT PRIMARY KEY,
+ acknowledged_sequence INTEGER NOT NULL,
+ updated_at TEXT NOT NULL
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/Migrations/079_PluginSettings.ts b/apps/server/src/persistence/Migrations/079_PluginSettings.ts
new file mode 100644
index 0000000000000..0dde99c4fb122
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/079_PluginSettings.ts
@@ -0,0 +1,37 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // Saved plugin setting values per installation, secrets excepted.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS plugin_settings (
+ installation_id TEXT NOT NULL,
+ key TEXT NOT NULL,
+ value_json TEXT NOT NULL,
+ PRIMARY KEY (installation_id, key)
+ )
+ `;
+ // Secrets whose value may be in the server secret store. A row is written before its file and
+ // deleted after it, so cleanup can always find the file; `saved` is 1 once the value is complete
+ // and 0 while it is written or deleted.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS plugin_setting_secrets (
+ installation_id TEXT NOT NULL,
+ key TEXT NOT NULL,
+ saved INTEGER NOT NULL,
+ PRIMARY KEY (installation_id, key)
+ )
+ `;
+ // Each installation's private key-value storage; `bytes` is the value's encoded size for quotas.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS plugin_storage (
+ installation_id TEXT NOT NULL,
+ key TEXT NOT NULL,
+ value_json TEXT NOT NULL,
+ bytes INTEGER NOT NULL,
+ PRIMARY KEY (installation_id, key)
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/reconcilePluginPreviewMigrations.test.ts b/apps/server/src/persistence/reconcilePluginPreviewMigrations.test.ts
new file mode 100644
index 0000000000000..c2544749db0b5
--- /dev/null
+++ b/apps/server/src/persistence/reconcilePluginPreviewMigrations.test.ts
@@ -0,0 +1,182 @@
+import { assert, describe, it } from "@effect/vitest";
+import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Migrator from "effect/sql/Migrator";
+import * as SqlClient from "effect/sql/SqlClient";
+
+import { migrationManifest, runMigrations } from "./Migrations.ts";
+import PluginInstallations from "./Migrations/077_PluginInstallations.ts";
+import PluginEventCursors from "./Migrations/078_PluginEventCursors.ts";
+import PluginSettings from "./Migrations/079_PluginSettings.ts";
+
+const previews = [
+ [67, "PluginInstallations", PluginInstallations],
+ [68, "PluginEventCursors", PluginEventCursors],
+ [69, "PluginSettings", PluginSettings],
+] as const;
+const seed = (through: number, start = 67) =>
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* runMigrations({ toMigrationInclusive: start - 1 });
+ yield* Migrator.make({})({
+ loader: Migrator.fromRecord(
+ Object.fromEntries(
+ previews
+ .filter(([id]) => start + id - 67 <= through)
+ .map(([id, name, migration]) => [`${start + id - 67}_${name}`, migration]),
+ ),
+ ),
+ });
+ yield* sql`INSERT INTO plugin_installations VALUES ('sentinel', '/plugins/sentinel', '{"consent":"retained"}')`;
+ if (through >= start + 1)
+ yield* sql`INSERT INTO plugin_event_cursors VALUES ('sentinel', 321, 'retained-time')`;
+ if (through >= start + 2) {
+ yield* sql`INSERT INTO plugin_settings VALUES ('sentinel', 'setting', '{"retained":true}')`;
+ yield* sql`INSERT INTO plugin_setting_secrets VALUES ('sentinel', 'secret', 1)`;
+ yield* sql`INSERT INTO plugin_storage VALUES ('sentinel', 'key', '{"retained":true}', 17)`;
+ }
+ yield* sql`UPDATE effect_sql_migrations SET created_at = '2026-10-01 00:00:00' WHERE migration_id >= ${start}`;
+ });
+const snapshot = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const schema = yield* sql<{
+ readonly name: string;
+ readonly type: string;
+ readonly sql: string | null;
+ }>`SELECT name, type, sql FROM sqlite_master ORDER BY name`;
+ const data = [];
+ for (const row of schema) {
+ if (
+ row.type === "table" &&
+ (row.name.startsWith("plugin_") || row.name === "effect_sql_migrations")
+ )
+ data.push([row.name, yield* sql.unsafe(`SELECT * FROM "${row.name}" ORDER BY 1`)]);
+ }
+ return { schema, data };
+});
+const assertCurrent = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ assert.deepStrictEqual(
+ (yield* sql<{
+ readonly migration_id: number;
+ readonly name: string;
+ }>`SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id`).map(
+ (row) => [row.migration_id, row.name] as const,
+ ),
+ migrationManifest,
+ );
+ assert.strictEqual(
+ (yield* sql`PRAGMA table_info(scheduled_tasks)`).some(
+ (row) => row.name === "last_run_webhook_delivery_id",
+ ),
+ true,
+ );
+ assert.strictEqual(
+ (yield* sql`SELECT name FROM sqlite_master WHERE name = 'orchestration_v2_projection_turn_items_stop_request_idx'`)
+ .length,
+ 1,
+ );
+});
+const memory = (test: Effect.Effect) =>
+ test.pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" })));
+
+describe("plugin preview upgrade", () => {
+ it.effect.each([0, 68, 75, 76])("upgrades fresh/main %s with canonical migration ids", (main) =>
+ memory(
+ Effect.gen(function* () {
+ if (main) yield* runMigrations({ toMigrationInclusive: main });
+ yield* runMigrations();
+ yield* assertCurrent;
+ const before = yield* snapshot;
+ assert.deepStrictEqual(yield* runMigrations(), []);
+ assert.deepStrictEqual(yield* snapshot, before);
+ }),
+ ),
+ );
+ it.effect.each([
+ [67, 67],
+ [68, 67],
+ [69, 69],
+ [70, 69],
+ [69, 67],
+ [71, 69],
+ [76, 76],
+ [77, 76],
+ [78, 76],
+ ] as const)("preserves preview %s data and timestamps and is reentrant", ([through, start]) =>
+ memory(
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* seed(through, start);
+ const before = (yield* snapshot).data.filter(([name]) => name !== "effect_sql_migrations");
+ yield* runMigrations();
+ yield* assertCurrent;
+ const after = yield* snapshot;
+ // A later canonical build can add tables, but existing rows stay byte-for-byte identical.
+ for (const table of before)
+ assert.deepStrictEqual(
+ after.data.find(([name]) => name === table[0]),
+ table,
+ );
+ assert.deepStrictEqual(
+ yield* sql`SELECT created_at FROM effect_sql_migrations WHERE migration_id BETWEEN 77 AND ${77 + through - start}`,
+ Array.from({ length: through - start + 1 }, () => ({
+ created_at: "2026-10-01 00:00:00",
+ })),
+ );
+ assert.deepStrictEqual(yield* runMigrations(), []);
+ assert.deepStrictEqual(yield* snapshot, after);
+ }),
+ ),
+ );
+ it.effect.each([
+ "unknown-history",
+ "unknown-schema",
+ "partial-upstream",
+ "partial-search",
+ "partial-project",
+ "partial-target",
+ ])("rejects %s unchanged", (variant) =>
+ memory(
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* seed(67);
+ if (variant === "unknown-history")
+ yield* sql`INSERT INTO effect_sql_migrations (migration_id, name) VALUES (68, 'UnknownFork')`;
+ if (variant === "unknown-schema")
+ yield* sql`ALTER TABLE plugin_installations ADD COLUMN unknown TEXT`;
+ if (variant === "partial-upstream")
+ yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN last_run_webhook_delivery_id TEXT`;
+ if (variant === "partial-search")
+ yield* sql`CREATE TABLE orchestration_v2_message_search (text TEXT)`;
+ if (variant === "partial-project")
+ yield* sql`ALTER TABLE projection_projects ADD COLUMN pinned_at TEXT`;
+ const before = yield* snapshot;
+ assert.ok(
+ Exit.isFailure(
+ yield* Effect.exit(
+ runMigrations(variant === "partial-target" ? { toMigrationInclusive: 75 } : {}),
+ ),
+ ),
+ );
+ assert.deepStrictEqual(yield* snapshot, before);
+ }),
+ ),
+ );
+ it.effect("rolls back ledger moves and upstream schema on SQL failure, then retries", () =>
+ memory(
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ yield* seed(67);
+ yield* sql`CREATE TRIGGER refuse_plugin_upgrade BEFORE INSERT ON effect_sql_migrations WHEN NEW.name = 'ShellStopRequestIndex' BEGIN SELECT RAISE(ABORT, 'injected failure'); END`;
+ const before = yield* snapshot;
+ assert.ok(Exit.isFailure(yield* Effect.exit(runMigrations())));
+ assert.deepStrictEqual(yield* snapshot, before);
+ yield* sql`DROP TRIGGER refuse_plugin_upgrade`;
+ yield* runMigrations();
+ yield* assertCurrent;
+ }),
+ ),
+ );
+});
diff --git a/apps/server/src/persistence/reconcilePluginPreviewMigrations.ts b/apps/server/src/persistence/reconcilePluginPreviewMigrations.ts
new file mode 100644
index 0000000000000..9c0cfe1ff08de
--- /dev/null
+++ b/apps/server/src/persistence/reconcilePluginPreviewMigrations.ts
@@ -0,0 +1,179 @@
+import * as Effect from "effect/Effect";
+import * as Migrator from "effect/sql/Migrator";
+import * as SqlClient from "effect/sql/SqlClient";
+
+import WebhookDedupe from "./Migrations/067_WebhookDedupe.ts";
+import ShellStopRequestIndex from "./Migrations/068_ShellStopRequestIndex.ts";
+import Migration69 from "./Migrations/069_ThreadMessageSearch.ts";
+import Migration70 from "./Migrations/070_PeerLinks.ts";
+import Migration71 from "./Migrations/071_PeerLinkFence.ts";
+import Migration72 from "./Migrations/072_WaitingLinkRequestIndex.ts";
+import Migration73 from "./Migrations/073_ActiveTurnItemIndexes.ts";
+import Migration74 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts";
+import Migration75 from "./Migrations/075_SubagentDeliveryState.ts";
+import Migration76 from "./Migrations/076_ProjectionProjectsPinArchive.ts";
+
+// These unmerged preview builds used main's next slots. Recognize their exact
+// histories and schemas; never reinterpret an unknown build's migration ledger.
+const legacyNames = ["PluginInstallations", "PluginEventCursors", "PluginSettings"] as const;
+const legacySchemas = [
+ [
+ [
+ "plugin_installations",
+ "CREATE TABLE plugin_installations ( installation_id TEXT PRIMARY KEY, directory TEXT NOT NULL UNIQUE, record_json TEXT NOT NULL )",
+ ],
+ ],
+ [
+ [
+ "plugin_event_cursors",
+ "CREATE TABLE plugin_event_cursors ( installation_id TEXT PRIMARY KEY, acknowledged_sequence INTEGER NOT NULL, updated_at TEXT NOT NULL )",
+ ],
+ ],
+ [
+ [
+ "plugin_settings",
+ "CREATE TABLE plugin_settings ( installation_id TEXT NOT NULL, key TEXT NOT NULL, value_json TEXT NOT NULL, PRIMARY KEY (installation_id, key) )",
+ ],
+ [
+ "plugin_setting_secrets",
+ "CREATE TABLE plugin_setting_secrets ( installation_id TEXT NOT NULL, key TEXT NOT NULL, saved INTEGER NOT NULL, PRIMARY KEY (installation_id, key) )",
+ ],
+ [
+ "plugin_storage",
+ "CREATE TABLE plugin_storage ( installation_id TEXT NOT NULL, key TEXT NOT NULL, value_json TEXT NOT NULL, bytes INTEGER NOT NULL, PRIMARY KEY (installation_id, key) )",
+ ],
+ ],
+] as const;
+const upstream = [
+ [67, "WebhookDedupe", WebhookDedupe],
+ [68, "ShellStopRequestIndex", ShellStopRequestIndex],
+ [69, "ThreadMessageSearch", Migration69],
+ [70, "PeerLinks", Migration70],
+ [71, "PeerLinkFence", Migration71],
+ [72, "WaitingLinkRequestIndex", Migration72],
+ [73, "ActiveTurnItemIndexes", Migration73],
+ [74, "ExcludeAttachmentPlaceholdersFromSearch", Migration74],
+ [75, "SubagentDeliveryState", Migration75],
+ [76, "ProjectionProjectsPinArchive", Migration76],
+] as const;
+const normalize = (sql: string) => sql.trim().replace(/\s+/g, " ").toLowerCase();
+const badState = () =>
+ new Migrator.MigrationError({
+ kind: "BadState",
+ message: "Cannot upgrade an unknown or ambiguous plugin preview migration history/schema.",
+ });
+
+export const reconcilePluginPreviewMigrations = Effect.fn("reconcilePluginPreviewMigrations")(
+ function* (manifest: ReadonlyArray, throughId?: number) {
+ const sql = yield* SqlClient.SqlClient;
+ return yield* sql.withTransaction(
+ Effect.gen(function* () {
+ const exists =
+ yield* sql`SELECT name FROM sqlite_master WHERE name = 'effect_sql_migrations'`;
+ if (exists.length === 0) return [];
+ const history = yield* sql<{ readonly migration_id: number; readonly name: string }>`
+ SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id
+ `;
+ const preview = history.find(
+ (row) => row.name === legacyNames[0] && [67, 69, 76].includes(row.migration_id),
+ );
+ if (!preview) return [];
+ const start = preview.migration_id;
+ const canonical = manifest.find(([, name]) => name === legacyNames[0])?.[0];
+ if (canonical === undefined) return yield* badState();
+ const tail = history.filter((row) => row.migration_id >= start);
+ const available = legacyNames.filter((name, offset) =>
+ manifest.some(([id, found]) => id === canonical + offset && found === name),
+ );
+ if (
+ tail.length === 0 ||
+ tail.length > available.length ||
+ tail.some(
+ (row, offset) => row.migration_id !== start + offset || row.name !== available[offset],
+ ) ||
+ (throughId !== undefined && throughId < canonical + tail.length - 1)
+ )
+ return yield* badState();
+ const earlier = history.filter((row) => row.migration_id < start);
+ const expected = manifest.filter(([id]) => id < start);
+ if (
+ earlier.length !== expected.length ||
+ earlier.some(
+ (row, offset) =>
+ row.migration_id !== expected[offset]?.[0] || row.name !== expected[offset]?.[1],
+ )
+ )
+ return yield* badState();
+ const schema = yield* sql<{
+ readonly name: string;
+ readonly tbl_name: string;
+ readonly sql: string | null;
+ }>`SELECT name, tbl_name, sql FROM sqlite_master`;
+ for (const [offset, tables] of legacySchemas.entries()) {
+ for (const [name, definition] of tables) {
+ const objects = schema.filter((row) => row.tbl_name === name && row.sql !== null);
+ if (offset < tail.length) {
+ if (
+ objects.length !== 1 ||
+ objects[0]?.name !== name ||
+ normalize(objects[0].sql!) !== normalize(definition)
+ )
+ return yield* badState();
+ } else if (objects.length !== 0) return yield* badState();
+ }
+ }
+ if (
+ start < 76 &&
+ schema.some((row) =>
+ [
+ "orchestration_v2_message_search",
+ "peer_environment_links",
+ "orchestration_v2_projection_turn_items_waiting_link_idx",
+ "orchestration_v2_projection_turn_items_active_ordinal_idx",
+ "orchestration_v2_projection_turn_items_active_run_idx",
+ "orchestration_v2_projection_subagents_settled_idx",
+ ].includes(row.name),
+ )
+ )
+ return yield* badState();
+ const subagentColumns = yield* sql<{
+ readonly name: string;
+ }>`PRAGMA table_info(orchestration_v2_projection_subagents)`;
+ if (start < 76 && subagentColumns.some((row) => row.name === "completion_delivery_state"))
+ return yield* badState();
+ const projectColumns = yield* sql<{
+ readonly name: string;
+ }>`PRAGMA table_info(projection_projects)`;
+ if (projectColumns.some((row) => row.name === "pinned_at" || row.name === "archived_at"))
+ return yield* badState();
+ const taskColumns = yield* sql<{
+ readonly name: string;
+ }>`PRAGMA table_info(scheduled_tasks)`;
+ if (
+ start === 67 &&
+ (taskColumns.some((row) => row.name === "last_run_webhook_delivery_id") ||
+ schema.some((row) =>
+ [
+ "scheduled_task_webhook_replay_keys",
+ "scheduled_task_webhook_pending",
+ "orchestration_v2_projection_turn_items_stop_request_idx",
+ ].includes(row.name),
+ ))
+ )
+ return yield* badState();
+ // Move the preview ledger before filling main's slots; the transaction preserves both on failure.
+ for (let offset = tail.length - 1; offset >= 0; offset--) {
+ yield* sql`UPDATE effect_sql_migrations SET migration_id = ${canonical + offset} WHERE migration_id = ${start + offset} AND name = ${legacyNames[offset]}`;
+ }
+ const executed: Array = [];
+ for (const [id, name, migration] of upstream) {
+ if (id < start) continue;
+ yield* migration;
+ yield* sql`INSERT INTO effect_sql_migrations (migration_id, name) VALUES (${id}, ${name})`;
+ executed.push([id, name]);
+ }
+ return executed;
+ }),
+ );
+ },
+);
diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
index 08c5094bf49c8..95c1b1059e9b0 100644
--- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
@@ -56,6 +56,9 @@ describe("V2 preview upgrade", () => {
[74, "ExcludeAttachmentPlaceholdersFromSearch"],
[75, "SubagentDeliveryState"],
[76, "ProjectionProjectsPinArchive"],
+ [77, "PluginInstallations"],
+ [78, "PluginEventCursors"],
+ [79, "PluginSettings"],
]);
assert.deepStrictEqual(yield* runMigrations(), []);
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -155,6 +158,9 @@ describe("V2 preview upgrade", () => {
[74, "ExcludeAttachmentPlaceholdersFromSearch"],
[75, "SubagentDeliveryState"],
[76, "ProjectionProjectsPinArchive"],
+ [77, "PluginInstallations"],
+ [78, "PluginEventCursors"],
+ [79, "PluginSettings"],
]);
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
);
diff --git a/apps/server/src/plugins/PluginActions.test.ts b/apps/server/src/plugins/PluginActions.test.ts
new file mode 100644
index 0000000000000..94693dec3fa1c
--- /dev/null
+++ b/apps/server/src/plugins/PluginActions.test.ts
@@ -0,0 +1,489 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { describe, expect, it } from "@effect/vitest";
+import {
+ PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT,
+ PLUGIN_ACTIONS_MAX_PER_PLUGIN,
+ PLUGIN_ACTIONS_SNAPSHOT_MAX_BYTES,
+ PluginActionId,
+ PluginActionInvokeInput,
+ PluginId,
+ PluginInstallationId,
+ ProjectId,
+ ThreadId,
+ type OrchestrationProjectShell,
+ type OrchestrationV2ThreadShell,
+ type PluginActionsSnapshot,
+ type PluginInstallation,
+} from "@t3tools/contracts";
+import * as HostProcess from "@t3tools/shared/HostProcess";
+import * as Cause from "effect/Cause";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+
+import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts";
+import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts";
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
+import * as PluginActions from "./PluginActions.ts";
+import * as PluginCatalog from "./PluginCatalog.ts";
+import { loadPluginDirectory } from "./PluginManifestLoader.ts";
+import * as PluginSupervisor from "./PluginSupervisor.ts";
+
+// Children run the real CLI entry, which routes `__plugin-host` to the child runtime.
+const BIN_PATH = `${import.meta.dirname}/../bin.ts`;
+const FIXTURE = `${import.meta.dirname}/testFixtures/actions`;
+
+const toJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
+const fromJson = Schema.decodeSync(
+ Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
+);
+
+const THREAD = ThreadId.make("thread-1");
+const PROJECT = ProjectId.make("project-1");
+
+/** One thread in one project; every other target does not exist. */
+const threadsAndProjects = Layer.merge(
+ Layer.mock(ThreadManagementService)({
+ getThreadShell: (threadId) =>
+ Effect.succeed(
+ threadId === THREAD
+ ? ({
+ projectId: PROJECT,
+ worktreePath: null,
+ branch: "main",
+ } as OrchestrationV2ThreadShell)
+ : null,
+ ),
+ }),
+ Layer.mock(ProjectStoreV2)({
+ getShell: (projectId) =>
+ Effect.succeed(
+ projectId === PROJECT
+ ? Option.some({ workspaceRoot: "/work/project" } as OrchestrationProjectShell)
+ : Option.none(),
+ ),
+ }),
+);
+
+const startActions = Effect.fn("startActions")(function* (scope: Scope.Scope) {
+ const supervisor = yield* PluginSupervisor.make({
+ heapLimitMb: 64,
+ activationTimeout: "10 seconds",
+ stopGrace: "1 second",
+ }).pipe(
+ Effect.provideService(HostProcess.Arguments, [process.execPath, BIN_PATH]),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ const catalog = yield* PluginCatalog.make().pipe(
+ Effect.provideService(PluginSupervisor.PluginSupervisor, supervisor),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ const actions = yield* PluginActions.make().pipe(
+ Effect.provideService(PluginCatalog.PluginCatalog, catalog),
+ Effect.provide(threadsAndProjects),
+ );
+ return { catalog, actions };
+});
+
+/** Copies the fixture (or a variant of its manifest) into a fresh directory. */
+const preparePlugin = Effect.fn("preparePlugin")(function* (
+ manifest?: (manifest: Record) => Record,
+) {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const directory = path.join(
+ yield* fs.makeTempDirectoryScoped({ prefix: "t3-plugin-actions-" }),
+ "plugin",
+ );
+ yield* fs.makeDirectory(directory);
+ yield* fs.writeFileString(
+ path.join(directory, "main.mjs"),
+ yield* fs.readFileString(path.join(FIXTURE, "main.mjs")),
+ );
+ const original = fromJson(yield* fs.readFileString(path.join(FIXTURE, "t3-plugin.json")));
+ yield* fs.writeFileString(
+ path.join(directory, "t3-plugin.json"),
+ toJson(manifest ? manifest(original) : original),
+ );
+ return directory;
+});
+
+const enablePlugin = Effect.fn("enablePlugin")(function* (
+ catalog: PluginCatalog.PluginCatalog["Service"],
+ directory: string,
+) {
+ const { installation } = yield* catalog.add({ directory });
+ yield* catalog.consent({
+ installationId: installation.installationId,
+ digest: installation.source!.digest,
+ });
+ return (yield* catalog.enable({ installationId: installation.installationId })).installation;
+});
+
+const awaitActions = (
+ actions: PluginActions.PluginActions["Service"],
+ predicate: (snapshot: PluginActionsSnapshot) => boolean,
+) =>
+ actions.subscribe.pipe(
+ Stream.filter(predicate),
+ Stream.runHead,
+ Effect.map((snapshot) => Option.getOrThrow(snapshot).actions),
+ );
+
+const awaitHostState = (
+ catalog: PluginCatalog.PluginCatalog["Service"],
+ installationId: PluginInstallationId,
+ tags: ReadonlyArray,
+) =>
+ catalog.subscribe.pipe(
+ Stream.filter((snapshot) =>
+ snapshot.installations.some(
+ (installation) =>
+ installation.installationId === installationId &&
+ tags.includes(installation.hostState?._tag ?? ""),
+ ),
+ ),
+ Stream.runHead,
+ );
+
+const withDatabase = (effect: Effect.Effect) =>
+ effect.pipe(Effect.provide(SqlitePersistence.layerMemory));
+
+it.layer(NodeServices.layer)("PluginActions", (it) => {
+ it.effect(
+ "lists an enabled plugin's actions without starting it, and drops them on disable",
+ () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { catalog, actions } = yield* startActions(yield* Scope.Scope);
+ const directory = yield* preparePlugin();
+ const installation = yield* enablePlugin(catalog, directory);
+ const { installationId } = installation;
+
+ const listed = yield* awaitActions(actions, (snapshot) => snapshot.actions.length > 0);
+ expect(listed.map((action) => [action.name, action.target, action.placements])).toEqual([
+ ["echo-target", "thread", ["command-palette", "thread-menu", "composer-slash"]],
+ ["say-hello", "environment", ["command-palette"]],
+ ["fail", "environment", ["command-palette"]],
+ ["wait", "environment", ["command-palette"]],
+ ]);
+ expect(listed[0]).toMatchObject({
+ pluginId: "test.actions",
+ pluginName: "Actions fixture",
+ title: "Echo target",
+ description: "Says which thread it ran on.",
+ });
+ // Listing read the manifest only: no process was started.
+ const [row] = (yield* catalog.list).installations;
+ expect(row?.hostState).toEqual({ _tag: "idle" });
+
+ const hello = listed.find((action) => action.name === "say-hello")!;
+ expect(
+ yield* actions.invoke({ actionId: hello.id, target: { _tag: "environment" } }),
+ ).toEqual({ message: "Hello from test.actions" });
+
+ yield* catalog.disable({ installationId });
+ expect(yield* awaitActions(actions, () => true)).toEqual([]);
+ const disabled = yield* actions
+ .invoke({ actionId: hello.id, target: { _tag: "environment" } })
+ .pipe(Effect.flip);
+ expect(disabled.reason).toBe("not-found");
+
+ // Enabled again: new ids, and the old one can never reach the new registration.
+ yield* catalog.enable({ installationId });
+ const relisted = yield* awaitActions(actions, (snapshot) => snapshot.actions.length > 0);
+ expect(relisted.find((action) => action.name === "say-hello")!.id).not.toBe(hello.id);
+ const stale = yield* actions
+ .invoke({ actionId: hello.id, target: { _tag: "environment" } })
+ .pipe(Effect.flip);
+ expect(stale.reason).toBe("stale");
+
+ yield* catalog.remove({ installationId });
+ expect(yield* awaitActions(actions, () => true)).toEqual([]);
+ }),
+ ),
+ );
+
+ it.effect("runs an action on its resolved target and reports typed failures", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { catalog, actions } = yield* startActions(yield* Scope.Scope);
+ const { installationId } = yield* enablePlugin(catalog, yield* preparePlugin());
+ const listed = yield* awaitActions(actions, (snapshot) => snapshot.actions.length > 0);
+ const byName = (name: string) => listed.find((action) => action.name === name)!.id;
+
+ expect(
+ yield* actions.invoke({
+ actionId: byName("echo-target"),
+ target: { _tag: "thread", threadId: THREAD },
+ }),
+ ).toEqual({ message: "thread thread-1 in /work/project" });
+
+ const failures = yield* Effect.forEach(
+ [
+ { actionId: byName("echo-target"), target: { _tag: "environment" as const } },
+ {
+ actionId: byName("echo-target"),
+ target: { _tag: "thread" as const, threadId: ThreadId.make("elsewhere") },
+ },
+ { actionId: byName("fail"), target: { _tag: "environment" as const } },
+ {
+ actionId: PluginActionId.make(`${installationId}:1:missing`),
+ target: { _tag: "environment" as const },
+ },
+ {
+ actionId: PluginActionId.make("not-an-action"),
+ target: { _tag: "environment" as const },
+ },
+ ],
+ (input) => actions.invoke(input).pipe(Effect.flip),
+ );
+ expect(failures.map((error) => [error.reason, error.message])).toEqual([
+ ["target-mismatch", "Echo target runs on a thread."],
+ ["target-not-found", "That thread does not exist here."],
+ ["failed", "The fixture failed on purpose."],
+ ["not-found", "That action is not available now."],
+ ["not-found", "That action is not available here."],
+ ]);
+
+ // A disable while the action runs revokes it at once.
+ const running = yield* actions
+ .invoke({ actionId: byName("wait"), target: { _tag: "environment" } })
+ .pipe(Effect.flip, Effect.forkChild({ startImmediately: true }));
+ yield* awaitHostState(catalog, installationId, ["starting", "running"]);
+ yield* catalog.disable({ installationId });
+ expect((yield* Fiber.join(running)).reason).toBe("stopped");
+ }),
+ ),
+ );
+
+ it.effect("refuses malformed ids the wire accepts with a typed error, never a defect", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { catalog, actions } = yield* startActions(yield* Scope.Scope);
+ const { installationId } = yield* enablePlugin(catalog, yield* preparePlugin());
+ const decode = Schema.decodeUnknownEffect(PluginActionInvokeInput);
+ const exits = yield* Effect.forEach(
+ [
+ ":1:go",
+ `${"x".repeat(65)}:1:go`,
+ "::",
+ `${installationId}::say-hello`,
+ `${installationId}:1.0:say-hello`,
+ `${installationId}:99999999999999999999:say-hello`,
+ `${installationId} :1:say-hello`,
+ ],
+ (actionId) =>
+ decode({ actionId, target: { _tag: "environment" } }).pipe(
+ Effect.flatMap((input) => actions.invoke(input).pipe(Effect.exit)),
+ ),
+ );
+ for (const exit of exits) {
+ expect(Exit.isFailure(exit) && !Cause.hasDies(exit.cause)).toBe(true);
+ if (Exit.isFailure(exit)) {
+ expect(Cause.squash(exit.cause)).toMatchObject({
+ _tag: "PluginActionError",
+ reason: "not-found",
+ });
+ }
+ }
+ }),
+ ),
+ );
+
+ it.effect("offers plugins whole up to the environment bound and runs only those", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { catalog, actions } = yield* startActions(yield* Scope.Scope);
+ const pluginCount = PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT / PLUGIN_ACTIONS_MAX_PER_PLUGIN + 1;
+ const installations = yield* Effect.forEach(
+ Array.from({ length: pluginCount }, (_, index) => index),
+ (index) =>
+ preparePlugin((manifest) => ({
+ ...manifest,
+ id: `test.actions-${index}`,
+ actions: Array.from({ length: PLUGIN_ACTIONS_MAX_PER_PLUGIN }, (_, action) => ({
+ name: `go-${action}`,
+ title: `Go ${action}`,
+ target: "environment",
+ placements: ["command-palette"],
+ })),
+ })).pipe(Effect.flatMap((directory) => enablePlugin(catalog, directory))),
+ );
+ const full = yield* actions.subscribe.pipe(
+ Stream.filter((snapshot) => snapshot.omitted !== undefined),
+ Stream.runHead,
+ Effect.map(Option.getOrThrow),
+ );
+ expect(full.actions).toHaveLength(PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT);
+ expect(full.omitted).toEqual({ plugins: 1, actions: PLUGIN_ACTIONS_MAX_PER_PLUGIN });
+ const listedIds = new Set(full.actions.map((action) => action.id.split(":")[0]));
+ const left = installations.find(
+ (installation) => !listedIds.has(installation.installationId),
+ )!;
+
+ const refused = yield* actions
+ .invoke({
+ actionId: PluginActionId.make(`${left.installationId}:${left.generation}:go-0`),
+ target: { _tag: "environment" },
+ })
+ .pipe(Effect.flip);
+ expect(refused.reason).toBe("not-found");
+ expect(refused.message).toContain("more actions than it shows");
+
+ // Disabling a listed plugin makes room, and the left-out one is offered whole.
+ const listed = installations.find((installation) => installation !== left)!;
+ yield* catalog.disable({ installationId: listed.installationId });
+ const after = yield* awaitActions(actions, (snapshot) => snapshot.omitted === undefined);
+ expect(after).toHaveLength(PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT);
+ expect(
+ after.filter((action) => action.id.startsWith(`${left.installationId}:`)),
+ ).toHaveLength(PLUGIN_ACTIONS_MAX_PER_PLUGIN);
+ }),
+ ),
+ );
+
+ it.effect("refuses declarations the host cannot honour", () =>
+ Effect.gen(function* () {
+ const reasons = yield* Effect.forEach(
+ [
+ (manifest: Record) => ({ ...manifest, capabilities: [] }),
+ (manifest: Record) => ({ ...manifest, proposedApi: false }),
+ (manifest: Record) => ({
+ ...manifest,
+ actions: [
+ { name: "same", title: "One", target: "environment", placements: ["thread-menu"] },
+ { name: "same", title: "Two", target: "environment", placements: ["thread-menu"] },
+ ],
+ }),
+ (manifest: Record) => ({
+ ...manifest,
+ actions: [
+ {
+ name: "twice",
+ title: "Twice",
+ target: "environment",
+ placements: ["thread-menu", "thread-menu"],
+ },
+ ],
+ }),
+ ],
+ (variant) =>
+ preparePlugin(variant).pipe(
+ Effect.flatMap(loadPluginDirectory),
+ Effect.flip,
+ Effect.map((error) => error.reason),
+ ),
+ );
+ expect(reasons).toEqual([
+ "it declares actions without the actions capability.",
+ "it declares actions, which need proposedApi: true.",
+ "it declares the action same twice.",
+ "the action twice repeats a placement.",
+ ]);
+ }),
+ );
+});
+
+describe("pluginActionsFromCatalog", () => {
+ const digest = `sha256:${"a".repeat(64)}`;
+ const installation: PluginInstallation = {
+ installationId: PluginInstallationId.make("installation-1"),
+ generation: 3,
+ directory: "/srv/plugins/actions",
+ manifest: {
+ id: PluginId.make("test.actions"),
+ name: "Actions",
+ version: "1.0.0",
+ capabilities: ["actions"],
+ proposedApi: true,
+ actions: [{ name: "go", title: "Go", target: "environment", placements: ["thread-menu"] }],
+ },
+ source: { digest, files: 2, bytes: 10 },
+ problem: null,
+ inspectedAt: "2026-10-04T00:00:00.000Z",
+ consent: { digest, capabilities: ["actions"], grantedAt: "2026-10-04T00:00:00.000Z" },
+ enabled: true,
+ hostState: { _tag: "running" },
+ addedAt: "2026-10-04T00:00:00.000Z",
+ };
+ const names = (installations: ReadonlyArray) =>
+ PluginActions.pluginActionsFromCatalog({ installations }).actions.map((action) => action.id);
+
+ const many = (count: number, description?: string) =>
+ Array.from({ length: count }, (_, index): PluginInstallation => {
+ const installationId = PluginInstallationId.make(`installation-${index}`);
+ return {
+ ...installation,
+ installationId,
+ manifest: {
+ ...installation.manifest!,
+ id: PluginId.make(`test.actions-${index}`),
+ actions: Array.from({ length: PLUGIN_ACTIONS_MAX_PER_PLUGIN }, (_, action) => ({
+ name: `go-${action}`,
+ title: `Go ${action}`,
+ ...(description === undefined ? {} : { description }),
+ target: "environment" as const,
+ placements: ["command-palette" as const],
+ })),
+ },
+ };
+ });
+ const frameBytes = (snapshot: PluginActionsSnapshot) =>
+ Buffer.byteLength(JSON.stringify(snapshot));
+
+ it("bounds the actions and bytes one environment offers, counting what it leaves out", () => {
+ const byCount = PluginActions.pluginActionsFromCatalog({ installations: many(1000) });
+ expect(byCount.actions).toHaveLength(PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT);
+ const keptPlugins = PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT / PLUGIN_ACTIONS_MAX_PER_PLUGIN;
+ expect(byCount.omitted).toEqual({
+ plugins: 1000 - keptPlugins,
+ actions: (1000 - keptPlugins) * PLUGIN_ACTIONS_MAX_PER_PLUGIN,
+ });
+ // The first plugins in catalogue order are the ones kept.
+ expect(new Set(byCount.actions.map((action) => action.id.split(":")[0]))).toEqual(
+ new Set(Array.from({ length: keptPlugins }, (_, index) => `installation-${index}`)),
+ );
+ expect(frameBytes(byCount)).toBeLessThanOrEqual(PLUGIN_ACTIONS_SNAPSHOT_MAX_BYTES);
+
+ // Escaped control characters make each description six times its length on the wire.
+ const byBytes = PluginActions.pluginActionsFromCatalog({
+ installations: many(1000, "\u0001".repeat(240)),
+ });
+ expect(byBytes.actions.length).toBeLessThan(PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT);
+ expect(byBytes.actions.length % PLUGIN_ACTIONS_MAX_PER_PLUGIN).toBe(0);
+ expect(byBytes.omitted?.actions).toBe(
+ 1000 * PLUGIN_ACTIONS_MAX_PER_PLUGIN - byBytes.actions.length,
+ );
+ expect(frameBytes(byBytes)).toBeLessThanOrEqual(PLUGIN_ACTIONS_SNAPSHOT_MAX_BYTES);
+
+ // Within the bounds nothing is left out and `omitted` is absent.
+ expect(
+ PluginActions.pluginActionsFromCatalog({ installations: many(keptPlugins) }),
+ ).not.toHaveProperty("omitted");
+ });
+
+ it("offers only actions that can run now", () => {
+ expect(names([installation])).toEqual(["installation-1:3:go"]);
+ // A state this server version does not know is unknown, not a reason to hide the action.
+ const { hostState: _hostState, ...unknownState } = installation;
+ expect(names([unknownState])).toEqual(["installation-1:3:go"]);
+ for (const hidden of [
+ { ...installation, enabled: false },
+ { ...installation, consent: null },
+ { ...installation, source: null, problem: "gone" },
+ { ...installation, hostState: { _tag: "quarantined" as const, failures: 3, reason: "x" } },
+ { ...installation, hostState: { _tag: "incompatible" as const, reason: "x" } },
+ { ...installation, manifest: { ...installation.manifest!, capabilities: [] } },
+ ]) {
+ expect(names([hidden])).toEqual([]);
+ }
+ });
+});
diff --git a/apps/server/src/plugins/PluginActions.ts b/apps/server/src/plugins/PluginActions.ts
new file mode 100644
index 0000000000000..c96befa8311e5
--- /dev/null
+++ b/apps/server/src/plugins/PluginActions.ts
@@ -0,0 +1,294 @@
+// Ids pin registration generations; stale or omitted actions must never reach a replacement.
+// Listing starts no plugin, and caller interruption cancels the invoked handler.
+import {
+ PLUGIN_ACTION_MESSAGE_MAX_LENGTH,
+ PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT,
+ PLUGIN_ACTIONS_MAX_PER_PLUGIN,
+ PLUGIN_ACTIONS_SNAPSHOT_MAX_BYTES,
+ PluginActionError,
+ PluginActionId,
+ PluginInstallationId,
+ pluginInstallationStatus,
+ type PluginAction,
+ type PluginActionInvokeInput,
+ type PluginActionInvokeResult,
+ type PluginActionsSnapshot,
+ type PluginActionTarget,
+ type PluginCatalogError,
+ type PluginCatalogSnapshot,
+ type PluginInstallation,
+ type ProjectId,
+ type ThreadId,
+} from "@t3tools/contracts";
+import * as Context from "effect/Context";
+import type * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
+
+import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts";
+import { ThreadManagementService } from "../orchestration-v2/ThreadManagementService.ts";
+import { PluginCatalog } from "./PluginCatalog.ts";
+import type { PluginInvokeError } from "./PluginSupervisor.ts";
+
+const PLUGIN_ACTION_TIMEOUT: Duration.Input = "30 seconds";
+
+/** What an action handler receives as `input.target`: the target the user picked, resolved. */
+type PluginActionTargetContext =
+ | { readonly kind: "environment" }
+ | { readonly kind: "project"; readonly projectId: string; readonly workspaceRoot: string }
+ | {
+ readonly kind: "thread";
+ readonly threadId: string;
+ readonly projectId: string;
+ /** The thread's worktree, or its project's workspace root. */
+ readonly cwd: string;
+ readonly branch: string | null;
+ };
+
+/** Looks a target up in this environment's projects and threads; none when it does not exist here. */
+const resolvePluginActionTargetFrom =
+ (lookups: {
+ readonly getThreadShell: (threadId: ThreadId) => Effect.Effect<
+ {
+ readonly projectId: ProjectId;
+ readonly worktreePath: string | null;
+ readonly branch: string | null;
+ } | null,
+ ThreadError
+ >;
+ readonly getProjectShell: (
+ projectId: ProjectId,
+ ) => Effect.Effect, ProjectError>;
+ }) =>
+ (
+ target: PluginActionTarget,
+ ): Effect.Effect, PluginActionError> =>
+ Effect.gen(function* () {
+ switch (target._tag) {
+ case "environment":
+ return Option.some({ kind: "environment" as const });
+ case "project": {
+ const project = yield* lookups.getProjectShell(target.projectId);
+ return Option.map(project, (shell) => ({
+ kind: "project" as const,
+ projectId: target.projectId,
+ workspaceRoot: shell.workspaceRoot,
+ }));
+ }
+ case "thread": {
+ const thread = yield* lookups.getThreadShell(target.threadId);
+ if (thread === null) return Option.none();
+ const project = yield* lookups.getProjectShell(thread.projectId);
+ if (Option.isNone(project)) return Option.none();
+ return Option.some({
+ kind: "thread" as const,
+ threadId: target.threadId,
+ projectId: thread.projectId,
+ cwd: thread.worktreePath ?? project.value.workspaceRoot,
+ branch: thread.branch,
+ });
+ }
+ }
+ }).pipe(
+ Effect.mapError(() => actionError("unavailable", "Could not look up the action's target.")),
+ );
+
+export class PluginActions extends Context.Service<
+ PluginActions,
+ {
+ /** The current actions now, then a new list whenever it changes. */
+ readonly subscribe: Stream.Stream;
+ readonly invoke: (
+ input: PluginActionInvokeInput,
+ ) => Effect.Effect;
+ }
+>()("t3/plugins/PluginActions") {}
+
+const pluginActionHandlerName = (name: string) => `action:${name}`;
+
+const actionId = (installationId: string, generation: number, name: string) =>
+ PluginActionId.make(`${installationId}:${generation}:${name}`);
+
+const decodeInstallationId = Schema.decodeUnknownOption(PluginInstallationId);
+
+/** The parts of an id this server issued; undefined for anything else. */
+const parseActionId = (id: PluginActionId) => {
+ const parts = id.split(":");
+ const name = parts.pop();
+ const generationText = parts.pop();
+ if (name === undefined || generationText === undefined || !/^[0-9]+$/.test(generationText))
+ return undefined;
+ const generation = Number(generationText);
+ if (!Number.isSafeInteger(generation)) return undefined;
+ const segment = parts.join(":");
+ const installationId = decodeInstallationId(segment);
+ if (Option.isNone(installationId) || installationId.value !== segment) return undefined;
+ return { installationId: installationId.value, generation, name };
+};
+
+/** The actions one installation offers on its own: enabled and able to run now. */
+const installationActions = (installation: PluginInstallation): ReadonlyArray => {
+ const { manifest } = installation;
+ if (manifest === null || pluginInstallationStatus(installation) !== "enabled") return [];
+ if (!manifest.capabilities.includes("actions")) return [];
+ // These wait for someone to resume them; an action could only fail.
+ const state = installation.hostState?._tag;
+ if (state === "quarantined" || state === "incompatible") return [];
+ return (manifest.actions ?? []).slice(0, PLUGIN_ACTIONS_MAX_PER_PLUGIN).map((declaration) => ({
+ id: actionId(installation.installationId, installation.generation, declaration.name),
+ pluginId: manifest.id,
+ pluginName: manifest.name,
+ name: declaration.name,
+ title: declaration.title,
+ ...(declaration.description === undefined ? {} : { description: declaration.description }),
+ target: declaration.target,
+ placements: declaration.placements,
+ }));
+};
+
+// Room for the frame around the actions: `{"actions":[`, `]`, and `omitted`.
+const SNAPSHOT_ENVELOPE_BYTES = 128;
+const ACTIONS_MAX_BYTES = PLUGIN_ACTIONS_SNAPSHOT_MAX_BYTES - SNAPSHOT_ENVELOPE_BYTES;
+
+/** Each action's JSON plus its separator. */
+const encodedBytes = (actions: ReadonlyArray) =>
+ actions.reduce((total, action) => total + Buffer.byteLength(JSON.stringify(action)) + 1, 0);
+
+/**
+ * The actions a catalogue snapshot offers. Plugins are taken whole, in
+ * catalogue order, until one would pass the environment's action or byte
+ * bound; it and every later plugin are counted in `omitted` instead.
+ */
+export const pluginActionsFromCatalog = (
+ snapshot: PluginCatalogSnapshot,
+): PluginActionsSnapshot => {
+ const actions: Array = [];
+ let bytes = 0;
+ const omitted = { plugins: 0, actions: 0 };
+ for (const installation of snapshot.installations) {
+ const offered = installationActions(installation);
+ if (offered.length === 0) continue;
+ if (omitted.plugins === 0) {
+ const size = encodedBytes(offered);
+ if (
+ actions.length + offered.length <= PLUGIN_ACTIONS_MAX_PER_ENVIRONMENT &&
+ bytes + size <= ACTIONS_MAX_BYTES
+ ) {
+ actions.push(...offered);
+ bytes += size;
+ continue;
+ }
+ }
+ omitted.plugins += 1;
+ omitted.actions += offered.length;
+ }
+ return omitted.plugins === 0 ? { actions } : { actions, omitted };
+};
+
+const actionError = (reason: string, message: string) =>
+ new PluginActionError({ reason, message: bound(message) });
+
+/** At most the message bound, without splitting a surrogate pair. */
+const bound = (text: string) => {
+ const characters = Array.from(text);
+ return characters.length <= PLUGIN_ACTION_MESSAGE_MAX_LENGTH
+ ? text
+ : `${characters.slice(0, PLUGIN_ACTION_MESSAGE_MAX_LENGTH - 1).join("")}…`;
+};
+
+/** A plugin's `{ message }`, if it returned one. */
+const resultMessage = (value: Schema.Json): string | null => {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return null;
+ const message = (value as { readonly [key: string]: Schema.Json })["message"];
+ if (typeof message !== "string" || message.trim() === "") return null;
+ return bound(message.trim());
+};
+
+const fromInvokeError = (
+ error: PluginCatalogError | PluginInvokeError,
+ title: string,
+): PluginActionError => {
+ switch (error._tag) {
+ case "PluginCatalogError":
+ return error.reason === "generation-changed"
+ ? actionError("stale", "The plugin was enabled again since this action was listed.")
+ : error.reason === "not-found"
+ ? actionError("not-found", "That plugin is no longer installed.")
+ : actionError("unavailable", error.message);
+ case "PluginTimeoutError":
+ return actionError("timeout", `${title} did not finish in time.`);
+ case "PluginBusyError":
+ return actionError("busy", `${title} could not start: the plugin is busy.`);
+ case "PluginStoppedError":
+ return actionError("stopped", "The plugin was disabled while the action ran.");
+ case "PluginCallFailedError":
+ return actionError("failed", error.reason);
+ default:
+ return actionError("unavailable", error.message);
+ }
+};
+
+export const make = Effect.fn("PluginActions.make")(function* () {
+ const catalog = yield* PluginCatalog;
+ const threads = yield* ThreadManagementService;
+ const projects = yield* ProjectStoreV2;
+ const resolveTarget = resolvePluginActionTargetFrom({
+ getThreadShell: threads.getThreadShell,
+ getProjectShell: projects.getShell,
+ });
+
+ const invoke = Effect.fn("PluginActions.invoke")(function* (input: PluginActionInvokeInput) {
+ const parsed = parseActionId(input.actionId);
+ if (parsed === undefined)
+ return yield* actionError("not-found", "That action is not available here.");
+ const snapshot = yield* catalog.list;
+ const installation = snapshot.installations.find(
+ (candidate) => candidate.installationId === parsed.installationId,
+ );
+ if (installation === undefined)
+ return yield* actionError("not-found", "That plugin is no longer installed.");
+ if (installation.generation !== parsed.generation)
+ return yield* actionError(
+ "stale",
+ "The plugin was enabled again since this action was listed.",
+ );
+ // Only what the list offers runs, so an action left out by the bounds cannot.
+ const action = pluginActionsFromCatalog(snapshot).actions.find(
+ (candidate) => candidate.id === input.actionId,
+ );
+ if (action === undefined)
+ return yield* installationActions(installation).some(
+ (candidate) => candidate.name === parsed.name,
+ )
+ ? actionError(
+ "not-found",
+ "That action is not offered: this environment's plugins declare more actions than it shows.",
+ )
+ : actionError("not-found", "That action is not available now.");
+ if (action.target !== input.target._tag)
+ return yield* actionError("target-mismatch", `${action.title} runs on a ${action.target}.`);
+ const target = yield* resolveTarget(input.target);
+ if (Option.isNone(target))
+ return yield* actionError("target-not-found", `That ${action.target} does not exist here.`);
+ const value = yield* catalog
+ .invoke(
+ installation.installationId,
+ pluginActionHandlerName(action.name),
+ { action: action.name, target: target.value },
+ { generation: parsed.generation, timeout: PLUGIN_ACTION_TIMEOUT },
+ )
+ .pipe(Effect.mapError((error) => fromInvokeError(error, action.title)));
+ return { message: resultMessage(value) };
+ });
+
+ return PluginActions.of({
+ // Process state changes reach the catalogue too; most leave the list as it was.
+ subscribe: catalog.subscribe.pipe(Stream.map(pluginActionsFromCatalog), Stream.changes),
+ invoke,
+ });
+});
+
+export const layer = Layer.effect(PluginActions, make());
diff --git a/apps/server/src/plugins/PluginActionsRpc.test.ts b/apps/server/src/plugins/PluginActionsRpc.test.ts
new file mode 100644
index 0000000000000..335d9a3eb5c54
--- /dev/null
+++ b/apps/server/src/plugins/PluginActionsRpc.test.ts
@@ -0,0 +1,120 @@
+import {
+ type AuthEnvironmentScope,
+ AuthOrchestrationOperateScope,
+ AuthOrchestrationReadScope,
+ AuthRelayReadScope,
+ AuthStandardClientScopes,
+ PluginActionId,
+ type PluginActionsSnapshot,
+ WS_METHODS,
+ WsRpcGroup,
+} from "@t3tools/contracts";
+import { describe, expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Stream from "effect/Stream";
+import * as RpcTest from "effect/rpc/RpcTest";
+
+import { RPC_REQUIRED_SCOPES } from "../auth/RpcAuthorization.ts";
+import * as RpcAuthorization from "../auth/RpcAuthorization.ts";
+
+type ActionMethod =
+ | typeof WS_METHODS.pluginActionsSubscribe
+ | typeof WS_METHODS.pluginActionsInvoke;
+const actionMethods: ReadonlySet = new Set([
+ WS_METHODS.pluginActionsSubscribe,
+ WS_METHODS.pluginActionsInvoke,
+]);
+
+const group = WsRpcGroup.omit(
+ ...[...WsRpcGroup.requests.keys()].filter(
+ (tag): tag is Exclude =>
+ !actionMethods.has(tag),
+ ),
+);
+
+const actionId = PluginActionId.make("installation-1:1:say-hello");
+const snapshot: PluginActionsSnapshot = {
+ actions: [
+ {
+ id: actionId,
+ pluginId: "test.actions",
+ pluginName: "Actions fixture",
+ name: "say-hello",
+ title: "Say hello",
+ target: "environment",
+ placements: ["command-palette"],
+ },
+ ],
+};
+const invoke = { actionId, target: { _tag: "environment" as const } };
+
+/** Serves the action RPCs through the real scope middleware; handlers record that they ran. */
+const makeClient = (scopes: ReadonlyArray, handled: Array) =>
+ RpcTest.makeClient(group).pipe(
+ Effect.provide(
+ Layer.mergeAll(
+ group.toLayerHandler(WS_METHODS.pluginActionsSubscribe, () =>
+ Stream.fromEffect(
+ Effect.sync(() => handled.push(WS_METHODS.pluginActionsSubscribe)).pipe(
+ Effect.as(snapshot),
+ ),
+ ),
+ ),
+ group.toLayerHandler(WS_METHODS.pluginActionsInvoke, () =>
+ Effect.sync(() => handled.push(WS_METHODS.pluginActionsInvoke)).pipe(
+ Effect.as({ message: "Hello" }),
+ ),
+ ),
+ RpcAuthorization.layer(scopes),
+ ),
+ ),
+ );
+
+describe("plugin action RPC scopes", () => {
+ it.effect("lets a standard pairing list and run the actions an administrator enabled", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient(AuthStandardClientScopes, handled);
+
+ expect(
+ yield* client[WS_METHODS.pluginActionsSubscribe]({}).pipe(
+ Stream.take(1),
+ Stream.runCollect,
+ ),
+ ).toEqual([snapshot]);
+ expect(yield* client[WS_METHODS.pluginActionsInvoke](invoke)).toEqual({ message: "Hello" });
+ expect(handled).toEqual([WS_METHODS.pluginActionsSubscribe, WS_METHODS.pluginActionsInvoke]);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses to run an action for a read-only session", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient([AuthOrchestrationReadScope], handled);
+
+ expect(yield* client[WS_METHODS.pluginActionsInvoke](invoke).pipe(Effect.flip)).toMatchObject(
+ {
+ _tag: "EnvironmentAuthorizationError",
+ requiredScope: AuthOrchestrationOperateScope,
+ },
+ );
+ expect(handled).toEqual([]);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses the list without the orchestration read scope", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient([AuthRelayReadScope], handled);
+
+ expect(
+ yield* client[WS_METHODS.pluginActionsSubscribe]({}).pipe(Stream.runCollect, Effect.flip),
+ ).toMatchObject({
+ _tag: "EnvironmentAuthorizationError",
+ requiredScope: AuthOrchestrationReadScope,
+ });
+ expect(handled).toEqual([]);
+ }).pipe(Effect.scoped),
+ );
+});
diff --git a/apps/server/src/plugins/PluginCatalog.test.ts b/apps/server/src/plugins/PluginCatalog.test.ts
new file mode 100644
index 0000000000000..5a1fee181e166
--- /dev/null
+++ b/apps/server/src/plugins/PluginCatalog.test.ts
@@ -0,0 +1,958 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { describe, expect, it } from "@effect/vitest";
+import {
+ EnvironmentId,
+ ThreadId,
+ PluginInstallationId,
+ pluginInstallationStatus,
+ type PluginId,
+ type PluginInstallation,
+} from "@t3tools/contracts";
+import * as HostProcess from "@t3tools/shared/HostProcess";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+import * as PubSub from "effect/PubSub";
+import * as Queue from "effect/Queue";
+import * as Schema from "effect/Schema";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+import * as SqlClient from "effect/sql/SqlClient";
+
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
+import * as PluginCatalog from "./PluginCatalog.ts";
+import type { PluginRegistration } from "./PluginManifestLoader.ts";
+import * as PluginSupervisor from "./PluginSupervisor.ts";
+import * as PluginTools from "./PluginTools.ts";
+
+// Children run the real CLI entry, which routes `__plugin-host` to the child runtime.
+const BIN_PATH = `${import.meta.dirname}/../bin.ts`;
+
+type Catalog = PluginCatalog.PluginCatalog["Service"];
+
+const toJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
+
+/** Starts a catalogue and its supervisor in `scope`, as one server start would. */
+const startCatalog = Effect.fn("startCatalog")(function* (scope: Scope.Scope) {
+ const supervisor = yield* PluginSupervisor.make({
+ heapLimitMb: 64,
+ activationTimeout: "10 seconds",
+ stopGrace: "1 second",
+ }).pipe(
+ Effect.provideService(HostProcess.Arguments, [process.execPath, BIN_PATH]),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ return yield* PluginCatalog.make().pipe(
+ Effect.provideService(PluginSupervisor.PluginSupervisor, supervisor),
+ Effect.provideService(Scope.Scope, scope),
+ );
+});
+
+/**
+ * Writes a plugin whose activation leaves a marker outside its own directory:
+ * a plugin writing into its directory changes its own digest.
+ */
+const preparePlugin = Effect.fn("preparePlugin")(function* (id: string) {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-plugin-catalog-" });
+ const directory = path.join(root, "plugin");
+ yield* fs.makeDirectory(directory);
+ const marker = path.join(root, "activated");
+ const entry = path.join(directory, "main.mjs");
+ yield* fs.writeFileString(
+ entry,
+ [
+ `import * as NodeFS from "node:fs";`,
+ `export function activate(context) {`,
+ ` NodeFS.writeFileSync(${toJson(marker)}, String(process.pid));`,
+ ` context.proposed.handle("ping", (input) => ({ pid: process.pid, input }));`,
+ `}`,
+ ``,
+ ].join("\n"),
+ );
+ yield* fs.writeFileString(
+ path.join(directory, "t3-plugin.json"),
+ toJson({
+ id,
+ name: id,
+ version: "1.0.0",
+ apiVersion: 1,
+ entry: "main.mjs",
+ proposedApi: true,
+ }),
+ );
+ const edit = (line: string) =>
+ fs
+ .readFileString(entry)
+ .pipe(Effect.flatMap((content) => fs.writeFileString(entry, `${content}// ${line}\n`)));
+ return { directory, marker, edit };
+});
+
+/** Waits, through the subscription, for a snapshot that satisfies `predicate`. */
+const awaitSnapshot = (
+ catalog: Catalog,
+ predicate: (installations: ReadonlyArray) => boolean,
+) =>
+ catalog.subscribe.pipe(
+ Stream.filter((snapshot) => predicate(snapshot.installations)),
+ Stream.runHead,
+ Effect.map((snapshot) => Option.getOrThrow(snapshot).installations),
+ );
+
+/** A step a test can stop at: `reached` completes when it is entered, `release` lets it go on. */
+interface Hold {
+ readonly reached: Deferred.Deferred;
+ readonly release: Deferred.Deferred;
+}
+
+const makeHold = Effect.gen(function* () {
+ const hold: Hold = {
+ reached: yield* Deferred.make(),
+ release: yield* Deferred.make(),
+ };
+ return hold;
+});
+
+const passHold = (hold: Hold | undefined) =>
+ hold === undefined
+ ? Effect.void
+ : Deferred.succeed(hold.reached, undefined).pipe(Effect.andThen(Deferred.await(hold.release)));
+
+/**
+ * A supervisor without processes that behaves like the real one at its
+ * boundary: registration by plugin id, revocation at the start of `disable`,
+ * and `invoke` answering with the registered directory. Tests can hold
+ * `enable` and `disable` open.
+ */
+const makeStubSupervisor = Effect.gen(function* () {
+ const registrations = new Map();
+ const invoked: Array = [];
+ const holds: { enable?: Hold; disable?: Hold } = {};
+ const events = yield* PubSub.unbounded();
+ const service = PluginSupervisor.PluginSupervisor.of({
+ enable: (registration) =>
+ Effect.suspend(() => {
+ const pluginId = registration.manifest.id;
+ if (registrations.has(pluginId))
+ return Effect.fail(new PluginSupervisor.PluginAlreadyEnabledError({ pluginId }));
+ registrations.set(pluginId, registration);
+ return passHold(holds.enable);
+ }),
+ disable: (pluginId) =>
+ Effect.suspend(() => {
+ registrations.delete(pluginId);
+ return passHold(holds.disable);
+ }),
+ resume: () => Effect.void,
+ invoke: (pluginId) =>
+ Effect.suspend(() => {
+ const registration = registrations.get(pluginId);
+ if (registration === undefined)
+ return Effect.fail(new PluginSupervisor.PluginNotEnabledError({ pluginId }));
+ invoked.push(registration.directory);
+ return Effect.succeed(registration.directory);
+ }),
+ state: (pluginId) =>
+ Effect.sync(() =>
+ registrations.has(pluginId) ? Option.some({ _tag: "idle" as const }) : Option.none(),
+ ),
+ subscribe: PubSub.subscribe(events),
+ serveHostMethod: () => Effect.void,
+ });
+ return { service, registrations, invoked, holds };
+});
+
+/** The real file system, except that the next read of a held directory waits for its hold. */
+const makeHoldingFileSystem = Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ let held: { readonly directory: string; readonly hold: Hold } | undefined;
+ const fileSystem: FileSystem.FileSystem = {
+ ...fs,
+ realPath: (target) =>
+ Effect.suspend(() => {
+ if (held === undefined || !target.startsWith(held.directory)) return fs.realPath(target);
+ const { hold } = held;
+ held = undefined;
+ return passHold(hold).pipe(Effect.andThen(fs.realPath(target)));
+ }),
+ };
+ const holdDirectory = Effect.fn("holdDirectory")(function* (directory: string) {
+ const hold = yield* makeHold;
+ held = { directory, hold };
+ return hold;
+ });
+ return { fileSystem, holdDirectory };
+});
+
+/** A catalogue over the stub supervisor, reading files through `fileSystem`. */
+const startStubCatalog = Effect.fn("startStubCatalog")(function* (
+ scope: Scope.Scope,
+ supervisor: PluginSupervisor.PluginSupervisor["Service"],
+ fileSystem?: FileSystem.FileSystem,
+) {
+ return yield* PluginCatalog.make().pipe(
+ Effect.provideService(PluginSupervisor.PluginSupervisor, supervisor),
+ Effect.provideService(FileSystem.FileSystem, fileSystem ?? (yield* FileSystem.FileSystem)),
+ Effect.provideService(Scope.Scope, scope),
+ );
+});
+
+const pidOf = (value: unknown) => (value as { readonly pid: number }).pid;
+
+const isProcessAlive = (pid: number) => {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+};
+
+// Each test gets its own database; the restart test shares one between two starts.
+const withDatabase = (effect: Effect.Effect) =>
+ effect.pipe(Effect.provide(SqlitePersistence.layerMemory));
+
+it.layer(NodeServices.layer)("PluginCatalog", (it) => {
+ it.effect("revokes changed code even when persisting its disabled record fails", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const stub = yield* makeStubSupervisor;
+ const catalog = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ const plugin = yield* preparePlugin("test.failed-revocation");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ yield* catalog.enable({ installationId });
+ yield* plugin.edit("changed after consent");
+ yield* sql`CREATE TRIGGER refuse_plugin_save BEFORE UPDATE ON plugin_installations BEGIN SELECT RAISE(FAIL, 'test write failure'); END`;
+ const failed = yield* catalog.refresh({ installationId }).pipe(Effect.flip);
+ expect(failed.reason).toBe("storage");
+ expect(stub.registrations.size).toBe(0);
+ const listed = (yield* catalog.list).installations[0]!;
+ expect(listed.enabled).toBe(false);
+ const published = yield* awaitSnapshot(catalog, ([row]) => row?.enabled === false);
+ expect(published[0]?.enabled).toBe(false);
+ const invoked = yield* catalog.invoke(installationId, "ping", null).pipe(Effect.flip);
+ expect(invoked._tag).toBe("PluginCatalogError");
+ expect(stub.invoked).toEqual([]);
+ yield* sql`DROP TRIGGER refuse_plugin_save`;
+ yield* catalog.refresh({ installationId });
+ expect((yield* catalog.list).installations[0]?.enabled).toBe(false);
+ const saved = yield* sql<{
+ readonly record_json: string;
+ }>`SELECT record_json FROM plugin_installations WHERE installation_id = ${installationId}`;
+ expect(JSON.parse(saved[0]!.record_json).enabled).toBe(false);
+ }),
+ ),
+ );
+
+ it.effect("persists a replacement's committed outcome independently of later consent", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const scope = yield* Scope.Scope;
+ const stub = yield* makeStubSupervisor;
+ const catalog = yield* startStubCatalog(scope, stub.service);
+ const plugin = yield* preparePlugin("test.committed-replacement");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ yield* plugin.edit("approved replacement");
+ const refreshed = yield* catalog.refresh({ installationId });
+ const digest = refreshed.installations[0]!.source!.digest;
+ yield* catalog.replace(
+ { installationId, digest },
+ { paths: [], swap: Effect.void, restore: Effect.die("unexpected rollback") },
+ );
+ yield* plugin.edit("later consent");
+ const later = yield* catalog.refresh({ installationId });
+ yield* catalog.consent({ installationId, digest: later.installations[0]!.source!.digest });
+ for (const current of [catalog, yield* startStubCatalog(scope, stub.service)]) {
+ let restored = false;
+ expect(
+ yield* current.settleReplace(
+ { installationId, digest },
+ {
+ paths: [],
+ restore: Effect.sync(() => {
+ restored = true;
+ }),
+ },
+ ),
+ ).toBe("committed");
+ expect(restored).toBe(false);
+ }
+ }),
+ ),
+ );
+
+ describe("consent", () => {
+ it.effect("runs nothing until the exact bytes are approved, then starts on first use", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const sql = yield* SqlClient.SqlClient;
+ const catalog = yield* startCatalog(yield* Scope.Scope);
+ const plugin = yield* preparePlugin("test.lazy");
+
+ const relative = yield* catalog.add({ directory: "plugins/lazy" }).pipe(Effect.flip);
+ expect(relative.reason).toBe("invalid-directory");
+
+ const { installation: added } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = added.installationId;
+ expect(pluginInstallationStatus(added)).toBe("needs-consent");
+ expect(added).toMatchObject({ enabled: false, consent: null, generation: 0 });
+ expect(added.manifest?.id).toBe("test.lazy");
+ expect(added.hostState).toBeUndefined();
+ const again = yield* catalog.add({ directory: plugin.directory }).pipe(Effect.flip);
+ expect(again.reason).toBe("already-added");
+
+ const unapproved = yield* catalog.enable({ installationId }).pipe(Effect.flip);
+ expect(unapproved.reason).toBe("consent-required");
+ const wrongDigest = yield* catalog
+ .consent({ installationId, digest: `sha256:${"0".repeat(64)}` })
+ .pipe(Effect.flip);
+ expect(wrongDigest.reason).toBe("source-changed");
+
+ const digest = added.source!.digest;
+ const { installation: approved } = yield* catalog.consent({ installationId, digest });
+ expect(pluginInstallationStatus(approved)).toBe("disabled");
+ expect(approved.consent).toMatchObject({ digest, capabilities: [] });
+
+ const { installation: enabled } = yield* catalog.enable({ installationId });
+ expect(pluginInstallationStatus(enabled)).toBe("enabled");
+ expect(enabled).toMatchObject({ generation: 1, hostState: { _tag: "idle" } });
+ expect(yield* fs.exists(plugin.marker)).toBe(false);
+
+ const pid = pidOf(yield* catalog.invoke(installationId, "ping", null));
+ expect(isProcessAlive(pid)).toBe(true);
+ yield* awaitSnapshot(catalog, ([row]) => row?.hostState?._tag === "running");
+
+ const { installation: disabled } = yield* catalog.disable({ installationId });
+ expect(isProcessAlive(pid)).toBe(false);
+ expect(pluginInstallationStatus(disabled)).toBe("disabled");
+ expect(disabled.hostState).toBeUndefined();
+ const stopped = yield* catalog.invoke(installationId, "ping", null).pipe(Effect.flip);
+ expect(stopped._tag).toBe("PluginCatalogError");
+
+ expect((yield* catalog.enable({ installationId })).installation.generation).toBe(2);
+ expect(yield* catalog.remove({ installationId })).toEqual({ installationId });
+ expect((yield* catalog.list).installations).toEqual([]);
+ // Settings promises that removing an added directory leaves it in place.
+ expect(yield* fs.exists(plugin.directory)).toBe(true);
+ expect(yield* sql`SELECT installation_id FROM plugin_installations`).toEqual([]);
+ }),
+ ),
+ );
+
+ it.effect("stops a plugin and asks again when its bytes change", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const catalog = yield* startCatalog(yield* Scope.Scope);
+ const plugin = yield* preparePlugin("test.changed");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ const firstDigest = installation.source!.digest;
+ yield* catalog.consent({ installationId, digest: firstDigest });
+ yield* catalog.enable({ installationId });
+ const pid = pidOf(yield* catalog.invoke(installationId, "ping", null));
+
+ yield* plugin.edit("changed while running");
+ const [refreshed] = (yield* catalog.refresh({ installationId })).installations;
+ expect(pluginInstallationStatus(refreshed!)).toBe("needs-consent");
+ expect(refreshed!.enabled).toBe(false);
+ expect(isProcessAlive(pid)).toBe(false);
+ const changed = yield* catalog.enable({ installationId }).pipe(Effect.flip);
+ expect(changed.reason).toBe("consent-required");
+ const stale = yield* catalog
+ .consent({ installationId, digest: firstDigest })
+ .pipe(Effect.flip);
+ expect(stale.reason).toBe("source-changed");
+
+ yield* catalog.consent({ installationId, digest: refreshed!.source!.digest });
+ yield* catalog.enable({ installationId });
+ // Changed again with no refresh: the check before a fresh process catches it.
+ yield* plugin.edit("changed before first use");
+ yield* fs.remove(plugin.marker);
+ const beforeStart = yield* catalog.invoke(installationId, "ping", null).pipe(Effect.flip);
+ expect(beforeStart).toMatchObject({ reason: "source-changed" });
+ expect(yield* fs.exists(plugin.marker)).toBe(false);
+ const [revoked] = (yield* catalog.list).installations;
+ expect(pluginInstallationStatus(revoked!)).toBe("needs-consent");
+ }),
+ ),
+ );
+
+ it.effect("checks the bytes when enabling an installation that is already enabled", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const catalog = yield* startCatalog(yield* Scope.Scope);
+ const plugin = yield* preparePlugin("test.enable-again");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ const approve = (digest: string) =>
+ catalog
+ .consent({ installationId, digest })
+ .pipe(Effect.andThen(catalog.enable({ installationId })));
+ const { installation: enabled } = yield* approve(installation.source!.digest);
+
+ // Unchanged: the same registration, not a new generation.
+ const { installation: same } = yield* catalog.enable({ installationId });
+ expect(same).toMatchObject({ enabled: true, generation: enabled.generation });
+
+ // Changed while registered but idle.
+ yield* plugin.edit("changed while idle");
+ const idle = yield* catalog.enable({ installationId }).pipe(Effect.flip);
+ expect(idle.reason).toBe("consent-required");
+ const [afterIdle] = (yield* catalog.list).installations;
+ expect(pluginInstallationStatus(afterIdle!)).toBe("needs-consent");
+ expect(afterIdle!.enabled).toBe(false);
+
+ // Changed while its process runs: enabling again stops it.
+ yield* approve(afterIdle!.source!.digest);
+ const pid = pidOf(yield* catalog.invoke(installationId, "ping", null));
+ yield* plugin.edit("changed while running");
+ const running = yield* catalog.enable({ installationId }).pipe(Effect.flip);
+ expect(running.reason).toBe("consent-required");
+ expect(isProcessAlive(pid)).toBe(false);
+ const [afterRunning] = (yield* catalog.list).installations;
+ expect(pluginInstallationStatus(afterRunning!)).toBe("needs-consent");
+ expect(afterRunning!.hostState).toBeUndefined();
+ }),
+ ),
+ );
+
+ it.effect("runs one directory per plugin id at a time", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const catalog = yield* startCatalog(yield* Scope.Scope);
+ const approve = Effect.fn(function* (directory: string) {
+ const { installation } = yield* catalog.add({ directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ return installationId;
+ });
+ const first = yield* approve((yield* preparePlugin("test.same")).directory);
+ const second = yield* approve((yield* preparePlugin("test.same")).directory);
+
+ yield* catalog.enable({ installationId: first });
+ const conflict = yield* catalog.enable({ installationId: second }).pipe(Effect.flip);
+ expect(conflict.reason).toBe("plugin-id-conflict");
+ yield* catalog.disable({ installationId: first });
+ const { installation } = yield* catalog.enable({ installationId: second });
+ expect(installation.enabled).toBe(true);
+ }),
+ ),
+ );
+
+ it.effect("keeps disable and remove available when the directory is gone", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const catalog = yield* startCatalog(yield* Scope.Scope);
+ const plugin = yield* preparePlugin("test.gone");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ yield* catalog.enable({ installationId });
+
+ yield* fs.remove(plugin.directory, { recursive: true });
+ const [missing] = (yield* catalog.refresh({})).installations;
+ expect(pluginInstallationStatus(missing!)).toBe("unavailable");
+ expect(missing).toMatchObject({ enabled: false, source: null });
+ expect(missing!.problem).toContain("does not exist");
+ expect(missing!.manifest?.id).toBe("test.gone");
+ expect((yield* catalog.enable({ installationId }).pipe(Effect.flip)).reason).toBe(
+ "unavailable",
+ );
+ expect((yield* catalog.disable({ installationId })).installation.enabled).toBe(false);
+ yield* catalog.remove({ installationId });
+ const unknown = yield* catalog
+ .disable({ installationId: PluginInstallationId.make("missing") })
+ .pipe(Effect.flip);
+ expect(unknown.reason).toBe("not-found");
+ }),
+ ),
+ );
+ });
+
+ describe("calls racing management", () => {
+ it.effect("fails a call whose installation is replaced while its bytes are checked", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const stub = yield* makeStubSupervisor;
+ const files = yield* makeHoldingFileSystem;
+ const catalog = yield* startStubCatalog(
+ yield* Scope.Scope,
+ stub.service,
+ files.fileSystem,
+ );
+ const approve = Effect.fn(function* (directory: string) {
+ const { installation } = yield* catalog.add({ directory });
+ yield* catalog.consent({
+ installationId: installation.installationId,
+ digest: installation.source!.digest,
+ });
+ return installation;
+ });
+ const first = yield* approve((yield* preparePlugin("test.same")).directory);
+ const second = yield* approve((yield* preparePlugin("test.same")).directory);
+ yield* catalog.enable({ installationId: first.installationId });
+
+ // The call stops in its byte check, before a fresh process would start.
+ const hold = yield* files.holdDirectory(first.directory);
+ const call = yield* catalog
+ .invoke(first.installationId, "ping", null)
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(hold.reached);
+ yield* catalog.disable({ installationId: first.installationId });
+ yield* catalog.enable({ installationId: second.installationId });
+ yield* Deferred.succeed(hold.release, undefined);
+
+ const refused = yield* Fiber.join(call).pipe(Effect.flip);
+ expect(refused).toMatchObject({ _tag: "PluginCatalogError", reason: "unavailable" });
+ expect(stub.invoked).toEqual([]);
+ expect(yield* catalog.invoke(second.installationId, "ping", null)).toBe(second.directory);
+ }),
+ ),
+ );
+
+ it.effect(
+ "fails a call when its installation is enabled again or names an old generation",
+ () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const stub = yield* makeStubSupervisor;
+ const files = yield* makeHoldingFileSystem;
+ const catalog = yield* startStubCatalog(
+ yield* Scope.Scope,
+ stub.service,
+ files.fileSystem,
+ );
+ const plugin = yield* preparePlugin("test.generation");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ const { installation: enabled } = yield* catalog.enable({ installationId });
+
+ const hold = yield* files.holdDirectory(installation.directory);
+ const call = yield* catalog
+ .invoke(installationId, "ping", null)
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(hold.reached);
+ yield* catalog.disable({ installationId });
+ const { installation: again } = yield* catalog.enable({ installationId });
+ yield* Deferred.succeed(hold.release, undefined);
+ expect(yield* Fiber.join(call).pipe(Effect.flip)).toMatchObject({
+ reason: "unavailable",
+ });
+ expect(stub.invoked).toEqual([]);
+
+ expect(again.generation).toBe(enabled.generation + 1);
+ const stale = yield* catalog
+ .invoke(installationId, "ping", null, { generation: enabled.generation })
+ .pipe(Effect.flip);
+ expect(stale).toMatchObject({ reason: "generation-changed" });
+ expect(stub.invoked).toEqual([]);
+ expect(
+ yield* catalog.invoke(installationId, "ping", null, { generation: again.generation }),
+ ).toBe(installation.directory);
+ }),
+ ),
+ );
+ });
+
+ describe("subscription", () => {
+ it.effect("sends a snapshot only when a step changed what it shows", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const stub = yield* makeStubSupervisor;
+ const catalog = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ const plugin = yield* preparePlugin("test.quiet");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ const digest = installation.source!.digest;
+ yield* catalog.consent({ installationId, digest });
+
+ const snapshots = yield* Queue.unbounded>();
+ yield* catalog.subscribe.pipe(
+ Stream.runForEach((snapshot) => Queue.offer(snapshots, snapshot.installations)),
+ Effect.forkScoped,
+ );
+ const [initial] = yield* Queue.take(snapshots);
+ expect(initial).toMatchObject({ enabled: false });
+
+ // Failures and steps that find nothing new.
+ yield* catalog.add({ directory: "relative" }).pipe(Effect.flip);
+ yield* catalog.add({ directory: plugin.directory }).pipe(Effect.flip);
+ yield* catalog
+ .consent({ installationId, digest: `sha256:${"0".repeat(64)}` })
+ .pipe(Effect.flip);
+ yield* catalog.resume({ installationId }).pipe(Effect.flip);
+ yield* catalog.refresh({});
+ yield* catalog.disable({ installationId });
+
+ yield* catalog.enable({ installationId });
+ const [enabled] = yield* Queue.take(snapshots);
+ expect(enabled).toMatchObject({ enabled: true, inspectedAt: initial!.inspectedAt });
+
+ yield* catalog.enable({ installationId });
+ yield* catalog.resume({ installationId });
+ yield* catalog.disable({ installationId });
+ const [disabled] = yield* Queue.take(snapshots);
+ expect(disabled).toMatchObject({ enabled: false });
+ }),
+ ),
+ );
+ });
+
+ describe("interrupted management", () => {
+ /** An approved, enabled installation in a catalogue that can be restarted on the same database. */
+ const enabledInStub = Effect.fn("enabledInStub")(function* (
+ fileSystem?: FileSystem.FileSystem,
+ ) {
+ const stub = yield* makeStubSupervisor;
+ const before = yield* Scope.make();
+ const catalog = yield* startStubCatalog(before, stub.service, fileSystem);
+ const plugin = yield* preparePlugin("test.interrupted");
+ const { installation } = yield* catalog.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ yield* catalog.enable({ installationId });
+ return { stub, before, catalog, installationId, directory: installation.directory };
+ });
+
+ /** Starts the catalogue again and waits until its startup re-registration has run. */
+ const restart = Effect.fn("restart")(function* () {
+ const stub = yield* makeStubSupervisor;
+ const catalog = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ // Management steps queue behind startup, so this returns after it.
+ const { installations } = yield* catalog.refresh({});
+ return { stub, installations };
+ });
+
+ const storedRows = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const rows = yield* sql<{ readonly record_json: string }>`
+ SELECT record_json FROM plugin_installations
+ `;
+ return rows.map((row) => JSON.parse(row.record_json) as { readonly enabled: boolean });
+ });
+
+ it.effect("keeps a disable whose caller left while the process stopped", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { stub, before, catalog, installationId } = yield* enabledInStub();
+ const stopping = yield* makeHold;
+ stub.holds.disable = stopping;
+ const disabling = yield* catalog
+ .disable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(stopping.reached);
+ yield* Fiber.interrupt(disabling);
+
+ const [row] = (yield* catalog.list).installations;
+ expect(row).toMatchObject({ enabled: false });
+ expect(row!.hostState).toBeUndefined();
+ expect(yield* storedRows).toMatchObject([{ enabled: false }]);
+ expect(stub.registrations.size).toBe(0);
+ yield* Deferred.succeed(stopping.release, undefined);
+ yield* Scope.close(before, Exit.void);
+
+ const after = yield* restart();
+ expect(after.installations).toMatchObject([{ installationId, enabled: false }]);
+ expect(after.stub.registrations.size).toBe(0);
+ }),
+ ),
+ );
+
+ it.effect("makes a repeated disable wait for the interrupted stop", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { stub, catalog, installationId } = yield* enabledInStub();
+ const stopping = yield* makeHold;
+ stub.holds.disable = stopping;
+ const disabling = yield* catalog
+ .disable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(stopping.reached);
+ yield* Fiber.interrupt(disabling);
+
+ const retry = yield* catalog
+ .disable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Effect.yieldNow;
+ expect(retry.pollUnsafe()).toBeUndefined();
+ expect(stub.registrations.size).toBe(0);
+
+ yield* Deferred.succeed(stopping.release, undefined);
+ const { installation } = yield* Fiber.join(retry);
+ expect(installation).toMatchObject({ enabled: false });
+ expect(installation.hostState).toBeUndefined();
+ expect(stub.registrations.size).toBe(0);
+ }),
+ ),
+ );
+
+ it.effect("waits for an interrupted stop before inspecting and enabling again", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* makeHoldingFileSystem;
+ const { stub, catalog, installationId, directory } = yield* enabledInStub(fs.fileSystem);
+ const stopping = yield* makeHold;
+ stub.holds.disable = stopping;
+ const disabling = yield* catalog
+ .disable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(stopping.reached);
+ yield* Fiber.interrupt(disabling);
+
+ const inspection = yield* fs.holdDirectory(directory);
+ const enabling = yield* catalog
+ .enable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ const inspectedBeforeStop = yield* Deferred.isDone(inspection.reached);
+ expect(stub.registrations.size).toBe(0);
+
+ yield* Deferred.succeed(stopping.release, undefined);
+ yield* Deferred.await(inspection.reached);
+ yield* Deferred.succeed(inspection.release, undefined);
+ const { installation } = yield* Fiber.join(enabling);
+ expect(inspectedBeforeStop).toBe(false);
+ expect(installation).toMatchObject({ enabled: true, generation: 2 });
+ expect(stub.registrations.size).toBe(1);
+ }),
+ ),
+ );
+
+ it.effect("keeps a remove whose caller left while the process stopped", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { stub, before, catalog, installationId } = yield* enabledInStub();
+ const stopping = yield* makeHold;
+ stub.holds.disable = stopping;
+ const removing = yield* catalog
+ .remove({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(stopping.reached);
+ yield* Fiber.interrupt(removing);
+
+ expect((yield* catalog.list).installations).toEqual([]);
+ expect(yield* storedRows).toEqual([]);
+ expect(stub.registrations.size).toBe(0);
+ yield* Deferred.succeed(stopping.release, undefined);
+ yield* Scope.close(before, Exit.void);
+
+ const after = yield* restart();
+ expect(after.installations).toEqual([]);
+ expect(after.stub.registrations.size).toBe(0);
+ }),
+ ),
+ );
+
+ it.effect("finishes an enable whose caller left during registration", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { stub, before, catalog, installationId } = yield* enabledInStub();
+ yield* catalog.disable({ installationId });
+ const registering = yield* makeHold;
+ stub.holds.enable = registering;
+ const enabling = yield* catalog
+ .enable({ installationId })
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Deferred.await(registering.reached);
+ const interrupting = yield* Fiber.interrupt(enabling).pipe(
+ Effect.forkChild({ startImmediately: true }),
+ );
+ yield* Deferred.succeed(registering.release, undefined);
+ yield* Fiber.join(interrupting);
+
+ // Registered, saved, and shown together: never one without the others.
+ const [row] = (yield* catalog.list).installations;
+ expect(row).toMatchObject({ enabled: true, generation: 2 });
+ expect(yield* storedRows).toMatchObject([{ enabled: true }]);
+ expect(stub.registrations.size).toBe(1);
+ yield* Scope.close(before, Exit.void);
+ }),
+ ),
+ );
+
+ it.effect("prepares tool grants after startup restores registration generations", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const before = yield* Scope.make();
+ const first = yield* startStubCatalog(before, (yield* makeStubSupervisor).service);
+ const { directory } = yield* preparePlugin("test.startup-tools");
+ yield* fs.writeFileString(
+ path.join(directory, "t3-plugin.json"),
+ toJson({
+ id: "test.startup-tools",
+ name: "Startup tools",
+ version: "1.0.0",
+ apiVersion: 1,
+ entry: "main.mjs",
+ proposedApi: true,
+ capabilities: ["tools"],
+ tools: [
+ {
+ name: "ping",
+ description: "Ping",
+ inputSchema: { type: "object" },
+ sideEffect: "read",
+ },
+ ],
+ }),
+ );
+ const { installation } = yield* first.add({ directory });
+ const installationId = installation.installationId;
+ yield* first.consent({ installationId, digest: installation.source!.digest });
+ yield* first.enable({ installationId });
+ yield* Scope.close(before, Exit.void);
+
+ const stub = yield* makeStubSupervisor;
+ const registering = yield* makeHold;
+ stub.holds.enable = registering;
+ const catalog = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ yield* Effect.addFinalizer(() => Deferred.succeed(registering.release, undefined));
+ yield* Deferred.await(registering.reached);
+ const tools = yield* PluginTools.make.pipe(
+ Effect.provideService(PluginCatalog.PluginCatalog, catalog),
+ );
+ const preparing = yield* tools.grants.pipe(Effect.forkChild({ startImmediately: true }));
+ expect(preparing.pollUnsafe()).toBeUndefined();
+ yield* Deferred.succeed(registering.release, undefined);
+ const grants = yield* Fiber.join(preparing);
+ expect(grants).toEqual([{ installationId, generation: 2 }]);
+ expect((yield* tools.list(grants)).tools).toHaveLength(1);
+ expect(
+ yield* tools.call(grants, {
+ tool: "test.startup-tools/ping",
+ input: {},
+ context: {
+ environmentId: EnvironmentId.make("test"),
+ threadId: ThreadId.make("test"),
+ },
+ }),
+ ).toBe(directory);
+ }),
+ ),
+ );
+
+ it.effect("holds a call made during startup until the plugin is registered again", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { before, installationId } = yield* enabledInStub();
+ yield* Scope.close(before, Exit.void);
+
+ const stub = yield* makeStubSupervisor;
+ const registering = yield* makeHold;
+ stub.holds.enable = registering;
+ const after = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ // Runs before the catalogue's finalizers, so a failing test can still close its scope.
+ yield* Effect.addFinalizer(() => Deferred.succeed(registering.release, undefined));
+ yield* Deferred.await(registering.reached);
+ const calling = yield* after
+ .invoke(installationId, "ping", null)
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ yield* Effect.yieldNow;
+ expect(calling.pollUnsafe()).toBeUndefined();
+
+ yield* Deferred.succeed(registering.release, undefined);
+ const [directory] = [...stub.registrations.values()].map((r) => r.directory);
+ expect(yield* Fiber.join(calling)).toBe(directory);
+ expect(stub.invoked).toEqual([directory]);
+ }),
+ ),
+ );
+
+ it.effect("keeps a remove made as the catalogue starts", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { before, installationId } = yield* enabledInStub();
+ yield* Scope.close(before, Exit.void);
+
+ const stub = yield* makeStubSupervisor;
+ const after = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ yield* after.remove({ installationId });
+ // A call waits for startup to finish restoring.
+ yield* after.invoke(installationId, "ping", null).pipe(Effect.flip);
+
+ expect((yield* after.list).installations).toEqual([]);
+ expect(yield* storedRows).toEqual([]);
+ expect(stub.registrations.size).toBe(0);
+ }),
+ ),
+ );
+
+ it.effect("keeps an enable made as the catalogue starts", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const { before, installationId } = yield* enabledInStub();
+ yield* Scope.close(before, Exit.void);
+
+ const stub = yield* makeStubSupervisor;
+ const after = yield* startStubCatalog(yield* Scope.Scope, stub.service);
+ yield* after.enable({ installationId });
+ const [directory] = [...stub.registrations.values()].map((r) => r.directory);
+ expect(yield* after.invoke(installationId, "ping", null)).toBe(directory);
+
+ const [row] = (yield* after.list).installations;
+ expect(row).toMatchObject({ enabled: true, generation: 2 });
+ expect(stub.registrations.size).toBe(1);
+ }),
+ ),
+ );
+ });
+
+ describe("server restart", () => {
+ it.effect("re-enables approved plugins without starting them and drops changed ones", () =>
+ withDatabase(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const kept = yield* preparePlugin("test.kept");
+ const changed = yield* preparePlugin("test.restart-changed");
+
+ const before = yield* Scope.make();
+ const first = yield* startCatalog(before);
+ const ids = yield* Effect.forEach([kept, changed], (plugin) =>
+ Effect.gen(function* () {
+ const { installation } = yield* first.add({ directory: plugin.directory });
+ const installationId = installation.installationId;
+ yield* first.consent({ installationId, digest: installation.source!.digest });
+ yield* first.enable({ installationId });
+ return installationId;
+ }),
+ );
+ const pid = pidOf(yield* first.invoke(ids[0]!, "ping", null));
+ yield* Scope.close(before, Exit.void);
+ expect(isProcessAlive(pid)).toBe(false);
+
+ yield* fs.remove(kept.marker);
+ yield* changed.edit("changed while the server was down");
+ const after = yield* startCatalog(yield* Scope.Scope);
+ const restarted = yield* awaitSnapshot(after, (rows) =>
+ rows.every((row) => row.hostState?._tag === "idle" || !row.enabled),
+ );
+ const keptRow = restarted.find((row) => row.installationId === ids[0]);
+ const changedRow = restarted.find((row) => row.installationId === ids[1]);
+ expect(keptRow).toMatchObject({ enabled: true, generation: 2 });
+ expect(pluginInstallationStatus(changedRow!)).toBe("needs-consent");
+ expect(changedRow!.enabled).toBe(false);
+ expect(yield* fs.exists(kept.marker)).toBe(false);
+
+ const restartedPid = pidOf(yield* after.invoke(ids[0]!, "ping", null));
+ expect(restartedPid).not.toBe(pid);
+ expect(yield* fs.exists(kept.marker)).toBe(true);
+ }),
+ ),
+ );
+ });
+});
diff --git a/apps/server/src/plugins/PluginCatalog.ts b/apps/server/src/plugins/PluginCatalog.ts
new file mode 100644
index 0000000000000..613c7d534cfeb
--- /dev/null
+++ b/apps/server/src/plugins/PluginCatalog.ts
@@ -0,0 +1,939 @@
+// Consent binds to exact directory bytes; edits revoke registration and require fresh consent.
+// The digest cannot constrain its OS-user owner between checks or turn trusted plugins into a sandbox.
+import {
+ PluginCatalogError,
+ PluginInstallation,
+ pluginInstallationStatus,
+ type PluginAddInput,
+ type PluginCatalogSnapshot,
+ type PluginConsentInput,
+ type PluginId,
+ type PluginInstallationInput,
+ type PluginInstallationManifest,
+ type PluginInstallationResult,
+ type PluginManifest,
+ type PluginRefreshInput,
+ type PluginRemoveResult,
+ type PluginSource,
+ PluginInstallationId,
+ PluginSourceDigest,
+} from "@t3tools/contracts";
+import * as Context from "effect/Context";
+import * as Crypto from "effect/Crypto";
+import * as DateTime from "effect/DateTime";
+import * as Equal from "effect/Equal";
+import * as Deferred from "effect/Deferred";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+import * as PubSub from "effect/PubSub";
+import * as Schema from "effect/Schema";
+import * as Semaphore from "effect/Semaphore";
+import * as Stream from "effect/Stream";
+import * as SqlClient from "effect/sql/SqlClient";
+
+import { PluginEventDelivery } from "./PluginEventDelivery.ts";
+import { loadPluginDirectory, type PluginRegistration } from "./PluginManifestLoader.ts";
+import {
+ defaultPluginSourceLimits,
+ digestPluginSource,
+ type PluginSourceLimits,
+} from "./pluginSource.ts";
+import { PluginSupervisor, type PluginInvokeError } from "./PluginSupervisor.ts";
+
+/** What is persisted: the wire record without the live process and delivery states. */
+const PluginInstallationRecord = PluginInstallation.mapFields(
+ ({ hostState: _hostState, eventDelivery: _eventDelivery, ...fields }) => ({
+ ...fields,
+ // A successful file swap stays committed when a later consent changes.
+ replacement: Schema.optionalKey(
+ Schema.Struct({
+ digest: PluginSourceDigest,
+ committed: Schema.Boolean,
+ attemptId: Schema.optionalKey(Schema.String),
+ }),
+ ),
+ }),
+);
+type PluginInstallationRecord = typeof PluginInstallationRecord.Type;
+
+const decodeRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(PluginInstallationRecord));
+const encodeRecord = Schema.encodeEffect(Schema.fromJsonString(PluginInstallationRecord));
+
+/** One registration with the supervisor. A re-enable makes a new one; it is never mutated. */
+interface Registration {
+ readonly pluginId: PluginId;
+ readonly generation: number;
+}
+
+interface Installation {
+ record: PluginInstallationRecord;
+ /** Revocation is visible even when storage fails; retry that write on refresh. */
+ unsaved?: boolean;
+ /** The registration this installation runs under, while enabled. */
+ registered: Registration | undefined;
+ /** Waits for the process of the last registration revoked here to exit. */
+ stopping: Fiber.Fiber | undefined;
+}
+
+type Inspection =
+ | {
+ readonly _tag: "ok";
+ readonly registration: PluginRegistration;
+ readonly source: PluginSource;
+ }
+ | { readonly _tag: "failed"; readonly reason: string };
+
+/**
+ * How long a call waits for startup to re-register enabled plugins before it
+ * proceeds without them, matching the supervisor's default activation timeout.
+ */
+const STARTUP_RESTORE_WAIT = Duration.seconds(10);
+
+/** What one admission attempt of `invoke` found. */
+type Admission =
+ | { readonly _tag: "called"; readonly value: Schema.Json }
+ | { readonly _tag: "revoked" }
+ /** A fresh process would start: the bytes must be checked against `consented` first. */
+ | { readonly _tag: "check"; readonly consented: string };
+
+/** What clients see of a manifest; also the summary of a downloaded npm update under review. */
+export const summarizePluginManifest = (manifest: PluginManifest): PluginInstallationManifest => ({
+ id: manifest.id,
+ name: manifest.name,
+ version: manifest.version,
+ ...(manifest.description === undefined ? {} : { description: manifest.description }),
+ capabilities: manifest.capabilities,
+ proposedApi: manifest.proposedApi,
+ ...(manifest.tools === undefined || manifest.tools.length === 0 ? {} : { tools: manifest.tools }),
+ ...(manifest.settings === undefined ? {} : { settings: manifest.settings }),
+ ...(manifest.actions === undefined || manifest.actions.length === 0
+ ? {}
+ : { actions: manifest.actions }),
+});
+
+const catalogError = (
+ reason: string,
+ message: string,
+ installationId?: PluginInstallationId,
+): PluginCatalogError =>
+ new PluginCatalogError({
+ reason,
+ message,
+ ...(installationId === undefined ? {} : { installationId }),
+ });
+
+const storageError = (cause: unknown) =>
+ Effect.logWarning("Plugin catalogue storage failed", { cause }).pipe(
+ Effect.andThen(Effect.fail(catalogError("storage", "Could not save the plugin catalogue."))),
+ );
+
+export class PluginCatalog extends Context.Service<
+ PluginCatalog,
+ {
+ readonly list: Effect.Effect;
+ /** Completes after enabled installations have their startup registration generations. */
+ readonly ready: Effect.Effect;
+ /** Changes whenever `list` could show different installation records, so a reader can cache what it derives from them. */
+ readonly revision: Effect.Effect;
+ /** One snapshot now, then a fresh one after every catalogue or plugin state change. */
+ readonly subscribe: Stream.Stream;
+ readonly add: (
+ input: PluginAddInput,
+ ) => Effect.Effect;
+ readonly refresh: (
+ input: PluginRefreshInput,
+ ) => Effect.Effect;
+ readonly consent: (
+ input: PluginConsentInput,
+ ) => Effect.Effect;
+ readonly enable: (
+ input: PluginInstallationInput,
+ ) => Effect.Effect;
+ readonly disable: (
+ input: PluginInstallationInput,
+ ) => Effect.Effect;
+ readonly remove: (
+ input: PluginInstallationInput,
+ ) => Effect.Effect;
+ /** Clears the process's backoff, quarantine, or incompatibility, and restarts stopped event delivery. */
+ readonly resume: (
+ input: PluginInstallationInput,
+ ) => Effect.Effect;
+ // Claim all move paths and keep registration revoked throughout the locked swap and consent commit.
+ // Restore on precommit failure; re-enable only if previously enabled and the final bytes have consent.
+ readonly replace: (
+ input: PluginConsentInput & { readonly attemptId?: string },
+ files: {
+ readonly paths: ReadonlyArray;
+ readonly swap: Effect.Effect;
+ readonly restore: Effect.Effect;
+ },
+ ) => Effect.Effect;
+ // The persisted attempt outcome decides commit versus rollback independently of later consent.
+ // Recovery claims paths, waits for stop, and never grants consent or enables the installation.
+ readonly settleReplace: (
+ input: PluginConsentInput & { readonly attemptId?: string },
+ files?: {
+ readonly paths: ReadonlyArray;
+ readonly restore: Effect.Effect;
+ },
+ ) => Effect.Effect<"committed" | "rolled-back", PluginCatalogError>;
+ // Claim paths before mutation; reject overlapping installations and await every process that used those files.
+ readonly changeFiles: (
+ paths: ReadonlyArray,
+ effect: Effect.Effect,
+ ) => Effect.Effect;
+ // Checks consent before cold starts; generation pins the registration so revocation or replacement never redirects a call.
+ readonly invoke: (
+ installationId: PluginInstallationId,
+ handler: string,
+ input: Schema.Json,
+ options?: { readonly timeout?: Duration.Input; readonly generation?: number },
+ ) => Effect.Effect;
+ }
+>()("t3/plugins/PluginCatalog") {}
+
+export const make = Effect.fn("PluginCatalog.make")(function* (
+ sourceLimits: PluginSourceLimits = defaultPluginSourceLimits,
+) {
+ const sql = yield* SqlClient.SqlClient;
+ const supervisor = yield* PluginSupervisor;
+ const crypto = yield* Crypto.Crypto;
+ const path = yield* Path.Path;
+ const fileSystem = yield* FileSystem.FileSystem;
+ const scope = yield* Effect.scope;
+ const eventDeliveries = yield* Effect.serviceOption(PluginEventDelivery);
+
+ const installations = new Map();
+ // Management is rare and each step may wait for a process to exit; one at a time keeps the
+ // catalogue, the table, and the supervisor in step.
+ const lock = yield* Semaphore.make(1);
+ /**
+ * Processes still exiting after their registration was revoked, by the
+ * directory they ran from. Kept apart from `registered`, since a disable
+ * or remove whose caller went away no longer waits for its process.
+ */
+ const exiting = new Map, string>();
+ const changes = yield* PubSub.sliding(1);
+ const notify = PubSub.publish(changes, undefined).pipe(Effect.asVoid);
+ // Counts changes a snapshot can show, so a step that changed nothing tells no one.
+ let revision = 0;
+ // Completes when startup has tried to re-register every enabled installation.
+ const restored = yield* Deferred.make();
+
+ const now = DateTime.now.pipe(Effect.map(DateTime.formatIso));
+
+ const save = (record: PluginInstallationRecord) =>
+ encodeRecord(record).pipe(
+ Effect.flatMap(
+ (json) => sql`
+ INSERT INTO plugin_installations (installation_id, directory, record_json)
+ VALUES (${record.installationId}, ${record.directory}, ${json})
+ ON CONFLICT (installation_id) DO UPDATE SET
+ directory = excluded.directory,
+ record_json = excluded.record_json
+ `,
+ ),
+ Effect.asVoid,
+ Effect.catch(storageError),
+ );
+
+ /** Saves `record` as enabled, with the event cursor its capabilities need, or neither. */
+ const saveEnabled = (record: PluginInstallationRecord, capabilities: ReadonlyArray) =>
+ Option.match(eventDeliveries, {
+ onNone: () => save(record),
+ onSome: (delivery) =>
+ sql
+ .withTransaction(
+ delivery
+ .begin(record.installationId, capabilities)
+ .pipe(Effect.catch(storageError), Effect.andThen(save(record))),
+ )
+ .pipe(Effect.catchTags({ SqlError: storageError })),
+ });
+
+ /** Saves `record` and then shows it, or neither. */
+ const commit = (installation: Installation, record: PluginInstallationRecord) =>
+ save(record).pipe(
+ Effect.andThen(
+ Effect.sync(() => {
+ installation.record = record;
+ installation.unsaved = false;
+ revision++;
+ }),
+ ),
+ Effect.uninterruptible,
+ );
+
+ const inspect = (directory: string): Effect.Effect =>
+ loadPluginDirectory(directory).pipe(
+ Effect.flatMap((registration) =>
+ digestPluginSource(registration.directory, sourceLimits).pipe(
+ Effect.map((source) => ({ _tag: "ok" as const, registration, source })),
+ ),
+ ),
+ Effect.catch((error) => Effect.succeed({ _tag: "failed" as const, reason: error.reason })),
+ Effect.provideService(FileSystem.FileSystem, fileSystem),
+ Effect.provideService(Path.Path, path),
+ );
+
+ const isReady = (record: PluginInstallationRecord) =>
+ pluginInstallationStatus({ ...record, enabled: true }) === "enabled";
+
+ // Revocation starts immediately; retries share the original stop fiber and cannot stop a replacement registration.
+ const unregister = Effect.fnUntraced(function* (installation: Installation) {
+ const registered = installation.registered;
+ if (registered === undefined) return installation.stopping;
+ installation.registered = undefined;
+ revision++;
+ const stopping = yield* supervisor
+ .disable(registered.pluginId)
+ .pipe(Effect.forkIn(scope, { startImmediately: true }));
+ installation.stopping = stopping;
+ exiting.set(stopping, installation.record.directory);
+ stopping.addObserver(() => exiting.delete(stopping));
+ return stopping;
+ });
+
+ /**
+ * Revokes the registration, saves `record` (normally with `enabled: false`),
+ * and waits for the process to exit. Everything but the wait finishes even
+ * if the caller goes away, so a disable that was cut short never comes
+ * back enabled at the next start. A failed save leaves the registration revoked, so changed bytes cannot run.
+ */
+ const revoke = (installation: Installation, record: PluginInstallationRecord) =>
+ Effect.uninterruptibleMask((restore) =>
+ Effect.gen(function* () {
+ const stopping = yield* unregister(installation);
+ if (record !== installation.record || installation.unsaved)
+ yield* commit(installation, record).pipe(
+ Effect.tapError(() =>
+ Effect.sync(() => {
+ installation.record = record;
+ installation.unsaved = true;
+ revision++;
+ }),
+ ),
+ );
+ if (stopping) yield* restore(Fiber.join(stopping));
+ }),
+ );
+
+ /**
+ * Inspects the directory again and records the result. An enabled
+ * installation whose bytes no longer match its consent is stopped and
+ * disabled.
+ */
+ const reinspect = Effect.fnUntraced(function* (installation: Installation) {
+ const inspection = yield* inspect(installation.record.directory);
+ const current = installation.record;
+ const found =
+ inspection._tag === "ok"
+ ? {
+ manifest: summarizePluginManifest(inspection.registration.manifest),
+ source: inspection.source,
+ problem: null,
+ }
+ : { manifest: current.manifest, source: null, problem: inspection.reason };
+ // The same result keeps the record as it was, `inspectedAt` included.
+ if (
+ Equal.equals(found, {
+ manifest: current.manifest,
+ source: current.source,
+ problem: current.problem,
+ })
+ ) {
+ if (installation.unsaved) yield* commit(installation, current);
+ return inspection;
+ }
+ const record: PluginInstallationRecord = { ...current, ...found, inspectedAt: yield* now };
+ if (record.enabled && !isReady(record)) {
+ yield* Effect.logWarning("Plugin source changed; disabling until consent is renewed", {
+ installationId: record.installationId,
+ directory: record.directory,
+ });
+ yield* revoke(installation, { ...record, enabled: false });
+ } else {
+ yield* commit(installation, record);
+ }
+ return inspection;
+ });
+
+ /**
+ * Registers a ready installation with the supervisor under a new generation
+ * and saves it as enabled. Both happen or neither, even if the caller is
+ * interrupted.
+ */
+ const register = Effect.fnUntraced(function* (
+ installation: Installation,
+ registration: PluginRegistration,
+ ) {
+ const pluginId = registration.manifest.id;
+ const holder = [...installations.values()].find(
+ (other) => other !== installation && other.registered?.pluginId === pluginId,
+ );
+ if (holder)
+ return yield* catalogError(
+ "plugin-id-conflict",
+ `Another enabled plugin already uses the id ${pluginId} (${holder.record.directory}). Disable it first.`,
+ installation.record.installationId,
+ );
+ yield* supervisor
+ .enable({ ...registration, installationId: installation.record.installationId })
+ .pipe(
+ Effect.mapError(() =>
+ catalogError(
+ "plugin-id-conflict",
+ `A plugin with the id ${pluginId} is already running.`,
+ installation.record.installationId,
+ ),
+ ),
+ );
+ const record = {
+ ...installation.record,
+ enabled: true,
+ generation: installation.record.generation + 1,
+ };
+ yield* saveEnabled(record, registration.manifest.capabilities).pipe(
+ Effect.tapError(() => supervisor.disable(pluginId)),
+ );
+ // Together, so an invoke never sees the new registration with the old generation.
+ installation.record = record;
+ installation.registered = { pluginId, generation: record.generation };
+ revision++;
+ }, Effect.uninterruptible);
+
+ const find = (installationId: PluginInstallationId) =>
+ Effect.suspend(() => {
+ const installation = installations.get(installationId);
+ return installation
+ ? Effect.succeed(installation)
+ : Effect.fail(
+ catalogError("not-found", "That plugin is not installed here.", installationId),
+ );
+ });
+
+ const toWire = Effect.fnUntraced(function* (installation: Installation) {
+ const { replacement: _replacement, ...record } = installation.record;
+ const registered = installation.registered;
+ if (registered === undefined) return record;
+ const hostState = yield* supervisor.state(registered.pluginId);
+ const eventDelivery = Option.isSome(eventDeliveries)
+ ? yield* eventDeliveries.value.state(
+ installation.record.installationId,
+ registered.generation,
+ )
+ : Option.none();
+ const wire: PluginInstallation = {
+ ...record,
+ ...(Option.isSome(hostState) ? { hostState: hostState.value } : {}),
+ ...(Option.isSome(eventDelivery) ? { eventDelivery: eventDelivery.value } : {}),
+ };
+ return wire;
+ });
+
+ const list = Effect.suspend(() =>
+ Effect.forEach(
+ [...installations.values()].sort(
+ (a, b) =>
+ a.record.addedAt.localeCompare(b.record.addedAt) ||
+ a.record.installationId.localeCompare(b.record.installationId),
+ ),
+ toWire,
+ ),
+ ).pipe(Effect.map((installations) => ({ installations })));
+
+ const result = (installation: Installation) =>
+ toWire(installation).pipe(Effect.map((wire) => ({ installation: wire })));
+
+ /** Runs a management step under the lock and tells subscribers if it changed anything, even when it then failed. */
+ const managed = (effect: Effect.Effect) =>
+ lock.withPermit(
+ Effect.suspend(() => {
+ const before = revision;
+ return effect.pipe(
+ Effect.ensuring(Effect.suspend(() => (revision === before ? Effect.void : notify))),
+ );
+ }),
+ );
+
+ const add = Effect.fn("PluginCatalog.add")(function* (input: PluginAddInput) {
+ if (!path.isAbsolute(input.directory))
+ return yield* catalogError(
+ "invalid-directory",
+ "Enter the plugin directory's absolute path on the server's machine.",
+ );
+ const inspection = yield* inspect(input.directory);
+ if (inspection._tag === "failed")
+ return yield* catalogError("invalid-directory", inspection.reason);
+ const directory = inspection.registration.directory;
+ const existing = [...installations.values()].find(
+ (installation) => installation.record.directory === directory,
+ );
+ if (existing)
+ return yield* catalogError(
+ "already-added",
+ `${directory} is already installed.`,
+ existing.record.installationId,
+ );
+ const installationId = PluginInstallationId.make(
+ yield* crypto.randomUUIDv4().pipe(Effect.orDie),
+ );
+ const at = yield* now;
+ const record: PluginInstallationRecord = {
+ installationId,
+ generation: 0,
+ directory,
+ manifest: summarizePluginManifest(inspection.registration.manifest),
+ source: inspection.source,
+ problem: null,
+ inspectedAt: at,
+ consent: null,
+ enabled: false,
+ addedAt: at,
+ };
+ const installation: Installation = { record, registered: undefined, stopping: undefined };
+ yield* save(record).pipe(
+ Effect.andThen(
+ Effect.sync(() => {
+ installations.set(installationId, installation);
+ revision++;
+ }),
+ ),
+ Effect.uninterruptible,
+ );
+ return yield* result(installation);
+ }, managed);
+
+ const refresh = Effect.fn("PluginCatalog.refresh")(function* (input: PluginRefreshInput) {
+ const targets =
+ input.installationId === undefined
+ ? [...installations.values()]
+ : [yield* find(input.installationId)];
+ yield* Effect.forEach(targets, reinspect, { discard: true });
+ return yield* list;
+ }, managed);
+
+ const consent = Effect.fn("PluginCatalog.consent")(function* (input: PluginConsentInput) {
+ const installation = yield* find(input.installationId);
+ const inspection = yield* reinspect(installation);
+ if (inspection._tag === "failed")
+ return yield* catalogError("unavailable", inspection.reason, input.installationId);
+ if (inspection.source.digest !== input.digest)
+ return yield* catalogError(
+ "source-changed",
+ "The plugin's files changed after they were reviewed. Review the current version.",
+ input.installationId,
+ );
+ yield* commit(installation, {
+ ...installation.record,
+ ...(installation.record.replacement?.digest === input.digest
+ ? { replacement: { ...installation.record.replacement, committed: true } }
+ : {}),
+ consent: {
+ digest: inspection.source.digest,
+ capabilities: inspection.registration.manifest.capabilities,
+ grantedAt: yield* now,
+ },
+ });
+ return yield* result(installation);
+ }, managed);
+
+ const enable = Effect.fn("PluginCatalog.enable")(function* (input: PluginInstallationInput) {
+ const installation = yield* find(input.installationId);
+ // A caller may have left disable while its process was still exiting.
+ if (installation.stopping) yield* Fiber.join(installation.stopping);
+ // Checked even when already enabled: changed bytes stop it and need consent again.
+ const inspection = yield* reinspect(installation);
+ if (inspection._tag === "failed")
+ return yield* catalogError("unavailable", inspection.reason, input.installationId);
+ if (!isReady(installation.record))
+ return yield* catalogError(
+ "consent-required",
+ installation.record.consent === null
+ ? "Review and approve the plugin before enabling it."
+ : "The plugin's files changed since they were approved. Review the current version.",
+ input.installationId,
+ );
+ // Already enabled with these bytes: keep the registration and its generation.
+ if (installation.registered !== undefined) return yield* result(installation);
+ yield* register(installation, inspection.registration);
+ return yield* result(installation);
+ }, managed);
+
+ const disableInstallation = (installation: Installation) =>
+ revoke(
+ installation,
+ installation.record.enabled
+ ? { ...installation.record, enabled: false }
+ : installation.record,
+ );
+
+ const overlaps = (a: string, b: string) =>
+ a === b || a.startsWith(`${b}${path.sep}`) || b.startsWith(`${a}${path.sep}`);
+
+ /**
+ * The one rule for moving or deleting files under `paths`, run inside a
+ * management step: no installation but `owner` may be rooted in or around
+ * them, `owner` is disabled, and every process that ran from files there
+ * has exited before this returns.
+ */
+ const claimFiles = Effect.fnUntraced(function* (
+ paths: ReadonlyArray,
+ owner?: Installation,
+ ) {
+ const other = [...installations.values()].find(
+ (installation) =>
+ installation !== owner &&
+ paths.some((target) => overlaps(target, installation.record.directory)),
+ );
+ if (other)
+ return yield* catalogError(
+ "already-added",
+ `${other.record.directory} is installed as a plugin, so its files were left in place.`,
+ other.record.installationId,
+ );
+ if (owner) yield* disableInstallation(owner);
+ yield* Effect.forEach(
+ [...exiting].filter(([, directory]) => paths.some((target) => overlaps(target, directory))),
+ ([stopping]) => Fiber.await(stopping),
+ { discard: true },
+ );
+ });
+
+ const disable = Effect.fn("PluginCatalog.disable")(function* (input: PluginInstallationInput) {
+ const installation = yield* find(input.installationId);
+ yield* disableInstallation(installation);
+ return yield* result(installation);
+ }, managed);
+
+ const remove = Effect.fn("PluginCatalog.remove")(function* (input: PluginInstallationInput) {
+ const installation = yield* find(input.installationId);
+ // Forgotten durably first, like disable, so an interrupted remove does not come back.
+ yield* Effect.uninterruptibleMask((restore) =>
+ Effect.gen(function* () {
+ yield* sql`DELETE FROM plugin_installations WHERE installation_id = ${input.installationId}`.pipe(
+ Effect.catch(storageError),
+ );
+ installations.delete(input.installationId);
+ revision++;
+ const stopping = yield* unregister(installation);
+ if (stopping) yield* restore(Fiber.join(stopping));
+ }),
+ );
+ return { installationId: input.installationId };
+ }, managed);
+
+ const replace = Effect.fn("PluginCatalog.replace")(
+ function* (
+ input: PluginConsentInput & { readonly attemptId?: string },
+ files: Parameters[1],
+ ) {
+ const installation = yield* find(input.installationId);
+ const wasEnabled = installation.record.enabled;
+ yield* claimFiles([installation.record.directory, ...files.paths], installation);
+ yield* commit(installation, {
+ ...installation.record,
+ replacement: {
+ digest: input.digest,
+ committed: false,
+ ...(input.attemptId === undefined ? {} : { attemptId: input.attemptId }),
+ },
+ });
+ const enableAgain = (inspection: Inspection) =>
+ wasEnabled && inspection._tag === "ok" && isReady(installation.record)
+ ? register(installation, inspection.registration)
+ : Effect.void;
+ const inspection = yield* files.swap.pipe(
+ Effect.andThen(reinspect(installation)),
+ Effect.filterOrFail(
+ (inspection): inspection is Extract =>
+ inspection._tag === "ok" && inspection.source.digest === input.digest,
+ () =>
+ catalogError(
+ "source-changed",
+ "The new files changed after they were reviewed, so the old ones were put back.",
+ input.installationId,
+ ),
+ ),
+ Effect.tap((inspection) =>
+ Effect.gen(function* () {
+ yield* commit(installation, {
+ ...installation.record,
+ replacement: {
+ digest: inspection.source.digest,
+ committed: true,
+ ...(input.attemptId === undefined ? {} : { attemptId: input.attemptId }),
+ },
+ consent: {
+ digest: inspection.source.digest,
+ capabilities: inspection.registration.manifest.capabilities,
+ grantedAt: yield* now,
+ },
+ });
+ }),
+ ),
+ Effect.tapError(() =>
+ files.restore.pipe(
+ Effect.andThen(reinspect(installation)),
+ Effect.flatMap(enableAgain),
+ Effect.catch((error) =>
+ Effect.logWarning("Could not put a plugin's old files back", {
+ installationId: input.installationId,
+ detail: error.message,
+ }),
+ ),
+ ),
+ ),
+ );
+ yield* enableAgain(inspection);
+ return yield* result(installation);
+ },
+ Effect.uninterruptible,
+ managed,
+ );
+
+ const settleReplace = Effect.fn("PluginCatalog.settleReplace")(
+ function* (
+ input: PluginConsentInput & { readonly attemptId?: string },
+ files: Parameters[1],
+ ) {
+ const installation = yield* find(input.installationId);
+ // New journals must match this attempt; only legacy journals use digest/consent recovery.
+ const replacement = installation.record.replacement;
+ const committed =
+ input.attemptId !== undefined
+ ? replacement?.attemptId === input.attemptId &&
+ replacement.digest === input.digest &&
+ replacement.committed
+ : replacement?.digest === input.digest
+ ? replacement.committed
+ : installation.record.consent?.digest === input.digest;
+ if (committed) return "committed" as const;
+ if (files !== undefined) {
+ yield* claimFiles([installation.record.directory, ...files.paths], installation);
+ yield* files.restore;
+ yield* reinspect(installation);
+ }
+ return "rolled-back" as const;
+ },
+ Effect.uninterruptible,
+ managed,
+ );
+
+ const changeFiles: PluginCatalog["Service"]["changeFiles"] = Effect.fn(
+ "PluginCatalog.changeFiles",
+ )(
+ function* (paths: ReadonlyArray, effect: Effect.Effect) {
+ yield* claimFiles(paths);
+ return yield* effect;
+ },
+ Effect.uninterruptible,
+ managed,
+ );
+
+ const resume = Effect.fn("PluginCatalog.resume")(function* (input: PluginInstallationInput) {
+ const installation = yield* find(input.installationId);
+ if (installation.registered === undefined)
+ return yield* catalogError("unavailable", "The plugin is not enabled.", input.installationId);
+ yield* supervisor.resume(installation.registered.pluginId).pipe(Effect.ignore);
+ if (Option.isSome(eventDeliveries)) yield* eventDeliveries.value.resume(input.installationId);
+ return yield* result(installation);
+ }, managed);
+
+ /**
+ * Re-checks that `registered` is still this installation's registration and, unless the call
+ * would start a fresh process with unchecked bytes, hands the call to the supervisor. It runs
+ * in a fiber that starts synchronously, so no management step can revoke or replace the
+ * registration between the check and the supervisor admitting the call.
+ */
+ const admit = (
+ installation: Installation,
+ registered: Registration,
+ verified: string | undefined,
+ call: (pluginId: PluginId) => Effect.Effect,
+ ) =>
+ Effect.uninterruptibleMask((restore) =>
+ Effect.forkChild(
+ Effect.suspend((): Effect.Effect => {
+ if (
+ installations.get(installation.record.installationId) !== installation ||
+ installation.registered !== registered ||
+ installation.record.consent === null
+ )
+ return Effect.succeed({ _tag: "revoked" });
+ const consented = installation.record.consent.digest;
+ return supervisor
+ .state(registered.pluginId)
+ .pipe(
+ Effect.flatMap((state): Effect.Effect =>
+ Option.isSome(state) && state.value._tag === "idle" && verified !== consented
+ ? Effect.succeed({ _tag: "check", consented })
+ : call(registered.pluginId).pipe(
+ Effect.map((value) => ({ _tag: "called", value })),
+ ),
+ ),
+ );
+ }),
+ { startImmediately: true },
+ ).pipe(
+ Effect.flatMap((fiber) =>
+ restore(Fiber.join(fiber)).pipe(Effect.onInterrupt(() => Fiber.interrupt(fiber))),
+ ),
+ ),
+ );
+
+ const invoke: PluginCatalog["Service"]["invoke"] = Effect.fn("PluginCatalog.invoke")(
+ function* (installationId, handler, input, options) {
+ // A call right after a restart waits for its installation to be registered again.
+ yield* Deferred.await(restored).pipe(Effect.timeoutOption(STARTUP_RESTORE_WAIT));
+ const installation = yield* find(installationId);
+ const registered = installation.registered;
+ if (registered === undefined)
+ return yield* catalogError("unavailable", "The plugin is not enabled.", installationId);
+ if (options?.generation !== undefined && options.generation !== registered.generation)
+ return yield* catalogError(
+ "generation-changed",
+ "The plugin was enabled again since this call was prepared.",
+ installationId,
+ );
+ const call = (pluginId: PluginId) =>
+ supervisor.invoke(
+ pluginId,
+ handler,
+ input,
+ options?.timeout === undefined ? undefined : { timeout: options.timeout },
+ );
+ let verified: string | undefined;
+ while (true) {
+ const admission = yield* admit(installation, registered, verified, call);
+ if (admission._tag === "called") return admission.value;
+ if (admission._tag === "revoked")
+ return yield* catalogError(
+ "unavailable",
+ "The plugin was disabled or replaced before the call started.",
+ installationId,
+ );
+ const inspection = yield* inspect(installation.record.directory);
+ if (inspection._tag === "ok" && inspection.source.digest === admission.consented) {
+ verified = admission.consented;
+ continue;
+ }
+ // Skip the record if it was disabled or removed meanwhile, so it is not written back.
+ yield* managed(
+ Effect.suspend(() =>
+ installations.get(installationId) === installation &&
+ installation.registered === registered
+ ? reinspect(installation)
+ : Effect.void,
+ ),
+ );
+ return yield* catalogError(
+ "source-changed",
+ "The plugin's files changed since they were approved, so it was disabled.",
+ installationId,
+ );
+ }
+ },
+ );
+
+ // Load what was installed before this start.
+ const rows = yield* sql<{ readonly record_json: string }>`
+ SELECT record_json FROM plugin_installations
+ `.pipe(Effect.orDie);
+ for (const row of rows) {
+ const decoded = yield* decodeRecord(row.record_json).pipe(Effect.option);
+ if (Option.isNone(decoded)) {
+ yield* Effect.logWarning("Skipping an unreadable plugin installation row");
+ continue;
+ }
+ installations.set(decoded.value.installationId, {
+ record: decoded.value,
+ registered: undefined,
+ stopping: undefined,
+ });
+ }
+
+ // Plugin state changes reach subscribers as fresh snapshots.
+ const supervisorEvents = yield* supervisor.subscribe;
+ yield* Stream.fromSubscription(supervisorEvents).pipe(
+ Stream.filter((event) => event._tag === "StateChanged"),
+ Stream.runForEach(() => notify),
+ Effect.forkScoped,
+ );
+
+ // So do changes of event delivery state.
+ if (Option.isSome(eventDeliveries)) {
+ const deliveryChanges = yield* eventDeliveries.value.changes;
+ yield* Stream.fromSubscription(deliveryChanges).pipe(
+ Stream.runForEach(() => notify),
+ Effect.forkScoped,
+ );
+ }
+
+ // Re-register what was enabled, off the startup path. Changed bytes are disabled here;
+ // nothing starts a process until it is used. Starting at once takes the lock before this
+ // returns, so no management step can act on an installation still waiting to be restored.
+ yield* managed(
+ Effect.forEach(
+ [...installations.values()].filter((installation) => installation.record.enabled),
+ (installation) =>
+ reinspect(installation).pipe(
+ Effect.flatMap((inspection) =>
+ inspection._tag === "ok" && installation.record.enabled
+ ? register(installation, inspection.registration)
+ : Effect.void,
+ ),
+ Effect.catch((error) =>
+ Effect.logWarning("Could not re-enable a plugin at startup", {
+ installationId: installation.record.installationId,
+ detail: error.message,
+ }).pipe(Effect.andThen(disableInstallation(installation).pipe(Effect.ignore))),
+ ),
+ ),
+ { discard: true },
+ ),
+ ).pipe(
+ Effect.ensuring(Deferred.succeed(restored, undefined)),
+ Effect.forkIn(scope, { startImmediately: true }),
+ );
+
+ return PluginCatalog.of({
+ list,
+ ready: Deferred.await(restored),
+ revision: Effect.sync(() => revision),
+ subscribe: Stream.unwrap(
+ // Subscribe before the first snapshot so a change in between is not lost.
+ PubSub.subscribe(changes).pipe(
+ Effect.map((subscription) =>
+ Stream.concat(
+ Stream.fromEffect(list),
+ Stream.fromSubscription(subscription).pipe(Stream.mapEffect(() => list)),
+ ).pipe(
+ // A plugin state event and the step that caused it can describe the same snapshot.
+ Stream.changes,
+ ),
+ ),
+ ),
+ ),
+ add,
+ refresh,
+ consent,
+ enable,
+ disable,
+ remove,
+ resume,
+ replace,
+ settleReplace,
+ changeFiles,
+ invoke,
+ });
+});
+
+export const layer = (sourceLimits?: PluginSourceLimits) =>
+ Layer.effect(PluginCatalog, make(sourceLimits));
diff --git a/apps/server/src/plugins/PluginCatalogRpc.test.ts b/apps/server/src/plugins/PluginCatalogRpc.test.ts
new file mode 100644
index 0000000000000..9cd64fc3d4301
--- /dev/null
+++ b/apps/server/src/plugins/PluginCatalogRpc.test.ts
@@ -0,0 +1,147 @@
+import {
+ AuthAccessWriteScope,
+ type AuthEnvironmentScope,
+ AuthOrchestrationReadScope,
+ AuthRelayReadScope,
+ AuthStandardClientScopes,
+ PluginCatalogError,
+ PluginInstallationId,
+ WS_METHODS,
+ WsRpcGroup,
+} from "@t3tools/contracts";
+import { describe, expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Stream from "effect/Stream";
+import * as RpcTest from "effect/rpc/RpcTest";
+
+import { RPC_REQUIRED_SCOPES } from "../auth/RpcAuthorization.ts";
+import * as RpcAuthorization from "../auth/RpcAuthorization.ts";
+
+const reads = [WS_METHODS.pluginsList, WS_METHODS.pluginsSubscribe] as const;
+const writes = [
+ WS_METHODS.pluginsAdd,
+ WS_METHODS.pluginsRefresh,
+ WS_METHODS.pluginsConsent,
+ WS_METHODS.pluginsEnable,
+ WS_METHODS.pluginsDisable,
+ WS_METHODS.pluginsRemove,
+ WS_METHODS.pluginsResume,
+] as const;
+type PluginMethod = (typeof reads)[number] | (typeof writes)[number];
+const pluginMethods: ReadonlySet = new Set([...reads, ...writes]);
+
+const group = WsRpcGroup.omit(
+ ...[...WsRpcGroup.requests.keys()].filter(
+ (tag): tag is Exclude =>
+ !pluginMethods.has(tag),
+ ),
+);
+
+const installationId = PluginInstallationId.make("fixture");
+const digest = `sha256:${"0".repeat(64)}`;
+
+/** Serves the plugin RPCs through the real scope middleware; handlers record that they ran. */
+const makeClient = (scopes: ReadonlyArray, handled: Array) => {
+ // Mutations answer with a catalogue error: reaching it proves the middleware let the call in.
+ const mutation = (method: string) => () =>
+ Effect.sync(() => handled.push(method)).pipe(
+ Effect.andThen(
+ Effect.fail(new PluginCatalogError({ reason: "not-found", message: "fixture" })),
+ ),
+ );
+ return RpcTest.makeClient(group).pipe(
+ Effect.provide(
+ Layer.mergeAll(
+ group.toLayerHandler(WS_METHODS.pluginsList, () =>
+ Effect.sync(() => handled.push(WS_METHODS.pluginsList)).pipe(
+ Effect.as({ installations: [] }),
+ ),
+ ),
+ group.toLayerHandler(WS_METHODS.pluginsSubscribe, () =>
+ Stream.fromEffect(
+ Effect.sync(() => handled.push(WS_METHODS.pluginsSubscribe)).pipe(
+ Effect.as({ installations: [] }),
+ ),
+ ),
+ ),
+ group.toLayerHandler(WS_METHODS.pluginsAdd, mutation(WS_METHODS.pluginsAdd)),
+ group.toLayerHandler(WS_METHODS.pluginsRefresh, mutation(WS_METHODS.pluginsRefresh)),
+ group.toLayerHandler(WS_METHODS.pluginsConsent, mutation(WS_METHODS.pluginsConsent)),
+ group.toLayerHandler(WS_METHODS.pluginsEnable, mutation(WS_METHODS.pluginsEnable)),
+ group.toLayerHandler(WS_METHODS.pluginsDisable, mutation(WS_METHODS.pluginsDisable)),
+ group.toLayerHandler(WS_METHODS.pluginsRemove, mutation(WS_METHODS.pluginsRemove)),
+ group.toLayerHandler(WS_METHODS.pluginsResume, mutation(WS_METHODS.pluginsResume)),
+ RpcAuthorization.layer(scopes),
+ ),
+ ),
+ );
+};
+
+/** Calls every management RPC and returns the tag each one failed with. */
+const callWrites = (client: Effect.Success>) =>
+ Effect.all([
+ client[WS_METHODS.pluginsAdd]({ directory: "/plugins/fixture" }).pipe(Effect.flip),
+ client[WS_METHODS.pluginsRefresh]({}).pipe(Effect.flip),
+ client[WS_METHODS.pluginsConsent]({ installationId, digest }).pipe(Effect.flip),
+ client[WS_METHODS.pluginsEnable]({ installationId }).pipe(Effect.flip),
+ client[WS_METHODS.pluginsDisable]({ installationId }).pipe(Effect.flip),
+ client[WS_METHODS.pluginsRemove]({ installationId }).pipe(Effect.flip),
+ client[WS_METHODS.pluginsResume]({ installationId }).pipe(Effect.flip),
+ ]);
+
+describe("plugin RPC scopes", () => {
+ it.effect("lets a read-only pairing read the catalogue but not change what runs", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient([AuthOrchestrationReadScope], handled);
+
+ expect(yield* client[WS_METHODS.pluginsList]({})).toEqual({ installations: [] });
+ expect(
+ yield* client[WS_METHODS.pluginsSubscribe]({}).pipe(Stream.take(1), Stream.runCollect),
+ ).toEqual([{ installations: [] }]);
+
+ const failures = yield* callWrites(client);
+ for (const failure of failures) {
+ expect(failure).toMatchObject({
+ _tag: "EnvironmentAuthorizationError",
+ requiredScope: AuthAccessWriteScope,
+ requiredPermission: AuthAccessWriteScope,
+ });
+ }
+ expect(handled).toEqual([WS_METHODS.pluginsList, WS_METHODS.pluginsSubscribe]);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("lets a standard pairing manage plugins", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient(AuthStandardClientScopes, handled);
+
+ const failures = yield* callWrites(client);
+ for (const failure of failures) expect(failure._tag).toBe("PluginCatalogError");
+ expect(handled).toEqual([...writes]);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses catalogue reads without the orchestration read scope", () =>
+ Effect.gen(function* () {
+ const handled: Array = [];
+ const client = yield* makeClient([AuthRelayReadScope], handled);
+
+ expect(yield* client[WS_METHODS.pluginsList]({}).pipe(Effect.flip)).toMatchObject({
+ _tag: "EnvironmentAuthorizationError",
+ requiredScope: AuthOrchestrationReadScope,
+ requiredPermission: AuthOrchestrationReadScope,
+ });
+ expect(
+ yield* client[WS_METHODS.pluginsSubscribe]({}).pipe(Stream.runCollect, Effect.flip),
+ ).toMatchObject({
+ _tag: "EnvironmentAuthorizationError",
+ requiredScope: AuthOrchestrationReadScope,
+ requiredPermission: AuthOrchestrationReadScope,
+ });
+ expect(handled).toEqual([]);
+ }).pipe(Effect.scoped),
+ );
+});
diff --git a/apps/server/src/plugins/PluginEventDelivery.ts b/apps/server/src/plugins/PluginEventDelivery.ts
new file mode 100644
index 0000000000000..b5f6a580b4f02
--- /dev/null
+++ b/apps/server/src/plugins/PluginEventDelivery.ts
@@ -0,0 +1,116 @@
+/**
+ * The part of plugin event delivery that the plugin catalogue calls into. It
+ * lets the catalogue start a plugin's event cursor, show how delivery is
+ * going, and resume it, without depending on the event feed, which itself
+ * depends on the catalogue.
+ */
+import {
+ PLUGIN_EVENTS_CAPABILITY,
+ type PluginEventDeliveryState,
+ type PluginInstallationId,
+} from "@t3tools/contracts";
+import * as Context from "effect/Context";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as Equal from "effect/Equal";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as PubSub from "effect/PubSub";
+import type * as Scope from "effect/Scope";
+import * as SqlClient from "effect/sql/SqlClient";
+import type * as SqlError from "effect/sql/SqlError";
+
+export class PluginEventDelivery extends Context.Service<
+ PluginEventDelivery,
+ {
+ // Starts at the event-log end in the enabling transaction; re-enables and restarts retain the cursor.
+ readonly begin: (
+ installationId: PluginInstallationId,
+ capabilities: ReadonlyArray,
+ ) => Effect.Effect;
+ /** The delivery state of one registration, if the feed reported one. */
+ readonly state: (
+ installationId: PluginInstallationId,
+ generation: number,
+ ) => Effect.Effect>;
+ /** Reports a registration's state; `undefined` when it no longer delivers. */
+ readonly report: (
+ installationId: PluginInstallationId,
+ generation: number,
+ state: PluginEventDeliveryState | undefined,
+ ) => Effect.Effect;
+ /** Signals each reported change of a state (never a cursor move). Sliding, 1. */
+ readonly changes: Effect.Effect, never, Scope.Scope>;
+ /** Restarts quarantined or retrying delivery from its cursor. Nothing else changes. */
+ readonly resume: (installationId: PluginInstallationId) => Effect.Effect;
+ /** Installs what `resume` runs, for as long as the scope is open. */
+ readonly handleResume: (
+ resume: (installationId: PluginInstallationId) => Effect.Effect,
+ ) => Effect.Effect;
+ }
+>()("t3/plugins/PluginEventDelivery") {}
+
+export const make = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const states = new Map<
+ PluginInstallationId,
+ { readonly generation: number; readonly state: PluginEventDeliveryState }
+ >();
+ const changes = yield* PubSub.sliding(1);
+ let resume: ((installationId: PluginInstallationId) => Effect.Effect) | undefined;
+
+ return PluginEventDelivery.of({
+ begin: Effect.fn("PluginEventDelivery.begin")((installationId, capabilities) =>
+ capabilities.includes(PLUGIN_EVENTS_CAPABILITY)
+ ? Effect.gen(function* () {
+ // One statement, so the end it reads is the end when the row is written.
+ yield* sql`
+ INSERT INTO plugin_event_cursors (installation_id, acknowledged_sequence, updated_at)
+ SELECT ${installationId}, COALESCE(MAX(sequence), 0), ${DateTime.formatIso(yield* DateTime.now)}
+ FROM orchestration_events
+ WHERE application_event_version = 2
+ AND aggregate_kind = 'thread'
+ ON CONFLICT (installation_id) DO NOTHING
+ `;
+ })
+ : Effect.void,
+ ),
+ state: Effect.fn("PluginEventDelivery.state")((installationId, generation) =>
+ Effect.sync(() => {
+ const current = states.get(installationId);
+ return current?.generation === generation ? Option.some(current.state) : Option.none();
+ }),
+ ),
+ report: Effect.fn("PluginEventDelivery.report")((installationId, generation, state) =>
+ Effect.suspend(() => {
+ const current = states.get(installationId);
+ if (state === undefined) {
+ if (current?.generation !== generation) return Effect.void;
+ states.delete(installationId);
+ } else {
+ if (current?.generation === generation && Equal.equals(current.state, state))
+ return Effect.void;
+ states.set(installationId, { generation, state });
+ }
+ return PubSub.publish(changes, undefined).pipe(Effect.asVoid);
+ }),
+ ),
+ changes: PubSub.subscribe(changes),
+ resume: Effect.fn("PluginEventDelivery.resume")((installationId) =>
+ Effect.suspend(() => resume?.(installationId) ?? Effect.void),
+ ),
+ handleResume: Effect.fn("PluginEventDelivery.handleResume")((handler) =>
+ Effect.acquireRelease(
+ Effect.sync(() => {
+ resume = handler;
+ }),
+ () =>
+ Effect.sync(() => {
+ if (resume === handler) resume = undefined;
+ }),
+ ),
+ ),
+ });
+});
+
+export const layer = Layer.effect(PluginEventDelivery, make);
diff --git a/apps/server/src/plugins/PluginEventFeed.test.ts b/apps/server/src/plugins/PluginEventFeed.test.ts
new file mode 100644
index 0000000000000..0e723eb110957
--- /dev/null
+++ b/apps/server/src/plugins/PluginEventFeed.test.ts
@@ -0,0 +1,965 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { describe, expect, it } from "@effect/vitest";
+import {
+ EnvironmentId,
+ EventId,
+ PluginCatalogError,
+ type PluginCatalogSnapshot,
+ type PluginEventDeliveryState,
+ PluginInstallationId,
+ ProjectId,
+ ProviderInstanceId,
+ RunId,
+ ThreadId,
+} from "@t3tools/contracts";
+import * as HostProcess from "@t3tools/shared/HostProcess";
+import * as DateTime from "effect/DateTime";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+import * as PubSub from "effect/PubSub";
+import * as Queue from "effect/Queue";
+import * as Schema from "effect/Schema";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+import * as SqlClient from "effect/sql/SqlClient";
+import * as TestClock from "effect/testing/TestClock";
+
+import { ServerEnvironment } from "../environment/ServerEnvironment.ts";
+import * as EventSink from "../orchestration-v2/EventSink.ts";
+import * as EventStore from "../orchestration-v2/EventStore.ts";
+import { LiveStreamBufferError } from "../orchestration-v2/LiveStreamBudget.ts";
+import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
+import * as PluginCatalog from "./PluginCatalog.ts";
+import * as PluginEventDelivery from "./PluginEventDelivery.ts";
+import * as PluginEventFeed from "./PluginEventFeed.ts";
+import * as PluginManifestLoader from "./PluginManifestLoader.ts";
+import * as PluginSupervisor from "./PluginSupervisor.ts";
+
+// Children run the real CLI entry, which routes `__plugin-host` to the child runtime.
+const BIN_PATH = `${import.meta.dirname}/../bin.ts`;
+
+const environmentId = EnvironmentId.make("environment:plugin-events");
+const threadId = ThreadId.make("thread:plugin-events");
+const projectId = ProjectId.make("project:plugin-events");
+const providerInstanceId = ProviderInstanceId.make("codex");
+
+const toJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
+
+type Receipts = PubSub.Subscription;
+
+type Catalog = PluginCatalog.PluginCatalog["Service"];
+
+/** A catalogue whose snapshots the feed sees only once `observed` completes. */
+const observedAfter =
+ (observed: Deferred.Deferred) =>
+ (catalog: Catalog): Catalog => ({
+ ...catalog,
+ subscribe: Stream.unwrap(Deferred.await(observed).pipe(Effect.as(catalog.subscribe))),
+ });
+
+type Sink = EventSink.EventSinkV2["Service"];
+
+/**
+ * Starts a supervisor, catalogue and event feed in `scope`, as one server start
+ * would. `feedSees` changes the catalogue or event sink the feed sees.
+ */
+const startServer = Effect.fn("startServer")(function* (
+ scope: Scope.Scope,
+ options: Partial = {},
+ feedSees: {
+ readonly catalog?: (catalog: Catalog) => Catalog;
+ readonly eventSink?: (eventSink: Sink) => Sink;
+ readonly sql?: (sql: SqlClient.SqlClient) => SqlClient.SqlClient;
+ } = {},
+) {
+ const delivery = yield* PluginEventDelivery.make;
+ const supervisor = yield* PluginSupervisor.make({
+ heapLimitMb: 64,
+ activationTimeout: "10 seconds",
+ stopGrace: "1 second",
+ }).pipe(
+ Effect.provideService(HostProcess.Arguments, [process.execPath, BIN_PATH]),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ const catalog = yield* PluginCatalog.make().pipe(
+ Effect.provideService(PluginSupervisor.PluginSupervisor, supervisor),
+ Effect.provideService(PluginEventDelivery.PluginEventDelivery, delivery),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ const feed = yield* PluginEventFeed.make(options).pipe(
+ Effect.provideService(
+ SqlClient.SqlClient,
+ feedSees.sql?.(yield* SqlClient.SqlClient) ?? (yield* SqlClient.SqlClient),
+ ),
+ Effect.provideService(PluginCatalog.PluginCatalog, feedSees.catalog?.(catalog) ?? catalog),
+ Effect.provideService(
+ EventSink.EventSinkV2,
+ feedSees.eventSink?.(yield* EventSink.EventSinkV2) ?? (yield* EventSink.EventSinkV2),
+ ),
+ Effect.provideService(PluginEventDelivery.PluginEventDelivery, delivery),
+ Effect.provideService(
+ ServerEnvironment,
+ ServerEnvironment.of({
+ getEnvironmentId: Effect.succeed(environmentId),
+ getDescriptor: Effect.die("unused"),
+ }),
+ ),
+ Effect.provideService(Scope.Scope, scope),
+ );
+ const receipts: Receipts = yield* feed.subscribe.pipe(Effect.provideService(Scope.Scope, scope));
+ return { catalog, feed, receipts };
+});
+
+/**
+ * Writes a plugin that appends each event it handles to `events.log` next to
+ * its directory, and fails every event while a `fail` file exists there.
+ */
+const preparePlugin = Effect.fn("preparePlugin")(function* (id: string, onEvent = true) {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-plugin-events-" });
+ const directory = path.join(root, "plugin");
+ yield* fs.makeDirectory(directory);
+ const log = path.join(root, "events.log");
+ const fail = path.join(root, "fail");
+ yield* fs.writeFileString(
+ path.join(directory, "main.mjs"),
+ [
+ `import * as NodeFS from "node:fs";`,
+ `export function activate(context) {`,
+ ` if (!${onEvent}) return;`,
+ ` context.proposed.onEvent((event) => {`,
+ ` if (NodeFS.existsSync(${toJson(fail)})) throw new Error("told to fail");`,
+ ` NodeFS.appendFileSync(${toJson(log)}, JSON.stringify(event) + "\\n");`,
+ ` });`,
+ `}`,
+ ``,
+ ].join("\n"),
+ );
+ yield* fs.writeFileString(
+ path.join(directory, "t3-plugin.json"),
+ toJson({
+ id,
+ name: id,
+ version: "1.0.0",
+ apiVersion: 1,
+ entry: "main.mjs",
+ capabilities: ["events"],
+ proposedApi: true,
+ }),
+ );
+ const handled = fs.exists(log).pipe(
+ Effect.flatMap((exists) => (exists ? fs.readFileString(log) : Effect.succeed(""))),
+ Effect.map((content) =>
+ content
+ .split("\n")
+ .filter((line) => line !== "")
+ .map((line) => JSON.parse(line) as Record),
+ ),
+ );
+ return {
+ directory,
+ handled,
+ failing: (on: boolean) => (on ? fs.writeFileString(fail, "") : fs.remove(fail)),
+ };
+});
+
+const install = Effect.fn("install")(function* (catalog: Catalog, directory: string) {
+ const { installation } = yield* catalog.add({ directory });
+ const installationId = installation.installationId;
+ yield* catalog.consent({ installationId, digest: installation.source!.digest });
+ yield* catalog.enable({ installationId });
+ return installationId;
+});
+
+/** Follows the event delivery states that catalogue subscribers see. */
+const deliveryStates = Effect.fn("deliveryStates")(function* (catalog: Catalog) {
+ const snapshots = yield* Queue.unbounded();
+ yield* catalog.subscribe.pipe(
+ Stream.runForEach((snapshot) => Queue.offer(snapshots, snapshot)),
+ Effect.forkScoped,
+ );
+ const stateOf = (snapshot: PluginCatalogSnapshot, installationId: PluginInstallationId) =>
+ snapshot.installations.find((installation) => installation.installationId === installationId)
+ ?.eventDelivery;
+ return {
+ /** Takes snapshots through the first that shows `tag`; returns each state that changed on the way. */
+ untilTag: (installationId: PluginInstallationId, tag: PluginEventDeliveryState["_tag"]) =>
+ Effect.gen(function* () {
+ const states: Array = [];
+ while (true) {
+ const state = stateOf(yield* Queue.take(snapshots), installationId);
+ if (state?._tag !== states.at(-1)?._tag) states.push(state);
+ if (state?._tag === tag) return states;
+ }
+ }),
+ /** Takes snapshots until one shows `tag` for the installation; earlier ones are dropped. */
+ next: (installationId: PluginInstallationId, tag: PluginEventDeliveryState["_tag"]) =>
+ Effect.gen(function* () {
+ while (true) {
+ const snapshot = yield* Queue.take(snapshots);
+ const state = snapshot.installations.find(
+ (installation) => installation.installationId === installationId,
+ )?.eventDelivery;
+ if (state?._tag === tag) return state;
+ }
+ }),
+ };
+});
+
+/** Takes receipts until one matches; earlier ones are dropped. */
+const next = (
+ receipts: Receipts,
+ tag: Tag,
+ installationId: PluginInstallationId,
+) =>
+ Effect.gen(function* () {
+ while (true) {
+ const receipt = yield* PubSub.take(receipts);
+ if (receipt._tag === tag && receipt.installationId === installationId)
+ return receipt as Extract;
+ }
+ });
+
+const seedThread = Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ yield* eventSink.write({
+ events: [
+ {
+ id: EventId.make("event:plugin-events:thread"),
+ type: "thread.created",
+ threadId,
+ providerInstanceId,
+ occurredAt: now,
+ payload: {
+ createdBy: "user",
+ creationSource: "web",
+ id: threadId,
+ projectId,
+ title: "Fix the login bug",
+ providerInstanceId,
+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ activeProviderThreadId: null,
+ lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
+ forkedFrom: null,
+ createdAt: now,
+ updatedAt: now,
+ archivedAt: null,
+ settledOverride: null,
+ settledAt: null,
+ lastVisitedAt: null,
+ deletedAt: null,
+ },
+ },
+ ],
+ });
+});
+
+/** Records a run's finalization the way RunFinalizationService does; returns its sequence. */
+const finalizeRun = (name: string, failedAt?: "refresh-workspace") =>
+ Effect.gen(function* () {
+ const eventSink = yield* EventSink.EventSinkV2;
+ const runId = RunId.make(`run:${name}`);
+ const envelope = {
+ id: EventId.make(`event:run-finalized:${runId}`),
+ threadId,
+ runId,
+ providerInstanceId,
+ occurredAt: yield* DateTime.now,
+ };
+ const [stored] = yield* eventSink.write({
+ events: [
+ failedAt === undefined
+ ? {
+ ...envelope,
+ type: "run.finalized",
+ payload: { runId, outcome: "completed", checkpointId: null },
+ }
+ : {
+ ...envelope,
+ type: "run.finalization-failed",
+ payload: { runId, operation: failedAt },
+ },
+ ],
+ });
+ return stored!.sequence;
+ });
+
+const storedCursor = (installationId: PluginInstallationId) =>
+ Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const rows = yield* sql<{ readonly acknowledged_sequence: number }>`
+ SELECT acknowledged_sequence FROM plugin_event_cursors
+ WHERE installation_id = ${installationId}
+ `;
+ return rows[0]?.acknowledged_sequence;
+ });
+
+// One database, event store and projection per test; "restarts" replace the plugin side.
+const withStores = (effect: Effect.Effect) => {
+ const stores = Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe(
+ Layer.provideMerge(SqlitePersistence.layerMemory),
+ );
+ return effect.pipe(Effect.provide(EventSink.layer.pipe(Layer.provideMerge(stores))));
+};
+
+it.layer(NodeServices.layer)("PluginEventFeed", (it) => {
+ describe("delivery", () => {
+ it.effect(
+ "keeps a newly enabled cursor when reconciliation sees an older catalogue snapshot",
+ () =>
+ withStores(
+ Effect.gen(function* () {
+ const captured = yield* Deferred.make();
+ const release = yield* Deferred.make();
+ const reconciled = yield* Deferred.make();
+ const scope = yield* Scope.Scope;
+ const { catalog } = yield* startServer(
+ scope,
+ {},
+ {
+ catalog: (catalog) => ({
+ ...catalog,
+ subscribe: Stream.fromEffect(catalog.list).pipe(
+ Stream.tap(() =>
+ Deferred.succeed(captured, undefined).pipe(
+ Effect.andThen(Deferred.await(release)),
+ ),
+ ),
+ Stream.concat(
+ Stream.fromEffect(Deferred.succeed(reconciled, undefined)).pipe(Stream.drain),
+ ),
+ Stream.concat(catalog.subscribe),
+ ),
+ }),
+ },
+ );
+ yield* Deferred.await(captured);
+ const plugin = yield* preparePlugin("test.concurrent-enable");
+ const installationId = yield* install(catalog, plugin.directory);
+ const cursor = yield* storedCursor(installationId);
+ expect(cursor).toBeDefined();
+ yield* seedThread;
+ yield* finalizeRun("after-enable-before-reconcile");
+ yield* Deferred.succeed(release, undefined);
+ yield* Deferred.await(reconciled);
+ expect(yield* storedCursor(installationId)).toBe(cursor);
+ }),
+ ),
+ );
+
+ it.effect("recovers from a reconciliation SQL failure using a fresh catalogue snapshot", () =>
+ withStores(
+ Effect.gen(function* () {
+ const failed = yield* Deferred.make();
+ let fail = true;
+ let reconciliations = 0;
+ const { catalog, receipts } = yield* startServer(
+ yield* Scope.Scope,
+ { maxRetryBackoff: "1 second" },
+ {
+ sql: (sql) =>
+ new Proxy(sql, {
+ apply: (target, self, args) => {
+ const parts = args[0] as ReadonlyArray