Revalidate stale connection health verdicts automatically

.changeset/health-verdict-revalidate.md

@@ -0,0 +1,11 @@
+---
+"@executor-js/sdk": patch
+---
+
+**Stale "unhealthy" verdicts no longer wait for a manual "Check now"**
+
+A connection's persisted health verdict was only ever re-checked from the web UI, so after one bad probe (a transient upstream error, a refresh that failed once) agents reading `connections.list` kept reporting "unhealthy, reconnect" for a connection that worked fine — invocation auto-refreshes OAuth tokens — until a human opened the page and clicked "Check now".
+
+Two repair paths make the verdict track reality on its own. The agent-facing `connections.list` now re-runs the same probe as "Check now" before reporting a non-healthy verdict older than a minute, so recovery shows on the next read while repeated lists collapse to one probe per window. And a successful tool invocation through a connection wearing a non-healthy verdict flips it back to healthy — real traffic is stronger evidence than any probe. Tool-sync failure verdicts and grants the authorization server has rejected as `invalid_grant` are deliberately left alone: the first is cleared only by a successful sync, and the second genuinely requires a reconnect.
+
+`PluginCtx.connections` gains `checkHealth`, the same probe-with-freshness-window the executor surface already exposed.

packages/core/sdk/src/connections.test.ts

@@ -14,6 +14,7 @@ import { createExecutor } from "./executor";
 import { definePlugin } from "./plugin";
 import type { CredentialProvider } from "./provider";
 import { makeTestConfig, makeTestExecutor } from "./testing";
+import { ToolResult } from "./tool-result";
 
 // removed: v1 connection-refresh lifecycle, ConnectionProvider.refresh,
 // SecretProvider, accessToken token-refresh + in-flight dedup tests — the v2
@@ -718,3 +719,192 @@ describe("execute over a connection", () => {
     }),
   );
 });
+
+// ---------------------------------------------------------------------------
+// Sticky-verdict repair: agents read `lastHealth` through coreTools
+// connections.list, and nothing else ever re-probes a persisted verdict, so a
+// transient failure used to read as "unhealthy, reconnect" until a human
+// clicked "Check now". These cover the two repair paths: read-time
+// revalidation on the agent list, and heal-on-use from a successful
+// invocation.
+// ---------------------------------------------------------------------------
+
+const CORE_LIST = ToolAddress.make("executor.coreTools.connections.list");
+const STALE_MS = 5 * 60 * 1000;
+
+type ListedConnections = {
+  readonly connections: readonly {
+    readonly name: string;
+    readonly lastHealth: { readonly status: string; readonly detail?: string } | null;
+  }[];
+};
+
+const makeHealthHarness = () => {
+  const counters = { probes: 0 };
+  const plugin = definePlugin(() => ({
+    id: "healthdemo" as const,
+    credentialProviders: [memoryProvider()],
+    storage: () => ({}),
+    resolveTools: () =>
+      Effect.succeed({ tools: [{ name: ToolName.make("deploy"), description: "deploy" }] }),
+    invokeTool: ({ toolRow, credential, args }) =>
+      Effect.succeed(
+        (args as { fail?: boolean }).fail === true
+          ? ToolResult.fail({ code: "upstream_error", message: "boom" })
+          : { ran: toolRow.name, value: credential.value },
+      ),
+    checkHealth: () =>
+      Effect.sync(() => {
+        counters.probes += 1;
+        return { status: "healthy" as const, checkedAt: Date.now(), detail: "probe ok" };
+      }),
+    extension: (ctx) => ({
+      seed: () =>
+        ctx.core.integrations.register({ slug: INTEG, description: "Vercel", config: {} }),
+    }),
+  }))();
+
+  return Effect.gen(function* () {
+    const config = makeTestConfig({
+      plugins: [plugin] as const,
+      coreTools: { webBaseUrl: "http://localhost:3000" },
+    });
+    const executor = yield* createExecutor(config);
+    yield* executor.healthdemo.seed();
+    yield* executor.connections.create({
+      owner: "org",
+      name: ConnectionName.make("main"),
+      integration: INTEG,
+      template: TEMPLATE,
+      value: "secret-token",
+    });
+    const stamp = (set: Record<string, unknown>) =>
+      Effect.promise(() =>
+        config.db.updateMany("connection", {
+          where: (b) => b.and(b("integration", "=", String(INTEG)), b("name", "=", "main")),
+          set,
+        }),
+      );
+    const persisted = () =>
+      executor.connections.get({
+        owner: "org",
+        integration: INTEG,
+        name: ConnectionName.make("main"),
+      });
+    return { executor, counters, stamp, persisted } as const;
+  });
+};
+
+describe("agent read revalidation (coreTools connections.list)", () => {
+  it.effect("re-probes a stale non-healthy verdict and reports + persists the fresh one", () =>
+    Effect.gen(function* () {
+      const { executor, counters, stamp, persisted } = yield* makeHealthHarness();
+      yield* stamp({
+        last_health: { status: "expired", checkedAt: Date.now() - STALE_MS, detail: "HTTP 401" },
+      });
+
+      const out = (yield* executor.execute(CORE_LIST, {})) as ListedConnections;
+      const listed = out.connections.find((c) => c.name === "main");
+      expect(listed?.lastHealth?.status).toBe("healthy");
+      expect(counters.probes).toBe(1);
+
+      const row = yield* persisted();
+      expect(row?.lastHealth?.status).toBe("healthy");
+    }),
+  );
+
+  it.effect("serves a fresh non-healthy verdict without re-probing", () =>
+    Effect.gen(function* () {
+      const { executor, counters, stamp } = yield* makeHealthHarness();
+      yield* stamp({
+        last_health: { status: "expired", checkedAt: Date.now(), detail: "HTTP 401" },
+      });
+
+      const out = (yield* executor.execute(CORE_LIST, {})) as ListedConnections;
+      const listed = out.connections.find((c) => c.name === "main");
+      expect(listed?.lastHealth?.status).toBe("expired");
+      expect(counters.probes).toBe(0);
+    }),
+  );
+
+  it.effect("never probes a healthy verdict, however old", () =>
+    Effect.gen(function* () {
+      const { executor, counters, stamp } = yield* makeHealthHarness();
+      yield* stamp({
+        last_health: { status: "healthy", checkedAt: Date.now() - STALE_MS, detail: "probe ok" },
+      });
+
+      const out = (yield* executor.execute(CORE_LIST, {})) as ListedConnections;
+      const listed = out.connections.find((c) => c.name === "main");
+      expect(listed?.lastHealth?.status).toBe("healthy");
+      expect(counters.probes).toBe(0);
+    }),
+  );
+
+  it.effect("leaves tool-sync failure verdicts for sync to clear", () =>
+    Effect.gen(function* () {
+      const { executor, counters, stamp } = yield* makeHealthHarness();
+      const detail = "Tool sync failing: plugin returned an incomplete tool catalog";
+      yield* stamp({
+        last_health: { status: "degraded", checkedAt: Date.now() - STALE_MS, detail },
+      });
+
+      const out = (yield* executor.execute(CORE_LIST, {})) as ListedConnections;
+      const listed = out.connections.find((c) => c.name === "main");
+      expect(listed?.lastHealth?.detail).toBe(detail);
+      expect(counters.probes).toBe(0);
+    }),
+  );
+});
+
+describe("heal-on-use", () => {
+  it.effect("a successful invocation flips a stale non-healthy verdict to healthy", () =>
+    Effect.gen(function* () {
+      const { executor, stamp, persisted } = yield* makeHealthHarness();
+      yield* stamp({
+        last_health: { status: "expired", checkedAt: Date.now() - STALE_MS, detail: "HTTP 401" },
+      });
+
+      yield* executor.execute(ToolAddress.make("tools.vercel.org.main.deploy"), {});
+
+      const row = yield* persisted();
+      expect(row?.lastHealth).toMatchObject({
+        status: "healthy",
+        detail: "Tool invocation succeeded.",
+      });
+    }),
+  );
+
+  it.effect("an explicit tool failure does not heal", () =>
+    Effect.gen(function* () {
+      const { executor, stamp, persisted } = yield* makeHealthHarness();
+      yield* stamp({
+        last_health: { status: "expired", checkedAt: Date.now() - STALE_MS, detail: "HTTP 401" },
+      });
+
+      yield* executor.execute(ToolAddress.make("tools.vercel.org.main.deploy"), { fail: true });
+
+      const row = yield* persisted();
+      expect(row?.lastHealth?.status).toBe("expired");
+    }),
+  );
+
+  it.effect("a grant recorded invalid_grant-dead is not healed by a lingering token", () =>
+    Effect.gen(function* () {
+      const { executor, stamp, persisted } = yield* makeHealthHarness();
+      yield* stamp({
+        provider_state: { oauthReauthRequiredAt: Date.now() },
+        last_health: {
+          status: "expired",
+          checkedAt: Date.now() - STALE_MS,
+          detail: "invalid_grant",
+        },
+      });
+
+      yield* executor.execute(ToolAddress.make("tools.vercel.org.main.deploy"), {});
+
+      const row = yield* persisted();
+      expect(row?.lastHealth?.status).toBe("expired");
+    }),
+  );
+});

packages/core/sdk/src/core-tools.ts

@@ -22,7 +22,7 @@ import {
   type Owner,
 } from "./ids";
 import { definePlugin, tool, type StaticToolSchema } from "./plugin";
-import { HealthCheckResult } from "./health-check";
+import { HealthCheckResult, isToolSyncHealth } from "./health-check";
 import { ToolPolicyActionSchema } from "./policies";
 import type { Tool } from "./tool";
 
@@ -408,6 +408,26 @@ const connectionToListItem = (connection: Connection, verbose: boolean) => ({
   ...(verbose ? { oauthScope: connection.oauthScope ?? null } : {}),
 });
 
+/** How long a non-healthy persisted verdict may be served to an agent before
+ *  it is re-verified. Verdicts are sticky — nothing re-probes them between UI
+ *  visits — so without read-time revalidation an agent keeps reporting
+ *  "unhealthy, reconnect" for a connection that recovered long ago (or was
+ *  never really down: invocation auto-refreshes OAuth tokens, so a stale
+ *  "expired" verdict often describes a working connection). The window is
+ *  short so recovery shows on the next read, but bounds repeated lists from
+ *  hammering a genuinely-down upstream. Healthy verdicts are deliberately
+ *  served as-is: they mislead no one into reconnect guidance, and the UI
+ *  owns their background revalidation. */
+const NON_HEALTHY_REVALIDATE_MS = 60 * 1000;
+
+/** Whether an agent read must re-verify a persisted verdict before reporting
+ *  it. Only probe-refutable non-healthy verdicts qualify: `unknown` and
+ *  missing verdicts carry no reconnect implication, and a tool-sync failure
+ *  verdict cannot be refuted by a credential probe (a successful sync clears
+ *  it instead). */
+const needsAgentReadRevalidation = (last: HealthCheckResult | null | undefined): boolean =>
+  (last?.status === "expired" || last?.status === "degraded") && !isToolSyncHealth(last);
+
 const toolToOutput = (toolRow: Tool) => ({
   address: String(toolRow.address),
   owner: toolRow.owner,
@@ -593,20 +613,45 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = {
           inputSchema: ConnectionsListInputStd,
           outputSchema: ConnectionsListOutputStd,
           execute: (input: typeof ConnectionsListInput.Type, { ctx }) =>
-            Effect.map(
-              ctx.connections.list({
+            Effect.gen(function* () {
+              const connections = yield* ctx.connections.list({
                 integration:
                   input.integration === undefined
                     ? undefined
                     : IntegrationSlug.make(input.integration),
                 owner: input.owner === undefined ? undefined : (input.owner as Owner),
-              }),
-              (connections) => ({
-                connections: connections.map((connection) =>
+              });
+              // Re-verify sticky non-healthy verdicts before reporting them
+              // (the same probe as the UI's "Check now", server-cached via
+              // `ifStaleMs` so repeated lists collapse to one probe per
+              // window). Quiet on probe failure: the persisted verdict is
+              // still the best known state, exactly like the UI surfaces.
+              const revalidated = yield* Effect.forEach(
+                connections,
+                (connection) =>
+                  needsAgentReadRevalidation(connection.lastHealth)
+                    ? ctx.connections
+                        .checkHealth(
+                          {
+                            owner: connection.owner,
+                            integration: connection.integration,
+                            name: connection.name,
+                          },
+                          { ifStaleMs: NON_HEALTHY_REVALIDATE_MS },
+                        )
+                        .pipe(
+                          Effect.map((health) => ({ ...connection, lastHealth: health })),
+                          Effect.catch(() => Effect.succeed(connection)),
+                        )
+                    : Effect.succeed(connection),
+                { concurrency: 4 },
+              );
+              return {
+                connections: revalidated.map((connection) =>
                   connectionToListItem(connection, input.verbose === true),
                 ),
-              }),
-            ),
+              };
+            }),
         }),
         tool({
           name: "connections.create",