fix(server): connects no longer wait seconds on a failing tailscale status

apps/server/src/environment/DirectEndpoints.test.ts

@@ -1,18 +1,27 @@
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { it as effectIt } from "@effect/vitest";
+import * as HostProcess from "@t3tools/shared/HostProcess";
+import * as NetService from "@t3tools/shared/Net";
 import type * as NodeOS from "node:os";
+import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
 import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
 import * as HttpClient from "effect/http/HttpClient";
 import * as HttpClientResponse from "effect/http/HttpClientResponse";
+import * as ChildProcess from "effect/process/ChildProcess";
 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
+import * as TestClock from "effect/testing/TestClock";
 import { describe, expect, it } from "vite-plus/test";
 
 import * as ServerConfig from "../config.ts";
 import * as DirectEndpoints from "./DirectEndpoints.ts";
 import { resolveBoundEndpoints } from "./DirectEndpoints.ts";
+import * as RemoteOpenTargets from "./RemoteOpenTargets.ts";
+import * as TailscaleStatusReader from "./TailscaleStatusReader.ts";
 
 const INTERFACES: ReturnType<typeof NodeOS.networkInterfaces> = {
   lo0: [
@@ -171,26 +180,31 @@ const TAILSCALE_STATUS_JSON = JSON.stringify({
   Self: { DNSName: "bb-1.tail1234.ts.net.", TailscaleIPs: ["100.64.1.2"] },
 });
 
-const layerTailscaleUp = Layer.succeed(
-  ChildProcessSpawner.ChildProcessSpawner,
-  ChildProcessSpawner.make(() =>
-    Effect.succeed(
-      ChildProcessSpawner.makeHandle({
-        pid: ChildProcessSpawner.ProcessId(1),
-        exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)),
-        isRunning: Effect.succeed(false),
-        kill: () => Effect.void,
-        unref: Effect.succeed(Effect.void),
-        stdin: Sink.drain,
-        stdout: Stream.make(new TextEncoder().encode(TAILSCALE_STATUS_JSON)),
-        stderr: Stream.empty,
-        all: Stream.empty,
-        getInputFd: () => Sink.drain,
-        getOutputFd: () => Stream.empty,
+// `tailscale status` exits non-zero while tailscaled is unavailable, and prints the node once it is up.
+const layerTailscale = (isUp: () => boolean) =>
+  Layer.succeed(
+    ChildProcessSpawner.ChildProcessSpawner,
+    ChildProcessSpawner.make(() =>
+      Effect.sync(() => {
+        const up = isUp();
+        return ChildProcessSpawner.makeHandle({
+          pid: ChildProcessSpawner.ProcessId(1),
+          exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(up ? 0 : 1)),
+          isRunning: Effect.succeed(false),
+          kill: () => Effect.void,
+          unref: Effect.succeed(Effect.void),
+          stdin: Sink.drain,
+          stdout: up ? Stream.make(new TextEncoder().encode(TAILSCALE_STATUS_JSON)) : Stream.empty,
+          stderr: Stream.empty,
+          all: Stream.empty,
+          getInputFd: () => Sink.drain,
+          getOutputFd: () => Stream.empty,
+        });
       }),
     ),
-  ),
-);
+  );
+
+const layerTailscaleUp = layerTailscale(() => true);
 
 const layerServeProbe = (status: number) =>
   Layer.succeed(
@@ -217,6 +231,7 @@ const resolveWithServe = (probeStatus: number) =>
   Effect.flatMap(DirectEndpoints.DirectEndpoints, (service) => service.resolve()).pipe(
     Effect.provide(
       DirectEndpoints.layer.pipe(
+        Layer.provide(TailscaleStatusReader.layer),
         Layer.provide(
           Layer.mergeAll(layerServeConfig, layerTailscaleUp, layerServeProbe(probeStatus)),
         ),
@@ -239,3 +254,103 @@ describe("DirectEndpoints Tailscale Serve", () => {
     }),
   );
 });
+
+// A CLI that cannot reach tailscaled: `tailscale status` never answers, so each read waits out its timeout.
+const layerTailscaleHung = (onStatusSpawn: () => void) =>
+  Layer.succeed(
+    ChildProcessSpawner.ChildProcessSpawner,
+    ChildProcessSpawner.make((command) => {
+      if (ChildProcess.isStandardCommand(command) && command.args[0] === "status") onStatusSpawn();
+      return Effect.succeed(
+        ChildProcessSpawner.makeHandle({
+          pid: ChildProcessSpawner.ProcessId(1),
+          exitCode: Effect.never,
+          isRunning: Effect.succeed(true),
+          kill: () => Effect.void,
+          unref: Effect.succeed(Effect.void),
+          stdin: Sink.drain,
+          stdout: Stream.empty,
+          stderr: Stream.empty,
+          all: Stream.empty,
+          getInputFd: () => Sink.drain,
+          getOutputFd: () => Stream.empty,
+        }),
+      );
+    }),
+  );
+
+const layerSshdListening = Layer.succeed(NetService.NetService, {
+  canListenOnHost: () => Effect.succeed(true),
+  isPortAvailableOnLoopback: () => Effect.succeed(true),
+  hasListenerOnHost: () => Effect.succeed(true),
+  reserveLoopbackPort: () => Effect.succeed(40_000),
+  findAvailablePort: (preferred) => Effect.succeed(preferred),
+});
+
+const MDNS_ONLY = [{ kind: "mdns", host: "bb-1.local", username: "t3-user" }];
+
+// What ws.loadServerConfig resolves for every client that connects. The layers live in the
+// test's scope, as the server's do, so the shared status read outlives each connect.
+const connectWithTailscale = (layerSpawner: Layer.Layer<ChildProcessSpawner.ChildProcessSpawner>) =>
+  Effect.gen(function* () {
+    const context = yield* Layer.build(
+      Layer.mergeAll(DirectEndpoints.layer, RemoteOpenTargets.layer).pipe(
+        Layer.provide(TailscaleStatusReader.layer),
+        Layer.provide(
+          Layer.mergeAll(layerServeConfig, layerSpawner, layerServeProbe(200), layerSshdListening),
+        ),
+      ),
+    );
+    const remoteOpenTargets = Context.get(context, RemoteOpenTargets.RemoteOpenTargets);
+    const directEndpoints = Context.get(context, DirectEndpoints.DirectEndpoints);
+    return Effect.all([remoteOpenTargets.resolveTargets(), directEndpoints.resolve()], {
+      concurrency: "unbounded",
+    }).pipe(
+      Effect.provideService(HostProcess.Hostname, "bb-1"),
+      Effect.provideService(HostProcess.Username, "t3-user"),
+    );
+  });
+
+describe("Tailscale status shared by DirectEndpoints and RemoteOpenTargets", () => {
+  effectIt.effect("reads `tailscale status` once per connect, not once per service", () =>
+    Effect.gen(function* () {
+      let statusReads = 0;
+      const connect = yield* connectWithTailscale(layerTailscaleHung(() => statusReads++));
+      const first = yield* Effect.forkChild(connect);
+      yield* TestClock.adjust("4 seconds");
+      expect(yield* Fiber.join(first)).toEqual([MDNS_ONLY, []]);
+      expect(statusReads, "tailscale status spawns for one connect").toBe(1);
+    }),
+  );
+
+  effectIt.effect(
+    "answers a later connect from the settled failed read without waiting again",
+    () =>
+      Effect.gen(function* () {
+        const connect = yield* connectWithTailscale(layerTailscaleHung(() => {}));
+        const first = yield* Effect.forkChild(connect);
+        yield* TestClock.adjust("4 seconds");
+        expect(yield* Fiber.join(first)).toEqual([MDNS_ONLY, []]);
+
+        const later = yield* connect.pipe(Effect.timeoutOption("1 second"), Effect.forkChild);
+        yield* TestClock.adjust("1 second");
+        expect(yield* Fiber.join(later), "later connect still waiting on tailscale status").toEqual(
+          Option.some([MDNS_ONLY, []]),
+        );
+      }),
+  );
+
+  effectIt.effect("answers a connect from a recovered read that settles within the grace", () =>
+    Effect.gen(function* () {
+      let tailscaleUp = false;
+      const connect = yield* connectWithTailscale(layerTailscale(() => tailscaleUp));
+      expect(yield* connect, "startup read fails").toEqual([MDNS_ONLY, []]);
+
+      tailscaleUp = true;
+      expect(yield* connect).toEqual([
+        [{ kind: "tailscale", host: "bb-1.tail1234.ts.net", username: "t3-user" }, ...MDNS_ONLY],
+        [{ kind: "tailnet", httpBaseUrl: "https://bb-1.tail1234.ts.net/" }],
+      ]);
+    }),
+  );
+});

apps/server/src/environment/DirectEndpoints.ts

@@ -1,16 +1,11 @@
 import type { ServerDirectEndpoint } from "@t3tools/contracts";
-import {
-  buildTailscaleHttpsBaseUrl,
-  probeTailscaleHttpsEndpoint,
-  readTailscaleStatus,
-} from "@t3tools/tailscale";
+import { buildTailscaleHttpsBaseUrl, probeTailscaleHttpsEndpoint } from "@t3tools/tailscale";
 import * as Context from "effect/Context";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Option from "effect/Option";
 import { HttpClient } from "effect/http";
-import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
 import * as NodeNet from "node:net";
 import * as NodeOS from "node:os";
 
@@ -24,6 +19,7 @@ import {
   isWildcardHost,
   VIRTUAL_INTERFACE,
 } from "../startupAccess.ts";
+import * as TailscaleStatusReader from "./TailscaleStatusReader.ts";
 
 type NetworkInterfacesMap = ReturnType<typeof NodeOS.networkInterfaces>;
 
@@ -109,7 +105,7 @@ const probeCloudMetadata = Effect.callback<boolean>((resume) => {
 /** @public */
 export const make = Effect.gen(function* () {
   const config = yield* ServerConfig.ServerConfig;
-  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  const tailscaleStatus = yield* TailscaleStatusReader.TailscaleStatusReader;
   const httpClient = yield* HttpClient.HttpClient;
   // Only the service default listens on every interface without being asked,
   // so only it checks once whether it runs in a cloud. The check starts now so
@@ -130,7 +126,7 @@ export const make = Effect.gen(function* () {
     ];
     // Serve setup can fail without stopping startup; advertise only after it answers as this server.
     if (config.tailscaleServeEnabled) {
-      const servedUrl = yield* readTailscaleStatus.pipe(
+      const servedUrl = yield* tailscaleStatus.read().pipe(
         Effect.map((status) => status.magicDnsName),
         Effect.flatMap((magicDnsName) => {
           if (magicDnsName === null) return Effect.succeed(null);
@@ -145,7 +141,6 @@ export const make = Effect.gen(function* () {
         Effect.timeoutOption(TAILSCALE_ENDPOINT_TIMEOUT),
         Effect.map(Option.flatMap(Option.fromNullishOr)),
         Effect.orElseSucceed(() => Option.none<string>()),
-        Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
         Effect.provideService(HttpClient.HttpClient, httpClient),
       );
       if (Option.isSome(servedUrl)) {

apps/server/src/environment/RemoteOpenTargets.test.ts

@@ -12,6 +12,7 @@ import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
 import { afterEach, describe, expect, vi } from "vite-plus/test";
 
 import * as RemoteOpenTargets from "./RemoteOpenTargets.ts";
+import * as TailscaleStatusReader from "./TailscaleStatusReader.ts";
 
 afterEach(() => {
   vi.restoreAllMocks();
@@ -74,6 +75,7 @@ const resolveTargets = (input: {
     Effect.provideService(HostProcess.Hostname, input.hostname),
     Effect.provide(
       RemoteOpenTargets.layer.pipe(
+        Layer.provide(TailscaleStatusReader.layer),
         Layer.provide(Layer.mergeAll(layerNet(input.sshd), layerSpawner(input.tailscale))),
       ),
     ),

apps/server/src/environment/RemoteOpenTargets.ts

@@ -1,12 +1,14 @@
 import { type RemoteOpenTarget } from "@t3tools/contracts";
 import * as HostProcess from "@t3tools/shared/HostProcess";
 import * as NetService from "@t3tools/shared/Net";
-import { readTailscaleSshEnabled, readTailscaleStatus } from "@t3tools/tailscale";
+import { readTailscaleSshEnabled } from "@t3tools/tailscale";
 import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
 
+import * as TailscaleStatusReader from "./TailscaleStatusReader.ts";
+
 const SSH_PORT = 22;
 
 export class RemoteOpenTargets extends Context.Service<
@@ -20,6 +22,7 @@ export class RemoteOpenTargets extends Context.Service<
 export const make = Effect.gen(function* () {
   const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
   const net = yield* NetService.NetService;
+  const tailscaleStatus = yield* TailscaleStatusReader.TailscaleStatusReader;
 
   const resolveTargets = Effect.gen(function* () {
     // Check both loopback families: sshd can be bound IPv6-only.
@@ -30,10 +33,9 @@ export const make = Effect.gen(function* () {
     );
     const username = yield* HostProcess.Username;
 
-    const magicDnsName = yield* readTailscaleStatus.pipe(
+    const magicDnsName = yield* tailscaleStatus.read().pipe(
       Effect.map((status) => status.magicDnsName),
       Effect.orElseSucceed(() => null),
-      Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
     );
 
     const targets: Array<RemoteOpenTarget> = [];

apps/server/src/environment/TailscaleStatusReader.ts

@@ -0,0 +1,95 @@
+import {
+  readTailscaleStatus,
+  type TailscaleCommandError,
+  type TailscaleStatus,
+  type TailscaleStatusParseError,
+} from "@t3tools/tailscale";
+import * as Context from "effect/Context";
+import * as Deferred from "effect/Deferred";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as PubSub from "effect/PubSub";
+import type * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
+
+type TailscaleStatusError = TailscaleCommandError | TailscaleStatusParseError;
+
+export class TailscaleStatusReader extends Context.Service<
+  TailscaleStatusReader,
+  {
+    readonly read: () => Effect.Effect<TailscaleStatus, TailscaleStatusError>;
+    /** Emits when a read settles to a different outcome than the previous settled read. */
+    readonly subscribeChanges: Effect.Effect<Stream.Stream<void>, never, Scope.Scope>;
+  }
+>()("t3/environment/TailscaleStatusReader") {}
+
+/** Answer from the last settled read without refreshing, for re-resolves that a settled change drives. */
+export const SettledReadOnly = Context.Reference<boolean>(
+  "t3/environment/TailscaleStatusReader/SettledReadOnly",
+  { defaultValue: () => false },
+);
+
+// A healthy CLI answers in tens of milliseconds, so a connect still gets a fresh read from it.
+const FRESH_READ_GRACE = Duration.millis(250);
+
+// Callers treat every failure alike, so failures compare equal regardless of cause.
+const outcomeKey = (exit: Exit.Exit<TailscaleStatus, TailscaleStatusError>) =>
+  Exit.isSuccess(exit) ? JSON.stringify(exit.value) : null;
+
+const make = Effect.gen(function* () {
+  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+  // The layer's scope, so a caller that times out or disconnects never interrupts the shared read.
+  const scope = yield* Effect.scope;
+  const changes = yield* PubSub.unbounded<void>();
+  let settled: Exit.Exit<TailscaleStatus, TailscaleStatusError> | undefined;
+  let inFlight: Deferred.Deferred<TailscaleStatus, TailscaleStatusError> | undefined;
+
+  const refresh = Effect.suspend(() => {
+    if (inFlight !== undefined) return Effect.succeed(inFlight);
+    const deferred = Deferred.makeUnsafe<TailscaleStatus, TailscaleStatusError>();
+    inFlight = deferred;
+    return readTailscaleStatus.pipe(
+      Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
+      Effect.onExit((exit) =>
+        Effect.suspend(() => {
+          const changed = settled === undefined || outcomeKey(settled) !== outcomeKey(exit);
+          settled = exit;
+          inFlight = undefined;
+          return Deferred.done(deferred, exit).pipe(
+            Effect.andThen(changed ? PubSub.publish(changes, undefined) : Effect.void),
+          );
+        }),
+      ),
+      Effect.interruptible,
+      Effect.forkIn(scope),
+      Effect.as(deferred),
+    );
+  }).pipe(
+    // Claiming `inFlight` and forking are one step: an interrupt between them would strand the claim.
+    Effect.uninterruptible,
+  );
+  yield* refresh;
+
+  return TailscaleStatusReader.of({
+    // A caller gets the refresh it starts or joins if that settles within the grace, else the
+    // previous read's result; a later outcome reaches open subscribers through `subscribeChanges`.
+    read: Effect.fn("TailscaleStatusReader.read")(function* () {
+      const previous = settled;
+      if (previous !== undefined && (yield* SettledReadOnly)) return yield* previous;
+      const pending = yield* refresh;
+      if (previous === undefined) return yield* Deferred.await(pending);
+      const fresh = yield* Deferred.await(pending).pipe(
+        Effect.exit,
+        Effect.timeoutOption(FRESH_READ_GRACE),
+      );
+      return yield* Option.getOrElse(fresh, () => previous);
+    }),
+    subscribeChanges: PubSub.subscribe(changes).pipe(Effect.map(Stream.fromSubscription)),
+  });
+});
+
+export const layer = Layer.effect(TailscaleStatusReader, make);