Spike: run the VS Code E2E suite inside a dev container

.github/workflows/build-extensions.yml

@@ -139,6 +139,48 @@ jobs:
       - name: Run VS Code E2E suite
         run: xvfb-run -a npm run test:e2e
 
+  vscode-e2e-devcontainer:
+    # The second E2E level: the workspace extension and suite run in a real
+    # remote extension host inside a dev container built from this checkout's
+    # feature, reached through a test-only resolver extension. Separate from
+    # `vscode-e2e` because it adds Docker and a second (server) download to
+    # the things that can fail for reasons unrelated to the code.
+    name: VS Code E2E in a dev container (spike)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Typecheck E2E suite
+        run: npm run typecheck:e2e
+
+      - name: Build UI Extension
+        run: npm run build -w src/vscode-ui-extension
+
+      - name: Build Workspace Extension
+        run: npm run build -w src/vscode-workspace-extension
+
+      - name: Build E2E suite
+        run: npm run build:e2e
+
+      - name: Generate test certificate fixture
+        run: npm run gen:test-cert
+
+      - name: Check E2E suite bundles load
+        run: npm run check:e2e-load
+
+      - name: Install xvfb
+        run: sudo apt-get update -y && sudo apt-get install -y --no-install-recommends xvfb
+
+      - name: Run VS Code E2E suite in a dev container
+        run: xvfb-run -a npm run test:e2e:devcontainer
+
   windows-certificate-validation:
     name: Windows Certificate Validation
     runs-on: windows-latest

AGENTS.md

@@ -126,6 +126,12 @@ If you change this, keep both halves. An env-var-only check would be a genuine e
 
 The alternative that avoids a product-code seam entirely is a resolver extension implementing `resolveAuthority` to populate `remoteName` for real. It is more faithful and would also unlock testing against a real container — but it rides a **proposed API**, so it needs `--enable-proposed-api` and can break between VS Code releases. Choosing between the two is the open question this spike exists to inform.
 
+### Devcontainer level (`test/vscode-e2e/runDevcontainer.mjs`) — spike status
+
+The resolver alternative above, prototyped as a second level rather than a replacement: the seam-based local level stays, because it is cheap and needs no Docker. `npm run test:e2e:devcontainer` stages `.out/devcontainer-e2e/` with this checkout's feature, `devcontainer up`s it with the open-source `@devcontainers/cli`, starts a VS Code server of the **same commit** as the downloaded desktop build inside the container via `devcontainer exec` (so it gets the CLI's userEnvProbe), and launches desktop VS Code on `vscode-remote://devcerts-e2e+e2e/...`. `test/vscode-e2e/resolver/` is a ~40-line test-only extension whose only job is `registerRemoteAuthorityResolver("devcerts-e2e")` → the server's published loopback port. The UI extension and resolver load locally; the workspace extension and `remote-suite.cjs` load from the container through `vscode-remote://` development/test paths.
+
+That closes the gaps the local level lists: the `getAllCertMaterialV3` payload crosses VS Code's real RPC, the gate opens on a real `remoteName` (the suite asserts the seam variable is **unset**), `install.sh` runs for real and its `SSL_CERT_DIR` must reach the extension host, and the final check is `openssl verify` inside the container using only that inherited `SSL_CERT_DIR`. Still not covered: the Dev Containers extension itself (closed-source; it would also install the marketplace builds the feature names over the checkout under test), and the dotnet dev-cert / reverse-sync paths. `--container-only` stops after `devcontainer up` plus an env probe — Docker but no VS Code download. Linux-only for now (server arch is read from the container; the published port assumes Docker on the same host).
+
 ## File Paths That Matter
 
 | Path (in container) | Purpose |

package.json

@@ -13,9 +13,11 @@
     "typecheck:e2e": "tsc --noEmit -p test/vscode-e2e/tsconfig.json",
     "build:e2e": "node test/vscode-e2e/esbuild.mjs",
     "check:e2e-load": "node test/vscode-e2e/loadCheck.cjs",
-    "test:e2e": "node test/vscode-e2e/runTests.mjs"
+    "test:e2e": "node test/vscode-e2e/runTests.mjs",
+    "test:e2e:devcontainer": "node test/vscode-e2e/runDevcontainer.mjs"
   },
   "devDependencies": {
+    "@devcontainers/cli": "^0.89.0",
     "@eslint/js": "^10.0.1",
     "@vscode/test-electron": "^2.5.2",
     "esbuild": "^0.28.0",

test/vscode-e2e/devcontainer/.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+// Template for the devcontainer E2E level. runDevcontainer.mjs stages this
+// folder into .out/devcontainer-e2e/ and copies the feature next to it as
+// .devcontainer/devcontainer-dev-certs, so the feature under test is the one
+// in this checkout rather than the published ghcr.io build.
+{
+  "name": "devcontainer-dev-certs-e2e",
+  "image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+  "remoteUser": "vscode",
+  // Explicit, because the CLI otherwise mounts the enclosing git checkout and
+  // points workspaceFolder at .out/devcontainer-e2e inside it. Only the staged
+  // folder should be visible to the container.
+  "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/devcerts-e2e,type=bind",
+  "workspaceFolder": "/workspaces/devcerts-e2e",
+  "features": {
+    "./devcontainer-dev-certs": {}
+  },
+  // The VS Code server listens on 8123 inside the container. Published on
+  // loopback only, on an ephemeral host port the launcher reads back with
+  // `docker port`, so parallel runs don't collide.
+  "runArgs": ["-p", "127.0.0.1::8123"],
+  "containerEnv": {
+    "DEVCERTS_E2E": "1"
+  }
+}

test/vscode-e2e/esbuild.mjs

@@ -11,10 +11,18 @@ const repoRoot = resolve(here, "..", "..");
 // node_modules. Everything else — including the shared package, which is
 // published as TypeScript source — is bundled in, so the suite is a single
 // self-contained file with no resolution surprises at runtime.
+//
+// Two bundles: `suite.cjs` for the local level (runTests.mjs) and
+// `remote-suite.cjs` for the devcontainer level (runDevcontainer.mjs), which
+// is copied into the container and loaded by the remote extension host.
 await esbuild.build({
-  entryPoints: [resolve(here, "src", "index.ts")],
+  entryPoints: {
+    suite: resolve(here, "src", "index.ts"),
+    "remote-suite": resolve(here, "src", "remote", "index.ts"),
+  },
   bundle: true,
-  outfile: resolve(repoRoot, ".out", "vscode-e2e", "suite.cjs"),
+  outdir: resolve(repoRoot, ".out", "vscode-e2e"),
+  outExtension: { ".js": ".cjs" },
   external: ["vscode"],
   format: "cjs",
   platform: "node",

test/vscode-e2e/launcherCommon.mjs

@@ -0,0 +1,95 @@
+/**
+ * Pieces shared by both E2E launchers: `runTests.mjs` (one local window, the
+ * remote-gate seam) and `runDevcontainer.mjs` (a real remote extension host
+ * inside a dev container). Plain Node, never loaded into VS Code.
+ */
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, writeFileSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const here = dirname(fileURLToPath(import.meta.url));
+export const repoRoot = resolve(here, "..", "..");
+
+export const USER_CERT_NAME = "e2e-user-cert";
+export const FIXTURE_DIR = join(repoRoot, ".out", "test-fixtures");
+export const FIXTURE_PEM = join(FIXTURE_DIR, "corp-wildcard.pem");
+export const FIXTURE_KEY = join(FIXTURE_DIR, "corp-wildcard.key");
+
+/** The fixture generator is cheap and deterministic; just run it if needed. */
+export function ensureFixture() {
+  if (existsSync(FIXTURE_PEM) && existsSync(FIXTURE_KEY)) return;
+  console.log("Generating test certificate fixture...");
+  const result = spawnSync(
+    process.execPath,
+    [join(repoRoot, "test", "generate-test-cert.mjs")],
+    { cwd: repoRoot, stdio: "inherit" }
+  );
+  if (result.status !== 0) {
+    throw new Error("test/generate-test-cert.mjs failed");
+  }
+}
+
+export function requireBuilt(label, path) {
+  if (!existsSync(path)) {
+    throw new Error(
+      `${label} is missing (${path}). Build it first — see the usage note in ` +
+        "test/vscode-e2e/runTests.mjs."
+    );
+  }
+  return path;
+}
+
+export function buildSandbox() {
+  const root = mkdtempSync(join(tmpdir(), "devcerts-e2e-"));
+  const home = join(root, "home");
+  const userDataDir = join(root, "user-data");
+  const extensionsDir = join(root, "extensions");
+  const workspaceDir = join(root, "workspace");
+  const trustDir = join(home, ".aspnet", "dev-certs", "trust");
+
+  for (const dir of [home, userDataDir, extensionsDir, workspaceDir, trustDir]) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  // Machine/user settings for the run. `userCertificates` is what makes the
+  // host serve a certificate at all; the rest keeps the window quiet and the
+  // flow deterministic:
+  //   - generateDotNetCert/autoProvision off  → the dev-cert branch (and its
+  //     modal consent prompt, and the OS trust-store write) never runs.
+  //   - autoInject off → the suite drives the pull explicitly instead of
+  //     racing extension activation.
+  //   - warnOnStaleDevCerts off → no post-install warning dialog.
+  mkdirSync(join(userDataDir, "User"), { recursive: true });
+  writeFileSync(
+    join(userDataDir, "User", "settings.json"),
+    JSON.stringify(
+      {
+        "devcontainerDevCerts.generateDotNetCert": false,
+        "devcontainerDevCerts.autoProvision": false,
+        "devcontainerDevCerts.autoInject": false,
+        "devcontainerDevCerts.warnOnStaleDevCerts": false,
+        "devcontainerDevCerts.installUserCertsToDotNetStore": false,
+        "devcontainerDevCerts.userCertificates": [
+          {
+            name: USER_CERT_NAME,
+            pemCertPath: FIXTURE_PEM,
+            pemKeyPath: FIXTURE_KEY,
+            trustInContainer: true,
+          },
+        ],
+        "security.workspace.trust.enabled": false,
+        "telemetry.telemetryLevel": "off",
+        "update.mode": "none",
+        "extensions.autoUpdate": false,
+        "extensions.autoCheckUpdates": false,
+        "workbench.startupEditor": "none",
+      },
+      null,
+      2
+    )
+  );
+
+  return { root, home, userDataDir, extensionsDir, workspaceDir, trustDir };
+}

test/vscode-e2e/loadCheck.cjs

@@ -26,11 +26,14 @@ const Module = require("node:module");
 const path = require("node:path");
 const fs = require("node:fs");
 
-const BUNDLE = path.resolve(__dirname, "..", "..", ".out", "vscode-e2e", "suite.cjs");
+const OUT = path.resolve(__dirname, "..", "..", ".out", "vscode-e2e");
+const BUNDLES = [path.join(OUT, "suite.cjs"), path.join(OUT, "remote-suite.cjs")];
 
-if (!fs.existsSync(BUNDLE)) {
-  console.error(`Suite bundle is missing (${BUNDLE}). Run: npm run build:e2e`);
-  process.exit(1);
+for (const bundle of BUNDLES) {
+  if (!fs.existsSync(bundle)) {
+    console.error(`Suite bundle is missing (${bundle}). Run: npm run build:e2e`);
+    process.exit(1);
+  }
 }
 
 /**
@@ -72,22 +75,25 @@ require.cache["vscode"] = {
   exports: vscodeStub,
 };
 
-try {
-  const suite = require(BUNDLE);
-  if (typeof suite.run !== "function") {
+for (const bundle of BUNDLES) {
+  const name = path.basename(bundle);
+  try {
+    const suite = require(bundle);
+    if (typeof suite.run !== "function") {
+      console.error(
+        `${name} loaded but does not export run(). VS Code's ` +
+          "--extensionTestsPath contract requires it."
+      );
+      process.exit(1);
+    }
+    console.log(`${name} loads cleanly and exports run().`);
+  } catch (err) {
+    console.error(`${name} threw during module initialization:`);
+    console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
     console.error(
-      "Suite bundle loaded but does not export run(). VS Code's " +
-        "--extensionTestsPath contract requires it."
+      "\nThis would fail identically inside the extension host, after the " +
+        "VS Code download. Fix it here."
     );
     process.exit(1);
   }
-  console.log("E2E suite bundle loads cleanly and exports run().");
-} catch (err) {
-  console.error("Suite bundle threw during module initialization:");
-  console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
-  console.error(
-    "\nThis would fail identically inside the extension host, after the " +
-      "VS Code download. Fix it here."
-  );
-  process.exit(1);
 }

test/vscode-e2e/resolver/extension.js

@@ -0,0 +1,46 @@
+// Test-only remote authority resolver for the devcontainer E2E level.
+//
+// It does exactly one thing: tell VS Code that the `devcerts-e2e` authority
+// lives at the VS Code server `runDevcontainer.mjs` started inside the dev
+// container. Everything after that is stock VS Code: the server runs the
+// workspace extension and the suite in a REAL remote extension host, so
+// `vscode.env.remoteName` is set for real (no DEVCONTAINER_DEV_CERTS_TEST_REMOTE
+// seam), and every cross-host executeCommand payload is genuinely serialized
+// over VS Code's RPC rather than passed by reference.
+//
+// `registerRemoteAuthorityResolver` is a proposed API (`resolvers`). It is
+// unlocked by `enabledApiProposals` in package.json plus
+// `--enable-proposed-api dnegstad.devcerts-e2e-resolver` on the command line,
+// which the launcher passes. Proposed APIs can change between VS Code
+// releases; if this file breaks on a new stable, compare against
+// src/vscode-dts/vscode.proposed.resolvers.d.ts in microsoft/vscode.
+//
+// Plain JS on purpose: there is nothing here worth a build step, and the
+// proposed .d.ts isn't in @types/vscode.
+"use strict";
+const vscode = require("vscode");
+
+function required(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(
+      `${name} is not set; this resolver only works under test/vscode-e2e/runDevcontainer.mjs`
+    );
+  }
+  return value;
+}
+
+exports.activate = function activate(context) {
+  context.subscriptions.push(
+    vscode.workspace.registerRemoteAuthorityResolver("devcerts-e2e", {
+      resolve(_authority) {
+        const host = required("DEVCERTS_E2E_SERVER_HOST");
+        const port = Number(required("DEVCERTS_E2E_SERVER_PORT"));
+        const token = required("DEVCERTS_E2E_SERVER_TOKEN");
+        return new vscode.ResolvedAuthority(host, port, token);
+      },
+    })
+  );
+};
+
+exports.deactivate = function deactivate() {};

test/vscode-e2e/resolver/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "devcerts-e2e-resolver",
+  "displayName": "devcontainer-dev-certs E2E remote resolver (test only)",
+  "publisher": "dnegstad",
+  "version": "0.0.0",
+  "private": true,
+  "engines": {
+    "vscode": "^1.100.0"
+  },
+  "extensionKind": [
+    "ui"
+  ],
+  "enabledApiProposals": [
+    "resolvers"
+  ],
+  "activationEvents": [
+    "onResolveRemoteAuthority:devcerts-e2e"
+  ],
+  "main": "./extension.js"
+}