Skip to content

chore(e2e): run .agents/run-e2e on macOS and fail when no spec ran #2138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chhoumann merged 2 commits into from
Oct 2, 2026
Merged

chore(e2e): run .agents/run-e2e on macOS and fail when no spec ran #2138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f366b28
chore(e2e): run .agents/run-e2e on macOS and fail when no spec ran
chhoumann Oct 2, 2026
3857b54
chore(e2e): don't let an inherited results variable name a file to de…
chhoumann Oct 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 34 additions & 8 deletions .agents/run-e2e
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@
# a Templater path: CI runs no native specs, so nothing else catches those.
set -euo pipefail

# The start-environment parsing below needs bash 4 (associative arrays,
# `[[ -v ]]`). macOS ships bash 3.2 as /bin/bash; Homebrew's bash on PATH works.
if ((BASH_VERSINFO[0] < 4)); then
echo "run-e2e needs bash 4 or newer; this is bash $BASH_VERSION. On macOS: brew install bash" >&2
exit 1
fi

readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
readonly REPO_ROOT="$(dirname -- "$SCRIPT_DIR")"
readonly ORIGINAL_HOME="$HOME"
Expand All @@ -25,9 +32,12 @@ fi
# source so a stale bundle can't pass or fail a run for code it doesn't hold.
pnpm run --silent build >/dev/null

# Set before the trap below can read it: an inherited `results` must not be removed.
results=""
cleanup() {
local status=$?
trap - EXIT INT TERM
rm -f -- "${results:-}"
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
if ! HOME="$ORIGINAL_HOME" pnpm run --silent stop:e2e-obsidian; then
echo "Failed to stop the isolated Obsidian E2E instance." >&2
if ((status == 0)); then status=1; fi
Expand Down Expand Up @@ -87,14 +97,22 @@ done
}
vault_path="${start_env[OBSIDIAN_E2E_VAULT_PATH]}"
obsidian_home="${start_env[OBSIDIAN_E2E_OBSIDIAN_HOME]}"
expected_vault_root="$(realpath -e -- "$REPO_ROOT/.obsidian-e2e-vaults")"
profile_root="$(realpath -e -- /tmp/quickadd-obsidian-e2e)"
if [[ ! -d "$vault_path" || "$(realpath -e -- "$vault_path")" != "$vault_path" || \
# The physical path of an existing directory, or nothing (`realpath -e` is
# GNU-only). The printed paths must already be physical, so a `..` or a link
# can't point the run elsewhere. Only the profile root is resolved first: on
# macOS /tmp is a link to /private/tmp, and the runner prints the /tmp form.
physical_dir() { (cd -P -- "$1" 2>/dev/null && pwd -P) || true; }
readonly PROFILE_ROOT=/tmp/quickadd-obsidian-e2e
expected_vault_root="$(physical_dir "$REPO_ROOT/.obsidian-e2e-vaults")"
profile_root="$(physical_dir "$PROFILE_ROOT")"
if [[ -z "$expected_vault_root" || -z "$profile_root" || \
"$(physical_dir "$vault_path")" != "$vault_path" || \
"$(dirname -- "$vault_path")" != "$expected_vault_root" || \
! -d "$obsidian_home" || "$(realpath -e -- "$obsidian_home")" != "$obsidian_home" || \
"$obsidian_home" != "$PROFILE_ROOT"/* || \
"$(physical_dir "$obsidian_home")" != "$profile_root${obsidian_home#"$PROFILE_ROOT"}" || \
"$(basename -- "$obsidian_home")" != "home" || \
"$(dirname -- "$(dirname -- "$obsidian_home")")" != "$profile_root" ]]; then
echo "Start environment paths are not canonical isolated E2E paths." >&2
"$(dirname -- "$(dirname -- "$obsidian_home")")" != "$PROFILE_ROOT" ]]; then
echo "Start environment paths are not canonical isolated E2E paths: vault $vault_path, home $obsidian_home." >&2
exit 1
fi
for name in "${!start_env[@]}"; do
Expand Down Expand Up @@ -125,7 +143,8 @@ if [[ "${OBSIDIAN_E2E_TEMPLATER:-}" == "1" ]]; then
"https://github.com/SilentVoid13/Templater/releases/download/$templater_version/$file"
mv "$part" "$templater_cache/$file"
fi
echo "${templater_sha256[$file]} $templater_cache/$file" | sha256sum --check --quiet
# `-` for stdin: macOS's sha256sum needs an operand.
echo "${templater_sha256[$file]} $templater_cache/$file" | sha256sum --check --quiet -
cp "$templater_cache/$file" "$templater_dir/$file"
done
# A reused instance may already run a Templater, and enabling a loaded
Expand All @@ -144,4 +163,11 @@ fi

# No `--`: pnpm passes it through, and Vitest ignores everything after it, so
# file and `-t` filters would silently run the whole suite.
pnpm run test:e2e "$@"
# Vitest also exits 0 when a `-t` filter matches no test, so count what ran.
results="$(mktemp)"
pnpm run test:e2e "$@" --reporter=default --reporter=json --outputFile.json="$results"
ran="$(node -e 'const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(String(r.numPassedTests + r.numFailedTests));' "$results")"
if ((ran == 0)); then
echo "No spec ran: the arguments matched no test ($*)." >&2
exit 1
fi
31 changes: 26 additions & 5 deletions tests/orb-setup.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,10 @@ exit 0
expect(fs.readFileSync(log, "utf8")).not.toContain("test:e2e");
});

it("passes its arguments to Vitest as filters", () => {
const instanceName = `orb-args-test-${process.pid}-${Date.now()}`;
// Runs run-e2e against a mock pnpm whose start prints this instance's paths
// and whose Vitest run reports `ran` passed tests.
function runWithMockVitest(name: string, args: string[], ran: number) {
const instanceName = `${name}-${process.pid}-${Date.now()}`;
const initialRoot = prepareProfileRoot();
const home = path.join(profileRoot, instanceName, "home");
fs.mkdirSync(home, { recursive: true });
Expand All @@ -131,19 +133,38 @@ if [[ "$*" == *"start:e2e-obsidian"* ]]; then
echo "export \${prefix}_E2E_OBSIDIAN_HOME='${home}'"
done
fi
if [[ "$*" == *"test:e2e"* ]]; then
for arg in "$@"; do
[[ "$arg" == --outputFile.json=* ]] && echo '{"numPassedTests":${ran},"numFailedTests":0}' > "\${arg#--outputFile.json=}"
done
fi
exit 0
`, { mode: 0o755 });

try {
const result = spawnSync(runE2E, ["tests/e2e/field-label.test.ts", "-t", "label"], {
const result = spawnSync(runE2E, args, {
env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
encoding: "utf8",
});
expect(result.status, result.stderr).toBe(0);
expect(fs.readFileSync(log, "utf8")).toContain("\nrun test:e2e tests/e2e/field-label.test.ts -t label\n");
return { result, calls: fs.readFileSync(log, "utf8") };
} finally {
fs.rmSync(vault, { recursive: true, force: true });
cleanProfileInstance(instanceName, initialRoot);
}
}

it("passes its arguments to Vitest as filters", () => {
const { result, calls } = runWithMockVitest("orb-args-test", ["tests/e2e/field-label.test.ts", "-t", "label"], 1);
expect(result.status, result.stderr).toBe(0);
expect(calls).toContain("\nrun test:e2e tests/e2e/field-label.test.ts -t label --reporter=default --reporter=json --outputFile.json=");
});

// Vitest exits 0 when a -t filter matches no test.
it("fails when its arguments match no test", () => {
const { result, calls } = runWithMockVitest("orb-no-spec-test", ["tests/e2e/field-label.test.ts", "-t", "no such test"], 0);
expect(result.status).toBe(1);
expect(result.stderr).toContain("No spec ran");
expect(calls).toContain("stop:e2e-obsidian");
});

it("rejects rather than evaluates unexpected start output", () => {
Expand Down
Loading