diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c65d62c27..b1689963e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,13 +88,54 @@ jobs: - uses: pnpm/action-setup@v5 with: version: 10 + # The lockfile-pruner tests exercising a real bun install skip themselves + # when bun is not on PATH, so without this step they would silently never + # run in CI. The version is pinned because the tests regenerate a + # committed bun.lock fixture and a future bun may serialize it + # differently. + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.11' - uses: actions/setup-node@v4 with: node-version: '20.x' cache: "pnpm" + # Provisions a corepack-managed yarn for the real-yarn lockfile-pruner + # tests (the fixture's packageManager field pins the exact Yarn Berry + # version). Scoped to yarn on purpose: a bare `corepack enable` would + # also shim pnpm into the setup-node bin dir, which precedes PNPM_HOME + # in PATH and would silently replace the pinned pnpm for every later + # step (this repo has no root packageManager field for corepack to + # resolve). + - run: corepack enable yarn + # Downloads the fixture-pinned Yarn versions (Berry 4 and 3) now, so + # a registry or corepack failure surfaces here with an obvious + # message instead of inside the pruner's timed subprocess during the + # test run. The env var suppresses corepack's download notice/prompt + # as defense in depth (Actions has no TTY, so corepack would not + # prompt anyway). + - run: yarn --version + working-directory: packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace + env: + COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' + - run: yarn --version + working-directory: packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace + env: + COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' - run: pnpm install --frozen-lockfile - run: pnpm run prepack - run: pnpm run test + env: + # Makes the test suite assert that bun is actually on PATH, so a + # lost setup-bun step fails loudly instead of silently skipping + # the real-bun lockfile-pruner tests. + CHECKLY_EXPECT_BUN: '1' + # Same assertion for the corepack-managed Yarn Berry. The suite's + # probe checks the resolved major version from the fixture + # directory, so a preinstalled Classic yarn shadowing the corepack + # shim fails the assertion rather than silently skipping the + # real-yarn lockfile-pruner tests. + CHECKLY_EXPECT_YARN: '1' - name: Save LLM rules as an artifact uses: actions/upload-artifact@v4 with: @@ -112,13 +153,34 @@ jobs: - uses: pnpm/action-setup@v5 with: version: 10 + # See test-ubuntu: provisions bun for the real-bun lockfile-pruner tests, + # which otherwise skip themselves. + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3.11' - uses: actions/setup-node@v4 with: node-version: '20.x' cache: "pnpm" + # See test-ubuntu: yarn-scoped so the corepack pnpm shim cannot shadow + # the pinned pnpm, and pre-downloaded so a corepack failure surfaces + # here instead of inside the timed pruner subprocess. + - run: corepack enable yarn + - run: yarn --version + working-directory: packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace + env: + COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' + - run: yarn --version + working-directory: packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace + env: + COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' - run: pnpm install --frozen-lockfile - run: pnpm run prepack - run: pnpm run test + env: + # See test-ubuntu. + CHECKLY_EXPECT_BUN: '1' + CHECKLY_EXPECT_YARN: '1' - name: Save LLM rules as an artifact uses: actions/upload-artifact@v4 with: diff --git a/CLAUDE.md b/CLAUDE.md index b8d3c735a..763461ad6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,3 +117,5 @@ Source in `src/ai-context/`, built during `prepare`. Generates examples from fix - `CHECKLY_ENV` — target environment (`production`, `staging`, `development`, `local`) - `CHECKLY_API_URL` — override API base URL (used when `CHECKLY_ENV=local`) - `CHECKLY_CLI_VERSION` — override reported CLI version (useful for testing `create-checkly`) +- `CHECKLY_CACHE_DIR` — override the CLI's cache directory (embedded-package tarball downloads) +- `CHECKLY_LOCKFILE_PRUNE` — set to `0` to disable pruning the bundled lockfile to the code bundle's contents diff --git a/packages/cli/src/ai-context/references/configure-playwright-checks.md b/packages/cli/src/ai-context/references/configure-playwright-checks.md index 56c397b85..984127829 100644 --- a/packages/cli/src/ai-context/references/configure-playwright-checks.md +++ b/packages/cli/src/ai-context/references/configure-playwright-checks.md @@ -12,9 +12,11 @@ - For private packages or custom registries, `.npmrc` is bundled automatically — the workspace-root `.npmrc` and any `.npmrc` beside a package's `package.json` are included by default. You do not need to add `.npmrc` to `include`. - The `.npmrc` should reference a Checkly environment variable such as `${NPM_TOKEN}`. Tell the user that the token must exist in Checkly before `deploy` or `trigger`. Because `.npmrc` is uploaded automatically, warn users to reference credentials via environment variables (`${NPM_TOKEN}`) rather than embedding plaintext tokens. - Use `installCommand` only when the default package-manager install command is not enough. -- Checkly caches installed dependencies between runs, keyed off the lock file, `package.json` and `.npmrc` contents, and the resolved `bundle.packages.embed` tarball set. To force a reinstall declaratively, set `caching.dependencyCache.version` (a string or a safe integer) at the top level of `checkly.config.ts` (not per check — one code bundle serves all Playwright Check Suites) and change its value whenever the cache should be invalidated; scheduled checks pick up the change on the next `checkly deploy`. Unset or empty-string values leave the cache key unchanged, so a dynamic value such as `version: process.env.DEPENDENCY_CACHE_VERSION` is safe when the variable is not always set. For a one-off reinstall during an ad-hoc run, use the `--refresh-cache` flag available on the run/test commands (`checkly test`, `checkly pw-test`, `checkly trigger`, `checkly checks run`) instead; the config value is the persistent knob that also applies to deployed, scheduled checks. +- For pnpm projects, the workspace-root pnpmfile (`.pnpmfile.cjs` / `.pnpmfile.mjs`) is bundled automatically so the remote install reproduces the lockfile's recorded `pnpmfileChecksum` — but only when two conditions hold. First, the lockfile must record a `pnpmfileChecksum` (pnpm writes one when `pnpm install` last ran with the pnpmfile in place); without one there is nothing for the remote install to reproduce and no pnpmfile is bundled. Second, the pnpmfile must be self-contained: it may load only side-effect-free Node.js builtins (`assert`, `buffer`, `crypto`, `events`, `path`, `punycode`, `querystring`, `string_decoder`, `url`, `util`) via literal `require('...')`/`import` specifiers, and must not reference `process`, `__dirname`, `__filename`, `import.meta`, `globalThis`, `global`, `eval`, `Function`, `require.resolve` or dynamically computed module paths — the remote install loads the pnpmfile before any dependencies exist and in a different environment. A pnpmfile that does not satisfy this (e.g. loads `fs`, local helper files or npm packages) is skipped with a warning and the remote install may re-resolve dependencies instead of using the lockfile; make the pnpmfile self-contained to avoid this. Since the pnpmfile is uploaded, avoid embedding secrets in it. +- In a workspace (monorepo) whose code bundle covers only part of the workspace, the bundled lockfile is pruned automatically: the CLI regenerates it (via `pnpm install --lockfile-only` / `npm install --package-lock-only` / `bun install --lockfile-only` / `yarn install --mode=update-lockfile` in a temp dir) so it only references the packages actually in the bundle — otherwise the remote install would try to fetch dependencies of workspace members that were omitted or shipped as dependency-free placeholder manifests, which fails outright for private packages. Supported for `pnpm-lock.yaml` versions 6/9, `package-lock.json` versions 2/3, the text `bun.lock` version 1 and Yarn Berry `yarn.lock` files (for bun projects, keep registry configuration in `.npmrc`, which bun reads: `bunfig.toml` is not carried into the regeneration — recorded resolutions keep their URLs, but whenever bun declines to reuse the lockfile — it is out of date with a manifest, or a workspace member's name collides with a registry dependency — bun re-resolves those entries against the wrong registry, disclosing the package names to it (typically the public registry), and pruning rejects the result with a warning; for yarn projects, `.yarnrc.yml` is likewise not carried into the regeneration, which is safe because Berry lockfiles are registry-agnostic and the regeneration reuses recorded resolutions without the network — settings like `approvedGitRepositories` and `npmScopes` only affect new resolutions, which pruning never performs — and the regeneration runs with yarn's network access disabled outright, since it never needs it: a lockfile that is out of date with a manifest then fails fast with a warning instead of resolving the missing package against the wrong registry and disclosing its name; yarn's hardened mode is disabled for the same reason; `yarn patch` files under `.yarn/patches` are bundled automatically because the regeneration reads them; a `yarn` binary that resolves to Yarn Classic on a Berry project is refused before it can run, because Classic would silently perform a full install); when a bundled lockfile over-describes a partial-workspace bundle but pruning cannot run — other lockfile formats, Yarn Classic v1 lockfiles, a `yarn` binary that resolves to Yarn Classic on a Berry project (set the `packageManager` field so Corepack provisions Yarn 2+), bun's binary `bun.lockb` (regenerate a text lockfile with `bun install --save-text-lockfile`), the package manager binary not being installed on the machine running the CLI, `excludeLinksFromLockfile`, a recorded pnpmfile checksum without a bundled pnpmfile, a workspace member whose version cannot be determined, among others — the original lockfile ships unchanged and the CLI prints a note saying so. Other skips are silent: nothing to prune (the bundle contains the full workspace, or regeneration produced identical bytes), no bundled lockfile to prune, or pruning disabled via `CHECKLY_LOCKFILE_PRUNE=0`; silent skip reasons are visible via `DEBUG='checkly:cli:services:check-parser:*'`. When pruning runs but cannot produce a provably pruned copy of the original — the lockfile is out of date with a `package.json`, the package manager could not run or timed out, the lockfile could not be read or written, or verification failed, among others — the original ships unchanged with a warning. Set `CHECKLY_LOCKFILE_PRUNE=0` to disable pruning. +- Checkly caches installed dependencies between runs, keyed off the workspace's lock file, every workspace member's `package.json` and `.npmrc` (whether or not the member is in the bundle), bundled pnpmfile contents, and the resolved `bundle.packages.embed` tarball set (filtered to what the pruned lockfile still references when pruning applied) — plus, as additional inputs, any synthesized placeholder manifests shipped in the bundle and the pruned lockfile when pruning applied. Because the bundle-specific inputs follow the bundle, the key can change without a file edit — e.g. when a different set of workspace members ends up in the bundle. To force a reinstall declaratively, set `caching.dependencyCache.version` (a string or a safe integer) at the top level of `checkly.config.ts` (not per check — one code bundle serves all Playwright Check Suites) and change its value whenever the cache should be invalidated; scheduled checks pick up the change on the next `checkly deploy`. Unset or empty-string values leave the cache key unchanged, so a dynamic value such as `version: process.env.DEPENDENCY_CACHE_VERSION` is safe when the variable is not always set. For a one-off reinstall during an ad-hoc run, use the `--refresh-cache` flag available on the run/test commands (`checkly test`, `checkly pw-test`, `checkly trigger`, `checkly checks run`) instead; the config value is the persistent knob that also applies to deployed, scheduled checks. - In Checkly CLI v8.0.0 and later, `include` patterns resolve relative to the Playwright config directory, not the project root. If `playwrightConfigPath` points to a subdirectory, adjust `include` globs. Example: `playwrightConfigPath: "./e2e/playwright.config.ts"` with a root fixture at `fixtures/data.json` needs `include: ["../fixtures/data.json"]`. -- If dependencies come from a private registry that Checkly's infrastructure cannot reach (for example an intranet-only Nexus mirror), list them in `bundle.packages.embed` in `checkly.config.ts` — a top-level section: `bundle: { packages: { embed: ['@acme/private-utils', 'legacy-private-pkg@2.1.0'] } }`. Each entry is a package name (embeds every version found in the lockfile) or an exact `name@version` pin; names may contain `*` wildcards (`@acme/*`, `acme-*`, `@acme/*-utils`) where each `*` matches any run of characters except `/` (never crossing the scope separator); as long as a spec matches at least one registry package, matches that cannot be embedded are skipped (workspace members silently, git/file/URL dependencies and integrity-less entries with a warning since the runner must fetch those itself), while a spec whose only matches cannot be embedded — or that matches nothing at all — is an error; a pattern embeds every lockfile version of every package it matches, so scope it to the packages the runner genuinely cannot fetch. List every unreachable package by name, including private packages that only appear as transitive dependencies of other private packages — dependencies of listed packages are not embedded automatically. The CLI resolves entries against the workspace-root lockfile (`pnpm-lock.yaml` or `package-lock.json`), reuses tarballs from local caches (its own, then npm's) or downloads them from the registry configured in `.npmrc`, verifies each against the lockfile's integrity hash, and ships them inside the code bundle at `.checkly/embedded-packages/*.tgz`, where the runner serves them through a local registry during install. Downloads are cached under the workspace root's `node_modules/.cache/checkly` (in a monorepo that is the repo root, not the member package; override with `CHECKLY_CACHE_DIR`; a per-user cache dir is the fallback when the project location isn't writable), so nothing lands in the project outside `node_modules`. CI setups that cache `node_modules` — or platforms that preserve `node_modules/.cache` — persist the tarballs automatically; otherwise persist `CHECKLY_CACHE_DIR` in CI to avoid re-downloading (note `npm ci` deletes `node_modules` wholesale, unlike incremental pnpm installs). The machine running `checkly deploy`/`test` needs registry access on a cold cache. Changing the resolved set of embedded packages invalidates the runner's dependency cache, so the next run reinstalls with the new tarballs. Applies to Playwright Check Suites only, not browser or multistep checks. +- If dependencies come from a private registry that Checkly's infrastructure cannot reach (for example an intranet-only Nexus mirror), list them in `bundle.packages.embed` in `checkly.config.ts` — a top-level section: `bundle: { packages: { embed: ['@acme/private-utils', 'legacy-private-pkg@2.1.0'] } }`. Each entry is a package name (embeds every version found in the lockfile) or an exact `name@version` pin; names may contain `*` wildcards (`@acme/*`, `acme-*`, `@acme/*-utils`) where each `*` matches any run of characters except `/` (never crossing the scope separator); as long as a spec matches at least one registry package, matches that cannot be embedded are skipped (workspace members silently, git/file/URL dependencies and integrity-less entries with a warning since the runner must fetch those itself), while a spec whose only matches cannot be embedded — or that matches nothing at all — is an error; a pattern embeds every lockfile version of every package it matches, so scope it to the packages the runner genuinely cannot fetch. List every unreachable package by name, including private packages that only appear as transitive dependencies of other private packages — dependencies of listed packages are not embedded automatically. The CLI resolves entries against the workspace-root lockfile (`pnpm-lock.yaml`, `package-lock.json`, the text `bun.lock` or a Yarn Berry `yarn.lock` — Yarn Classic v1 lockfiles are not supported), reuses tarballs from local caches (its own, then npm's) or downloads them from the registry configured in `.npmrc` (only `.npmrc` — bun or yarn users whose registry credentials live solely in `bunfig.toml` or `.yarnrc.yml` must duplicate them into `.npmrc`, or downloads fail with an auth error), verifies each against the lockfile's integrity hash, and ships them inside the code bundle at `.checkly/embedded-packages/*.tgz`, where the runner serves them through a local registry during install. Yarn Berry lockfiles record no npm tarball integrity (Berry checksums cover yarn's own cache format), so the CLI resolves the tarball integrity from the registry's package metadata instead — one small metadata request per embedded package on every deploy (the per-version route, falling back to the full packument), even when the tarballs themselves come from a warm cache, so a yarn embed needs registry reachability at deploy time even on a warm cache. When the bundled lockfile is pruned to the code bundle's contents (see the pruning bullet above), the embedded set follows it: packages the pruned lockfile no longer references — dependencies of workspace members that are not part of the bundle — are neither embedded nor downloaded, even if an entry matches them. If a package unexpectedly stops being embedded, the usual cause is that only a workspace member outside the bundle depends on it, in which case the runner never installs it and nothing is wrong; if the checks genuinely need it, make the depending member part of the bundle (import it from check code) rather than disabling pruning — `CHECKLY_LOCKFILE_PRUNE=0` restores the unfiltered set but reintroduces the over-describing lockfile that pruning exists to prevent, so treat it as a last resort. Downloads are cached under the workspace root's `node_modules/.cache/checkly` (in a monorepo that is the repo root, not the member package; override with `CHECKLY_CACHE_DIR`; a per-user cache dir is the fallback when the project location isn't writable), so nothing lands in the project outside `node_modules`. CI setups that cache `node_modules` — or platforms that preserve `node_modules/.cache` — persist the tarballs automatically; otherwise persist `CHECKLY_CACHE_DIR` in CI to avoid re-downloading (note `npm ci` deletes `node_modules` wholesale, unlike incremental pnpm installs). The machine running `checkly deploy`/`test` needs registry access on a cold cache — but only for the tarballs actually shipped, not for pruned-away ones. Changing the resolved set of embedded packages invalidates the runner's dependency cache, so the next run reinstalls with the new tarballs. Applies to Playwright Check Suites only, not browser or multistep checks. ## Install troubleshooting diff --git a/packages/cli/src/commands/debug/parse-project.ts b/packages/cli/src/commands/debug/parse-project.ts index b14e03078..a673d2b6b 100644 --- a/packages/cli/src/commands/debug/parse-project.ts +++ b/packages/cli/src/commands/debug/parse-project.ts @@ -169,6 +169,7 @@ export default class ParseProjectCommand extends Command { const bundler = await Bundler.createForWorkspace(Session.workspace.unwrap(), { dependencyCacheVersion: checklyConfig.caching?.dependencyCache?.version, embeddedPackagesMaterializer: Session.getEmbeddedPackagesMaterializer(), + packageManager: Session.packageManager, }) const bundleStartedAt = performance.now() diff --git a/packages/cli/src/commands/deploy.ts b/packages/cli/src/commands/deploy.ts index d7da311c1..61f2bea7e 100644 --- a/packages/cli/src/commands/deploy.ts +++ b/packages/cli/src/commands/deploy.ts @@ -211,6 +211,7 @@ export default class Deploy extends AuthCommand { const bundler = await Bundler.createForWorkspace(Session.workspace.unwrap(), { dependencyCacheVersion: checklyConfig.caching?.dependencyCache?.version, embeddedPackagesMaterializer: Session.getEmbeddedPackagesMaterializer(), + packageManager: Session.packageManager, }) this.style.actionStart('Bundling project resources') diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index f570512d1..5697f0164 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -268,6 +268,7 @@ export default class PwTestCommand extends AuthCommand { const bundler = await Bundler.createForWorkspace(Session.workspace.unwrap(), { dependencyCacheVersion: checklyConfig.caching?.dependencyCache?.version, embeddedPackagesMaterializer: Session.getEmbeddedPackagesMaterializer(), + packageManager: Session.packageManager, }) this.style.actionStart('Bundling project resources') diff --git a/packages/cli/src/commands/test.ts b/packages/cli/src/commands/test.ts index 7dcd4c1d9..85f48026e 100644 --- a/packages/cli/src/commands/test.ts +++ b/packages/cli/src/commands/test.ts @@ -293,6 +293,7 @@ export default class Test extends AuthCommand { const bundler = await Bundler.createForWorkspace(Session.workspace.unwrap(), { dependencyCacheVersion: checklyConfig.caching?.dependencyCache?.version, embeddedPackagesMaterializer: Session.getEmbeddedPackagesMaterializer(), + packageManager: Session.packageManager, }) this.style.actionStart('Bundling project resources') diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/README.md b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/README.md index b3813417f..36d41a2b8 100644 --- a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/README.md +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/README.md @@ -5,6 +5,14 @@ in `playwright-check.spec.ts`. Their sha512 integrities are hardcoded in the `pnpm-lock.yaml` files of the `test-embedded-packages*` fixtures, so the tarball bytes and the lockfile entries must change together. +`ms@2.1.3.tgz` is different: it is the genuine registry artifact for +`ms@2.1.3` (from https://registry.npmjs.org/ms/-/ms-2.1.3.tgz), used by the +`test-bundling-workspace-lockfile-prune-embed` fixture, whose lockfile must +survive a real offline `pnpm install --lockfile-only` regeneration — a fake +package would 404 when pnpm re-resolves a stale lockfile, so the kept-side +embedded package has to be real. Its integrity in that fixture's lockfile is +the real registry integrity. + To regenerate (and then update the `resolution.integrity` values the script prints into the fixture lockfiles): diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/ms@2.1.3.tgz b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/ms@2.1.3.tgz new file mode 100644 index 000000000..c7670dc8c Binary files /dev/null and b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/embedded-tarballs/ms@2.1.3.tgz differ diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/package.json new file mode 100644 index 000000000..a5c6310ab --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/package.json @@ -0,0 +1,5 @@ +{ + "name": "workspace-lockfile-prune-embed-bundle-test", + "version": "1.0.0", + "private": true +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/checkly.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/checkly.config.ts new file mode 100644 index 000000000..6eec32ad3 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/checkly.config.ts @@ -0,0 +1,27 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.ts', + ignoreDirectoriesMatch: [], + playwrightConfigPath: './playwright.config.ts', + playwrightChecks: [ + { + logicalId: 'playwright-check-suite', + name: 'Playwright Check Suite', + } + ], + }, + bundle: { + packages: { + embed: [ + '@acme/private-utils', + 'ms' + ], + }, + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/package.json new file mode 100644 index 000000000..65c03285b --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/package.json @@ -0,0 +1,10 @@ +{ + "name": "@fixture-embed/c", + "version": "1.0.0", + "private": true, + "dependencies": { + "@playwright/test": "^1.55.1", + "@fixture-embed/used": "workspace:*", + "@fixture-embed/shimmed": "workspace:*" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/playwright.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/playwright.config.ts new file mode 100644 index 000000000..eed093e29 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/playwright.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + timeout: 30000, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/tests/example.spec.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/tests/example.spec.ts new file mode 100644 index 000000000..69046578f --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/c/tests/example.spec.ts @@ -0,0 +1,7 @@ +import { test, expect } from '@playwright/test' + +import { entry } from '@fixture-embed/used' + +test('uses the workspace member', async () => { + expect(entry()).toBe('used') +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/shimmed/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/shimmed/package.json new file mode 100644 index 000000000..b7f34767a --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/shimmed/package.json @@ -0,0 +1,8 @@ +{ + "name": "@fixture-embed/shimmed", + "version": "1.0.0", + "private": true, + "dependencies": { + "@acme/private-utils": "1.2.3" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/package.json new file mode 100644 index 000000000..bfd2b88bb --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/package.json @@ -0,0 +1,9 @@ +{ + "name": "@fixture-embed/used", + "version": "1.0.0", + "private": true, + "main": "src/index.js", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/src/index.js b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/src/index.js new file mode 100644 index 000000000..6199e199c --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/packages/used/src/index.js @@ -0,0 +1,3 @@ +module.exports.entry = function entry () { + return 'used' +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-lock.yaml b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-lock.yaml new file mode 100644 index 000000000..abbed167a --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-lock.yaml @@ -0,0 +1,82 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: {} + + packages/c: + dependencies: + '@fixture-embed/shimmed': + specifier: workspace:* + version: link:../shimmed + '@fixture-embed/used': + specifier: workspace:* + version: link:../used + '@playwright/test': + specifier: ^1.55.1 + version: 1.62.1 + + packages/shimmed: + dependencies: + '@acme/private-utils': + specifier: 1.2.3 + version: 1.2.3 + + packages/used: + dependencies: + ms: + specifier: 2.1.3 + version: 2.1.3 + +packages: + + '@acme/private-utils@1.2.3': + resolution: {integrity: sha512-dnkm3WedrIfH8+nRoHESfj0/DDeZdBTCpP2B5ZUSR/6YsMiOtYmauw1FRb2hDNC00ZLWu8Ya8sZfR2D/s1VhTQ==} + + '@playwright/test@1.62.1': + resolution: {integrity: sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==} + engines: {node: '>=20'} + hasBin: true + + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + playwright-core@1.62.1: + resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} + engines: {node: '>=20'} + hasBin: true + + playwright@1.62.1: + resolution: {integrity: sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==} + engines: {node: '>=20'} + hasBin: true + +snapshots: + + '@acme/private-utils@1.2.3': {} + + '@playwright/test@1.62.1': + dependencies: + playwright: 1.62.1 + + fsevents@2.3.2: + optional: true + + ms@2.1.3: {} + + playwright-core@1.62.1: {} + + playwright@1.62.1: + dependencies: + playwright-core: 1.62.1 + optionalDependencies: + fsevents: 2.3.2 diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-workspace.yaml b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-workspace.yaml new file mode 100644 index 000000000..924b55f42 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune-embed/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - packages/* diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/package.json new file mode 100644 index 000000000..38a0c009e --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/package.json @@ -0,0 +1,5 @@ +{ + "name": "workspace-lockfile-prune-bundle-test", + "version": "1.0.0", + "private": true +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/absent/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/absent/package.json new file mode 100644 index 000000000..687582914 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/absent/package.json @@ -0,0 +1,8 @@ +{ + "name": "@fixture-prune/absent", + "version": "1.0.0", + "private": true, + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/checkly.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/checkly.config.ts new file mode 100644 index 000000000..3dede8ef2 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/checkly.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from 'checkly' + +const config = defineConfig({ + projectName: 'Check Fixture', + logicalId: 'check-fixture', + checks: { + checkMatch: '**/*.check.ts', + ignoreDirectoriesMatch: [], + playwrightConfigPath: './playwright.config.ts', + playwrightChecks: [ + { + logicalId: 'playwright-check-suite', + name: 'Playwright Check Suite', + } + ], + }, +}) + +export default config diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/package.json new file mode 100644 index 000000000..b5bd95972 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/package.json @@ -0,0 +1,13 @@ +{ + "name": "@fixture-prune/c", + "version": "1.0.0", + "private": true, + "dependencies": { + "@playwright/test": "^1.55.1", + "@fixture-prune/used": "workspace:*", + "@fixture-prune/shimmed": "workspace:*" + }, + "optionalDependencies": { + "@fixture-prune/opt": "workspace:*" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/playwright.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/playwright.config.ts new file mode 100644 index 000000000..eed093e29 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/playwright.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: './tests', + timeout: 30000, +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/tests/example.spec.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/tests/example.spec.ts new file mode 100644 index 000000000..ed39d3ebd --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/c/tests/example.spec.ts @@ -0,0 +1,7 @@ +import { test, expect } from '@playwright/test' + +import { entry } from '@fixture-prune/used' + +test('uses the workspace member', async () => { + expect(entry()).toBe('used') +}) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/opt/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/opt/package.json new file mode 100644 index 000000000..e53435302 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/opt/package.json @@ -0,0 +1,8 @@ +{ + "name": "@fixture-prune/opt", + "version": "1.0.0", + "private": true, + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/shimmed/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/shimmed/package.json new file mode 100644 index 000000000..76df71b12 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/shimmed/package.json @@ -0,0 +1,8 @@ +{ + "name": "@fixture-prune/shimmed", + "version": "1.0.0", + "private": true, + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/package.json b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/package.json new file mode 100644 index 000000000..5e306ded5 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/package.json @@ -0,0 +1,9 @@ +{ + "name": "@fixture-prune/used", + "version": "1.0.0", + "private": true, + "main": "src/index.js", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/src/index.js b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/src/index.js new file mode 100644 index 000000000..6199e199c --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/packages/used/src/index.js @@ -0,0 +1,3 @@ +module.exports.entry = function entry () { + return 'used' +} diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-lock.yaml b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-lock.yaml new file mode 100644 index 000000000..837c02f3f --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-lock.yaml @@ -0,0 +1,103 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: {} + + packages/absent: + dependencies: + ee-first: + specifier: 1.1.1 + version: 1.1.1 + + packages/c: + dependencies: + '@fixture-prune/shimmed': + specifier: workspace:* + version: link:../shimmed + '@fixture-prune/used': + specifier: workspace:* + version: link:../used + '@playwright/test': + specifier: ^1.55.1 + version: 1.62.1 + optionalDependencies: + '@fixture-prune/opt': + specifier: workspace:* + version: link:../opt + + packages/opt: + dependencies: + ms: + specifier: 2.1.3 + version: 2.1.3 + + packages/shimmed: + dependencies: + isarray: + specifier: 2.0.5 + version: 2.0.5 + + packages/used: + dependencies: + ms: + specifier: 2.1.3 + version: 2.1.3 + +packages: + + '@playwright/test@1.62.1': + resolution: {integrity: sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==} + engines: {node: '>=20'} + hasBin: true + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + playwright-core@1.62.1: + resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} + engines: {node: '>=20'} + hasBin: true + + playwright@1.62.1: + resolution: {integrity: sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==} + engines: {node: '>=20'} + hasBin: true + +snapshots: + + '@playwright/test@1.62.1': + dependencies: + playwright: 1.62.1 + + ee-first@1.1.1: {} + + fsevents@2.3.2: + optional: true + + isarray@2.0.5: {} + + ms@2.1.3: {} + + playwright-core@1.62.1: {} + + playwright@1.62.1: + dependencies: + playwright-core: 1.62.1 + optionalDependencies: + fsevents: 2.3.2 diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-workspace.yaml b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-workspace.yaml new file mode 100644 index 000000000..924b55f42 --- /dev/null +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-bundling-workspace-lockfile-prune/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - packages/* diff --git a/packages/cli/src/constructs/__tests__/playwright-check.spec.ts b/packages/cli/src/constructs/__tests__/playwright-check.spec.ts index e6e02de20..493055692 100644 --- a/packages/cli/src/constructs/__tests__/playwright-check.spec.ts +++ b/packages/cli/src/constructs/__tests__/playwright-check.spec.ts @@ -9,8 +9,13 @@ import { list } from 'tar' import { FixtureSandbox, RunOptions } from '../../testing/fixture-sandbox.js' import { ParseProjectOutput } from '../../commands/debug/parse-project.js' import { TarballCache } from '../../services/embedded-packages/cache.js' +import { composeWorkspaceCacheHash, loadWorkspaceCacheHashInputs } from '../../services/check-parser/cache-hash.js' +import { PNpmDetector } from '../../services/check-parser/package-files/package-manager.js' -async function parseProject (fixt: FixtureSandbox, ...args: string[]): Promise { +async function parseProject ( + fixt: FixtureSandbox, + ...args: string[] +): Promise { return await parseProjectWithOptions(fixt, {}, ...args) } @@ -18,7 +23,7 @@ async function parseProjectWithOptions ( fixt: FixtureSandbox, options: RunOptions, ...args: string[] -): Promise { +): Promise { const result = await fixt.run('pnpm', [ 'checkly', 'debug', @@ -37,7 +42,7 @@ async function parseProjectWithOptions ( const output: ParseProjectOutput = JSON.parse(result.stdout) - return output + return { ...output, stderr: String(result.stderr ?? '') } } async function listTarFiles (filePath: string): Promise { @@ -68,6 +73,25 @@ async function listTarEntries (filePath: string): Promise { return entries } +async function readTarEntryContent (filePath: string, entryPath: string): Promise { + const chunks: Buffer[] = [] + let found = false + await list({ + file: filePath, + onReadEntry: entry => { + if (entry.path !== entryPath) { + return + } + found = true + entry.on('data', chunk => chunks.push(chunk)) + }, + }) + if (!found) { + throw new Error(`Archive entry not found: ${entryPath}`) + } + return Buffer.concat(chunks).toString('utf8') +} + /** * Asserts the one thing an archive must never contain: a symlink with entries * beneath it. A path cannot be both a symlink and a directory, and tar refuses @@ -1311,9 +1335,11 @@ describe('PlaywrightCheck', () => { }) it('should bundle members selectively and keep the workspace links resolvable', async () => { + // DEBUG is scoped to the bundling namespaces so the lockfile-prune + // skip assertion below can observe the (debug-only) prune activity. const result = await fixt.run('pnpm', [ 'checkly', 'debug', 'parse-project', '--config', 'packages/c/checkly.config.ts', - ]) + ], { env: { DEBUG: 'checkly:cli:services:check-parser:*' } }) expect(result.exitCode).toBe(0) const output: ParseProjectOutput = JSON.parse(result.stdout) @@ -1370,7 +1396,210 @@ describe('PlaywrightCheck', () => { // Nothing lands at through-link spellings. expect(files.filter(file => file.includes('node_modules/@scope/x/'))).toEqual([]) + + // Every workspace member's real manifest is in this bundle, so + // lockfile pruning is skipped — it must not even attempt to run — and + // the original lockfile ships byte-for-byte. The skip is only visible + // through the scoped DEBUG output enabled above: the skip reason must + // appear, and the prune command (recognizable by its --lockfile-only + // flag) must never be spawned. + expect(String(result.stderr)).toContain('Lockfile pruning skipped: the bundle contains the full workspace') + expect(String(result.stderr)).not.toContain('--lockfile-only') + expect(String(result.stderr)).not.toContain('could not prune the bundled lockfile') + const archivedLockfile = await readTarEntryContent(codeBundlePath, 'pnpm-lock.yaml') + const originalLockfile = await fs.readFile(fixt.abspath('pnpm-lock.yaml'), 'utf8') + expect(archivedLockfile).toEqual(originalLockfile) + }, DEFAULT_TEST_TIMEOUT) + }) + + describe('bundling a pnpm workspace with a pruned lockfile', () => { + let fixt: FixtureSandbox + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'playwright-check', 'test-cases', 'test-bundling-workspace-lockfile-prune'), + }) + }, DEFAULT_TEST_TIMEOUT) + + afterAll(async () => { + await fixt?.destroy() + }) + + it('prunes the bundled lockfile to the bundle contents and updates the cache hash', async () => { + const output = await parseProject(fixt, '--config', 'packages/c/checkly.config.ts') + + const { + codeBundlePath, + cacheHash, + } = output.payload.resources[0].payload as any + + const files = await listTarFiles(codeBundlePath) + + // Members: `used` is imported (real files), `shimmed` is declared but + // unimported (faux manifest only), `opt` is referenced only through + // `optionalDependencies` — invisible to the parser's faux mechanism, + // so its manifest reaches the bundle only via the pruner's backfill — + // and `absent` is referenced by nobody (nothing in the bundle). + expect(files).toEqual(expect.arrayContaining([ + 'pnpm-lock.yaml', + 'packages/used/package.json', + 'packages/used/src/index.js', + 'packages/shimmed/package.json', + 'packages/opt/package.json', + ])) + expect(files.filter(file => file.startsWith('packages/absent/'))).toEqual([]) + + // The backfilled manifest is a faux shim carrying the real version. + const optManifest = await readTarEntryContent(codeBundlePath, 'packages/opt/package.json') + expect(JSON.parse(optManifest)).toMatchObject({ + name: '@fixture-prune/opt', + version: '1.0.0', + private: true, + }) + + const lockfile = await readTarEntryContent(codeBundlePath, 'pnpm-lock.yaml') + + // Kept: the imported member's importer and dependency, the shimmed + // member's importer (its manifest ships as a dep-free shim), and the + // backfilled member's importer. + expect(lockfile).toContain('packages/used') + expect(lockfile).toContain('ms@2.1.3') + expect(lockfile).toContain('packages/shimmed') + expect(lockfile).toContain('packages/opt') + + // Dropped: the absent member's importer, and the dependencies of both + // the shimmed and the absent member. + expect(lockfile).not.toContain('packages/absent') + expect(lockfile).not.toContain('isarray') + expect(lockfile).not.toContain('ee-first') + + // The cache hash must reflect the bundle's actual install inputs: the + // workspace inputs plus the faux manifests (including the backfilled + // one) and the pruned lockfile exactly as archived. Recomputing the + // expected value from the archive contents pins that both bundle-time + // record types are wired through. + const shimmedManifest = await readTarEntryContent(codeBundlePath, 'packages/shimmed/package.json') + const workspace = await new PNpmDetector().lookupWorkspace(fixt.root) + const expectedHash = composeWorkspaceCacheHash(await loadWorkspaceCacheHashInputs(workspace!), { + fauxPackageJsons: [ + { path: 'packages/opt/package.json', raw: Buffer.from(optManifest, 'utf8') }, + { path: 'packages/shimmed/package.json', raw: Buffer.from(shimmedManifest, 'utf8') }, + ], + prunedLockfile: { + name: 'pnpm-lock.yaml', + hash: createHash('sha256').update(lockfile).digest(), + }, + }) + expect(cacheHash).toEqual(expectedHash) + }, DEFAULT_TEST_TIMEOUT) + + it('still hashes faux manifests when pruning is disabled', async () => { + const output = await parseProjectWithOptions( + fixt, + { env: { CHECKLY_LOCKFILE_PRUNE: '0' } }, + '--config', 'packages/c/checkly.config.ts', + ) + + const { + codeBundlePath, + cacheHash, + } = output.payload.resources[0].payload as any + + // The original lockfile ships unchanged, but the faux manifest is + // still an install input and must still reach the cache hash. With + // pruning disabled, no backfill runs either. + const lockfile = await readTarEntryContent(codeBundlePath, 'pnpm-lock.yaml') + const originalLockfile = await fs.readFile(fixt.abspath('pnpm-lock.yaml'), 'utf8') + expect(lockfile).toEqual(originalLockfile) + const files = await listTarFiles(codeBundlePath) + expect(files).not.toContain('packages/opt/package.json') + + const shimmedManifest = await readTarEntryContent(codeBundlePath, 'packages/shimmed/package.json') + const workspace = await new PNpmDetector().lookupWorkspace(fixt.root) + const expectedHash = composeWorkspaceCacheHash(await loadWorkspaceCacheHashInputs(workspace!), { + fauxPackageJsons: [ + { path: 'packages/shimmed/package.json', raw: Buffer.from(shimmedManifest, 'utf8') }, + ], + }) + expect(cacheHash).toEqual(expectedHash) + }, DEFAULT_TEST_TIMEOUT) + }) + + describe('bundling a pnpm workspace with a pruned lockfile and embedded packages', () => { + const MS_INTEGRITY = 'sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==' + + let fixt: FixtureSandbox + let cacheDir: string + + beforeAll(async () => { + fixt = await FixtureSandbox.create({ + source: path.join(__dirname, 'fixtures', 'playwright-check', 'test-cases', 'test-bundling-workspace-lockfile-prune-embed'), + }) + // Only the kept package's tarball is seeded: the dropped package's + // bytes exist in no cache and on no registry, so any attempt to + // materialize it would fail the run loudly — this test passing proves + // it was never requested. + cacheDir = await seedTarballCache('ms@2.1.3.tgz') + }, DEFAULT_TEST_TIMEOUT) + + afterAll(async () => { + await fixt?.destroy() + if (cacheDir) { + await fs.rm(cacheDir, { recursive: true, force: true }) + } + }) + + it('embeds and downloads only the tarballs the pruned lockfile still references', async () => { + const output = await parseProjectWithOptions( + fixt, + { env: { CHECKLY_CACHE_DIR: cacheDir } }, + '--config', 'packages/c/checkly.config.ts', + ) + expect(output.diagnostics.fatal).toBe(false) + + const { + codeBundlePath, + cacheHash, + } = output.payload.resources[0].payload as any + + // The shimmed member ships as a dep-free faux manifest, so pruning + // drops its dependency @acme/private-utils from the lockfile — and + // with it, the embedded tarball for it. + const lockfile = await readTarEntryContent(codeBundlePath, 'pnpm-lock.yaml') + expect(lockfile).not.toContain('@acme/private-utils') + expect(lockfile).toContain('ms@2.1.3') + + const files = await listTarFiles(codeBundlePath) + expect(files).toContain('.checkly/embedded-packages/ms@2.1.3.tgz') + expect(files).not.toContain('.checkly/embedded-packages/@acme+private-utils@1.2.3.tgz') + + // A successful prune-and-embed produces no user-facing prune output; + // progress is debug-only for now. + expect(output.stderr).not.toContain('could not prune the bundled lockfile') + expect(output.stderr).not.toContain('the bundled lockfile was not pruned') + + // The cache hash reflects exactly what ships: the kept-only embedded + // set, the faux manifest and the pruned lockfile. + const shimmedManifest = await readTarEntryContent(codeBundlePath, 'packages/shimmed/package.json') + const workspace = await new PNpmDetector().lookupWorkspace(fixt.root) + const expectedHash = composeWorkspaceCacheHash(await loadWorkspaceCacheHashInputs(workspace!), { + embeddedPackages: [ + { name: 'ms', version: '2.1.3', integrity: MS_INTEGRITY }, + ], + fauxPackageJsons: [ + { path: 'packages/shimmed/package.json', raw: Buffer.from(shimmedManifest, 'utf8') }, + ], + prunedLockfile: { + name: 'pnpm-lock.yaml', + hash: createHash('sha256').update(lockfile).digest(), + }, + }) + expect(cacheHash).toEqual(expectedHash) }, DEFAULT_TEST_TIMEOUT) + + // The CHECKLY_LOCKFILE_PRUNE=0 direction (full planned set ships, full-set + // hash) is covered hermetically in bundler.spec.ts — with pruning off no + // pnpm interaction is involved, so a sandbox run would add nothing. }) describe('bundling with testDir through a symlink', () => { diff --git a/packages/cli/src/constructs/playwright-check-bundle.ts b/packages/cli/src/constructs/playwright-check-bundle.ts index 521f887f6..fc0c12459 100644 --- a/packages/cli/src/constructs/playwright-check-bundle.ts +++ b/packages/cli/src/constructs/playwright-check-bundle.ts @@ -1,5 +1,5 @@ import { Bundle } from './construct.js' -import { BundlePathMarker } from '../services/check-parser/bundler.js' +import { BundlePathMarker, CacheHashMarker } from '../services/check-parser/bundler.js' import { PlaywrightCheck } from './playwright-check.js' import { Ref } from './ref.js' @@ -7,7 +7,7 @@ export interface PlaywrightCheckBundleProps { groupId?: Ref codeBundlePath: BundlePathMarker browsers?: string[] - cacheHash?: string + cacheHash?: CacheHashMarker playwrightVersion?: string installCommand?: string testCommand: string @@ -19,7 +19,7 @@ export class PlaywrightCheckBundle implements Bundle { groupId?: Ref codeBundlePath: BundlePathMarker browsers?: string[] - cacheHash?: string + cacheHash?: CacheHashMarker playwrightVersion?: string installCommand?: string testCommand: string diff --git a/packages/cli/src/constructs/project.ts b/packages/cli/src/constructs/project.ts index bd95a00b8..eeea225aa 100644 --- a/packages/cli/src/constructs/project.ts +++ b/packages/cli/src/constructs/project.ts @@ -122,9 +122,12 @@ export class Project extends Construct { /** * Validates the project-wide `bundle.packages.embed` option once per - * project (individual checks share the session-level materializer). Only + * project (validation and the Bundler share one session-level + * materializer: one plan, one materialization at finalize). Only * local checks run here — resolving the configured specs against the - * lockfile — no tarballs are fetched until bundling. Skipped when the + * lockfile — no tarballs are fetched until the bundle is finalized (the + * shipped set is filtered to what the possibly-pruned bundled lockfile + * still references, and only that set is downloaded). Skipped when the * project has no Playwright checks: the option only affects Playwright * code bundles, and no bundling (or materialization) happens without one. * Deliberately ignores testOnly flags and the session check filter — a diff --git a/packages/cli/src/constructs/session.ts b/packages/cli/src/constructs/session.ts index 23bb52ec9..fcf6e887f 100644 --- a/packages/cli/src/constructs/session.ts +++ b/packages/cli/src/constructs/session.ts @@ -236,7 +236,8 @@ export class Session { /** * The materializer for the project's `bundle.packages.embed` option, or * undefined when the option is not set. Memoized so that validation and - * every concurrently bundling check share one plan and one download run. + * bundling share one plan; tarballs are materialized once, at bundle + * finalize time, after the bundled lockfile has been pruned. */ static getEmbeddedPackagesMaterializer (): EmbeddedPackagesMaterializer | undefined { const specs = this.embeddedPackages diff --git a/packages/cli/src/services/__tests__/playwright-project-bundler.spec.ts b/packages/cli/src/services/__tests__/playwright-project-bundler.spec.ts index 212ba9b07..e0e80f0db 100644 --- a/packages/cli/src/services/__tests__/playwright-project-bundler.spec.ts +++ b/packages/cli/src/services/__tests__/playwright-project-bundler.spec.ts @@ -115,16 +115,38 @@ describe('getAutoIncludes()', () => { expect(result).toEqual(['patches/*.patch']) }) + it('should return patches/*.patch for bun', () => { + const result = getAutoIncludes(basePath, basePath, makePm('bun'), []) + expect(result).toEqual(['patches/*.patch']) + }) + it('should return empty for npm', () => { const result = getAutoIncludes(basePath, basePath, makePm('npm'), []) expect(result).toEqual([]) }) - it('should return empty for yarn', () => { + it('should return .yarn/patches/*.patch for yarn', () => { const result = getAutoIncludes(basePath, basePath, makePm('yarn'), []) + expect(result).toEqual(['.yarn/patches/*.patch']) + }) + + it('should skip when user already includes .yarn/patches/*.patch', () => { + const result = getAutoIncludes(basePath, basePath, makePm('yarn'), ['.yarn/patches/*.patch']) expect(result).toEqual([]) }) + it('should not let a sibling directory suppress the pnpm auto-include', () => { + // `patches-archive` shares the `patches` prefix but is a different + // directory; only includes at or under the patches dir count. + const result = getAutoIncludes(basePath, basePath, makePm('pnpm'), ['patches-archive/*.patch']) + expect(result).toEqual(['patches/*.patch']) + }) + + it('should not let a sibling directory suppress the yarn auto-include', () => { + const result = getAutoIncludes(basePath, basePath, makePm('yarn'), ['.yarn/patches-archive/*.patch']) + expect(result).toEqual(['.yarn/patches/*.patch']) + }) + it('should skip when user already includes patches/*.patch', () => { const result = getAutoIncludes(basePath, basePath, makePm('pnpm'), ['patches/*.patch']) expect(result).toEqual([]) diff --git a/packages/cli/src/services/check-parser/__tests__/bundler.spec.ts b/packages/cli/src/services/check-parser/__tests__/bundler.spec.ts index 29ebab1e1..b3d42fb17 100644 --- a/packages/cli/src/services/check-parser/__tests__/bundler.spec.ts +++ b/packages/cli/src/services/check-parser/__tests__/bundler.spec.ts @@ -1,13 +1,18 @@ +import { createHash } from 'node:crypto' import fs from 'node:fs/promises' import os from 'node:os' import path from 'node:path' +import { list } from 'tar' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { BundleArchive, BundleTooLargeError, Bundler, FinalizedBundleArchive } from '../bundler.js' +import { BundleArchive, BundleTooLargeError, Bundler, embeddedPackageHashInputs, FinalizedBundleArchive } from '../bundler.js' +import { composeWorkspaceCacheHash, loadWorkspaceCacheHashInputs } from '../cache-hash.js' +import { CNpmDetector, npmPackageManager, PNpmDetector, Runnable } from '../package-files/package-manager.js' import { Package, Workspace } from '../package-files/workspace.js' import { Err, Ok } from '../package-files/result.js' -import { EmbeddedPackagesMaterializer } from '../../embedded-packages/materializer.js' +import { EmbeddedPackageError, EmbeddedPackagesMaterializer } from '../../embedded-packages/materializer.js' +import { TarballCache } from '../../embedded-packages/cache.js' import { PayloadTooLargeError } from '../../../rest/errors.js' const uploadCodeBundle = vi.hoisted(() => vi.fn()) @@ -185,8 +190,48 @@ describe('Bundler.createForWorkspace', () => { configFile: Err(new Error('no config file')), }) - const without = await Bundler.createForWorkspace(workspace) + const without = await Bundler.createForWorkspace(workspace, { + packageManager: npmPackageManager, + }) + const withFoo = await Bundler.createForWorkspace(workspace, { + packageManager: npmPackageManager, + embeddedPackagesMaterializer: new EmbeddedPackagesMaterializer({ + specs: ['@acme/foo'], + lockfilePath, + workspaceRoot: dir, + }), + }) + + expect(without.cacheHash.toJSON()).not.toBe(withFoo.cacheHash.toJSON()) + }) + + it('mixes a yarn.lock embed plan into the cache hash', async () => { + const lockfilePath = path.join(dir, 'yarn.lock') + await fs.writeFile(lockfilePath, [ + `__metadata:`, + ` version: 10`, + ` cacheKey: 10c0`, + ``, + `"@acme/foo@npm:1.2.3":`, + ` version: 1.2.3`, + ` resolution: "@acme/foo@npm:1.2.3"`, + ` checksum: 10c0/aaa`, + ` languageName: node`, + ` linkType: hard`, + ``, + ].join('\n')) + const workspace = new Workspace({ + root: new Package({ name: 'fixture-root', path: dir }), + packages: [], + lockfile: Ok(lockfilePath), + configFile: Err(new Error('no config file')), + }) + + const without = await Bundler.createForWorkspace(workspace, { + packageManager: npmPackageManager, + }) const withFoo = await Bundler.createForWorkspace(workspace, { + packageManager: npmPackageManager, embeddedPackagesMaterializer: new EmbeddedPackagesMaterializer({ specs: ['@acme/foo'], lockfilePath, @@ -194,6 +239,466 @@ describe('Bundler.createForWorkspace', () => { }), }) - expect(without.cacheHash).not.toBe(withFoo.cacheHash) + expect(without.cacheHash.toJSON()).not.toBe(withFoo.cacheHash.toJSON()) + }) +}) + +describe('embeddedPackageHashInputs()', () => { + it('uses the SRI integrity when present and the Berry checksum otherwise', () => { + // yarn.lock plans carry no npm tarball integrity, so their hash records + // must carry the lockfile's own checksum — a content pin that is known + // at plan time, keeping the eager and finalize hashes consistent. + expect(embeddedPackageHashInputs([ + { name: 'bar', version: '2.0.0', integrity: 'sha512-bbb', archiveFilename: 'bar@2.0.0.tgz' }, + { name: 'ms', version: '2.1.3', lockfileChecksum: '10c0/aaa', archiveFilename: 'ms@2.1.3.tgz' }, + ])).toEqual([ + { name: 'bar', version: '2.0.0', integrity: 'sha512-bbb' }, + { name: 'ms', version: '2.1.3', integrity: '10c0/aaa' }, + ]) + }) + + it('passes undefined through for an absent plan', () => { + expect(embeddedPackageHashInputs(undefined)).toBeUndefined() + }) +}) + +describe('Bundler.finalize() lockfile prune reporting', () => { + let dir: string + let stderrWrites: string[] + + beforeEach(async () => { + dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-bundler-'))) + await fs.mkdir(path.join(dir, 'packages/m'), { recursive: true }) + await fs.writeFile(path.join(dir, 'package.json'), JSON.stringify({ + name: 'fixture-root', + private: true, + dependencies: { '@fixture/m': 'workspace:*' }, + })) + await fs.writeFile(path.join(dir, 'packages/m/package.json'), JSON.stringify({ + name: '@fixture/m', + version: '1.0.0', + })) + await fs.writeFile(path.join(dir, 'pnpm-lock.yaml'), `lockfileVersion: '9.0'\n`) + stderrWrites = [] + vi.spyOn(process.stderr, 'write').mockImplementation(chunk => { + // The debug library also writes to stderr when DEBUG is enabled; only + // the CLI's own user-facing writes are under test. + const text = String(chunk) + if (!text.includes('checkly:cli:')) { + stderrWrites.push(text) + } + return true + }) + vi.stubEnv('CHECKLY_LOCKFILE_PRUNE', '') + }) + + afterEach(async () => { + vi.restoreAllMocks() + vi.unstubAllEnvs() + await fs.rm(dir, { recursive: true, force: true }) + }) + + const makeWorkspace = () => new Workspace({ + root: new Package({ name: 'fixture-root', path: dir }), + packages: [new Package({ name: '@fixture/m', path: path.join(dir, 'packages/m'), version: '1.0.0' })], + lockfile: Ok(path.join(dir, 'pnpm-lock.yaml')), + configFile: Err(new Error('no config file')), + }) + + it('prints a note when pruning is needed but unavailable', async () => { + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + // cnpm has no lockfile-only install, so a partial-workspace bundle + // must surface the unpruned lockfile instead of skipping silently. + // (Not yarn: yarn gained a lockfile-only install, so it would spawn + // a real package manager here.) + packageManager: new CNpmDetector(), + }) + bundler.registerFiles( + { filePath: path.join(dir, 'package.json'), physical: true }, + { filePath: path.join(dir, 'pnpm-lock.yaml'), physical: true }, + { filePath: path.join(dir, 'packages/m/package.json'), physical: false, content: '{"name":"@fixture/m","version":"1.0.0"}' }, + ) + await bundler.finalize() + + const output = stderrWrites.join('') + expect(output).toContain('Note: the bundled lockfile was not pruned') + expect(output).toContain('CHECKLY_LOCKFILE_PRUNE=0') + }) + + it('stays silent when the bundle contains the full workspace', async () => { + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: new CNpmDetector(), + }) + bundler.registerFiles( + { filePath: path.join(dir, 'package.json'), physical: true }, + { filePath: path.join(dir, 'pnpm-lock.yaml'), physical: true }, + { filePath: path.join(dir, 'packages/m/package.json'), physical: true }, + ) + await bundler.finalize() + + expect(stderrWrites.join('')).toEqual('') + }) +}) + +describe('Bundler.finalize() embedded package materialization', () => { + let dir: string + let cacheDir: string + let homeDir: string + let stderrWrites: string[] + + // Any accidental download attempt must fail deterministically and + // instantly instead of reaching the public registry. + const UNREACHABLE_REGISTRY = 'http://127.0.0.1:9/' + + const keptBytes = Buffer.from('kept-tarball-bytes') + const droppedBytes = Buffer.from('dropped-tarball-bytes') + const integrityOf = (bytes: Buffer) => `sha512-${createHash('sha512').update(bytes).digest('base64')}` + const keptIntegrity = integrityOf(keptBytes) + const droppedIntegrity = integrityOf(droppedBytes) + + const FAUX_MANIFEST = '{"name":"@fixture/m","version":"1.0.0"}' + + const originalLockfile = () => [ + `lockfileVersion: '9.0'`, + ``, + `importers:`, + ``, + ` .:`, + ` dependencies:`, + ` '@fixture/m':`, + ` specifier: workspace:*`, + ` version: link:packages/m`, + ``, + ` packages/m:`, + ` dependencies:`, + ` '@acme/dropped':`, + ` specifier: 1.0.0`, + ` version: 1.0.0`, + ` '@acme/kept':`, + ` specifier: 1.0.0`, + ` version: 1.0.0`, + ``, + `packages:`, + ``, + ` '@acme/dropped@1.0.0':`, + ` resolution: {integrity: ${droppedIntegrity}}`, + ``, + ` '@acme/kept@1.0.0':`, + ` resolution: {integrity: ${keptIntegrity}}`, + ``, + `snapshots:`, + ``, + ` '@acme/dropped@1.0.0': {}`, + ``, + ` '@acme/kept@1.0.0': {}`, + ``, + ].join('\n') + + // The "regenerated" lockfile a stub prune produces when every embedded + // package's referent was pruned away. + const prunedLockfileAllDropped = () => [ + `lockfileVersion: '9.0'`, + ``, + `importers:`, + ``, + ` .:`, + ` dependencies:`, + ` '@fixture/m':`, + ` specifier: workspace:*`, + ` version: link:packages/m`, + ``, + ` packages/m: {}`, + ``, + ].join('\n') + + // The "regenerated" lockfile the stub prune produces: the dropped + // package's entries removed everywhere, everything else intact. + const prunedLockfile = () => [ + `lockfileVersion: '9.0'`, + ``, + `importers:`, + ``, + ` .:`, + ` dependencies:`, + ` '@fixture/m':`, + ` specifier: workspace:*`, + ` version: link:packages/m`, + ``, + ` packages/m:`, + ` dependencies:`, + ` '@acme/kept':`, + ` specifier: 1.0.0`, + ` version: 1.0.0`, + ``, + `packages:`, + ``, + ` '@acme/kept@1.0.0':`, + ` resolution: {integrity: ${keptIntegrity}}`, + ``, + `snapshots:`, + ``, + ` '@acme/kept@1.0.0': {}`, + ``, + ].join('\n') + + beforeEach(async () => { + dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-bundler-embed-'))) + cacheDir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-bundler-embed-cache-')) + homeDir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-bundler-embed-home-')) + await fs.mkdir(path.join(dir, 'packages/m'), { recursive: true }) + await fs.writeFile(path.join(dir, 'package.json'), JSON.stringify({ + name: 'embed-fixture-root', + private: true, + dependencies: { '@fixture/m': 'workspace:*' }, + })) + await fs.writeFile(path.join(dir, 'packages/m/package.json'), JSON.stringify({ + name: '@fixture/m', + version: '1.0.0', + dependencies: { '@acme/kept': '1.0.0', '@acme/dropped': '1.0.0' }, + })) + await fs.writeFile(path.join(dir, 'pnpm-lock.yaml'), originalLockfile()) + stderrWrites = [] + vi.spyOn(process.stderr, 'write').mockImplementation(chunk => { + // The debug library also writes to stderr when DEBUG is enabled; only + // the CLI's own user-facing writes are under test. + const text = String(chunk) + if (!text.includes('checkly:cli:')) { + stderrWrites.push(text) + } + return true + }) + vi.stubEnv('CHECKLY_LOCKFILE_PRUNE', '') + }) + + afterEach(async () => { + vi.restoreAllMocks() + vi.unstubAllEnvs() + for (const tempDir of [dir, cacheDir, homeDir]) { + await fs.rm(tempDir, { recursive: true, force: true }) + } + }) + + const materializerEnv = () => ({ + CHECKLY_CACHE_DIR: cacheDir, + npm_config_registry: UNREACHABLE_REGISTRY, + }) + + const seedCache = async (...tarballs: Buffer[]) => { + const cache = TarballCache.default(materializerEnv(), dir, process.platform, homeDir) + for (const bytes of tarballs) { + await cache.put(integrityOf(bytes), bytes) + } + } + + const makeWorkspace = () => new Workspace({ + root: new Package({ name: 'embed-fixture-root', path: dir }), + packages: [new Package({ name: '@fixture/m', path: path.join(dir, 'packages/m'), version: '1.0.0' })], + lockfile: Ok(path.join(dir, 'pnpm-lock.yaml')), + configFile: Err(new Error('no config file')), + }) + + const makeMaterializer = () => new EmbeddedPackagesMaterializer({ + specs: ['@acme/kept', '@acme/dropped'], + lockfilePath: path.join(dir, 'pnpm-lock.yaml'), + workspaceRoot: dir, + env: materializerEnv(), + homedir: homeDir, + }) + + // A package manager whose lockfile-only install replaces the lockfile + // with the given pruned variant, standing in for a real pnpm run. + const stubPruningPackageManager = async (prunedContent: string = prunedLockfile()) => { + await fs.writeFile(path.join(dir, 'pruned-lock.yaml'), prunedContent) + const scriptPath = path.join(dir, 'prune.cjs') + await fs.writeFile(scriptPath, [ + `const fs = require('fs')`, + `fs.writeFileSync('pnpm-lock.yaml', fs.readFileSync(${JSON.stringify(path.join(dir, 'pruned-lock.yaml'))}, 'utf8'))`, + ].join('\n')) + return Object.assign(Object.create(new PNpmDetector()), { + lockfileOnlyInstallCommand: () => new Runnable('node', [scriptPath]), + }) + } + + const registerPartialWorkspace = (bundler: Bundler) => { + bundler.registerFiles( + { filePath: path.join(dir, 'package.json'), physical: true }, + { filePath: path.join(dir, 'pnpm-lock.yaml'), physical: true }, + { filePath: path.join(dir, 'packages/m/package.json'), physical: false, content: FAUX_MANIFEST }, + ) + } + + const listEntries = async (archiveFile: string): Promise => { + const entries: string[] = [] + await list({ file: archiveFile, onReadEntry: entry => { + entries.push(entry.path) + } }) + return entries + } + + const expectedHash = async (options: { + embedded: Array<{ name: string, version: string, integrity: string }> + // The pruned lockfile content the stub prune produced, or false when + // pruning did not run. + pruned: string | false + }) => { + return composeWorkspaceCacheHash(await loadWorkspaceCacheHashInputs(makeWorkspace()), { + embeddedPackages: options.embedded, + fauxPackageJsons: [{ path: 'packages/m/package.json', raw: Buffer.from(FAUX_MANIFEST, 'utf8') }], + prunedLockfile: options.pruned !== false + ? { name: 'pnpm-lock.yaml', hash: createHash('sha256').update(options.pruned).digest() } + : undefined, + }) + } + + it('materializes only the tarballs the pruned lockfile still references, without downloading the rest', async () => { + // Only the kept tarball is seeded: the dropped one exists in no cache + // and the registry is unreachable, so this passing proves the dropped + // tarball was never fetched. + await seedCache(keptBytes) + + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(), + embeddedPackagesMaterializer: makeMaterializer(), + }) + registerPartialWorkspace(bundler) + const archive = await bundler.finalize() + + const entries = await listEntries(archive.archiveFile) + expect(entries).toContain('.checkly/embedded-packages/@acme+kept@1.0.0.tgz') + expect(entries).not.toContain('.checkly/embedded-packages/@acme+dropped@1.0.0.tgz') + + // Progress is debug-only for now: a successful prune-and-materialize + // writes nothing user-facing. + expect(stderrWrites.join('')).toEqual('') + + expect(bundler.cacheHash.toJSON()).toEqual(await expectedHash({ + embedded: [{ name: '@acme/kept', version: '1.0.0', integrity: keptIntegrity }], + pruned: prunedLockfile(), + })) + expect(bundler.cacheHash.toJSON()).not.toEqual(await expectedHash({ + embedded: [ + { name: '@acme/dropped', version: '1.0.0', integrity: droppedIntegrity }, + { name: '@acme/kept', version: '1.0.0', integrity: keptIntegrity }, + ], + pruned: prunedLockfile(), + })) + }) + + it('erases the embedded cache-hash records when pruning drops every planned tarball', async () => { + // Nothing is seeded: with every embedded package dropped, no tarball + // may be requested at all. + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(prunedLockfileAllDropped()), + embeddedPackagesMaterializer: makeMaterializer(), + }) + registerPartialWorkspace(bundler) + const archive = await bundler.finalize() + + const entries = await listEntries(archive.archiveFile) + expect(entries.filter(entry => entry.startsWith('.checkly/embedded-packages/'))).toEqual([]) + expect(stderrWrites.join('')).toEqual('') + + // An all-dropped set must hash as [] (no embedded-package records), not + // fall back to the full planned set. + expect(bundler.cacheHash.toJSON()).toEqual(await expectedHash({ + embedded: [], + pruned: prunedLockfileAllDropped(), + })) + }) + + it('materializes the full planned set when pruning is disabled', async () => { + vi.stubEnv('CHECKLY_LOCKFILE_PRUNE', '0') + await seedCache(keptBytes, droppedBytes) + + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(), + embeddedPackagesMaterializer: makeMaterializer(), + }) + registerPartialWorkspace(bundler) + const archive = await bundler.finalize() + + const entries = await listEntries(archive.archiveFile) + expect(entries).toContain('.checkly/embedded-packages/@acme+kept@1.0.0.tgz') + expect(entries).toContain('.checkly/embedded-packages/@acme+dropped@1.0.0.tgz') + + expect(stderrWrites.join('')).toEqual('') + + expect(bundler.cacheHash.toJSON()).toEqual(await expectedHash({ + embedded: [ + { name: '@acme/dropped', version: '1.0.0', integrity: droppedIntegrity }, + { name: '@acme/kept', version: '1.0.0', integrity: keptIntegrity }, + ], + pruned: false, + })) + }) + + it('ships no embedded tarballs when the bundler has no materializer', async () => { + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(), + }) + registerPartialWorkspace(bundler) + const archive = await bundler.finalize() + + const entries = await listEntries(archive.archiveFile) + expect(entries.filter(entry => entry.startsWith('.checkly/embedded-packages/'))).toEqual([]) + // The hash carries no embedded-package records either. + expect(bundler.cacheHash.toJSON()).toEqual(await expectedHash({ + embedded: [], + pruned: prunedLockfile(), + })) + }) + + it('does not materialize anything for an empty bundle', async () => { + // Every command calls finalize() unconditionally, including on bundles + // no check registered files into — an empty bundle must not trigger + // downloads. Nothing is seeded and the registry is unreachable, so a + // regressed guard turns into a rejected finalize, not a silent pass. + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(), + embeddedPackagesMaterializer: makeMaterializer(), + }) + await expect(bundler.finalize()).resolves.toBeDefined() + expect(stderrWrites.join('')).toEqual('') + + // Nothing ships from an empty bundle, so nothing may reach the hash + // either: the finalize-time digest carries no embedded-package records, + // rather than falling back to the full planned set. + expect(bundler.cacheHash.toJSON()).toEqual( + composeWorkspaceCacheHash(await loadWorkspaceCacheHashInputs(makeWorkspace()), {}), + ) + expect(bundler.cacheHash.toJSON()).not.toEqual(await expectedHash({ + embedded: [ + { name: '@acme/dropped', version: '1.0.0', integrity: droppedIntegrity }, + { name: '@acme/kept', version: '1.0.0', integrity: keptIntegrity }, + ], + pruned: false, + })) + }) + + it('rejects a plan whose specs all failed to resolve, even when nothing would ship', async () => { + // The materializer's plan-issues backstop must still fire for a + // non-empty bundle whose embed specs resolved to nothing at all — + // matching the pre-deferral behavior where bundling threw. + const materializer = new EmbeddedPackagesMaterializer({ + specs: ['no-such-package'], + lockfilePath: path.join(dir, 'pnpm-lock.yaml'), + workspaceRoot: dir, + env: materializerEnv(), + homedir: homeDir, + }) + const bundler = await Bundler.createForWorkspace(makeWorkspace(), { + tempDir: path.join(dir, 'out'), + packageManager: await stubPruningPackageManager(), + embeddedPackagesMaterializer: materializer, + }) + registerPartialWorkspace(bundler) + await expect(bundler.finalize()).rejects.toThrow(EmbeddedPackageError) }) }) diff --git a/packages/cli/src/services/check-parser/__tests__/cache-hash.spec.ts b/packages/cli/src/services/check-parser/__tests__/cache-hash.spec.ts index c788d86e5..6c3d37643 100644 --- a/packages/cli/src/services/check-parser/__tests__/cache-hash.spec.ts +++ b/packages/cli/src/services/check-parser/__tests__/cache-hash.spec.ts @@ -245,6 +245,82 @@ describe('composeCacheHash', () => { })) }) + test('adding a pnpmfile changes the hash', () => { + const root = buf('{"name":"root"}') + const lockfile = { name: 'pnpm-lock.yaml', hash: sha256('lock') } + const without = composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + excludedFields: ['version'], + }) + const withPnpmfile = composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + pnpmfiles: [{ path: '.pnpmfile.cjs', hash: sha256('module.exports = {}') }], + excludedFields: ['version'], + }) + expect(without).not.toBe(withPnpmfile) + expect(without).toBe(composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + pnpmfiles: [], + excludedFields: ['version'], + })) + }) + + test('changing pnpmfile content changes the hash', () => { + const root = buf('{"name":"root"}') + const lockfile = { name: 'pnpm-lock.yaml', hash: sha256('lock') } + const compose = (contents: string) => composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + pnpmfiles: [{ path: '.pnpmfile.cjs', hash: sha256(contents) }], + excludedFields: ['version'], + }) + expect(compose('module.exports = {}')).toBe(compose('module.exports = {}')) + expect(compose('module.exports = {}')).not.toBe(compose('module.exports = { hooks: {} }')) + }) + + test('faux package.json records change the hash, including their version', () => { + const root = buf('{"name":"root"}') + const lockfile = { name: 'pnpm-lock.yaml', hash: sha256('lock') } + const compose = (fauxContent?: string) => composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + ...fauxContent !== undefined + ? { fauxPackageJsons: [{ path: 'packages/member/package.json', raw: buf(fauxContent) }] } + : {}, + excludedFields: ['version'], + }) + expect(compose()).toBe(composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + fauxPackageJsons: [], + excludedFields: ['version'], + })) + expect(compose()).not.toBe(compose('{"name":"m","version":"1.0.0"}')) + // Unlike on-disk manifests, the faux version is load-bearing for the + // install and must affect the hash. + expect(compose('{"name":"m","version":"1.0.0"}')).not.toBe(compose('{"name":"m","version":"2.0.0"}')) + }) + + test('a pruned lockfile record changes the hash', () => { + const root = buf('{"name":"root"}') + const lockfile = { name: 'pnpm-lock.yaml', hash: sha256('lock') } + const base = composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + excludedFields: ['version'], + }) + const pruned = composeCacheHash({ + lockfile, + packageJsons: [{ path: 'package.json', raw: root }], + prunedLockfile: { name: 'pnpm-lock.yaml', hash: sha256('pruned-lock') }, + excludedFields: ['version'], + }) + expect(base).not.toBe(pruned) + }) + test('lockfile content change changes the hash', () => { const root = buf('{"name":"root"}') const a = composeCacheHash({ @@ -406,14 +482,18 @@ describe('composeCacheHash', () => { // fixture, mirror the change in the TF provider's test suite. // // NOTE: composeCacheHash also hashes `npmrc:` records (added for .npmrc - // bundling), `embedded-package:` records (the resolved - // bundle.packages.embed tarball set), and a `dependency-cache-version` - // record (the user-provided caching.dependencyCache.version config - // value). This fixture uses none of them so the digest is unchanged, but - // projects that DO have an .npmrc, embed packages, or set a dependency - // cache version will hash differently until the TF provider mirrors - // those record types. The fixture two tests down pins all three optional - // record groups together, in order. + // bundling), a `pnpmfile:` record (the workspace root's .pnpmfile.cjs), + // `embedded-package:` records (the resolved bundle.packages.embed tarball + // set, filtered to what the shipped — possibly pruned — bundled lockfile + // still references), a `dependency-cache-version` record (the user-provided + // caching.dependencyCache.version config value), `faux-package.json:` + // records (synthesized workspace member manifests shipped in the bundle), + // and a `pruned-lockfile:` record (when lockfile pruning replaced the + // bundled lockfile — the last two only occur for bundles that are a + // subset of the workspace). This fixture uses none of them so the digest + // is unchanged, but projects that DO use them will hash differently until + // the TF provider mirrors those record types. The 'every record group' + // fixture below pins all optional record groups together, in order. test('matches the cross-language parity fixture digest', () => { const lockfileBytes = buf('{"lockfileVersion":3}\n') const rootPackageJson = buf([ @@ -491,12 +571,12 @@ describe('composeCacheHash', () => { expect(digest).toBe('344f037a55163ba59146d9cdb71ef702782e3690a9f666067c0ccdf091214eb6') }) - // Same parity contract as above, but with all four optional record groups - // present, pinning the full record order (npmrc records, then - // embedded-package records, then the dependency-cache-version record) and - // the sort of the `name@version` record labels. The TF provider must - // produce this exact digest once it mirrors the embedded-package record - // type. + // Same parity contract as above, but with the npmrc, embedded-package and + // dependency-cache-version record groups present, pinning their relative + // order and the sort of the `name@version` record labels. The TF provider + // must produce this exact digest once it mirrors the embedded-package + // record type. (The pnpmfile record group is pinned separately by the next + // fixture.) test('matches the cross-language parity fixture digest with embedded packages', () => { const lockfileBytes = buf('{"lockfileVersion":3}\n') const rootPackageJson = buf([ @@ -532,6 +612,100 @@ describe('composeCacheHash', () => { expect(digest).toBe('4d9ce4b49fe543b4ec303ae17b1e98c7c9d0e37d8e17c57e78b36555abdf5207') }) + + // Same parity contract as above, pinning the pnpmfile record's position + // between the npmrc records and the dependency-cache-version record. The + // TF provider must produce this exact digest once it mirrors the pnpmfile + // record type. (The exhaustive every-record-group fixture is the next + // test.) + test('matches the cross-language parity fixture digest with a pnpmfile', () => { + const lockfileBytes = buf('{"lockfileVersion":3}\n') + const rootPackageJson = buf([ + '{', + ' "name": "fixture-root",', + ' "version": "0.0.0-SNAPSHOT",', + ' "private": true,', + ' "dependencies": {', + ' "@acme/foo": "1.2.3"', + ' }', + '}', + '', + ].join('\n')) + + const digest = composeCacheHash({ + lockfile: { + name: 'pnpm-lock.yaml', + hash: createHash('sha256').update(lockfileBytes).digest(), + }, + packageJsons: [ + { path: 'package.json', raw: rootPackageJson }, + ], + npmrcs: [ + { path: '.npmrc', hash: sha256('registry=https://registry.example.com/\n') }, + ], + pnpmfiles: [ + { path: '.pnpmfile.cjs', hash: sha256('module.exports = { hooks: {} }\n') }, + ], + embeddedPackages: [ + { name: '@acme/foo', version: '1.2.3', integrity: 'sha512-aaa' }, + ], + excludedFields: ['version'], + dependencyCacheVersion: '2', + }) + + expect(digest).toBe('9a5183603a1164e37065bfcc817cf25f978a9e408e153f42b477ad3fae4d1e67') + }) + + // Same parity contract as above, with every optional record group present + // at once, pinning the full record order: npmrc records, then the + // pnpmfile record, then embedded-package records, then the + // dependency-cache-version record, then faux-package.json records, then + // the pruned-lockfile record. The TF provider must produce this exact + // digest once it mirrors all record types. + test('matches the cross-language parity fixture digest with every record group', () => { + const lockfileBytes = buf('{"lockfileVersion":3}\n') + const rootPackageJson = buf([ + '{', + ' "name": "fixture-root",', + ' "version": "0.0.0-SNAPSHOT",', + ' "private": true,', + ' "dependencies": {', + ' "@acme/member": "workspace:*"', + ' }', + '}', + '', + ].join('\n')) + + const digest = composeCacheHash({ + lockfile: { + name: 'pnpm-lock.yaml', + hash: createHash('sha256').update(lockfileBytes).digest(), + }, + packageJsons: [ + { path: 'package.json', raw: rootPackageJson }, + ], + npmrcs: [ + { path: '.npmrc', hash: sha256('registry=https://registry.example.com/\n') }, + ], + pnpmfiles: [ + { path: '.pnpmfile.cjs', hash: sha256('module.exports = { hooks: {} }\n') }, + ], + embeddedPackages: [ + { name: '@acme/embedded', version: '3.0.0', integrity: 'sha512-ccc' }, + ], + fauxPackageJsons: [ + { path: 'packages/member/package.json', raw: buf('{"name":"@acme/member","version":"1.2.3"}') }, + ], + prunedLockfile: { + name: 'pnpm-lock.yaml', + hash: sha256('pruned-lockfile-bytes'), + }, + excludedFields: ['version'], + dependencyCacheVersion: '2', + }) + + expect(digest).toBe('0dcfaca5d3b62fd83b03f6165b972e352209f674260457b8778795ffe52ac34f') + }) }) describe('computeWorkspaceCacheHash', () => { @@ -575,6 +749,43 @@ describe('computeWorkspaceCacheHash', () => { embeddedPackages: [{ name: '@acme/foo', version: '1.2.3', integrity: 'sha512-aaa' }], })) }) + + test('a bundleable workspace pnpmfile flows into the hash', async () => { + const workspace = await makeWorkspace() + const base = await computeWorkspaceCacheHash(workspace) + + const pnpmfilePath = path.join(workspace.root.path, '.pnpmfile.cjs') + await fs.writeFile(pnpmfilePath, 'module.exports = {}\n') + const withPnpmfile = new Workspace({ + root: workspace.root, + packages: [], + lockfile: Err(new Error('no lockfile')), + configFile: Err(new Error('no config file')), + pnpmfiles: [{ path: pnpmfilePath }], + }) + const first = await computeWorkspaceCacheHash(withPnpmfile) + expect(base).not.toBe(first) + + await fs.writeFile(pnpmfilePath, 'module.exports = { hooks: {} }\n') + expect(first).not.toBe(await computeWorkspaceCacheHash(withPnpmfile)) + }) + + test('a non-bundleable pnpmfile does not affect the hash', async () => { + const workspace = await makeWorkspace() + const base = await computeWorkspaceCacheHash(workspace) + + const withSkippedPnpmfile = new Workspace({ + root: workspace.root, + packages: [], + lockfile: Err(new Error('no lockfile')), + configFile: Err(new Error('no config file')), + pnpmfiles: [{ + path: path.join(workspace.root.path, '.pnpmfile.cjs'), + skipReason: 'loads modules beyond Node.js builtins', + }], + }) + expect(base).toBe(await computeWorkspaceCacheHash(withSkippedPnpmfile)) + }) }) describe('normalizeDependencyCacheVersion', () => { diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/npm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/npm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs new file mode 100644 index 000000000..30c919cbe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/npm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs @@ -0,0 +1,7 @@ +module.exports = { + hooks: { + readPackage (pkg) { + return pkg + }, + }, +} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs new file mode 100644 index 000000000..30c919cbe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/.pnpmfile.cjs @@ -0,0 +1,7 @@ +module.exports = { + hooks: { + readPackage (pkg) { + return pkg + }, + }, +} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/pnpm-lock.yaml b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/pnpm-lock.yaml index fc4d52e52..03265935d 100644 --- a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/pnpm-lock.yaml +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-depend-on-workspace-package-in-root-package/pnpm-lock.yaml @@ -1,5 +1,7 @@ lockfileVersion: '9.0' +pnpmfileChecksum: sha256-fixture-checksum + settings: autoInstallPeers: true excludeLinksFromLockfile: false diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/.pnpmfile.cjs b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/.pnpmfile.cjs new file mode 100644 index 000000000..30c919cbe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/.pnpmfile.cjs @@ -0,0 +1,7 @@ +module.exports = { + hooks: { + readPackage (pkg) { + return pkg + }, + }, +} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/package.json b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/package.json new file mode 100644 index 000000000..9d813a7dc --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/package.json @@ -0,0 +1,5 @@ +{ + "name": "test-pnpm-pnpmfile-no-checksum", + "version": "1.0.0", + "private": true +} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-lock.yaml b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-lock.yaml new file mode 100644 index 000000000..6a932beee --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-lock.yaml @@ -0,0 +1,5 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-workspace.yaml b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-workspace.yaml new file mode 100644 index 000000000..3334c0e43 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpm-pnpmfile-no-checksum/pnpm-workspace.yaml @@ -0,0 +1 @@ +packages: [] diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/.pnpmfile.cjs b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/.pnpmfile.cjs new file mode 100644 index 000000000..48ee92748 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/.pnpmfile.cjs @@ -0,0 +1,2 @@ +try { require('./local-overrides.cjs') } catch {} +module.exports = {} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/package.json b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/package.json new file mode 100644 index 000000000..55b8f9a76 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/package.json @@ -0,0 +1,4 @@ +{ + "name": "pnpmfile-bundling", + "private": true +} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/tests/foo.spec.js b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/tests/foo.spec.js new file mode 100644 index 000000000..4ba52ba2c --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/check-parser-fixtures/pnpmfile-bundling/tests/foo.spec.js @@ -0,0 +1 @@ +module.exports = {} diff --git a/packages/cli/src/services/check-parser/__tests__/check-parser.spec.ts b/packages/cli/src/services/check-parser/__tests__/check-parser.spec.ts index a3d2f5fe6..a9304885e 100644 --- a/packages/cli/src/services/check-parser/__tests__/check-parser.spec.ts +++ b/packages/cli/src/services/check-parser/__tests__/check-parser.spec.ts @@ -1,13 +1,14 @@ import fs from 'node:fs' import path from 'node:path' -import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Parser } from '../parser.js' import { FixtureSandbox } from '../../../testing/fixture-sandbox.js' import { BunDetector, NpmDetector, PNpmDetector } from '../package-files/package-manager.js' import { FAUX_PACKAGE_DESCRIPTION } from '../faux-package.js' -import { Workspace } from '../package-files/workspace.js' +import { Package, Workspace } from '../package-files/workspace.js' +import { Err } from '../package-files/result.js' import { PlaywrightConfig } from '../../playwright-config.js' import { pathToPosix } from '../../util.js' @@ -91,10 +92,24 @@ describe('dependency-parser - parser()', () => { } } + // Faux manifests must carry the member's real name and version (so that + // workspace specifiers still resolve to the workspace package during the + // remote install), and nothing else besides the placeholder description + // and the private flag. const fauxFileEntry = (filePath: string) => { + const realPackageJson = JSON.parse(fs.readFileSync(filePath, 'utf8')) return { filePath, - content: expect.stringContaining(FAUX_PACKAGE_DESCRIPTION), + content: JSON.stringify( + { + name: realPackageJson.name, + version: realPackageJson.version, + description: FAUX_PACKAGE_DESCRIPTION, + private: true, + }, + undefined, + 2, + ), } } @@ -122,6 +137,7 @@ describe('dependency-parser - parser()', () => { const { dependencies } = await parser.parse(toAbsolutePath('apps/main/tests/foo.spec.js')) const got = dependencies.sort((a, b) => a.filePath.localeCompare(b.filePath)) const want = [ + realFileEntry(toAbsolutePath('.pnpmfile.cjs')), realFileEntry(toAbsolutePath('apps/depended-on-by-main-and-root/index.js')), realFileEntry(toAbsolutePath('apps/depended-on-by-main-and-root/package.json')), realFileEntry(toAbsolutePath('apps/depended-on-by-main/index.js')), @@ -137,6 +153,86 @@ describe('dependency-parser - parser()', () => { expect(got.map(({ filePath }) => filePath)).toEqual(want.map(({ filePath }) => filePath)) expect(got).toEqual(want) }) + + it('does not bundle the pnpmfile in restricted mode', async () => { + const parser = new Parser({ + checkUnsupportedModules: false, + restricted: true, + workspace, + }) + const { dependencies } = await parser.parse(toAbsolutePath('apps/main/tests/foo.spec.js')) + expect(dependencies.map(({ filePath }) => filePath)).not.toContain(toAbsolutePath('.pnpmfile.cjs')) + }) + }) + + describe('pnpm-pnpmfile-no-checksum', () => { + it('reports no pnpmfiles when the lockfile records no checksum', async () => { + // The fixture has a .pnpmfile.cjs, but its lockfile records no + // pnpmfileChecksum — so there is nothing for a remote install to + // reproduce and the workspace must not pick up the pnpmfile. + const workspace = await new PNpmDetector().lookupWorkspace(fixt.abspath('pnpm-pnpmfile-no-checksum')) + expect(workspace).toBeDefined() + expect(workspace?.pnpmfiles).toEqual([]) + }) + }) + }) + + describe('pnpmfile bundling', () => { + // The fixture's .pnpmfile.cjs contains an optional require that would + // be reported as a missing dependency if the file were ever parsed as + // check code. The Workspace is built by hand so the pnpmfile entries + // can be controlled directly, independent of the discovery-time + // self-containedness analysis. + const root = () => fixt.abspath('pnpmfile-bundling') + + const makeWorkspace = (pnpmfiles: { path: string, skipReason?: string }[]) => { + return new Workspace({ + root: new Package({ name: 'pnpmfile-bundling', path: root() }), + packages: [], + lockfile: Err(new Error('no lockfile')), + configFile: Err(new Error('no config file')), + pnpmfiles, + }) + } + + it('bundles the pnpmfile verbatim without parsing it as check code', async () => { + const parser = new Parser({ + checkUnsupportedModules: false, + restricted: false, + workspace: makeWorkspace([{ path: path.join(root(), '.pnpmfile.cjs') }]), + }) + const { dependencies } = await parser.parse(path.join(root(), 'tests', 'foo.spec.js')) + const paths = dependencies.map(({ filePath }) => filePath) + expect(paths).toContain(path.join(root(), '.pnpmfile.cjs')) + // The pnpmfile's own require must not be treated as a dependency. + expect(paths).not.toContain(path.join(root(), 'local-overrides.cjs')) + }) + + it('does not bundle a pnpmfile with a skip reason, and warns once', async () => { + const warnings: string[] = [] + const stderrSpy = vi.spyOn(process.stderr, 'write') + .mockImplementation((chunk: any) => { + warnings.push(String(chunk)) + return true + }) + try { + const parser = new Parser({ + checkUnsupportedModules: false, + restricted: false, + workspace: makeWorkspace([{ + path: path.join(root(), '.pnpmfile.cjs'), + skipReason: 'not self-contained', + }]), + }) + const { dependencies } = await parser.parse(path.join(root(), 'tests', 'foo.spec.js')) + expect(dependencies.map(({ filePath }) => filePath)) + .not.toContain(path.join(root(), '.pnpmfile.cjs')) + } finally { + stderrSpy.mockRestore() + } + const pnpmfileWarnings = warnings.filter(w => w.includes('not bundling pnpmfile')) + expect(pnpmfileWarnings).toHaveLength(1) + expect(pnpmfileWarnings[0]).toContain('not self-contained') }) }) @@ -178,6 +274,20 @@ describe('dependency-parser - parser()', () => { expect(got.map(({ filePath }) => filePath)).toEqual(want.map(({ filePath }) => filePath)) expect(got).toEqual(want) }) + + it('ignores a leftover .pnpmfile.cjs in a non-pnpm workspace', async () => { + // The fixture has a root .pnpmfile.cjs, but pnpmfile discovery is + // gated on the workspace using pnpm, so it must be neither + // registered on the workspace nor bundled. + expect(workspace!.pnpmfiles).toEqual([]) + const parser = new Parser({ + checkUnsupportedModules: false, + restricted: false, + workspace, + }) + const { dependencies } = await parser.parse(toAbsolutePath('apps/main/tests/foo.spec.js')) + expect(dependencies.map(({ filePath }) => filePath)).not.toContain(toAbsolutePath('.pnpmfile.cjs')) + }) }) }) diff --git a/packages/cli/src/services/check-parser/__tests__/faux-package.spec.ts b/packages/cli/src/services/check-parser/__tests__/faux-package.spec.ts new file mode 100644 index 000000000..3330c0324 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/faux-package.spec.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from 'vitest' + +import { + createFauxPackageFiles, + FAUX_PACKAGE_DESCRIPTION, + FAUX_PACKAGE_FALLBACK_VERSION, +} from '../faux-package.js' +import { Package } from '../package-files/workspace.js' + +describe('createFauxPackageFiles()', () => { + const contentOf = (pkg: Package): any => { + const files = createFauxPackageFiles(pkg) + expect(files).toHaveLength(1) + expect(files[0]).toMatchObject({ + filePath: pkg.packageJsonPath, + physical: false, + }) + return JSON.parse(files[0].content) + } + + it('carries the real version from the package', () => { + const pkg = new Package({ name: '@test/with-version', path: '/ws/with-version', version: '1.2.3' }) + expect(contentOf(pkg)).toEqual({ + name: '@test/with-version', + version: '1.2.3', + description: FAUX_PACKAGE_DESCRIPTION, + private: true, + }) + }) + + it('falls back when the package has no version', () => { + const pkg = new Package({ name: '@test/no-version', path: '/ws/no-version' }) + expect(contentOf(pkg).version).toEqual(FAUX_PACKAGE_FALLBACK_VERSION) + }) + + it('falls back when the version is empty', () => { + const pkg = new Package({ name: '@test/empty-version', path: '/ws/empty-version', version: '' }) + expect(contentOf(pkg).version).toEqual(FAUX_PACKAGE_FALLBACK_VERSION) + }) + + it('falls back when the version is not a string', () => { + // The version originates from a plain JSON.parse of the member's manifest, + // so despite the declared type it may be any JSON value at runtime. + const pkg = new Package({ name: '@test/numeric-version', path: '/ws/numeric-version', version: 1 as any }) + expect(contentOf(pkg).version).toEqual(FAUX_PACKAGE_FALLBACK_VERSION) + }) +}) diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/bun.lock b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/bun.lock new file mode 100644 index 000000000..50296b40c --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/bun.lock @@ -0,0 +1,48 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "lockfile-pruner-bun-fixture", + "dependencies": { + "@fixture/absent": "workspace:*", + "@fixture/shimmed": "workspace:*", + "@fixture/used": "workspace:*", + }, + }, + "packages/absent": { + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1", + }, + }, + "packages/shimmed": { + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5", + }, + }, + "packages/used": { + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3", + }, + }, + }, + "packages": { + "@fixture/absent": ["@fixture/absent@workspace:packages/absent"], + + "@fixture/shimmed": ["@fixture/shimmed@workspace:packages/shimmed"], + + "@fixture/used": ["@fixture/used@workspace:packages/used"], + + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], + + "isarray": ["isarray@2.0.5", "", {}, "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/package.json new file mode 100644 index 000000000..3db5468d0 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/package.json @@ -0,0 +1,12 @@ +{ + "name": "lockfile-pruner-bun-fixture", + "private": true, + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@fixture/used": "workspace:*", + "@fixture/shimmed": "workspace:*", + "@fixture/absent": "workspace:*" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/absent/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/absent/package.json new file mode 100644 index 000000000..2f5151309 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/absent/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/shimmed/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/shimmed/package.json new file mode 100644 index 000000000..0c1157782 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/shimmed/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/used/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/used/package.json new file mode 100644 index 000000000..014958abe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/bun-workspace/packages/used/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package-lock.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package-lock.json new file mode 100644 index 000000000..85deabdb5 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package-lock.json @@ -0,0 +1,69 @@ +{ + "name": "lockfile-pruner-npm-fixture", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "lockfile-pruner-npm-fixture", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@fixture/absent": "^1.0.0", + "@fixture/shimmed": "^1.0.0", + "@fixture/used": "^1.0.0" + } + }, + "node_modules/@fixture/absent": { + "resolved": "packages/absent", + "link": true + }, + "node_modules/@fixture/shimmed": { + "resolved": "packages/shimmed", + "link": true + }, + "node_modules/@fixture/used": { + "resolved": "packages/used", + "link": true + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "packages/absent": { + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } + }, + "packages/shimmed": { + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } + }, + "packages/used": { + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } + } + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package.json new file mode 100644 index 000000000..38b8f3385 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/package.json @@ -0,0 +1,12 @@ +{ + "name": "lockfile-pruner-npm-fixture", + "private": true, + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@fixture/used": "^1.0.0", + "@fixture/shimmed": "^1.0.0", + "@fixture/absent": "^1.0.0" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/absent/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/absent/package.json new file mode 100644 index 000000000..2f5151309 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/absent/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/shimmed/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/shimmed/package.json new file mode 100644 index 000000000..0c1157782 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/shimmed/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/used/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/used/package.json new file mode 100644 index 000000000..014958abe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/npm-workspace/packages/used/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/package.json new file mode 100644 index 000000000..dc036a4c9 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/package.json @@ -0,0 +1,9 @@ +{ + "name": "lockfile-pruner-fixture", + "private": true, + "dependencies": { + "@fixture/used": "workspace:*", + "@fixture/shimmed": "workspace:*", + "@fixture/absent": "workspace:*" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/absent/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/absent/package.json new file mode 100644 index 000000000..2f5151309 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/absent/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/shimmed/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/shimmed/package.json new file mode 100644 index 000000000..0c1157782 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/shimmed/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/used/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/used/package.json new file mode 100644 index 000000000..014958abe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/packages/used/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-lock.yaml b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-lock.yaml new file mode 100644 index 000000000..846b733b9 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-lock.yaml @@ -0,0 +1,56 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@fixture/absent': + specifier: workspace:* + version: link:packages/absent + '@fixture/shimmed': + specifier: workspace:* + version: link:packages/shimmed + '@fixture/used': + specifier: workspace:* + version: link:packages/used + + packages/absent: + dependencies: + ee-first: + specifier: 1.1.1 + version: 1.1.1 + + packages/shimmed: + dependencies: + isarray: + specifier: 2.0.5 + version: 2.0.5 + + packages/used: + dependencies: + ms: + specifier: 2.1.3 + version: 2.1.3 + +packages: + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + +snapshots: + + ee-first@1.1.1: {} + + isarray@2.0.5: {} + + ms@2.1.3: {} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-workspace.yaml b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-workspace.yaml new file mode 100644 index 000000000..924b55f42 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/pnpm-workspace/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - packages/* diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/package.json new file mode 100644 index 000000000..9f982fcc8 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/package.json @@ -0,0 +1,13 @@ +{ + "name": "lockfile-pruner-yarn-fixture", + "private": true, + "packageManager": "yarn@4.18.0", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@fixture/absent": "workspace:*", + "@fixture/shimmed": "workspace:*", + "@fixture/used": "workspace:*" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/absent/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/absent/package.json new file mode 100644 index 000000000..2f5151309 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/absent/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/shimmed/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/shimmed/package.json new file mode 100644 index 000000000..0c1157782 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/shimmed/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/used/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/used/package.json new file mode 100644 index 000000000..014958abe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/packages/used/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/yarn.lock b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/yarn.lock new file mode 100644 index 000000000..76259d5de --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn-workspace/yarn.lock @@ -0,0 +1,61 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 10 + cacheKey: 10c0 + +"@fixture/absent@workspace:*, @fixture/absent@workspace:packages/absent": + version: 0.0.0-use.local + resolution: "@fixture/absent@workspace:packages/absent" + dependencies: + ee-first: "npm:1.1.1" + languageName: unknown + linkType: soft + +"@fixture/shimmed@workspace:*, @fixture/shimmed@workspace:packages/shimmed": + version: 0.0.0-use.local + resolution: "@fixture/shimmed@workspace:packages/shimmed" + dependencies: + isarray: "npm:2.0.5" + languageName: unknown + linkType: soft + +"@fixture/used@workspace:*, @fixture/used@workspace:packages/used": + version: 0.0.0-use.local + resolution: "@fixture/used@workspace:packages/used" + dependencies: + ms: "npm:2.1.3" + languageName: unknown + linkType: soft + +"ee-first@npm:1.1.1": + version: 1.1.1 + resolution: "ee-first@npm:1.1.1" + checksum: 10c0/b5bb125ee93161bc16bfe6e56c6b04de5ad2aa44234d8f644813cc95d861a6910903132b05093706de2b706599367c4130eb6d170f6b46895686b95f87d017b7 + languageName: node + linkType: hard + +"isarray@npm:2.0.5": + version: 2.0.5 + resolution: "isarray@npm:2.0.5" + checksum: 10c0/4199f14a7a13da2177c66c31080008b7124331956f47bca57dd0b6ea9f11687aa25e565a2c7a2b519bc86988d10398e3049a1f5df13c9f6b7664154690ae79fd + languageName: node + linkType: hard + +"lockfile-pruner-yarn-fixture@workspace:.": + version: 0.0.0-use.local + resolution: "lockfile-pruner-yarn-fixture@workspace:." + dependencies: + "@fixture/absent": "workspace:*" + "@fixture/shimmed": "workspace:*" + "@fixture/used": "workspace:*" + languageName: unknown + linkType: soft + +"ms@npm:2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/d924b57e7312b3b63ad21fc5b3dc0af5e78d61a1fc7cfb5457edaf26326bf62be5307cc87ffb6862ef1c2b33b0233cdb5d4f01c4c958cc0d660948b65a287a48 + languageName: node + linkType: hard diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/package.json new file mode 100644 index 000000000..e583b3d42 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/package.json @@ -0,0 +1,13 @@ +{ + "name": "lockfile-pruner-yarn3-fixture", + "private": true, + "packageManager": "yarn@3.8.7", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@fixture/absent": "workspace:*", + "@fixture/shimmed": "workspace:*", + "@fixture/used": "workspace:*" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/absent/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/absent/package.json new file mode 100644 index 000000000..2f5151309 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/absent/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/absent", + "version": "1.0.0", + "dependencies": { + "ee-first": "1.1.1" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/shimmed/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/shimmed/package.json new file mode 100644 index 000000000..0c1157782 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/shimmed/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/shimmed", + "version": "1.0.0", + "dependencies": { + "isarray": "2.0.5" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/used/package.json b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/used/package.json new file mode 100644 index 000000000..014958abe --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/packages/used/package.json @@ -0,0 +1,7 @@ +{ + "name": "@fixture/used", + "version": "1.0.0", + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/yarn.lock b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/yarn.lock new file mode 100644 index 000000000..a6210f079 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner-fixtures/yarn3-workspace/yarn.lock @@ -0,0 +1,61 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 6 + cacheKey: 8 + +"@fixture/absent@workspace:*, @fixture/absent@workspace:packages/absent": + version: 0.0.0-use.local + resolution: "@fixture/absent@workspace:packages/absent" + dependencies: + ee-first: 1.1.1 + languageName: unknown + linkType: soft + +"@fixture/shimmed@workspace:*, @fixture/shimmed@workspace:packages/shimmed": + version: 0.0.0-use.local + resolution: "@fixture/shimmed@workspace:packages/shimmed" + dependencies: + isarray: 2.0.5 + languageName: unknown + linkType: soft + +"@fixture/used@workspace:*, @fixture/used@workspace:packages/used": + version: 0.0.0-use.local + resolution: "@fixture/used@workspace:packages/used" + dependencies: + ms: 2.1.3 + languageName: unknown + linkType: soft + +"ee-first@npm:1.1.1": + version: 1.1.1 + resolution: "ee-first@npm:1.1.1" + checksum: 1b4cac778d64ce3b582a7e26b218afe07e207a0f9bfe13cc7395a6d307849cfe361e65033c3251e00c27dd060cab43014c2d6b2647676135e18b77d2d05b3f4f + languageName: node + linkType: hard + +"isarray@npm:2.0.5": + version: 2.0.5 + resolution: "isarray@npm:2.0.5" + checksum: bd5bbe4104438c4196ba58a54650116007fa0262eccef13a4c55b2e09a5b36b59f1e75b9fcc49883dd9d4953892e6fc007eef9e9155648ceea036e184b0f930a + languageName: node + linkType: hard + +"lockfile-pruner-yarn3-fixture@workspace:.": + version: 0.0.0-use.local + resolution: "lockfile-pruner-yarn3-fixture@workspace:." + dependencies: + "@fixture/absent": "workspace:*" + "@fixture/shimmed": "workspace:*" + "@fixture/used": "workspace:*" + languageName: unknown + linkType: soft + +"ms@npm:2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d + languageName: node + linkType: hard diff --git a/packages/cli/src/services/check-parser/__tests__/lockfile-pruner.spec.ts b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner.spec.ts new file mode 100644 index 000000000..6daaf1697 --- /dev/null +++ b/packages/cli/src/services/check-parser/__tests__/lockfile-pruner.spec.ts @@ -0,0 +1,2458 @@ +import { spawnSync } from 'node:child_process' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { describe, it, expect, afterAll, vi } from 'vitest' + +import { + pruneBundledLockfile, + selectMaterializationEntries, + shouldPruneLockfile, +} from '../lockfile-pruner.js' +import { createFauxPackageFiles } from '../faux-package.js' +import { BunDetector, NpmDetector, PackageManager, PNpmDetector, Runnable, YarnDetector } from '../package-files/package-manager.js' +import { Package, Workspace } from '../package-files/workspace.js' +import { Err, Ok } from '../package-files/result.js' +import { File } from '../parser.js' + +const PNPM_FIXTURE_ROOT = path.join(__dirname, 'lockfile-pruner-fixtures', 'pnpm-workspace') +const NPM_FIXTURE_ROOT = path.join(__dirname, 'lockfile-pruner-fixtures', 'npm-workspace') +const BUN_FIXTURE_ROOT = path.join(__dirname, 'lockfile-pruner-fixtures', 'bun-workspace') +const YARN_FIXTURE_ROOT = path.join(__dirname, 'lockfile-pruner-fixtures', 'yarn-workspace') +const YARN3_FIXTURE_ROOT = path.join(__dirname, 'lockfile-pruner-fixtures', 'yarn3-workspace') + +// Unlike pnpm and npm, bun is not part of the repo's own toolchain, so the +// tests that run a real bun install skip themselves when it is not on PATH. +// A dedicated CI-only test asserts that bun IS provisioned there, so losing +// the provisioning step fails the job with a self-describing message +// instead of silently removing the coverage. +const bunAvailable = spawnSync('bun', ['--version']).status === 0 + +// Same for yarn, which must additionally resolve to the Yarn Berry major the +// fixture pins via its packageManager field (a Corepack-managed yarn does; a +// standalone Yarn Classic prints 1.x and the real-yarn tests skip). The +// probe needs a shell: Corepack's Windows shim is yarn.cmd, which a +// shell-less spawn cannot resolve even though the pruner's own spawn (execa +// via cross-spawn) can. +const probeYarnMajor = (fixtureRoot: string, major: string): boolean => { + const probe = spawnSync('yarn --version', { + cwd: fixtureRoot, + shell: true, + encoding: 'utf8', + env: { ...process.env, COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' }, + // Bounds the blocking probe during test collection; a timeout counts + // as unavailable, so the gated tests skip instead of stalling. Kept + // short deliberately: a cold corepack cache then skips locally rather + // than downloading toolchains at import time — CI pre-downloads both + // pinned versions in a dedicated workflow step and asserts coverage + // via CHECKLY_EXPECT_YARN. + timeout: 15_000, + }) + return probe.status === 0 && probe.stdout?.trim().startsWith(`${major}.`) === true +} +const yarnBerryAvailable = probeYarnMajor(YARN_FIXTURE_ROOT, '4') +const yarn3Available = probeYarnMajor(YARN3_FIXTURE_ROOT, '3') + +// Generates a stub package-manager script that rewrites the materialized +// lockfile through string replacements — parse-and-rewrite is not an option +// for bun.lock (JSONC with trailing commas, which node's JSON.parse rejects, +// and the prune temp dir has no node_modules to load a JSON5 parser from), +// so every format is rewritten the same way. A search string that no longer +// matches (e.g. after a fixture change) fails the script rather than +// silently leaving the lockfile unmodified, which would let some tests +// pass vacuously. +const rewriteLockfileScript = (lockfileName: string, ...replacements: Array<[string, string]>): string => ` + const fs = require('fs') + let content = fs.readFileSync(${JSON.stringify(lockfileName)}, 'utf8') + ${replacements.map(([from, to]) => ` + if (!content.includes(${JSON.stringify(from)})) { + console.error('rewriteLockfileScript: no match for ' + ${JSON.stringify(from)}) + process.exit(93) + } + content = content.split(${JSON.stringify(from)}).join(${JSON.stringify(to)})`).join('\n')} + fs.writeFileSync(${JSON.stringify(lockfileName)}, content) +` +const rewriteBunLockScript = (...replacements: Array<[string, string]>): string => + rewriteLockfileScript('bun.lock', ...replacements) +const rewriteYarnLockScript = (...replacements: Array<[string, string]>): string => + rewriteLockfileScript('yarn.lock', ...replacements) + +// Ambient values (particularly CHECKLY_LOCKFILE_PRUNE) must not leak into +// test outcomes. +const testEnv = (): NodeJS.ProcessEnv => ({ ...process.env, CHECKLY_LOCKFILE_PRUNE: undefined }) + +// A PackageManager whose lockfile-only install command is replaced, for +// exercising failure paths without a real package manager. +const stubPackageManager = (runnable: Runnable | undefined): PackageManager => { + return Object.assign(Object.create(new PNpmDetector()), { + lockfileOnlyInstallCommand: () => runnable, + }) +} + +describe('lockfile-pruner', () => { + const tempDirs: string[] = [] + + afterAll(async () => { + await Promise.all(tempDirs.map(dir => fs.rm(dir, { recursive: true, force: true, maxRetries: 3 }))) + }) + + const makeTempDir = async (): Promise => { + const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'lockfile-pruner-spec-'))) + tempDirs.push(dir) + return dir + } + + // Builds the workspace + bundle file map for a fixture, simulating a + // bundle where the `used` member is imported (real manifest), the `shimmed` + // member is declared but unimported (faux manifest), and the `absent` + // member is missing entirely. + const makeScenario = (root: string, lockfileName: string) => { + const used = new Package({ name: '@fixture/used', path: path.join(root, 'packages/used'), version: '1.0.0' }) + const shimmed = new Package({ name: '@fixture/shimmed', path: path.join(root, 'packages/shimmed'), version: '1.0.0' }) + const absent = new Package({ name: '@fixture/absent', path: path.join(root, 'packages/absent'), version: '1.0.0' }) + + const configFile = lockfileName === 'pnpm-lock.yaml' + ? Ok(path.join(root, 'pnpm-workspace.yaml')) + : Err(new Error('no config file')) + + const workspace = new Workspace({ + root: new Package({ name: 'lockfile-pruner-fixture', path: root }), + packages: [used, shimmed, absent], + lockfile: Ok(path.join(root, lockfileName)), + configFile, + }) + + const physical = (archivePath: string): [string, File] => [ + archivePath, + { filePath: path.join(root, ...archivePath.split('/')), physical: true }, + ] + + const files = new Map([ + physical('package.json'), + physical(lockfileName), + physical('packages/used/package.json'), + ['packages/shimmed/package.json', createFauxPackageFiles(shimmed)[0]], + ]) + if (lockfileName === 'pnpm-lock.yaml') { + files.set(...physical('pnpm-workspace.yaml')) + } + + return { workspace, files, used, shimmed, absent } + } + + const makePnpmScenario = (root: string = PNPM_FIXTURE_ROOT) => makeScenario(root, 'pnpm-lock.yaml') + const makeNpmScenario = (root: string = NPM_FIXTURE_ROOT) => makeScenario(root, 'package-lock.json') + const makeBunScenario = (root: string = BUN_FIXTURE_ROOT) => makeScenario(root, 'bun.lock') + const makeYarnScenario = (root: string = YARN_FIXTURE_ROOT) => makeScenario(root, 'yarn.lock') + const makeYarn3Scenario = (root: string = YARN3_FIXTURE_ROOT) => makeScenario(root, 'yarn.lock') + + describe('shouldPruneLockfile()', () => { + it('skips when disabled via CHECKLY_LOCKFILE_PRUNE=0', () => { + const { workspace, files } = makePnpmScenario() + const decision = shouldPruneLockfile(workspace, files, { CHECKLY_LOCKFILE_PRUNE: '0' }) + expect(decision).toMatchObject({ prune: false, reason: expect.stringContaining('CHECKLY_LOCKFILE_PRUNE') }) + }) + + it('skips when the workspace has no lockfile', () => { + const { workspace, files } = makePnpmScenario() + const noLockfile = new Workspace({ + root: workspace.root, + packages: workspace.packages, + lockfile: Err(new Error('no lockfile')), + configFile: workspace.configFile, + }) + expect(shouldPruneLockfile(noLockfile, files, {})).toMatchObject({ prune: false }) + }) + + it('skips when the bundle does not contain the lockfile', () => { + const { workspace, files } = makePnpmScenario() + files.delete('pnpm-lock.yaml') + expect(shouldPruneLockfile(workspace, files, {})).toMatchObject({ + prune: false, + reason: expect.stringContaining('does not contain the lockfile'), + }) + }) + + it('skips when the bundle contains the full workspace', () => { + const { workspace, files } = makePnpmScenario() + files.set('packages/shimmed/package.json', { + filePath: path.join(PNPM_FIXTURE_ROOT, 'packages/shimmed/package.json'), + physical: true, + }) + files.set('packages/absent/package.json', { + filePath: path.join(PNPM_FIXTURE_ROOT, 'packages/absent/package.json'), + physical: true, + }) + expect(shouldPruneLockfile(workspace, files, {})).toMatchObject({ + prune: false, + reason: expect.stringContaining('full workspace'), + }) + }) + + it('skips when a faux member version is unknown', () => { + const { workspace, files, shimmed } = makePnpmScenario() + shimmed.version = undefined + expect(shouldPruneLockfile(workspace, files, {})).toMatchObject({ + prune: false, + reason: expect.stringContaining('@fixture/shimmed'), + }) + }) + + it('prunes when the bundle differs from the workspace', () => { + const { workspace, files } = makePnpmScenario() + expect(shouldPruneLockfile(workspace, files, {})).toEqual({ + prune: true, + lockfileArchivePath: 'pnpm-lock.yaml', + }) + }) + }) + + describe('selectMaterializationEntries()', () => { + it('selects manifests, package manager config, patches and the lockfile', () => { + const files = new Map([ + ['package.json', { filePath: '/ws/package.json', physical: true }], + ['pnpm-lock.yaml', { filePath: '/ws/pnpm-lock.yaml', physical: true }], + ['pnpm-workspace.yaml', { filePath: '/ws/pnpm-workspace.yaml', physical: true }], + ['.npmrc', { filePath: '/ws/.npmrc', physical: true }], + ['.pnpmfile.cjs', { filePath: '/ws/.pnpmfile.cjs', physical: true }], + ['patches/left-pad.patch', { filePath: '/ws/patches/left-pad.patch', physical: true }], + ['packages/a/package.json', { filePath: '/ws/packages/a/package.json', physical: false, content: '{}' }], + ['tests/foo.spec.ts', { filePath: '/ws/tests/foo.spec.ts', physical: true }], + ['node_modules/dep/package.json', { filePath: '/ws/node_modules/dep/package.json', physical: true }], + ['.checkly/embedded-packages/a.tgz', { filePath: '/ws/.checkly/embedded-packages/a.tgz', physical: true }], + ['../outside/package.json', { filePath: '/outside/package.json', physical: true }], + ['packages/link/package.json', { + filePath: '/ws/packages/link/package.json', + physical: true, + symlinkTarget: '../real', + }], + ]) + + const selected = selectMaterializationEntries(files, 'pnpm-lock.yaml').map(([archivePath]) => archivePath) + expect(selected.sort()).toEqual([ + '.npmrc', + '.pnpmfile.cjs', + 'package.json', + 'packages/a/package.json', + 'patches/left-pad.patch', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml', + ]) + }) + + it('always selects the lockfile, even as a symlink entry', () => { + const files = new Map([ + ['pnpm-lock.yaml', { + filePath: '/ws/pnpm-lock.yaml', + physical: true, + symlinkTarget: 'config/pnpm-lock.yaml', + }], + ]) + const selected = selectMaterializationEntries(files, 'pnpm-lock.yaml').map(([archivePath]) => archivePath) + expect(selected).toEqual(['pnpm-lock.yaml']) + }) + }) + + describe('pruneBundledLockfile()', () => { + it('skips notably for an unsupported package manager even when the lockfile is unreadable', async () => { + // Pins the ordering invariant in pruneBundledLockfile: the capability + // check runs before the lockfile read, so an unsupported package + // manager never surfaces a read error as a 'failed' warning that + // implies pruning was attempted. + const { workspace: base, files } = makePnpmScenario() + const missingLockfile = path.join(PNPM_FIXTURE_ROOT, 'missing-pnpm-lock.yaml') + const workspace = new Workspace({ + root: base.root, + packages: base.packages, + lockfile: Ok(missingLockfile), + configFile: Ok(path.join(PNPM_FIXTURE_ROOT, 'pnpm-workspace.yaml')), + }) + files.delete('pnpm-lock.yaml') + files.set('missing-pnpm-lock.yaml', { filePath: missingLockfile, physical: true }) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(undefined), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('lockfile-only'), + notable: true, + }) + }) + + it('skips notably when the executable does not exist', async () => { + // A lockfile can be committed without its package manager being + // installed where the CLI runs (e.g. a bun.lock deployed from a + // node-only CI image). That is a skip with an accurate reason, not a + // failure implying the lockfile itself is broken. + const { workspace, files } = makePnpmScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('checkly-no-such-executable-xyz', [])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('checkly-no-such-executable-xyz is not installed'), + notable: true, + }) + }) + + it('fails when the command times out', async () => { + const { workspace, files } = makePnpmScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', 'setInterval(() => {}, 1000)'])), + files, + timeoutMs: 500, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'failed', reason: expect.stringContaining('timed out') }) + }, 30_000) + + it('skips when the regenerated lockfile is identical and nothing was backfilled', async () => { + const { workspace, files } = makePnpmScenario() + // With the absent member's real manifest in the bundle there is + // nothing to backfill, so an unchanged lockfile means nothing to do. + files.set('packages/absent/package.json', { + filePath: path.join(PNPM_FIXTURE_ROOT, 'packages/absent/package.json'), + physical: true, + }) + const result = await pruneBundledLockfile({ + workspace, + // A command that does nothing leaves the materialized lockfile as-is. + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'skipped', reason: expect.stringContaining('identical') }) + }) + + it('reports a prune with the original content when only backfill is needed', async () => { + const { workspace, files } = makePnpmScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + // The absent member must be backfilled even though the lockfile itself + // did not change — a lockfile importer without a bundled manifest + // breaks the remote install. + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('@fixture/absent') + }) + + it('strips behavior-altering npm_config env vars but keeps everything else', async () => { + const { workspace, files } = makePnpmScenario() + const outFile = path.join(await makeTempDir(), 'env.json') + const script = `require('fs').writeFileSync(${JSON.stringify(outFile)}, JSON.stringify(process.env))` + // On win32, node's spawn sorts env keys and deduplicates them + // case-insensitively keeping the first — uppercase sorts before + // lowercase, so an ambient case-variant (NPM_CONFIG_REGISTRY on the + // CI runner) would silently displace the lowercase sentinel below. + // Drop every ambient variant first so exactly one casing exists. + const baseEnv = testEnv() + for (const key of Object.keys(baseEnv)) { + if (key.toLowerCase() === 'npm_config_registry') { + delete baseEnv[key] + } + } + await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: { + ...baseEnv, + npm_config_frozen_lockfile: 'true', + npm_config_dry_run: 'true', + NPM_CONFIG_LOCKFILE: 'false', + npm_config_package_lock: 'false', + npm_config_ignore_workspace: 'true', + npm_config_lockfile_dir: '/tmp/decoy', + npm_config_registry: 'https://registry.example.com/', + }, + }) + const childEnv = JSON.parse(await fs.readFile(outFile, 'utf8')) + expect(childEnv.npm_config_frozen_lockfile).toBeUndefined() + expect(childEnv.npm_config_dry_run).toBeUndefined() + expect(childEnv.NPM_CONFIG_LOCKFILE).toBeUndefined() + expect(childEnv.npm_config_package_lock).toBeUndefined() + expect(childEnv.npm_config_ignore_workspace).toBeUndefined() + expect(childEnv.npm_config_lockfile_dir).toBeUndefined() + expect(childEnv.npm_config_registry).toEqual('https://registry.example.com/') + expect(childEnv.COREPACK_ENABLE_STRICT).toEqual('0') + // Yarn's network access is always disabled: a stale lockfile would + // otherwise resolve missing descriptors against the public registry, + // disclosing private package names. Scripts likewise (defense in + // depth), and rc loading is pointed at a nonexistent filename so an + // uncontrolled .yarnrc.yml in an ancestor of the temp dir (e.g. a + // world-writable /tmp) cannot inject yarnPath code execution or + // redirect the lockfile write. (YARN_ENABLE_HARDENED_MODE is + // deliberately NOT set here — yarn 3 rejects the unknown setting — + // and is covered by the yarn-generation tests instead.) + expect(childEnv.YARN_ENABLE_NETWORK).toEqual('0') + expect(childEnv.YARN_ENABLE_SCRIPTS).toEqual('0') + expect(childEnv.YARN_IGNORE_PATH).toEqual('1') + // The rc filename must be random so no ancestor .yarnrc.yml can be + // pre-created under a known name to re-open the yarnPath channel. + expect(childEnv.YARN_RC_FILENAME).toMatch(/^\.checkly-lockfile-prune-no-rc-[0-9a-f-]+\.yml$/) + expect(childEnv.YARN_ENABLE_HARDENED_MODE).toBeUndefined() + }) + + it('fails when a workspace link is no longer a link after regeneration', async () => { + const { workspace, files } = makePnpmScenario() + const script = ` + const fs = require('fs') + const content = fs.readFileSync('pnpm-lock.yaml', 'utf8') + fs.writeFileSync('pnpm-lock.yaml', content.split('link:packages/used').join('9.9.9')) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('no longer a workspace link'), + }) + }) + + it('fails when the regenerated lockfile resolves new entries', async () => { + const { workspace, files } = makePnpmScenario() + const script = ` + const fs = require('fs') + const content = fs.readFileSync('pnpm-lock.yaml', 'utf8') + fs.writeFileSync('pnpm-lock.yaml', content + '\\n safe-buffer@5.2.1:\\n resolution: {integrity: sha512-x}\\n') + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('not present in the original'), + }) + }) + + it('fails when the lockfile format version changes', async () => { + const { workspace, files } = makePnpmScenario() + const script = ` + const fs = require('fs') + const content = fs.readFileSync('pnpm-lock.yaml', 'utf8') + fs.writeFileSync('pnpm-lock.yaml', content.replace("lockfileVersion: '9.0'", "lockfileVersion: '6.0'")) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('lockfile version changed'), + }) + }) + + it('fails when a bundled importer disappears from the lockfile', async () => { + const { workspace, files } = makePnpmScenario() + const script = ` + const fs = require('fs') + const content = fs.readFileSync('pnpm-lock.yaml', 'utf8') + // Rename the used member's importer so it effectively disappears. + fs.writeFileSync('pnpm-lock.yaml', content.replace(' packages/used:', ' packages/renamed:')) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining(`lost the importer 'packages/used'`), + }) + }) + + it('skips when the lockfile records a pnpmfile checksum but no pnpmfile is bundled', async () => { + const root = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'pnpm-lock.yaml') + await fs.appendFile(lockfilePath, '\npnpmfileChecksum: sha256-abcdef\n') + + const { workspace, files } = makePnpmScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('pnpmfile checksum'), + }) + }) + + it('skips when the lockfile is written with excludeLinksFromLockfile', async () => { + const root = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'pnpm-lock.yaml') + const content = await fs.readFile(lockfilePath, 'utf8') + await fs.writeFile(lockfilePath, content.replace( + 'excludeLinksFromLockfile: false', + 'excludeLinksFromLockfile: true', + )) + + const { workspace, files } = makePnpmScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('excludeLinksFromLockfile'), + }) + }) + + it('fails when the regenerated lockfile drops the pnpmfile checksum', async () => { + const root = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'pnpm-lock.yaml') + await fs.appendFile(lockfilePath, '\npnpmfileChecksum: sha256-abcdef\n') + await fs.writeFile(path.join(root, '.pnpmfile.cjs'), 'module.exports = {}\n') + + const { workspace, files } = makePnpmScenario(root) + files.set('.pnpmfile.cjs', { filePath: path.join(root, '.pnpmfile.cjs'), physical: true }) + + const script = ` + const fs = require('fs') + const content = fs.readFileSync('pnpm-lock.yaml', 'utf8') + fs.writeFileSync('pnpm-lock.yaml', content.split('\\n').filter(l => !l.startsWith('pnpmfileChecksum')).join('\\n')) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('pnpmfile checksum'), + }) + }) + + it('skips when a backfilled member version is unknown', async () => { + const { workspace, files, absent } = makePnpmScenario() + absent.version = undefined + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('@fixture/absent'), + }) + }) + + it('redacts credentials and truncates long output in failure reasons', async () => { + const { workspace, files } = makePnpmScenario() + // The script goes through a file so the credential appears only in the + // child's output, not in the displayed command line. + const scriptPath = path.join(await makeTempDir(), 'fail.cjs') + await fs.writeFile(scriptPath, ` + process.stdout.write('GET https://alice:sup3rsecret@registry.example.com/pkg failed ' + 'x'.repeat(600)) + process.exit(1) + `) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', [scriptPath])), + files, + env: testEnv(), + }) + expect(result.status).toEqual('failed') + if (result.status !== 'failed') { + return + } + expect(result.reason).not.toContain('sup3rsecret') + expect(result.reason).toContain('registry.example.com') + expect(result.reason).toContain('…') + }) + + it('drops the importer of a member no bundled manifest references', async () => { + // The pruner's primary outcome: a workspace member that nothing in the + // bundle depends on loses its importer (and its dependencies) without + // failing the prune. + const root = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, root, { recursive: true }) + const rootManifestPath = path.join(root, 'package.json') + const rootManifest = JSON.parse(await fs.readFile(rootManifestPath, 'utf8')) + delete rootManifest.dependencies['@fixture/absent'] + await fs.writeFile(rootManifestPath, JSON.stringify(rootManifest, undefined, 2)) + + const { workspace, files } = makePnpmScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: new PNpmDetector(), + files, + env: testEnv(), + }) + + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(0) + expect(result.content).not.toContain('packages/absent') + expect(result.content).not.toContain('ee-first') + expect(result.content).toContain('link:packages/used') + expect(result.content).toContain('link:packages/shimmed') + }, 60_000) + + // Plants a lockfile-dir setting pointing at decoyDir in both config + // channels: pnpm <= 10 reads the setting from .npmrc while pnpm 11 only + // honors lockfileDir in pnpm-workspace.yaml. + const plantLockfileDirDecoys = async (files: Map, decoyDir: string) => { + files.set('.npmrc', { + filePath: path.join(PNPM_FIXTURE_ROOT, '.npmrc'), + physical: false, + content: `lockfile-dir=${decoyDir}\n`, + }) + const workspaceYaml = await fs.readFile(path.join(PNPM_FIXTURE_ROOT, 'pnpm-workspace.yaml'), 'utf8') + files.set('pnpm-workspace.yaml', { + filePath: path.join(PNPM_FIXTURE_ROOT, 'pnpm-workspace.yaml'), + physical: false, + content: `${workspaceYaml}lockfileDir: ${decoyDir}\n`, + }) + } + + it('pins the lockfile write to the temp dir despite a config lockfile-dir, with real pnpm', async () => { + const { workspace, files } = makePnpmScenario() + // The explicit --lockfile-dir flag on the prune command must outrank + // a lockfile-dir setting from a materialized config file — otherwise + // the subprocess could write over a lockfile outside the temp dir. + const decoyDir = await makeTempDir() + await plantLockfileDirDecoys(files, decoyDir) + const result = await pruneBundledLockfile({ + workspace, + packageManager: new PNpmDetector(), + files, + env: testEnv(), + }) + expect(result.status).toEqual('pruned') + await expect(fs.access(path.join(decoyDir, 'pnpm-lock.yaml'))).rejects.toThrow() + }, 60_000) + + it('decoy control: pnpm honors a config lockfile-dir when the flag is absent', async () => { + // Positive control for the test above: proves the planted channels + // actually reach the pnpm on PATH. If a future pnpm major stops + // reading both channels, this fails and the decoys need updating — + // without it, the pinning test could pass vacuously. + const { workspace, files } = makePnpmScenario() + const decoyDir = await makeTempDir() + await plantLockfileDirDecoys(files, decoyDir) + // Seed the decoy with the original lockfile so the redirected run can + // reuse its resolutions instead of needing registry access, and so + // the assertion below can detect that pnpm rewrote it. + const seedContent = await fs.readFile(path.join(PNPM_FIXTURE_ROOT, 'pnpm-lock.yaml'), 'utf8') + await fs.writeFile(path.join(decoyDir, 'pnpm-lock.yaml'), seedContent) + // Derive the unpinned command from the production one so a future + // change to the real argument list flows into this control instead of + // silently diverging from it. + const pinned = new PNpmDetector().lockfileOnlyInstallCommand() + const unpinned = new Runnable(pinned.executable, pinned.args.filter((arg, i, args) => { + return arg !== '--lockfile-dir' && args[i - 1] !== '--lockfile-dir' + })) + await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(unpinned), + files, + env: testEnv(), + }) + const decoyContent = await fs.readFile(path.join(decoyDir, 'pnpm-lock.yaml'), 'utf8') + expect(decoyContent).not.toEqual(seedContent) + }, 60_000) + + it('prunes the lockfile with real pnpm, backfilling link-referenced members', async () => { + const { workspace, files } = makePnpmScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new PNpmDetector(), + files, + env: testEnv(), + }) + + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + + expect(result.archivePath).toEqual('pnpm-lock.yaml') + + // The root manifest declares @fixture/absent as workspace:*, so a faux + // manifest must have been backfilled for it. + expect(result.backfilledManifests).toHaveLength(1) + expect(result.backfilledManifests[0].filePath) + .toEqual(path.join(PNPM_FIXTURE_ROOT, 'packages/absent/package.json')) + expect(JSON.parse(result.backfilledManifests[0].content)).toMatchObject({ + name: '@fixture/absent', + version: '1.0.0', + }) + + // Kept: the imported member and its dependency, and every workspace link. + expect(result.content).toContain('link:packages/used') + expect(result.content).toContain('link:packages/shimmed') + expect(result.content).toContain('link:packages/absent') + expect(result.content).toContain('ms@2.1.3') + + // Dropped: dependencies of the shimmed and absent members. + expect(result.content).not.toContain('isarray') + expect(result.content).not.toContain('ee-first') + }, 60_000) + + it('fails when npm silently replaces a workspace link with a registry package', async () => { + const { workspace, files } = makeNpmScenario() + // Simulate npm's registry substitution: the link entry for the used + // member becomes a plain registry resolution that exists nowhere in + // the original lockfile. + const script = ` + const fs = require('fs') + const doc = JSON.parse(fs.readFileSync('package-lock.json', 'utf8')) + doc.packages['node_modules/@fixture/used'] = { + version: '1.0.1', + resolved: 'https://registry.npmjs.org/@fixture/used/-/used-1.0.1.tgz', + integrity: 'sha512-x', + } + fs.writeFileSync('package-lock.json', JSON.stringify(doc, null, 2)) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('not present in the original'), + }) + }) + + it('fails when an npm dependency is re-resolved to a different version', async () => { + const { workspace, files } = makeNpmScenario() + const script = ` + const fs = require('fs') + const doc = JSON.parse(fs.readFileSync('package-lock.json', 'utf8')) + doc.packages['node_modules/ms'].version = '2.0.0' + fs.writeFileSync('package-lock.json', JSON.stringify(doc, null, 2)) + ` + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('not present in the original'), + }) + }) + + it('skips unsupported lockfile formats', async () => { + const npmV1Root = await makeTempDir() + await fs.cp(NPM_FIXTURE_ROOT, npmV1Root, { recursive: true }) + await fs.writeFile( + path.join(npmV1Root, 'package-lock.json'), + JSON.stringify({ name: 'x', lockfileVersion: 1, dependencies: {} }), + ) + const npmScenario = makeNpmScenario(npmV1Root) + expect(await pruneBundledLockfile({ + workspace: npmScenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files: npmScenario.files, + env: testEnv(), + })).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported'), + notable: true, + }) + + // A structurally valid pnpm lockfile at an out-of-range version must + // fail closed rather than passing the verification vacuously. + const pnpmV5Root = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, pnpmV5Root, { recursive: true }) + const v5LockfilePath = path.join(pnpmV5Root, 'pnpm-lock.yaml') + const v5Content = await fs.readFile(v5LockfilePath, 'utf8') + await fs.writeFile(v5LockfilePath, v5Content.replace('lockfileVersion: \'9.0\'', 'lockfileVersion: \'5.4\'')) + const pnpmV5Scenario = makePnpmScenario(pnpmV5Root) + expect(await pruneBundledLockfile({ + workspace: pnpmV5Scenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files: pnpmV5Scenario.files, + env: testEnv(), + })).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported'), + notable: true, + }) + + const pnpmRoot = await makeTempDir() + await fs.cp(PNPM_FIXTURE_ROOT, pnpmRoot, { recursive: true }) + await fs.writeFile(path.join(pnpmRoot, 'pnpm-lock.yaml'), 'just a string') + const pnpmScenario = makePnpmScenario(pnpmRoot) + expect(await pruneBundledLockfile({ + workspace: pnpmScenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files: pnpmScenario.files, + env: testEnv(), + })).toMatchObject({ status: 'skipped', reason: expect.stringContaining('could not parse') }) + }) + + it('does not mark explicitly disabled skips as notable', async () => { + const { workspace, files } = makePnpmScenario() + const disabled = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: { ...testEnv(), CHECKLY_LOCKFILE_PRUNE: '0' }, + }) + expect(disabled.status).toEqual('skipped') + expect(disabled.status === 'skipped' && disabled.notable).toBeFalsy() + }) + + // Replaces the root manifest with a virtual one that references the + // absent member through peerDependencies only. Shared between the peer + // backfill tests so they stay a controlled comparison. + const setRootManifestWithAbsentPeer = (files: Map, peerDependenciesMeta?: object) => { + files.set('package.json', { + filePath: path.join(PNPM_FIXTURE_ROOT, 'package.json'), + physical: false, + content: JSON.stringify({ + name: 'lockfile-pruner-fixture', + private: true, + dependencies: { + '@fixture/used': 'workspace:*', + '@fixture/shimmed': 'workspace:*', + }, + peerDependencies: { + '@fixture/absent': 'workspace:*', + }, + // JSON.stringify omits undefined-valued properties, so a call + // without meta produces a manifest without the key. + peerDependenciesMeta, + }), + }) + } + + it('backfills members referenced only through peerDependencies', async () => { + const { workspace, files } = makePnpmScenario() + // The root manifest becomes virtual, so the root package needs a + // known version to pass the unknown-version guard. + workspace.root.version = '1.0.0' + setRootManifestWithAbsentPeer(files) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('@fixture/absent') + }) + + it('backfills workspace peers even when marked optional, with real pnpm', async () => { + const { workspace, files } = makePnpmScenario() + workspace.root.version = '1.0.0' + // pnpm resolves a workspace: peer spec regardless of + // peerDependenciesMeta.optional when auto-install-peers is on (the + // default), so without the backfill the install fails with + // ERR_PNPM_WORKSPACE_PKG_NOT_FOUND. + setRootManifestWithAbsentPeer(files, { '@fixture/absent': { optional: true } }) + const result = await pruneBundledLockfile({ + workspace, + packageManager: new PNpmDetector(), + files, + env: testEnv(), + }) + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('@fixture/absent') + }, 60_000) + + it('prunes the lockfile with real npm, backfilling link-referenced members', async () => { + const { workspace, files } = makeNpmScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new NpmDetector(), + files, + env: testEnv(), + }) + + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + + expect(result.archivePath).toEqual('package-lock.json') + + // The root manifest declares @fixture/absent with a semver range and + // the original lockfile resolves it as a link, so a faux manifest must + // have been backfilled for it. + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content)).toMatchObject({ + name: '@fixture/absent', + version: '1.0.0', + }) + + const doc = JSON.parse(result.content) + // Kept: every workspace link and the imported member's dependency. + for (const name of ['@fixture/used', '@fixture/shimmed', '@fixture/absent']) { + expect(doc.packages[`node_modules/${name}`]).toMatchObject({ link: true }) + } + expect(doc.packages['node_modules/ms']).toBeDefined() + + // Dropped: dependencies of the shimmed and absent members. + expect(doc.packages['node_modules/isarray']).toBeUndefined() + expect(doc.packages['node_modules/ee-first']).toBeUndefined() + }, 60_000) + + // The bun stub scripts below mutate the materialized bun.lock via string + // replacement rather than parse-and-rewrite: bun.lock is JSONC with + // trailing commas, which node's JSON.parse rejects, and the prune temp + // dir has no node_modules to load a JSON5 parser from. + + it('reports a bun prune with the original content when only backfill is needed', async () => { + const { workspace, files } = makeBunScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned', archivePath: 'bun.lock' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('@fixture/absent') + }) + + it('fails when a kept bun package entry is rewritten to a different registry URL', async () => { + // Registry config in the environment makes bun rewrite tarball URLs + // inside otherwise-unchanged tuples, offline and with exit 0. The + // rewritten tuple exists nowhere in the original, so the subset check + // must reject it. + const { workspace, files } = makeBunScenario() + const script = rewriteBunLockScript( + ['["ms@2.1.3", "",', '["ms@2.1.3", "https://mirror.example.com/ms/-/ms-2.1.3.tgz",'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('not present in the original'), + }) + }) + + it('accepts a bun package entry re-keyed under a different hoist key', async () => { + // Pruning the member that owns a hoisted key makes bun re-key the + // surviving member-scoped entry with an unchanged tuple. The subset + // check is keyed by tuple content, so the rename alone must not fail + // the verification. + const { workspace, files } = makeBunScenario() + const script = rewriteBunLockScript( + ['"ms": ["ms@2.1.3"', '"@fixture/used/ms": ["ms@2.1.3"'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + }) + + it('treats an absent bun configVersion as 0', async () => { + // bun writes an explicit `configVersion: 0` when regenerating a + // lockfile that lacks the field, so absent-vs-0 must not be reported + // as a version change. + const root = await makeTempDir() + await fs.cp(BUN_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'bun.lock') + const content = await fs.readFile(lockfilePath, 'utf8') + // No newline in the search string: a Windows checkout may carry CRLF. + await fs.writeFile(lockfilePath, content.replace('"configVersion": 1,', '')) + + const { workspace, files } = makeBunScenario(root) + const script = rewriteBunLockScript( + ['"lockfileVersion": 1,', '"lockfileVersion": 1,\n "configVersion": 0,'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + }) + + it('fails when the bun configVersion changes', async () => { + const { workspace, files } = makeBunScenario() + const script = rewriteBunLockScript( + ['"configVersion": 1,', '"configVersion": 2,'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('lockfile version changed'), + }) + if (result.status !== 'failed') { + return + } + expect(result.reason).toContain('configVersion') + }) + + it('skips unsupported bun lockfiles notably', async () => { + // An unknown lockfileVersion fails closed before any command runs. + const versionRoot = await makeTempDir() + await fs.cp(BUN_FIXTURE_ROOT, versionRoot, { recursive: true }) + const versionLockfilePath = path.join(versionRoot, 'bun.lock') + const versionContent = await fs.readFile(versionLockfilePath, 'utf8') + await fs.writeFile(versionLockfilePath, versionContent.replace('"lockfileVersion": 1,', '"lockfileVersion": 2,')) + const versionScenario = makeBunScenario(versionRoot) + expect(await pruneBundledLockfile({ + workspace: versionScenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files: versionScenario.files, + env: testEnv(), + })).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported'), + notable: true, + }) + + // The binary lockfile format is rejected by basename, with a remedy. + const binaryRoot = await makeTempDir() + await fs.cp(BUN_FIXTURE_ROOT, binaryRoot, { recursive: true }) + await fs.writeFile(path.join(binaryRoot, 'bun.lockb'), Buffer.from([0x62, 0x75, 0x6e, 0x00, 0x01, 0x02])) + const binaryScenario = makeScenario(binaryRoot, 'bun.lockb') + expect(await pruneBundledLockfile({ + workspace: binaryScenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files: binaryScenario.files, + env: testEnv(), + })).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('bun install --save-text-lockfile'), + notable: true, + }) + }) + + // Builds a workspace where the `ms` member is consumed through a bare + // semver range — which bun records verbatim in the importer while + // resolving it to a member-scoped workspace tuple — AND a same-named + // registry package is consumed by the root. Link classification for + // these edges depends entirely on the per-edge member-scoped-then- + // hoisted packages-key probe: a name-global answer would misclassify + // one of the two edges. The lockfile is hand-built (plain JSON is valid + // JSONC) mirroring bun 1.3.11's real layout for this shape, with + // single-line entries so stub scripts can rewrite it via string + // replacement. + const makeMixedBunScenario = async () => { + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/a'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/ms'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'mixed-bun-fixture', + private: true, + workspaces: ['packages/*'], + dependencies: { ms: '2.1.3' }, + })) + await fs.writeFile(path.join(root, 'packages/a/package.json'), JSON.stringify({ + name: 'a', + version: '1.0.0', + dependencies: { ms: '^1.0.0' }, + })) + await fs.writeFile(path.join(root, 'packages/ms/package.json'), JSON.stringify({ + name: 'ms', + version: '1.0.0', + })) + await fs.writeFile(path.join(root, 'bun.lock'), `{ + "lockfileVersion": 1, + "workspaces": { + "": { "name": "mixed-bun-fixture", "dependencies": { "ms": "2.1.3" } }, + "packages/a": { "name": "a", "version": "1.0.0", "dependencies": { "ms": "^1.0.0" } }, + "packages/ms": { "name": "ms", "version": "1.0.0" } + }, + "packages": { + "a": ["a@workspace:packages/a"], + "a/ms": ["ms@workspace:packages/ms"], + "ms": ["ms@2.1.3", "", {}, "sha512-mmm"] + } +}`) + + const a = new Package({ name: 'a', path: path.join(root, 'packages/a'), version: '1.0.0' }) + const msMember = new Package({ name: 'ms', path: path.join(root, 'packages/ms'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'mixed-bun-fixture', path: root }), + packages: [a, msMember], + lockfile: Ok(path.join(root, 'bun.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['bun.lock', { filePath: path.join(root, 'bun.lock'), physical: true }], + ['packages/a/package.json', { filePath: path.join(root, 'packages/a/package.json'), physical: true }], + // The ms member's manifest is deliberately not in the bundle. + ]) + return { workspace, files, a } + } + + it('backfills a bun member consumed through a bare semver range', async () => { + const { workspace, files } = await makeMixedBunScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + // The manifest spec is '^1.0.0', not 'workspace:*', so the backfill + // can only trigger through the lockfile's per-edge link resolution + // (the member-scoped 'a/ms' workspace tuple), never through the spec + // prefix. + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('ms') + }) + + it('backfills a bun member consumed through a bare-semver peer dependency', async () => { + // Unlike pnpm importers, bun workspace entries record peerDependencies + // as their own group, and the snapshot parser must walk it: a peer + // edge resolved to a workspace tuple triggers backfill exactly like a + // regular dependency edge. The hand-built lockfile mirrors what bun + // 1.3.11 emits for this shape: the peer-consumed member holds the + // hoisted packages key as a workspace tuple. + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/a'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/core'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'peer-bun-fixture', + private: true, + workspaces: ['packages/*'], + })) + await fs.writeFile(path.join(root, 'packages/a/package.json'), JSON.stringify({ + name: 'a', + version: '1.0.0', + peerDependencies: { core: '^1.0.0' }, + })) + await fs.writeFile(path.join(root, 'packages/core/package.json'), JSON.stringify({ + name: 'core', + version: '1.0.0', + })) + await fs.writeFile(path.join(root, 'bun.lock'), `{ + "lockfileVersion": 1, + "workspaces": { + "": { "name": "peer-bun-fixture" }, + "packages/a": { "name": "a", "version": "1.0.0", "peerDependencies": { "core": "^1.0.0" } }, + "packages/core": { "name": "core", "version": "1.0.0" } + }, + "packages": { + "a": ["a@workspace:packages/a"], + "core": ["core@workspace:packages/core"] + } +}`) + const a = new Package({ name: 'a', path: path.join(root, 'packages/a'), version: '1.0.0' }) + const core = new Package({ name: 'core', path: path.join(root, 'packages/core'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'peer-bun-fixture', path: root }), + packages: [a, core], + lockfile: Ok(path.join(root, 'bun.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['bun.lock', { filePath: path.join(root, 'bun.lock'), physical: true }], + ['packages/a/package.json', { filePath: path.join(root, 'packages/a/package.json'), physical: true }], + // The core member's manifest is deliberately not in the bundle: the + // backfill must trigger through the peer edge's link resolution. + ]) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('core') + }) + + it('skips notably when bun leaves no regenerated lockfile behind', async () => { + // Bun deletes a lockfile that would describe no packages ("No + // packages! Deleted empty lockfile"); the pruner must not blame the + // user's lockfile for that. + const { workspace, files } = makeBunScenario() + const script = 'require(\'fs\').unlinkSync(\'bun.lock\')' + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('was not found after the command completed'), + notable: true, + }) + }) + + it('fails when a bare-semver bun workspace resolution becomes a registry package', async () => { + // The bun analog of npm's silent registry substitution: the + // member-scoped workspace tuple disappears, so the edge that used to + // resolve to the workspace member now resolves to the same-named + // hoisted registry package. + const { workspace, files } = await makeMixedBunScenario() + const script = rewriteBunLockScript( + ['"a/ms": ["ms@workspace:packages/ms"],', ''], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining(`'ms' is no longer a workspace link`), + }) + }) + + it('does not mistake a bun registry edge for a link when the same-named member is pruned away', async () => { + // With member a shimmed to a dependency-free manifest, pruning + // legitimately drops both a's dependency on the ms member and the + // member-scoped workspace tuple — while the root keeps its registry + // ms. A name-global link classification would mark the root's + // registry edge as a link in the original and fail verification with + // a spurious "no longer a workspace link"; the per-edge probe must + // accept this prune. + const { workspace, files, a } = await makeMixedBunScenario() + files.set('packages/a/package.json', createFauxPackageFiles(a)[0]) + const script = rewriteBunLockScript( + ['"a/ms": ["ms@workspace:packages/ms"],', ''], + [', "dependencies": { "ms": "^1.0.0" }', ''], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.content).not.toContain('a/ms') + expect(result.content).toContain('ms@2.1.3') + }) + + it('fails when a bundled bun importer disappears from the lockfile', async () => { + const { workspace, files } = makeBunScenario() + const script = rewriteBunLockScript( + ['"packages/used": {', '"packages/renamed": {'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining(`lost the importer 'packages/used'`), + }) + }) + + it('skips notably for bun when the temp dir sits inside a workspace', async () => { + // A workspace ancestor of the temp dir can capture bun's root + // resolution — bun walks up and re-roots at a matching workspaces + // glob, then writes the regenerated lockfile at THAT root, outside + // the sandbox and over a real file. The pruner must refuse to run + // there. (The skip fires before any command is spawned, so this test + // needs no real bun.) + const outer = await makeTempDir() + // Deliberately JSONC (comment + trailing comma): bun's own + // package.json parser accepts this, so the ancestor scan must too — + // strict JSON.parse would miss the workspace and let bun escape. + await fs.writeFile(path.join(outer, 'package.json'), `{ + // ancestor workspace + "name": "ancestor", + "private": true, + "workspaces": ["**"], +}`) + const tmpInside = path.join(outer, 'tmp') + await fs.mkdir(tmpInside) + const { workspace, files } = makeBunScenario() + vi.stubEnv('TMPDIR', tmpInside) + vi.stubEnv('TEMP', tmpInside) + vi.stubEnv('TMP', tmpInside) + try { + const result = await pruneBundledLockfile({ + workspace, + packageManager: new BunDetector(), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('outside any workspace to enable pruning'), + notable: true, + }) + } finally { + vi.unstubAllEnvs() + } + }) + + it('skips notably for bun when an unparseable manifest shadows the temp dir', async () => { + // A package.json that fails even JSONC parsing cannot be ruled out as + // a workspace root (bun's own parser might still accept it), so the + // scan fails safe — with a reason that names the real cause instead + // of asserting a workspace exists. + const outer = await makeTempDir() + await fs.writeFile(path.join(outer, 'package.json'), 'not a manifest {{{') + const tmpInside = path.join(outer, 'tmp') + await fs.mkdir(tmpInside) + const { workspace, files } = makeBunScenario() + vi.stubEnv('TMPDIR', tmpInside) + vi.stubEnv('TEMP', tmpInside) + vi.stubEnv('TMP', tmpInside) + try { + const result = await pruneBundledLockfile({ + workspace, + packageManager: new BunDetector(), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('could not be ruled out as a workspace root'), + notable: true, + }) + } finally { + vi.unstubAllEnvs() + } + }) + + it.skipIf(process.env.CHECKLY_EXPECT_BUN === undefined)('bun is provisioned when CHECKLY_EXPECT_BUN is set', () => { + // The repo's own CI workflow sets CHECKLY_EXPECT_BUN after + // provisioning bun; keying off that flag (rather than the generic CI + // variable) keeps this from failing for users who run the suite with + // CI=true on machines that legitimately lack bun. + expect( + bunAvailable, + 'CHECKLY_EXPECT_BUN is set but bun is missing from PATH, so the real-bun pruner tests were' + + ' skipped. Restore the oven-sh/setup-bun step in .github/workflows/test.yml.', + ).toBe(true) + }) + + it.skipIf(!bunAvailable)('prunes the lockfile with real bun, backfilling link-referenced members', async () => { + const { workspace, files } = makeBunScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new BunDetector(), + files, + env: testEnv(), + }) + + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + + expect(result.archivePath).toEqual('bun.lock') + + // The root manifest declares @fixture/absent as workspace:*, so a faux + // manifest must have been backfilled for it. + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content)).toMatchObject({ + name: '@fixture/absent', + version: '1.0.0', + }) + + // Kept: every workspace member entry (the shimmed and backfilled + // members keep dependency-free importers) and the imported member's + // dependency. + expect(result.content).toContain('@fixture/used@workspace:packages/used') + expect(result.content).toContain('@fixture/shimmed@workspace:packages/shimmed') + expect(result.content).toContain('@fixture/absent@workspace:packages/absent') + expect(result.content).toContain('ms@2.1.3') + + // Dropped: dependencies of the shimmed and absent members. + expect(result.content).not.toContain('isarray') + expect(result.content).not.toContain('ee-first') + }, 60_000) + + // The yarn stub scripts below mutate the materialized yarn.lock via the + // same string-replacement helper as the bun ones. Search strings on the + // COMMITTED fixture must stay single-line: a Windows checkout may carry + // CRLF while hand-built lockfiles written by the tests are always LF. + + it('reports a yarn prune with the original content when only backfill is needed', async () => { + const { workspace, files } = makeYarnScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned', archivePath: 'yarn.lock' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('@fixture/absent') + }) + + it('fails when a kept yarn entry changes content', async () => { + // Any change WITHIN an entry (here the resolved version) makes its + // serialized value unknown to the original, which the subset check + // must reject as a fresh resolution. + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + ['version: 2.1.3', 'version: 2.1.4'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('not present in the original'), + }) + }) + + it('fails when the regenerated yarn metadata version is not a supported one', async () => { + // A different __metadata.version means the lockfile was rewritten by + // a different yarn generation (e.g. a newer yarn migrating the + // format); an unknown version fails the allowlist closed. + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + ['version: 10', 'version: 11'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('unsupported yarn.lock metadata version 11'), + }) + }) + + it('fails when the yarn metadata version changes between supported versions', async () => { + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + ['version: 10', 'version: 8'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('lockfile version changed from 10 to 8'), + }) + }) + + it('skips an unsupported yarn metadata version notably before any command runs', async () => { + const root = await makeTempDir() + await fs.cp(YARN_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'yarn.lock') + const content = await fs.readFile(lockfilePath, 'utf8') + // No newline in the search string: a Windows checkout may carry CRLF. + await fs.writeFile(lockfilePath, content.replace('version: 10', 'version: 11')) + const { workspace, files } = makeYarnScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported yarn.lock metadata version 11'), + notable: true, + }) + }) + + it('skips notably for a yarn metadata version that collides with an Object prototype key', async () => { + // The version allowlist must fail closed even for a corrupted + // lockfile whose version equals an inherited property name like + // 'toString', which a naive `in` check would wrongly accept. + const root = await makeTempDir() + await fs.cp(YARN_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'yarn.lock') + const content = await fs.readFile(lockfilePath, 'utf8') + await fs.writeFile(lockfilePath, content.replace('version: 10', 'version: toString')) + const { workspace, files } = makeYarnScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported yarn.lock metadata version toString'), + notable: true, + }) + }) + + it('fails when the yarn cacheKey changes', async () => { + // The cacheKey names the checksum scheme; a regeneration under a + // different scheme rewrote every checksum, which is a format change, + // not a prune. + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + ['cacheKey: 10c0', 'cacheKey: 8'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('cacheKey changed from 10c0 to 8'), + }) + }) + + it('accepts a regenerated yarn lockfile that dropped the cacheKey entirely', async () => { + // Yarn 3 omits the cacheKey when a lockfile resolves no registry + // packages, which a prune that removes the last registry entry + // legitimately arrives at — absent-on-one-side must not be treated + // as a scheme change. + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + [' cacheKey: 10c0', ''], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + }) + + it('skips a Yarn Classic lockfile notably', async () => { + // Realistic Classic content: an entry with a nested `dependencies:` + // block does NOT parse as YAML (plain scalars followed by a mapping), + // so Classic must be recognized by its header before parsing — a + // parse-failure message would wrongly imply a broken lockfile. The + // skip fires before any command runs. + const root = await makeTempDir() + await fs.cp(YARN_FIXTURE_ROOT, root, { recursive: true }) + await fs.writeFile(path.join(root, 'yarn.lock'), `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +debug@4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +`) + const { workspace, files } = makeYarnScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('Yarn Classic'), + notable: true, + }) + }) + + it('fails when a bundled yarn importer disappears from the lockfile', async () => { + const { workspace, files } = makeYarnScenario() + const script = rewriteYarnLockScript( + ['@workspace:packages/used', '@workspace:packages/renamed'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining(`lost the importer 'packages/used'`), + }) + }) + + // Builds a workspace where the `ms` member is consumed through a bare + // semver range — which yarn keys under BOTH the npm-range descriptor and + // the workspace descriptor ("ms@npm:^1.0.0, ms@workspace:packages/ms") + // — AND a same-named registry package is consumed by the root. Link + // classification for these edges depends entirely on the per-descriptor + // probe: a name-global answer would misclassify one of the two edges. + // The isarray entry carries two descriptors so that pruning its second + // consumer exercises descriptor re-keying (the key shrinks, the value + // does not). The lockfile is hand-built (always LF) mirroring yarn + // 4.18.0's real layout for this shape. + const makeMixedYarnScenario = async () => { + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/a'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/ms'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'mixed-yarn-fixture', + private: true, + workspaces: ['packages/*'], + dependencies: { isarray: '2.0.5', ms: '2.1.3' }, + })) + await fs.writeFile(path.join(root, 'packages/a/package.json'), JSON.stringify({ + name: 'a', + version: '1.0.0', + dependencies: { isarray: '^2.0.0', ms: '^1.0.0' }, + })) + await fs.writeFile(path.join(root, 'packages/ms/package.json'), JSON.stringify({ + name: 'ms', + version: '1.0.0', + })) + await fs.writeFile(path.join(root, 'yarn.lock'), `__metadata: + version: 10 + cacheKey: 10c0 + +"a@workspace:packages/a": + version: 0.0.0-use.local + resolution: "a@workspace:packages/a" + dependencies: + isarray: "npm:^2.0.0" + ms: "npm:^1.0.0" + languageName: unknown + linkType: soft + +"isarray@npm:2.0.5, isarray@npm:^2.0.0": + version: 2.0.5 + resolution: "isarray@npm:2.0.5" + checksum: 10c0/iii + languageName: node + linkType: hard + +"mixed-yarn-fixture@workspace:.": + version: 0.0.0-use.local + resolution: "mixed-yarn-fixture@workspace:." + dependencies: + isarray: "npm:2.0.5" + ms: "npm:2.1.3" + languageName: unknown + linkType: soft + +"ms@npm:2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/mmm + languageName: node + linkType: hard + +"ms@npm:^1.0.0, ms@workspace:packages/ms": + version: 0.0.0-use.local + resolution: "ms@workspace:packages/ms" + languageName: unknown + linkType: soft +`) + + const a = new Package({ name: 'a', path: path.join(root, 'packages/a'), version: '1.0.0' }) + const msMember = new Package({ name: 'ms', path: path.join(root, 'packages/ms'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'mixed-yarn-fixture', path: root }), + packages: [a, msMember], + lockfile: Ok(path.join(root, 'yarn.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['yarn.lock', { filePath: path.join(root, 'yarn.lock'), physical: true }], + ['packages/a/package.json', { filePath: path.join(root, 'packages/a/package.json'), physical: true }], + // The ms member's manifest is deliberately not in the bundle. + ]) + return { workspace, files, a } + } + + it('backfills a yarn member consumed through a bare semver range', async () => { + const { workspace, files } = await makeMixedYarnScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + // The manifest spec is '^1.0.0', not 'workspace:*', so the backfill + // can only trigger through the lockfile's per-descriptor link + // resolution (the "ms@npm:^1.0.0" descriptor resolving to a workspace + // entry), never through the spec prefix. + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('ms') + }) + + it('fails when a bare-semver yarn workspace resolution becomes a registry package', async () => { + // The yarn analog of npm's silent registry substitution: the npm-range + // descriptor no longer resolves to the workspace entry, so the edge + // that used to be a link is not one anymore. + const { workspace, files } = await makeMixedYarnScenario() + const script = rewriteYarnLockScript( + ['"ms@npm:^1.0.0, ms@workspace:packages/ms":', '"ms@workspace:packages/ms":'], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining(`'ms' is no longer a workspace link`), + }) + }) + + it('does not mistake a yarn registry edge for a link when the same-named member is unlinked by a prune', async () => { + // With member a shimmed to a dependency-free manifest, pruning + // legitimately drops a's edges, the npm-range descriptor on the ms + // member's key, and isarray's second descriptor — while the root keeps + // its registry ms and isarray. A name-global link classification would + // mark the root's registry ms edge as a link in the original and fail + // verification with a spurious "no longer a workspace link"; a + // key-based subset check would reject the shrunk isarray key despite + // its unchanged value. The per-descriptor probe and the value-based + // subset check must both accept this prune. + const { workspace, files, a } = await makeMixedYarnScenario() + files.set('packages/a/package.json', createFauxPackageFiles(a)[0]) + const script = rewriteYarnLockScript( + ['"ms@npm:^1.0.0, ms@workspace:packages/ms":', '"ms@workspace:packages/ms":'], + ['"isarray@npm:2.0.5, isarray@npm:^2.0.0":', '"isarray@npm:2.0.5":'], + ['\n dependencies:\n isarray: "npm:^2.0.0"\n ms: "npm:^1.0.0"', ''], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.content).not.toContain('ms@npm:^1.0.0') + expect(result.content).toContain('ms@npm:2.1.3') + }) + + // Builds a workspace where member a peer-depends on the core member with + // the same bare range that member b really depends on, so the lockfile + // keys core's entry under the shared range descriptor — and b is shimmed + // away in the bundle, so a realistic regeneration drops both b's edge + // and the shared descriptor while a's peer stays recorded (peers are + // manifest echoes, never resolved on their own). Peer edges must + // therefore never be probed against descriptors: classifying a's peer + // edge as a link in the original would fail this correct prune with a + // spurious 'no longer a workspace link'. Parameterized by lockfile + // generation, because the descriptor/spec spelling differs: metadata + // version 6 (yarn 3) records bare ranges, 8+ record the npm: protocol. + const makeSharedPeerDescriptorScenario = async ( + metadataVersion: number, cacheKey: string, specPrefix: string, + ) => { + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/a'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/b'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/core'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'peer-yarn-fixture', + private: true, + workspaces: ['packages/*'], + dependencies: { a: 'workspace:*', b: 'workspace:*' }, + })) + await fs.writeFile(path.join(root, 'packages/a/package.json'), JSON.stringify({ + name: 'a', + version: '1.0.0', + peerDependencies: { core: '^1.0.0' }, + })) + await fs.writeFile(path.join(root, 'packages/b/package.json'), JSON.stringify({ + name: 'b', + version: '1.0.0', + dependencies: { core: '^1.0.0' }, + })) + await fs.writeFile(path.join(root, 'packages/core/package.json'), JSON.stringify({ + name: 'core', + version: '1.0.0', + })) + const coreSpec = specPrefix === '' ? '^1.0.0' : `"${specPrefix}^1.0.0"` + await fs.writeFile(path.join(root, 'yarn.lock'), `__metadata: + version: ${metadataVersion} + cacheKey: ${cacheKey} + +"a@workspace:*, a@workspace:packages/a": + version: 0.0.0-use.local + resolution: "a@workspace:packages/a" + peerDependencies: + core: ^1.0.0 + languageName: unknown + linkType: soft + +"b@workspace:*, b@workspace:packages/b": + version: 0.0.0-use.local + resolution: "b@workspace:packages/b" + dependencies: + core: ${coreSpec} + languageName: unknown + linkType: soft + +"core@${specPrefix}^1.0.0, core@workspace:packages/core": + version: 0.0.0-use.local + resolution: "core@workspace:packages/core" + languageName: unknown + linkType: soft + +"peer-yarn-fixture@workspace:.": + version: 0.0.0-use.local + resolution: "peer-yarn-fixture@workspace:." + dependencies: + a: "workspace:*" + b: "workspace:*" + languageName: unknown + linkType: soft +`) + const a = new Package({ name: 'a', path: path.join(root, 'packages/a'), version: '1.0.0' }) + const b = new Package({ name: 'b', path: path.join(root, 'packages/b'), version: '1.0.0' }) + const core = new Package({ name: 'core', path: path.join(root, 'packages/core'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'peer-yarn-fixture', path: root }), + packages: [a, b, core], + lockfile: Ok(path.join(root, 'yarn.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['yarn.lock', { filePath: path.join(root, 'yarn.lock'), physical: true }], + ['packages/a/package.json', { filePath: path.join(root, 'packages/a/package.json'), physical: true }], + ['packages/b/package.json', createFauxPackageFiles(b)[0]], + // The core member's manifest is deliberately not in the bundle: it + // must be backfilled through b's real dependency edge. + ]) + // Simulates the realistic regeneration: the shared descriptor and b's + // dependency block disappear. + const script = rewriteYarnLockScript( + [`"core@${specPrefix}^1.0.0, core@workspace:packages/core":`, '"core@workspace:packages/core":'], + [`\n dependencies:\n core: ${coreSpec}`, ''], + ) + return { workspace, files, script } + } + + const expectPeerPruneAccepted = async ( + scenario: { workspace: Workspace, files: Map, script: string }, + ) => { + const result = await pruneBundledLockfile({ + workspace: scenario.workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', scenario.script])), + files: scenario.files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('core') + } + + it('accepts a prune that unlinks a peer edge sharing a descriptor with a pruned dependency', async () => { + await expectPeerPruneAccepted(await makeSharedPeerDescriptorScenario(10, '10c0', 'npm:')) + }) + + it('handles yarn 3 (metadata version 6) lockfiles, whose specs carry no npm: prefix', async () => { + // The as-written probe must classify b's prefix-less dependency edge + // as a link (backfilling core through it) on yarn 3 shapes too. + await expectPeerPruneAccepted(await makeSharedPeerDescriptorScenario(6, '8', '')) + }) + + it('classifies a bare numeric yarn 3 range that YAML would coerce to a number', async () => { + // Yarn 3 writes bare numeric ranges unquoted (`two: 2`), which the + // default YAML schema turns into a number; the parser must read the + // lockfile with the failsafe schema so the edge survives, resolves + // to the workspace descriptor and triggers the backfill. + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/a'), { recursive: true }) + await fs.mkdir(path.join(root, 'packages/two'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'numeric-fixture', + private: true, + workspaces: ['packages/*'], + dependencies: { a: 'workspace:*' }, + })) + await fs.writeFile(path.join(root, 'packages/a/package.json'), JSON.stringify({ + name: 'a', + version: '1.0.0', + dependencies: { two: '2' }, + })) + await fs.writeFile(path.join(root, 'packages/two/package.json'), JSON.stringify({ + name: 'two', + version: '2.0.0', + })) + await fs.writeFile(path.join(root, 'yarn.lock'), `__metadata: + version: 6 + cacheKey: 8 + +"a@workspace:*, a@workspace:packages/a": + version: 0.0.0-use.local + resolution: "a@workspace:packages/a" + dependencies: + two: 2 + languageName: unknown + linkType: soft + +"numeric-fixture@workspace:.": + version: 0.0.0-use.local + resolution: "numeric-fixture@workspace:." + dependencies: + a: "workspace:*" + languageName: unknown + linkType: soft + +"two@2, two@workspace:packages/two": + version: 0.0.0-use.local + resolution: "two@workspace:packages/two" + languageName: unknown + linkType: soft +`) + const a = new Package({ name: 'a', path: path.join(root, 'packages/a'), version: '1.0.0' }) + const two = new Package({ name: 'two', path: path.join(root, 'packages/two'), version: '2.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'numeric-fixture', path: root }), + packages: [a, two], + lockfile: Ok(path.join(root, 'yarn.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['yarn.lock', { filePath: path.join(root, 'yarn.lock'), physical: true }], + ['packages/a/package.json', { filePath: path.join(root, 'packages/a/package.json'), physical: true }], + // The two member's manifest is deliberately not in the bundle: the + // backfill can only trigger through the numeric-range edge. + ]) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('two') + }) + + it('skips notably when a yarn entry has an unexpected shape', async () => { + // The per-entry shape check is a load-bearing fail-closed guard: a + // future format that restructures entries must skip, not silently + // produce an empty snapshot. + const root = await makeTempDir() + await fs.cp(YARN_FIXTURE_ROOT, root, { recursive: true }) + const lockfilePath = path.join(root, 'yarn.lock') + const content = await fs.readFile(lockfilePath, 'utf8') + // No newline in the search string: a Windows checkout may carry CRLF. + await fs.writeFile(lockfilePath, content.replace(' resolution: "ms@npm:2.1.3"', '')) + const { workspace, files } = makeYarnScenario(root) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('unsupported yarn.lock entry shape'), + notable: true, + }) + }) + + it('backfills a yarn member consumed through the portal: protocol', async () => { + // portal:/link: specs count as links via their prefix (their entries + // have no @workspace: resolution to probe). + const root = await makeTempDir() + await fs.mkdir(path.join(root, 'packages/x'), { recursive: true }) + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'portal-root', + private: true, + dependencies: { x: 'portal:./packages/x' }, + })) + await fs.writeFile(path.join(root, 'packages/x/package.json'), JSON.stringify({ + name: 'x', + version: '1.0.0', + })) + await fs.writeFile(path.join(root, 'yarn.lock'), `__metadata: + version: 10 + cacheKey: 10c0 + +"portal-root@workspace:.": + version: 0.0.0-use.local + resolution: "portal-root@workspace:." + dependencies: + x: "portal:./packages/x" + languageName: unknown + linkType: soft + +"x@portal:./packages/x::locator=portal-root%40workspace%3A.": + version: 0.0.0-use.local + resolution: "x@portal:./packages/x::locator=portal-root%40workspace%3A." + languageName: node + linkType: soft +`) + const x = new Package({ name: 'x', path: path.join(root, 'packages/x'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'portal-root', path: root }), + packages: [x], + lockfile: Ok(path.join(root, 'yarn.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['yarn.lock', { filePath: path.join(root, 'yarn.lock'), physical: true }], + // The x member's manifest is deliberately not in the bundle. + ]) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', ''])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content).name).toEqual('x') + }) + + it('keeps a yarn patch: entry intact through a prune', async () => { + // patch: entries are ordinary non-workspace entries in the subset + // set; a prune that leaves them untouched must pass verification + // with the patched resolution intact. + const root = await makeTempDir() + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify({ + name: 'patch-root', + private: true, + dependencies: { isarray: '2.0.5', ms: 'patch:ms@npm%3A2.1.3#~/.yarn/patches/ms.patch' }, + })) + await fs.writeFile(path.join(root, 'yarn.lock'), `__metadata: + version: 10 + cacheKey: 10c0 + +"isarray@npm:2.0.5": + version: 2.0.5 + resolution: "isarray@npm:2.0.5" + checksum: 10c0/iii + languageName: node + linkType: hard + +"ms@npm:2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/mmm + languageName: node + linkType: hard + +"ms@patch:ms@npm%3A2.1.3#~/.yarn/patches/ms.patch": + version: 2.1.3 + resolution: "ms@patch:ms@npm%3A2.1.3#~/.yarn/patches/ms.patch::version=2.1.3&hash=125495" + checksum: 10c0/ppp + languageName: node + linkType: hard + +"patch-root@workspace:.": + version: 0.0.0-use.local + resolution: "patch-root@workspace:." + dependencies: + isarray: "npm:2.0.5" + ms: "patch:ms@npm%3A2.1.3#~/.yarn/patches/ms.patch" + languageName: unknown + linkType: soft +`) + const member = new Package({ name: 'unused-member', path: path.join(root, 'packages/none'), version: '1.0.0' }) + const workspace = new Workspace({ + root: new Package({ name: 'patch-root', path: root }), + packages: [member], + lockfile: Ok(path.join(root, 'yarn.lock')), + configFile: Err(new Error('no config file')), + }) + const files = new Map([ + ['package.json', { filePath: path.join(root, 'package.json'), physical: true }], + ['yarn.lock', { filePath: path.join(root, 'yarn.lock'), physical: true }], + ]) + const script = rewriteYarnLockScript( + ['"isarray@npm:2.0.5":\n version: 2.0.5\n resolution: "isarray@npm:2.0.5"\n checksum: 10c0/iii\n languageName: node\n linkType: hard\n\n', ''], + ['\n isarray: "npm:2.0.5"', ''], + ) + const result = await pruneBundledLockfile({ + workspace, + packageManager: stubPackageManager(new Runnable('node', ['-e', script])), + files, + env: testEnv(), + }) + expect(result).toMatchObject({ status: 'pruned' }) + if (result.status !== 'pruned') { + return + } + expect(result.content).toContain('ms@patch:ms@npm%3A2.1.3') + expect(result.content).not.toContain('isarray') + }) + + // Writes a fake `yarn` (POSIX shell script) onto a fresh PATH dir and + // returns the env to hand the pruner. `versionBody` runs for + // `yarn --version`, `installBody` for everything else; the default + // install body fails, so a test asserting a pre-spawn skip would see a + // 'failed' result instead if the install were (wrongly) reached. + const makeFakeYarnEnv = async ( + versionBody: string, + installBody = 'exit 1', + ): Promise => { + const binDir = await makeTempDir() + const fakeYarn = path.join(binDir, 'yarn') + await fs.writeFile(fakeYarn, `#!/bin/sh\nif [ "$1" = "--version" ]; then ${versionBody}; fi\n${installBody}\n`) + await fs.chmod(fakeYarn, 0o755) + return { ...testEnv(), PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ''}` } + } + + it.skipIf(process.platform === 'win32')('skips notably when yarn resolves to Yarn Classic', async () => { + // The pruner must refuse BEFORE spawning the install: Classic + // silently ignores --mode=update-lockfile and performs a full + // install, scripts included. + const { workspace, files } = makeYarnScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: await makeFakeYarnEnv('echo 1.22.22; exit 0'), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('Yarn Classic (1.x)'), + notable: true, + }) + }) + + it.skipIf(process.platform !== 'win32')('skips notably when yarn resolves to Yarn Classic on Windows', async () => { + // The Windows variant matters in its own right: the guard must + // resolve a `yarn.cmd` shim (which shell-less spawns cannot) and + // tolerate CRLF-terminated probe output. `%~1` strips the quotes + // cross-spawn wraps every cmd.exe argument in, so the shim matches + // `--version` exactly as the real corepack yarn.cmd (which forwards + // %* to node, whose argv parser strips them) would. + const binDir = await makeTempDir() + await fs.writeFile( + path.join(binDir, 'yarn.cmd'), + '@echo off\r\nif "%~1"=="--version" (\r\n echo 1.22.22\r\n exit /b 0\r\n)\r\nexit /b 1\r\n', + ) + const { workspace, files } = makeYarnScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: { ...testEnv(), PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ''}` }, + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('Yarn Classic (1.x)'), + notable: true, + }) + }) + + it.skipIf(process.platform === 'win32')('attempts the install when the yarn version probe fails', async () => { + // The probe fails OPEN by design: an unidentifiable yarn must not + // block a working prune, and the install's own error carries the + // real detail. This pins the choice; the trade-off (a Classic yarn + // whose --version somehow fails would still install) is accepted. + // The ambient PATH is stubbed too: the post-failure missing-binary + // classification (PathLookup) reads process.env, and this test must + // not depend on the host having a real yarn there. + const { workspace, files } = makeYarnScenario() + const env = await makeFakeYarnEnv('echo probe broken >&2; exit 7', 'echo install ran >&2; exit 9') + vi.stubEnv('PATH', env.PATH!) + try { + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env, + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('install ran'), + }) + } finally { + vi.unstubAllEnvs() + } + }) + + it.skipIf(process.platform === 'win32')('skips notably when the yarn generation does not match the lockfile', async () => { + // Yarn only reuses a lockfile written by its own generation; handed + // an older one it re-resolves everything, which the network guard + // blocks. The version mismatch must surface as an actionable skip, + // not a blocked-registry failure. A yarn-3 lockfile with a yarn-4 + // binary: + const { workspace, files } = makeYarn3Scenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: await makeFakeYarnEnv('echo 4.18.0; exit 0'), + }) + expect(result).toMatchObject({ + status: 'skipped', + reason: expect.stringContaining('written by yarn 3 (metadata version 6) but yarn resolves to 4.18.0'), + notable: true, + }) + }) + + it.skipIf(process.platform === 'win32')('disables hardened mode only for a confirmed yarn 4', async () => { + // YARN_ENABLE_HARDENED_MODE must reach a yarn-4 install (hardened + // mode is auto-enabled on PR CI and would trip the network guard), + // but must NOT be set for yarn 3, which rejects the unknown setting + // with a usage error. + // POSIX-gated test, so the temp path is already shell-safe. + const outFile = path.join(await makeTempDir(), 'env.txt') + const recordEnv = `echo "hardened=$YARN_ENABLE_HARDENED_MODE" > "${outFile}"; exit 1` + + const v10 = makeYarnScenario() + await pruneBundledLockfile({ + workspace: v10.workspace, + packageManager: new YarnDetector(), + files: v10.files, + env: await makeFakeYarnEnv('echo 4.18.0; exit 0', recordEnv), + }) + expect((await fs.readFile(outFile, 'utf8')).trim()).toEqual('hardened=0') + + const v6 = makeYarn3Scenario() + await pruneBundledLockfile({ + workspace: v6.workspace, + packageManager: new YarnDetector(), + files: v6.files, + env: await makeFakeYarnEnv('echo 3.8.7; exit 0', recordEnv), + }) + expect((await fs.readFile(outFile, 'utf8')).trim()).toEqual('hardened=') + }) + + it.skipIf(process.platform === 'win32')('fails clearly when the prune timeout is below the yarn floor', async () => { + // A caller-supplied timeout too small to run any install (the version + // probe returns instantly, so the budget itself is the problem) is + // reported as a misconfiguration, not a toolchain-provisioning delay. + const { workspace, files } = makeYarnScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: await makeFakeYarnEnv('echo 4.18.0; exit 0'), + timeoutMs: 100, + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('below the minimum needed to run yarn'), + }) + }) + + it.skipIf(process.platform === 'win32')('fails clearly when the yarn version probe eats the prune budget', async () => { + // A probe (e.g. a first-use corepack download) slow enough to leave + // less than the install floor is attributed to provisioning, not a + // second install timeout. + const { workspace, files } = makeYarnScenario() + // The probe sleeps well within the timeout (so it never itself times + // out) but leaves under the 1s install floor: timeout 3000 − ~1500 + // sleep = ~1500 remaining is above the floor's own comparison only + // if the sleep is longer, so sleep 2.2s → ~800ms remaining. + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: await makeFakeYarnEnv('sleep 2.2; echo 4.18.0; exit 0'), + timeoutMs: 3000, + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('provisioning the yarn toolchain used up the prune time budget'), + }) + }, 15_000) + + // The network guard's own error names the user's "configuration + // settings", which reads like a broken setup; the pruner must name the + // real causes (stale lockfile, or a same-generation yarn that still + // declines to reuse it) instead. Real yarn prints YN0080 on STDOUT, but + // an install that also emits unrelated stderr must still be recognized, + // so the rewrite scans both streams. The ambient PATH is stubbed for + // the same reason as in the probe-failure test above. + const blockedMessage = + `YN0080: ms@npm:2.1.3: Request to 'https://registry.yarnpkg.com/ms' has been blocked because of your configuration settings` + for (const stream of ['stdout', 'stdout-with-stderr-noise']) { + it.skipIf(process.platform === 'win32')( + `rewrites yarn blocked-request failures into an actionable reason (${stream})`, async () => { + const { workspace, files } = makeYarnScenario() + const noise = stream === 'stdout-with-stderr-noise' ? 'echo "warning: some unrelated notice" >&2; ' : '' + const env = await makeFakeYarnEnv('echo 4.18.0; exit 0', `${noise}echo "${blockedMessage}"; exit 1`) + vi.stubEnv('PATH', env.PATH!) + try { + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env, + }) + expect(result).toMatchObject({ + status: 'failed', + reason: expect.stringContaining('pin it via the packageManager field'), + }) + if (result.status !== 'failed') { + return + } + // Yarn's own output stays attached (the descriptor is otherwise + // unrecoverable), but only after the actionable explanation. + expect(result.reason.indexOf('pin it via')).toBeLessThan(result.reason.indexOf('YN0080')) + } finally { + vi.unstubAllEnvs() + } + }) + } + + // Both real-yarn generations run the same end-to-end shape; the only + // differences are the fixture (and hence the pinned yarn) and the + // checksum spelling (yarn 3 writes bare hex without the cacheKey + // prefix). Environment-level incompatibilities have already differed + // between the generations (the hardened-mode setting does not exist + // before yarn 4), so both must stay covered. + for (const generation of [ + { + name: 'yarn 4', + fixture: 'yarn-workspace', + makeScenario: makeYarnScenario, + available: yarnBerryAvailable, + checksumMarker: 'checksum: 10c0/', + }, + { + name: 'yarn 3', + fixture: 'yarn3-workspace', + makeScenario: makeYarn3Scenario, + available: yarn3Available, + checksumMarker: 'checksum:', + }, + ]) { + it.skipIf(process.env.CHECKLY_EXPECT_YARN === undefined)( + `${generation.name} is provisioned when CHECKLY_EXPECT_YARN is set`, () => { + // The repo's own CI workflow sets CHECKLY_EXPECT_YARN after + // running `corepack enable yarn` and pre-downloading both pinned + // versions. The probe checks the resolved major from the fixture + // directory, so a preinstalled Yarn Classic shadowing the + // corepack shim fails here instead of silently skipping the + // real-yarn tests below. + expect( + generation.available, + `CHECKLY_EXPECT_YARN is set but yarn does not resolve to the ${generation.fixture}` + + ` fixture's pinned version, so the real-${generation.name} pruner tests were skipped.` + + ' Restore the `corepack enable yarn` step in .github/workflows/test.yml.', + ).toBe(true) + }) + + it.skipIf(!generation.available)( + `prunes the lockfile with real ${generation.name}, backfilling link-referenced members`, async () => { + const { workspace, files } = generation.makeScenario() + const result = await pruneBundledLockfile({ + workspace, + packageManager: new YarnDetector(), + files, + env: testEnv(), + }) + + expect(result.status).toEqual('pruned') + if (result.status !== 'pruned') { + return + } + + expect(result.archivePath).toEqual('yarn.lock') + + // The root manifest declares @fixture/absent as workspace:*, so a + // faux manifest must have been backfilled for it. + expect(result.backfilledManifests).toHaveLength(1) + expect(JSON.parse(result.backfilledManifests[0].content)).toMatchObject({ + name: '@fixture/absent', + version: '1.0.0', + }) + + // Kept: every workspace member entry (the shimmed and backfilled + // members keep dependency-free importers) and the imported + // member's dependency with its checksum. Byte-identity of kept + // entries is guaranteed by the pruned status (the value-based + // subset check). + expect(result.content).toContain('@fixture/used@workspace:packages/used') + expect(result.content).toContain('@fixture/shimmed@workspace:packages/shimmed') + expect(result.content).toContain('@fixture/absent@workspace:packages/absent') + expect(result.content).toContain('ms@npm:2.1.3') + expect(result.content).toContain(generation.checksumMarker) + + // Dropped: dependencies of the shimmed and absent members. + expect(result.content).not.toContain('isarray') + expect(result.content).not.toContain('ee-first') + }, 60_000) + } + }) +}) diff --git a/packages/cli/src/services/check-parser/bundler.ts b/packages/cli/src/services/check-parser/bundler.ts index 50edba193..e53d65188 100644 --- a/packages/cli/src/services/check-parser/bundler.ts +++ b/packages/cli/src/services/check-parser/bundler.ts @@ -9,10 +9,23 @@ import type { Archiver } from 'archiver' import Debug from 'debug' import * as uuid from 'uuid' +import { createHash } from 'node:crypto' + import { checklyStorage } from '../../rest/api.js' import { PayloadTooLargeError } from '../../rest/errors.js' -import { EMBEDDED_PACKAGES_ARCHIVE_DIR, EmbeddedPackagesMaterializer } from '../embedded-packages/materializer.js' -import { computeWorkspaceCacheHash, ComputeWorkspaceCacheHashOptions } from './cache-hash.js' +import { EMBEDDED_PACKAGES_ARCHIVE_DIR, EmbeddedPackagesMaterializer, PlannedTarball } from '../embedded-packages/materializer.js' +import { filterTarballsByLockfile } from '../embedded-packages/lockfile-filter.js' +import { + composeWorkspaceCacheHash, + ComposeWorkspaceCacheHashOptions, + ComputeWorkspaceCacheHashOptions, + EmbeddedPackageInput, + FauxPackageJsonInput, + loadWorkspaceCacheHashInputs, + LockfileInput, +} from './cache-hash.js' +import { pruneBundledLockfile } from './lockfile-pruner.js' +import { PackageManager } from './package-files/package-manager.js' import { File } from './parser.js' import { Workspace } from './package-files/workspace.js' import { pathToPosix } from '../util.js' @@ -450,26 +463,100 @@ export type CreateBundlerForWorkspaceOptions = & { /** * The materializer for the project's `bundle.packages.embed` option, - * when set. Its resolved tarball set (name, version, integrity) is mixed - * into the cache hash: embedded tarballs change the runner's install-step - * inputs without necessarily touching the lockfile, so a changed embed - * set must invalidate the dependency cache. + * when set. The tarballs are materialized during finalize(), after the + * bundled lockfile has been pruned, so only tarballs the shipped + * lockfile still references are downloaded and shipped. That same + * filtered set (name, version, integrity) is mixed into the cache hash: + * embedded tarballs change the runner's install-step inputs without + * necessarily touching the lockfile, so a changed embed set must + * invalidate the dependency cache. */ embeddedPackagesMaterializer?: EmbeddedPackagesMaterializer + + /** + * The workspace's package manager. When it supports a lockfile-only + * install, the bundled lockfile is pruned during finalize() to match the + * bundle's actual set of manifests. + */ + packageManager: PackageManager } +interface PrunedLockfile extends LockfileInput { + /** + * The pruned lockfile bytes, needed to decide which embedded package + * tarballs the shipped lockfile still references. Inert for the cache + * hash, which only consumes the name and hash. + */ + content: string +} + +/** + * Bundle-time cache-hash inputs. `embeddedPackages` is a required key + * (though its value may be undefined) so that no hash computation can + * silently omit the shipped embedded set — an omission would not change + * the bundle's bytes, only desync the runner's dependency cache key. + */ +type BundleTimeCacheHashInputs = + Pick + & { embeddedPackages: ComposeWorkspaceCacheHashOptions['embeddedPackages'] } + +/** + * Maps planned tarballs to cache-hash records. yarn.lock plans carry no SRI + * tarball integrity (it is resolved from registry metadata only at + * materialization time), so their records use the lockfile's own checksum — + * an equally stable content pin that is known at plan time, keeping the + * eager placeholder hash and the finalize hash consistent. The parsers + * guarantee one of the two hashes is always present; the empty-string + * fallback only satisfies the type. + */ +export function embeddedPackageHashInputs (tarballs: PlannedTarball[] | undefined): EmbeddedPackageInput[] | undefined { + return tarballs?.map(({ name, version, integrity, lockfileChecksum }) => ({ + name, + version, + integrity: integrity ?? lockfileChecksum ?? '', + })) +} + +/** + * Everything finalize() needs to prune the lockfile and recompute the cache + * hash from the bundle's actual contents. + */ +interface WorkspaceBundleContext { + workspace: Workspace + packageManager: PackageManager + /** + * The embedded-packages materializer, carried to finalize() so the + * tarballs the (possibly pruned) bundled lockfile still references can be + * materialized there — after pruning, so pruned-away tarballs are never + * downloaded. The planned set is re-derived from the materializer's + * memoized plan(). + */ + embeddedPackagesMaterializer?: EmbeddedPackagesMaterializer + /** + * Composes the cache hash from the workspace inputs captured at + * construction time, plus the given bundle-time inputs — including the + * embedded set actually shipped, which every caller passes explicitly. + * Capturing the composition (rather than its ingredients) keeps + * createForWorkspace and finalize() from having to spell the same + * argument list twice. + */ + composeCacheHash: (extra: BundleTimeCacheHashInputs) => string +} + interface BundlerOptions { tempDir?: string cacheHash: string stripPrefix?: string + workspaceContext?: WorkspaceBundleContext } export class Bundler { #id: string #marker: BundlePathMarker - #cacheHash: string + #cacheHashMarker: CacheHashMarker #tempDir?: string #stripPrefix?: string + #workspaceContext?: WorkspaceBundleContext #files = new Map() private constructor (options: BundlerOptions) { @@ -477,15 +564,24 @@ export class Bundler { tempDir, cacheHash, stripPrefix, + workspaceContext, } = options this.#id = uuid.v4() this.#marker = new BundlePathMarker(`bundle:${this.#id}`) - this.#cacheHash = cacheHash + this.#cacheHashMarker = new CacheHashMarker(cacheHash) this.#stripPrefix = stripPrefix this.#tempDir = tempDir + this.#workspaceContext = workspaceContext } + /** + * Creates a bundler without a workspace context. Workspace-dependent + * finalize() behavior — lockfile pruning, embedded package + * materialization (`bundle.packages.embed`) and the cache-hash recompute + * — only happens for bundlers built with {@link createForWorkspace}; a + * plain bundler archives exactly the files registered into it. + */ // eslint-disable-next-line require-await static async create (options: CreateBundlerOptions): Promise { debug(`Creating bundler`) @@ -494,7 +590,7 @@ export class Bundler { static async createForWorkspace ( workspace: Workspace, - options: CreateBundlerForWorkspaceOptions = {}, + options: CreateBundlerForWorkspaceOptions, ): Promise { debug(`Creating bundler for workspace`) @@ -502,16 +598,34 @@ export class Bundler { tempDir, dependencyCacheVersion, embeddedPackagesMaterializer, + packageManager, } = options const embeddedPackages = (await embeddedPackagesMaterializer?.plan())?.tarballs - const cacheHash = await computeWorkspaceCacheHash(workspace, { dependencyCacheVersion, embeddedPackages }) + // The composition is captured so finalize() can recompute the hash with + // bundle-time additions (faux manifests, a pruned lockfile) from the + // same workspace inputs, loaded once. The eager value below is only a + // placeholder for the window before finalize() runs — finalize() always + // recomputes it. + const cacheHashInputs = await loadWorkspaceCacheHashInputs(workspace) + const composeCacheHash = (extra: BundleTimeCacheHashInputs): string => { + return composeWorkspaceCacheHash(cacheHashInputs, { + dependencyCacheVersion, + ...extra, + }) + } return new Bundler({ tempDir, - cacheHash, + cacheHash: composeCacheHash({ embeddedPackages: embeddedPackageHashInputs(embeddedPackages) }), stripPrefix: workspace?.root.path, + workspaceContext: { + workspace, + packageManager, + embeddedPackagesMaterializer, + composeCacheHash, + }, }) } @@ -523,8 +637,15 @@ export class Bundler { this.#marker.updateValue(newValue) } - get cacheHash (): string { - return this.#cacheHash + /** + * The dependency cache hash. A mutable holder rather than a plain string: + * consumers copy it into check payloads during bundle(), but the final + * value — reflecting faux manifests and a possibly pruned lockfile — is + * only known once finalize() has run, the same ordering problem + * {@link BundlePathMarker} solves for the archive path. + */ + get cacheHash (): CacheHashMarker { + return this.#cacheHashMarker } /** @@ -555,7 +676,160 @@ export class Bundler { } } + /** + * Prunes the bundled lockfile to match the bundle's final file set, + * materializes the embedded package tarballs the (possibly pruned) + * lockfile still references, and recomputes the cache hash from the + * bundle's actual install inputs (faux manifests, the pruned lockfile and + * the shipped embedded set). Runs at finalize time because all of it + * depends on the complete file set, which only exists once every check + * has registered its files — and because materializing after pruning is + * what keeps pruned-away tarballs from ever being downloaded. + */ + async #refreshWorkspaceBundle (): Promise { + const context = this.#workspaceContext + if (context === undefined) { + return + } + + const pruned = await this.#pruneLockfile(context) + const embeddedPackages = await this.#materializeEmbeddedPackages(context, pruned) + + const fauxPackageJsons: FauxPackageJsonInput[] = [] + for (const [archivePath, file] of this.#files) { + if (file.physical || path.posix.basename(archivePath) !== 'package.json') { + continue + } + fauxPackageJsons.push({ path: archivePath, raw: Buffer.from(file.content, 'utf8') }) + } + + // Unconditional: with no faux manifests, no pruned lockfile and an + // unfiltered embedded set this reproduces the exact digest computed in + // createForWorkspace (empty record groups write nothing). + this.#cacheHashMarker.updateValue(context.composeCacheHash({ + fauxPackageJsons, + prunedLockfile: pruned, + embeddedPackages: embeddedPackageHashInputs(embeddedPackages), + })) + } + + /** + * Materializes the embedded package tarballs into the bundle: the full + * planned set, or — when the bundled lockfile was pruned — only the + * tarballs the pruned lockfile still references, so pruned-away packages + * are never downloaded. Returns the shipped set for the cache hash. + */ + async #materializeEmbeddedPackages ( + context: WorkspaceBundleContext, + pruned: PrunedLockfile | undefined, + ): Promise { + const materializer = context.embeddedPackagesMaterializer + if (materializer === undefined) { + return undefined + } + + // plan() is memoized and was already awaited in createForWorkspace, so + // this resolves the same promise without extra work. + const { tarballs: embeddedPackages } = await materializer.plan() + + // The empty-bundle arm is load-bearing: every command calls finalize() + // unconditionally, including on bundles no check registered files into, + // and an empty bundle must not trigger downloads (nor the materializer's + // plan-issues backstop, which validation never ran for a project + // without Playwright checks). Nothing ships from an empty bundle, so + // nothing reaches the hash either. + if (this.isEmpty) { + return undefined + } + + let kept = embeddedPackages + if (pruned !== undefined) { + const filtered = filterTarballsByLockfile(embeddedPackages, pruned.content, pruned.name) + kept = filtered.kept + if (filtered.dropped.length > 0) { + debug(`Embedded packages dropped with the pruned lockfile: ${ + filtered.dropped.map(tarball => `${tarball.name}@${tarball.version}`).join(', ')}`) + } + } + + // Debug rather than user-facing output for now: a bare stderr line has + // no good home in the current CLI output UX. Revisit when finalize-time + // work gets proper progress reporting. + if (kept.length > 0) { + debug(`Preparing ${kept.length} embedded package tarball(s)`) + } + + // Deliberately called even for an empty kept set: the materializer's + // plan-issues backstop must still reject a plan whose specs all failed + // to resolve. + const materialized = await materializer.materializeTarballs(kept) + this.registerFiles(...materialized.map(tarball => ({ + filePath: tarball.filePath, + physical: true as const, + archivePath: tarball.archivePath, + }))) + + return kept + } + + async #pruneLockfile (context: WorkspaceBundleContext): Promise { + const result = await pruneBundledLockfile({ + workspace: context.workspace, + packageManager: context.packageManager, + files: this.#files, + }) + + if (result.status === 'failed') { + process.stderr.write( + `Warning: could not prune the bundled lockfile: ${result.reason}. ` + + `Falling back to the original lockfile; it may reference workspace packages and ` + + `dependencies that are not part of the bundle. If the lockfile is out of date, ` + + `run your package manager's install to refresh it; set CHECKLY_LOCKFILE_PRUNE=0 ` + + `to disable pruning.\n`, + ) + return + } + if (result.status === 'skipped') { + if (result.notable) { + // The bundle is a partial workspace, so the unpruned lockfile + // over-describes it — say so instead of failing silently on the + // remote install. + process.stderr.write( + `Note: the bundled lockfile was not pruned (${result.reason}); it may reference ` + + `workspace packages and dependencies that are not part of the bundle. If this ` + + `setup cannot be pruned, set CHECKLY_LOCKFILE_PRUNE=0 to opt out of pruning ` + + `(and this note) entirely.\n`, + ) + } else { + debug(`Lockfile pruning skipped: ${result.reason}`) + } + return + } + + // Set entries directly rather than through registerFiles: its + // prefer-physical dedup would keep the original lockfile, and would drop + // a backfilled manifest whose path is occupied by a symlink entry — + // desyncing the bundle from the lockfile the prune was computed against. + for (const manifest of result.backfilledManifests) { + this.#files.set(archivePath(manifest, this.#stripPrefix), manifest) + } + this.#files.set(result.archivePath, { + filePath: context.workspace.lockfile.unwrap(), + physical: false, + content: result.content, + }) + debug(`Pruned bundled lockfile ${result.archivePath}`) + + return { + name: path.posix.basename(result.archivePath), + hash: createHash('sha256').update(result.content).digest(), + content: result.content, + } + } + async finalize (): Promise { + await this.#refreshWorkspaceBundle() + const archive = await BundleArchive.create({ tempDir: this.#tempDir, stripPrefix: this.#stripPrefix, @@ -601,3 +875,28 @@ export class BundlePathMarker { return this.#value } } + +/** + * Mutable holder for the dependency cache hash, serialized as a plain + * string. See {@link Bundler.cacheHash} for why a holder is needed. + * + * Deliberately a standalone class rather than a subclass of + * {@link BundlePathMarker}: the own `#value` field makes the two marker + * types nominally incompatible, so a bundle path cannot be passed where the + * cache hash is expected (or vice versa) without a compile error. + */ +export class CacheHashMarker { + #value: string + + constructor (initialValue: string) { + this.#value = initialValue + } + + updateValue (newValue: string) { + this.#value = newValue + } + + toJSON (): string { + return this.#value + } +} diff --git a/packages/cli/src/services/check-parser/cache-hash.ts b/packages/cli/src/services/check-parser/cache-hash.ts index ee16caf08..bf9417389 100644 --- a/packages/cli/src/services/check-parser/cache-hash.ts +++ b/packages/cli/src/services/check-parser/cache-hash.ts @@ -37,26 +37,64 @@ export interface NpmrcInput { hash: Buffer } +export interface PnpmfileInput { + /** + * Forward-slash relative path matching the pnpmfile's location in the + * eventual archive (e.g. ".pnpmfile.cjs" or ".pnpmfile.mjs"). + */ + path: string + /** + * Raw 32-byte SHA-256 digest of the pnpmfile contents. + */ + hash: Buffer +} + export interface EmbeddedPackageInput { /** Package name as recorded in the lockfile, e.g. `@acme/foo`. */ name: string /** Exact version, e.g. `1.2.3`. */ version: string - /** The lockfile's recorded integrity for the artifact (SRI string). */ + /** + * The lockfile's recorded content pin for the artifact: an SRI integrity + * string, or for Yarn Berry lockfiles (which record no npm tarball + * integrity) yarn's own checksum value — equally stable per content. + */ integrity: string } +export interface FauxPackageJsonInput { + /** + * Forward-slash relative path matching the faux manifest's location in the + * eventual archive (e.g. "packages/member/package.json"). + */ + path: string + /** + * The faux manifest's raw content bytes. + */ + raw: Buffer +} + export interface ComposeCacheHashInput { lockfile?: LockfileInput packageJsons: PackageJsonInput[] npmrcs?: NpmrcInput[] + /** + * The workspace root's bundleable pnpmfiles (pnpm workspaces only). pnpm's + * install hooks change resolution results without necessarily touching the + * lockfile (the recorded `pnpmfileChecksum` only updates when the user + * reinstalls), so the bundled files must contribute to the hash. An empty + * or absent list writes no records, leaving the digest identical to one + * computed before this input existed. + */ + pnpmfiles?: PnpmfileInput[] /** * The resolved set of embedded package tarballs shipped in the bundle - * (`bundle.packages.embed` after lockfile resolution). Embedded tarballs - * change the runner's install-step inputs without necessarily touching the - * lockfile, so they must contribute to the hash. An empty or absent list - * writes no records, leaving the digest identical to one computed before - * this input existed. + * (`bundle.packages.embed` after lockfile resolution, filtered to what + * the shipped — possibly pruned — bundled lockfile still references). + * Embedded tarballs change the runner's install-step inputs without + * necessarily touching the lockfile, so they must contribute to the hash. + * An empty or absent list writes no records, leaving the digest identical + * to one computed before this input existed. */ embeddedPackages?: EmbeddedPackageInput[] excludedFields: string[] @@ -66,6 +104,24 @@ export interface ComposeCacheHashInput { * so the digest stays identical to one computed without this input. */ dependencyCacheVersion?: string + /** + * Every synthesized (non-physical) `package.json` actually shipped in the + * bundle — in practice the faux workspace member manifests. Unlike + * on-disk manifests — whose `version` is excluded because the pinned + * lockfile absorbs it — a synthesized manifest's full content including + * its version is load-bearing for the remote install (it decides whether + * a specifier resolves to the workspace link, the registry, or fails), so + * these are hashed verbatim, exactly as the bytes ship. An empty or + * absent list writes no records, leaving the digest unchanged. + */ + fauxPackageJsons?: FauxPackageJsonInput[] + /** + * The pruned lockfile actually shipped in the bundle, when lockfile + * pruning replaced the original. The pruned bytes are the runner's real + * install input, so they must contribute to the hash. Absent when the + * original lockfile ships unchanged, writing no record. + */ + prunedLockfile?: LockfileInput } const PACKAGE_JSON_EXCLUDED_FIELDS = ['version'] @@ -179,17 +235,27 @@ export function canonicalizePackageJson (raw: Buffer, excludedFields: string[]): * 3. One record per .npmrc sorted by path, labeled * `npmrc:`, whose content is the raw 32-byte SHA-256 * digest of the .npmrc contents. - * 4. One record per embedded package sorted by `name@version`, labeled + * 4. One record per bundleable pnpmfile sorted by path, labeled + * `pnpmfile:`, whose content is the raw 32-byte SHA-256 + * digest of the pnpmfile contents. + * 5. One record per embedded package sorted by `name@version`, labeled * `embedded-package:`, whose content is the raw UTF-8 - * bytes of the lockfile's integrity string for the artifact. Callers - * must pass at most one entry per `name@version` (the materializer - * already de-duplicates); the record order among duplicate keys is - * undefined. - * 5. The dependency cache version record (if set to a non-empty string), + * bytes of the lockfile's content pin for the artifact — its SRI + * integrity string, or for Yarn Berry lockfiles (which record no npm + * tarball integrity) yarn's own checksum value. Callers must pass at + * most one entry per `name@version` (the materializer already + * de-duplicates); the record order among duplicate keys is undefined. + * 6. The dependency cache version record (if set to a non-empty string), * labeled `dependency-cache-version`, whose content is the raw UTF-8 * bytes of the user-provided value. An empty string is treated as * absent so that e.g. an unset environment variable interpolated into * the config leaves the digest unchanged. + * 7. One record per faux workspace member manifest sorted by path, + * labeled `faux-package.json:`, whose content is the + * manifest's raw UTF-8 bytes. + * 8. The pruned lockfile record (if present), labeled + * `pruned-lockfile:`, whose content is the raw 32-byte + * SHA-256 digest of the pruned lockfile contents. * * All sorts compare strings by UTF-16 code unit (JavaScript's `<`/`>`), * which coincides with byte-wise UTF-8 order for ASCII inputs — the only @@ -224,6 +290,12 @@ export function composeCacheHash (input: ComposeCacheHashInput): string { writeRecord(`npmrc:${entry.path}`, entry.hash) } + const sortedPnpmfiles = [...(input.pnpmfiles ?? [])].sort((a, b) => compareStrings(a.path, b.path)) + + for (const entry of sortedPnpmfiles) { + writeRecord(`pnpmfile:${entry.path}`, entry.hash) + } + const sortedEmbedded = (input.embeddedPackages ?? []) .map(entry => ({ key: `${entry.name}@${entry.version}`, integrity: entry.integrity })) .sort((a, b) => compareStrings(a.key, b.key)) @@ -236,6 +308,16 @@ export function composeCacheHash (input: ComposeCacheHashInput): string { writeRecord('dependency-cache-version', Buffer.from(input.dependencyCacheVersion, 'utf8')) } + const sortedFaux = [...(input.fauxPackageJsons ?? [])].sort((a, b) => compareStrings(a.path, b.path)) + + for (const entry of sortedFaux) { + writeRecord(`faux-package.json:${entry.path}`, entry.raw) + } + + if (input.prunedLockfile) { + writeRecord(`pruned-lockfile:${input.prunedLockfile.name}`, input.prunedLockfile.hash) + } + return hash.digest('hex') } @@ -263,10 +345,16 @@ function uint64BE (n: number): Buffer { * lockfile) invalidates the bundle cache. Packages without an `.npmrc` * contribute nothing, so a workspace with no `.npmrc` produces a hash * identical to before this input existed. + * + * The workspace's bundleable pnpmfiles (see {@link Workspace.pnpmfiles}) are + * hashed for the same reason: their install hooks change resolution results, + * and the lockfile's recorded `pnpmfileChecksum` only updates when the user + * reinstalls, so the lockfile bytes alone do not cover them. A workspace + * without bundleable pnpmfiles contributes nothing. */ export async function loadWorkspaceCacheHashInputs ( workspace: Workspace, -): Promise<{ lockfile?: LockfileInput, packageJsons: PackageJsonInput[], npmrcs: NpmrcInput[] }> { +): Promise { const allPackages = [workspace.root, ...workspace.packages] const packageJsons = await Promise.all(allPackages.map(async pkg => { @@ -311,7 +399,21 @@ export async function loadWorkspaceCacheHashInputs ( } } - return { lockfile, packageJsons, npmrcs } + // Hash exactly the pnpmfiles that get bundled (Workspace.pnpmfiles is the + // shared source of truth), so the cache key always reflects the bundle + // contents. A read error here must surface: silently dropping a file that + // would still be bundled would desync the cache key from the bundle. + const pnpmfiles = await Promise.all(workspace.pnpmfiles + .filter(info => info.skipReason === undefined) + .map(async (info): Promise => { + const bytes = await fs.readFile(info.path) + return { + path: path.relative(workspace.root.path, info.path).split(path.sep).join('/'), + hash: createHash('sha256').update(bytes).digest(), + } + })) + + return { lockfile, packageJsons, npmrcs, pnpmfiles } } export interface ComputeWorkspaceCacheHashOptions { @@ -352,6 +454,44 @@ export function normalizeDependencyCacheVersion (version: string | number | unde return version } +export interface WorkspaceCacheHashInputs { + lockfile?: LockfileInput + packageJsons: PackageJsonInput[] + npmrcs: NpmrcInput[] + pnpmfiles: PnpmfileInput[] +} + +export interface ComposeWorkspaceCacheHashOptions extends ComputeWorkspaceCacheHashOptions { + /** + * See {@link ComposeCacheHashInput.fauxPackageJsons}. + */ + fauxPackageJsons?: FauxPackageJsonInput[] + /** + * See {@link ComposeCacheHashInput.prunedLockfile}. + */ + prunedLockfile?: LockfileInput +} + +/** + * Composes the workspace cache hash from pre-loaded inputs, with the + * standard set of excluded package.json fields. Lets callers that need to + * recompute the hash later (with bundle-time inputs like faux manifests or + * a pruned lockfile) reuse inputs loaded once. + */ +export function composeWorkspaceCacheHash ( + inputs: WorkspaceCacheHashInputs, + options?: ComposeWorkspaceCacheHashOptions, +): string { + return composeCacheHash({ + ...inputs, + embeddedPackages: options?.embeddedPackages, + fauxPackageJsons: options?.fauxPackageJsons, + prunedLockfile: options?.prunedLockfile, + excludedFields: PACKAGE_JSON_EXCLUDED_FIELDS, + dependencyCacheVersion: normalizeDependencyCacheVersion(options?.dependencyCacheVersion), + }) +} + /** * Convenience wrapper that loads workspace inputs and composes the cache * hash with the standard set of excluded package.json fields. @@ -361,10 +501,5 @@ export async function computeWorkspaceCacheHash ( options?: ComputeWorkspaceCacheHashOptions, ): Promise { const inputs = await loadWorkspaceCacheHashInputs(workspace) - return composeCacheHash({ - ...inputs, - embeddedPackages: options?.embeddedPackages, - excludedFields: PACKAGE_JSON_EXCLUDED_FIELDS, - dependencyCacheVersion: normalizeDependencyCacheVersion(options?.dependencyCacheVersion), - }) + return composeWorkspaceCacheHash(inputs, options) } diff --git a/packages/cli/src/services/check-parser/faux-package.ts b/packages/cli/src/services/check-parser/faux-package.ts index 44ef0d84c..397581820 100644 --- a/packages/cli/src/services/check-parser/faux-package.ts +++ b/packages/cli/src/services/check-parser/faux-package.ts @@ -5,14 +5,23 @@ export const FAUX_PACKAGE_DESCRIPTION = `This is a placeholder for an ` + `otherwise unused package that Checkly determined to be needed during ` + `the installation step.` +/** + * The version used when the package's real version cannot be determined + * (e.g. its package.json is unreadable or has no version field). + */ +export const FAUX_PACKAGE_FALLBACK_VERSION = '0.0.0' + export function createFauxPackageFiles (pkg: Package): VirtualFile[] { + // Carry the package's real version so that specifiers like `workspace:^1.2.3` + // (pnpm) or plain semver ranges (npm) still resolve to the workspace package + // rather than failing or falling back to a registry lookup. return [{ filePath: pkg.packageJsonPath, physical: false, content: JSON.stringify( { name: pkg.name, - version: '0.0.0', + version: pkg.version ?? FAUX_PACKAGE_FALLBACK_VERSION, description: FAUX_PACKAGE_DESCRIPTION, private: true, }, diff --git a/packages/cli/src/services/check-parser/lockfile-pruner.ts b/packages/cli/src/services/check-parser/lockfile-pruner.ts new file mode 100644 index 000000000..00a61ba9c --- /dev/null +++ b/packages/cli/src/services/check-parser/lockfile-pruner.ts @@ -0,0 +1,1301 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' + +import Debug from 'debug' +import { execa } from 'execa' +import JSON5 from 'json5' +import { parse as parseYaml } from 'yaml' + +import { createFauxPackageFiles } from './faux-package.js' +import { isPnpmfilePath } from './package-files/pnpmfile.js' +import { lineage } from './package-files/walk.js' +import { PackageManager, PathLookup } from './package-files/package-manager.js' +import { Package, Workspace } from './package-files/workspace.js' +import { File, VirtualFile } from './parser.js' +import { pathToPosix } from '../util.js' + +const debug = Debug('checkly:cli:services:check-parser:lockfile-pruner') + +export interface PruneBundledLockfileOptions { + workspace: Workspace + packageManager: PackageManager + /** + * The bundle's final file set, keyed by archive path (posix, relative to + * the workspace root) — the bundler's own registry. + */ + files: ReadonlyMap + timeoutMs?: number + env?: NodeJS.ProcessEnv +} + +export type PruneBundledLockfileResult = + | { + status: 'pruned' + /** The lockfile's archive path. */ + archivePath: string + /** The regenerated lockfile contents. */ + content: string + /** + * Faux manifests synthesized for workspace members that bundled + * manifests reference as links but that have no manifest in the bundle + * (see {@link pruneBundledLockfile}). The caller must register these + * into the bundle so that the bundle and the pruned lockfile stay + * consistent. + */ + backfilledManifests: VirtualFile[] + } + | { + status: 'skipped' + reason: string + /** + * True when the skip means "pruning was needed but is unavailable" — + * the bundle is a partial workspace whose lockfile over-describes it, + * yet this setup cannot be pruned. Callers should surface these; skips + * where there is simply nothing to do stay quiet. + */ + notable?: boolean + } + | { status: 'failed', reason: string } + +// A legitimate prune reuses resolutions from the lockfile and takes seconds +// (measured 1-4s even against multi-thousand-package lockfiles); only a +// stale lockfile behind an unreachable registry runs long, and that path +// ends in a fallback anyway because the subset check rejects fresh +// resolutions — so waiting minutes buys nothing. +const DEFAULT_TIMEOUT_MS = 30_000 + +// The yarn version probe shares the install's budget; if it leaves the +// install less than this, the prune is abandoned with a provisioning +// message rather than spawning an install doomed to time out. +const YARN_PROBE_MIN_INSTALL_BUDGET_MS = 1_000 + +const MAX_FAILURE_DETAIL_LENGTH = 400 + +/** + * Environment keys that alter the very behavior the prune command pins with + * explicit flags. Everything else (registry and auth configuration in + * particular) is passed through. + */ +const STRIPPED_ENV_KEYS = new Set([ + 'npm_config_frozen_lockfile', + 'npm_config_lockfile', + 'npm_config_package_lock', + 'npm_config_dry_run', + 'npm_config_ignore_workspace', + // Redirects the lockfile write outside the temp dir. The explicit + // --lockfile-dir flag on the pnpm command outranks this anyway; stripped + // as defense in depth. + 'npm_config_lockfile_dir', +]) + +function manifestArchivePath (workspace: Workspace, pkg: Package): string { + return pathToPosix(path.relative(workspace.root.path, pkg.packageJsonPath)) +} + +function unknownVersionReason (pkg: Package): string { + return `the version of workspace package '${pkg.name}' could not be determined` +} + +function lockfileArchivePath (workspace: Workspace): string | undefined { + if (!workspace.lockfile.isOk()) { + return undefined + } + return pathToPosix(path.relative(workspace.root.path, workspace.lockfile.unwrap())) +} + +export type ShouldPruneResult = + | { prune: true, lockfileArchivePath: string } + | { prune: false, reason: string, notable?: boolean } + +/** + * Decides whether the bundle's lockfile needs pruning at all. Pruning only + * matters when the bundle differs from the full workspace — when every + * workspace member's real manifest is in the bundle, the original lockfile + * already describes the bundle exactly. + */ +export function shouldPruneLockfile ( + workspace: Workspace, + files: ReadonlyMap, + env: NodeJS.ProcessEnv = process.env, +): ShouldPruneResult { + // '0' is the documented spelling; 'false' is a tolerated alias for the + // common boolean-env habit and must keep working. + if (env.CHECKLY_LOCKFILE_PRUNE === '0' || env.CHECKLY_LOCKFILE_PRUNE === 'false') { + return { prune: false, reason: `disabled via CHECKLY_LOCKFILE_PRUNE=${env.CHECKLY_LOCKFILE_PRUNE}` } + } + + const archivePath = lockfileArchivePath(workspace) + if (archivePath === undefined) { + return { prune: false, reason: 'the workspace has no lockfile' } + } + + const lockfileEntry = files.get(archivePath) + if (lockfileEntry === undefined) { + return { prune: false, reason: 'the bundle does not contain the lockfile' } + } + + let bundleMatchesWorkspace = true + for (const pkg of [workspace.root, ...workspace.packages]) { + const manifest = files.get(manifestArchivePath(workspace, pkg)) + if (manifest === undefined || !manifest.physical) { + bundleMatchesWorkspace = false + } + // A faux manifest for a member with an unknown version carries the + // 0.0.0 fallback, which could make specifiers resolve differently than + // they did for the user; do not feed it into resolution. + if (manifest !== undefined && !manifest.physical && pkg.version === undefined) { + return { prune: false, reason: unknownVersionReason(pkg), notable: true } + } + } + + if (bundleMatchesWorkspace) { + return { prune: false, reason: 'the bundle contains the full workspace' } + } + + return { prune: true, lockfileArchivePath: archivePath } +} + +// Deliberately NOT materialized: .yarnrc.yml (and bunfig.toml). Both can +// hold registry auth secrets and neither is part of the bundle today. For +// yarn the omission is verified safe for --mode=update-lockfile: Berry +// lockfiles are registry-agnostic (npm: protocol, content checksums — no +// URLs to rewrite), locked git entries reuse without approvedGitRepositories, +// packageExtensions does not change entry serialization, and any resolution +// that WOULD need registry config is blocked by the child env's network +// guard and fails closed. +const MATERIALIZED_BASENAMES = new Set(['package.json', '.npmrc', 'pnpm-workspace.yaml']) + +/** + * Selects the bundle entries that affect dependency resolution: manifests, + * package manager configuration, patches and the lockfile itself. This + * deliberately mirrors what the remote install will see, so the pruned + * lockfile matches the remote install's inputs. + */ +export function selectMaterializationEntries ( + files: ReadonlyMap, + lockfileArchivePath: string, +): Array<[string, File]> { + const selected: Array<[string, File]> = [] + + for (const [archivePath, file] of files) { + // The lockfile is always selected — without it the "regeneration" would + // be a from-scratch registry resolution. (Even for a symlink entry, the + // materialization copies the file the entry's own path points at.) + if (archivePath === lockfileArchivePath) { + selected.push([archivePath, file]) + continue + } + + if (file.physical && file.symlinkTarget !== undefined) { + continue + } + + const segments = archivePath.split('/') + // Entries can resolve outside the workspace root (a '..' archive path) — + // never write those into the temp dir. node_modules content and embedded + // package tarballs play no part in lockfile resolution. + if (segments.includes('..') || segments.includes('node_modules')) { + continue + } + if (segments[0] === '.checkly') { + continue + } + + const basename = segments[segments.length - 1] + if ( + MATERIALIZED_BASENAMES.has(basename) + || isPnpmfilePath(basename) + || basename.endsWith('.patch') + ) { + selected.push([archivePath, file]) + } + } + + return selected +} + +interface LockfileEdge { + /** Bare package name of the dependency. */ + name: string + /** Whether the edge resolves to a workspace link. */ + isLink: boolean +} + +/** + * The parts of a lockfile the pruner reasons about, format-agnostic: every + * dependency edge (keyed uniquely per format), every resolution entry, the + * importer set and pnpm's recorded settings. + */ +interface LockfileSnapshot { + lockfileVersion?: string + /** + * Yarn Berry's checksum-scheme identifier (`__metadata.cacheKey`). + * Optional on BOTH sides even within one lockfile generation: yarn 3 + * omits it when the lockfile resolves no registry packages, so it is + * only compared when both snapshots record one. + */ + cacheKey?: string + pnpmfileChecksum?: string + excludeLinksFromLockfile: boolean + /** Dependency edges by a format-specific unique key. */ + edges: Map + /** + * Resolution entries: key → recorded version (or empty when the key + * itself pins the version, as in pnpm). Used for the subset check. + */ + resolutions: Map + /** Importer directories, relative to the root ('.' for the root itself). */ + importers: Set +} + +class UnsupportedLockfileFormatError extends Error {} + +// Yarn Berry lockfile metadata versions this parser handles, mapped to the +// yarn major that writes them — the parser allowlist and the +// generation-mismatch check both derive from this one table so adding a +// version cannot leave them out of sync. Version 6 is yarn 3; 8 (early +// yarn 4) and 10 (current) are yarn 4. 6 records dependency specs without +// the npm: protocol prefix; 8+ record it. +const YARN_METADATA_VERSION_TO_MAJOR: Record = { 6: 3, 8: 4, 10: 4 } + +const PNPM_DEPENDENCY_GROUPS = ['dependencies', 'devDependencies', 'optionalDependencies'] + +// Manifests can also reference a workspace member through peerDependencies +// (plugin-style monorepos), even though lockfile importer sections do not +// have a peer group of their own. +const MANIFEST_DEPENDENCY_GROUPS = [...PNPM_DEPENDENCY_GROUPS, 'peerDependencies'] + +function parseLockfileSnapshot (content: string, lockfileName: string): LockfileSnapshot { + const snapshot: LockfileSnapshot = { + excludeLinksFromLockfile: false, + edges: new Map(), + resolutions: new Map(), + importers: new Set(), + } + + if (lockfileName === 'pnpm-lock.yaml') { + const doc = parseYaml(content) + if (doc === null || typeof doc !== 'object') { + throw new UnsupportedLockfileFormatError(`could not parse ${lockfileName}`) + } + snapshot.lockfileVersion = doc.lockfileVersion !== undefined ? String(doc.lockfileVersion) : undefined + // Fail closed on unknown formats: a future pnpm schema could rename the + // sections this parser reads, silently emptying every check. + const pnpmMajor = snapshot.lockfileVersion?.split('.')[0] + if (pnpmMajor === undefined || !['6', '9'].includes(pnpmMajor)) { + throw new UnsupportedLockfileFormatError( + `unsupported ${lockfileName} version ${snapshot.lockfileVersion}`, + ) + } + snapshot.pnpmfileChecksum = typeof doc.pnpmfileChecksum === 'string' ? doc.pnpmfileChecksum : undefined + snapshot.excludeLinksFromLockfile = doc.settings?.excludeLinksFromLockfile === true + + for (const [importer, groups] of Object.entries(doc.importers ?? {})) { + snapshot.importers.add(importer) + if (groups === null || typeof groups !== 'object') { + continue + } + for (const group of PNPM_DEPENDENCY_GROUPS) { + const entries = groups[group] + if (entries === null || typeof entries !== 'object') { + continue + } + for (const [name, entry] of Object.entries(entries)) { + // v9 entries are `{ specifier, version }` objects; older formats + // use a plain version string. + const version = typeof entry === 'string' ? entry : entry?.version + if (typeof version !== 'string') { + continue + } + snapshot.edges.set(`${importer}\0${group}\0${name}`, { + name, + isLink: version.startsWith('link:'), + }) + } + } + } + + // Package keys pin exact versions (and peer suffixes), so presence alone + // is what the subset check needs. + for (const section of ['packages', 'snapshots']) { + for (const key of Object.keys(doc[section] ?? {})) { + snapshot.resolutions.set(`${section}\0${key}`, '') + } + } + + return snapshot + } + + if (lockfileName === 'package-lock.json') { + let doc: any + try { + doc = JSON.parse(content) + } catch { + throw new UnsupportedLockfileFormatError(`could not parse ${lockfileName}`) + } + snapshot.lockfileVersion = doc?.lockfileVersion !== undefined ? String(doc.lockfileVersion) : undefined + const packages = doc?.packages + // lockfileVersion 1 has no `packages` section; without it neither the + // link check nor the subset check can see anything. Unknown future + // versions fail closed for the same reason. + if ( + !['2', '3'].includes(snapshot.lockfileVersion ?? '') + || packages === null || typeof packages !== 'object' + ) { + throw new UnsupportedLockfileFormatError( + `unsupported ${lockfileName} version ${snapshot.lockfileVersion}`, + ) + } + + for (const [key, entry] of Object.entries(packages)) { + if (entry === null || typeof entry !== 'object') { + continue + } + const nodeModulesIndex = key.lastIndexOf('node_modules/') + if (nodeModulesIndex === -1) { + // The root project's entry is keyed '' in package-lock.json; + // normalize to '.' so the importer-preservation check treats it + // like pnpm's root importer. + snapshot.importers.add(key === '' ? '.' : key) + continue + } + // Covers nested installs too (`packages/a/node_modules/foo`), which + // npm emits when a member's version conflicts with a hoisted one. + const name = key.slice(nodeModulesIndex + 'node_modules/'.length) + snapshot.edges.set(key, { + name, + isLink: entry.link === true, + }) + if (entry.link !== true) { + snapshot.resolutions.set(key, String(entry.version ?? entry.resolved ?? '')) + } + } + + return snapshot + } + + if (lockfileName === 'bun.lock') { + // bun.lock is JSONC (bun writes trailing commas), hence JSON5. + let doc: any + try { + doc = JSON5.parse(content) + } catch { + throw new UnsupportedLockfileFormatError(`could not parse ${lockfileName}`) + } + const version = doc?.lockfileVersion !== undefined ? String(doc.lockfileVersion) : undefined + const workspaces = doc?.workspaces + const packages = doc?.packages + // Fail closed on unknown formats, like the parsers above. + if ( + version !== '1' + || workspaces === null || typeof workspaces !== 'object' + || packages === null || typeof packages !== 'object' + ) { + throw new UnsupportedLockfileFormatError( + `unsupported ${lockfileName} version ${version}`, + ) + } + // configVersion is folded into the version so the verification step also + // catches a regeneration that changed it. Bun preserves an existing value + // and treats an absent one as 0 rather than upgrading it, so the fold is + // stable for lockfiles written by older bun versions too. The format is + // self-describing because the value surfaces verbatim in the "lockfile + // version changed" failure reason. + snapshot.lockfileVersion = `${version} (configVersion ${doc.configVersion ?? 0})` + + // A dependency edge resolves to a workspace link if either its spec says + // so or the package entry it resolves to is a workspace tuple; the latter + // covers bare semver specs that bun resolved to a workspace member. The + // entry must be resolved per edge — member-scoped key first, hoisted key + // second, as in parseBunLockfileVersion — because a workspace member's + // name may also be consumed from the registry by a different importer, + // and a name-global answer would misclassify one of the two edges. + const resolvesToWorkspace = (memberName: unknown, depName: string): boolean => { + const keys = typeof memberName === 'string' && memberName !== '' + ? [`${memberName}/${depName}`, depName] + : [depName] + for (const key of keys) { + const tuple = packages[key] + if (Array.isArray(tuple) && typeof tuple[0] === 'string') { + return tuple[0].includes('@workspace:') + } + } + return false + } + + for (const [dir, entry] of Object.entries(workspaces)) { + // The root importer is keyed '' in bun.lock; normalize to '.' so the + // importer-preservation check treats it like pnpm's root importer. + const importer = dir === '' ? '.' : dir + snapshot.importers.add(importer) + if (entry === null || typeof entry !== 'object') { + continue + } + // Unlike pnpm importers, bun workspace entries mirror all four manifest + // dependency groups, peerDependencies included. + for (const group of MANIFEST_DEPENDENCY_GROUPS) { + const entries = entry[group] + if (entries === null || typeof entries !== 'object') { + continue + } + for (const [name, spec] of Object.entries(entries)) { + if (typeof spec !== 'string') { + continue + } + snapshot.edges.set(`${importer}\0${group}\0${name}`, { + name, + isLink: spec.startsWith('workspace:') + || spec.startsWith('link:') + || resolvesToWorkspace(entry.name, name), + }) + } + } + } + + // Package values are resolution tuples (name@version, then registry URL, + // dependencies and integrity in a kind-dependent arity). Key the subset + // check by the whole serialized tuple rather than by the lockfile key: + // pruning the member that owns a hoisted key re-keys the surviving + // member-scoped entry (e.g. `b/ms` becomes `ms`) with an unchanged tuple, + // which a key-based check would falsely reject — while any change WITHIN + // a tuple (a registry rewrite of the tarball URL, a version bump) must + // still fail the check. Serialization is stable because both sides are + // parsed from bun's own deterministic output by this same function. + for (const tuple of Object.values(packages)) { + snapshot.resolutions.set(JSON.stringify(tuple), '') + } + + return snapshot + } + + if (lockfileName === 'yarn.lock') { + // Yarn Classic (v1) files must be recognized BEFORE the YAML parse: + // realistic Classic lockfiles do not parse as YAML at all (an entry + // with a nested `dependencies:` block mixes plain scalars and a + // mapping, which the parser rejects), so without the header check a + // Classic user would get a "could not parse" message implying a broken + // lockfile. Every yarn-1-generated lockfile carries this header. + if (/^# yarn lockfile v1$/m.test(content)) { + throw new UnsupportedLockfileFormatError( + `${lockfileName} is a Yarn Classic (v1) lockfile, which is not supported`, + ) + } + let doc: any + try { + // The failsafe schema keeps every scalar a string: yarn 3 writes + // bare numeric ranges unquoted (`two: 2`), which the default schema + // would coerce to numbers — dropping those edges (and losing `1.0` + // as written, so String() could not undo it). + doc = parseYaml(content, { schema: 'failsafe' }) + } catch { + throw new UnsupportedLockfileFormatError(`could not parse ${lockfileName}`) + } + if (doc === null || typeof doc !== 'object') { + throw new UnsupportedLockfileFormatError(`could not parse ${lockfileName}`) + } + const metadata = doc.__metadata + if (metadata === null || typeof metadata !== 'object' || metadata.version === undefined) { + throw new UnsupportedLockfileFormatError( + `${lockfileName} is not a Yarn Berry lockfile (Yarn Classic lockfiles are not supported)`, + ) + } + // Fail closed on unknown metadata versions, like the parsers above + // (see YARN_METADATA_VERSION_TO_MAJOR). + const version = String(metadata.version) + // hasOwnProperty, not `in`: a corrupted lockfile whose version equals an + // Object.prototype key ('toString', '__proto__') must still fail closed. + if (!Object.prototype.hasOwnProperty.call(YARN_METADATA_VERSION_TO_MAJOR, version)) { + throw new UnsupportedLockfileFormatError( + `unsupported ${lockfileName} metadata version ${version}`, + ) + } + snapshot.lockfileVersion = version + // The cacheKey names the checksum scheme; a regeneration under a + // different scheme must fail verification. Compared as its own field — + // not folded into the version — because yarn 3 omits cacheKey entirely + // when a lockfile resolves no registry packages, so a prune that + // removes the last registry entry legitimately goes from "cacheKey: 8" + // to no cacheKey at all. + snapshot.cacheKey = metadata.cacheKey !== undefined ? String(metadata.cacheKey) : undefined + + // First pass: validate the entry shape and collect the descriptors (the + // comma-joined parts of each entry key) that name workspace entries, so + // edges can be classified per descriptor below. Splitting on ', ' is + // safe: npm semver ranges cannot contain a comma, and yarn itself joins + // descriptor lists with this exact separator. + const workspaceDescriptors = new Set() + const entries: Array<[string, any]> = [] + for (const [key, entry] of Object.entries(doc)) { + if (key === '__metadata') { + continue + } + if (entry === null || typeof entry !== 'object' || typeof entry.resolution !== 'string') { + throw new UnsupportedLockfileFormatError( + `unsupported ${lockfileName} entry shape for '${key}'`, + ) + } + entries.push([key, entry]) + if (entry.resolution.includes('@workspace:')) { + for (const descriptor of key.split(', ')) { + workspaceDescriptors.add(descriptor) + } + } + } + + // A regular dependency edge resolves to a workspace link if either its + // spec says so or its descriptor is one the lockfile keys a workspace + // entry under; the latter covers bare semver specs that yarn resolved + // to a workspace member (the member's entry is then keyed under both + // the range descriptor and the workspace descriptor). Classified per + // descriptor, because a member's name may also be consumed from the + // registry by a different importer. The spec is probed as written — + // metadata version 6 records `^1.0.0` where 8+ record `npm:^1.0.0`, + // and the keys follow the same convention, so no prefix juggling is + // needed. Peer edges are deliberately NEVER probed: a peer only shares + // a descriptor with some other importer's real dependency, and pruning + // that importer away legitimately removes the descriptor — probing + // would then classify the surviving peer edge as a link that + // "degraded", failing a correct prune. Peers are never resolved on + // their own (the consumer's ancestors provide them), so there is no + // silent-substitution channel to catch either; a `workspace:` peer + // spec still counts as a link via its prefix. + const resolvesToWorkspace = (name: string, spec: string): boolean => { + return workspaceDescriptors.has(`${name}@${spec}`) + } + + for (const [, entry] of entries) { + const resolution: string = entry.resolution + // lastIndexOf, not a simple split: scoped names contain '@'. + const workspaceMarker = resolution.lastIndexOf('@workspace:') + if (workspaceMarker === -1) { + // Non-workspace entries (registry, git, patch, portal, ...) feed the + // subset check. Key it by the whole serialized entry rather than the + // lockfile key: pruning a consumer shrinks a multi-descriptor key + // (e.g. "b@npm:^1.0.0, b@workspace:packages/b" loses its npm range) + // with an unchanged value, which a key-based check would falsely + // reject — while any change WITHIN an entry (version, checksum, + // dependencies) must still fail the check. Serialization is stable + // because both sides are parsed from yarn's own deterministic + // output by this same function. + snapshot.resolutions.set(JSON.stringify(entry), '') + continue + } + // Workspace entries are the importers: their resolution carries the + // member directory ('.' for the root), and their dependencies maps + // carry the importer's edges — devDependencies and + // optionalDependencies are merged into `dependencies` by yarn, and + // peerDependencies stays its own group. Their content changes when a + // member is shimmed, which is exactly what pruning does, so they must + // NOT feed the subset check above. + const importer = resolution.slice(workspaceMarker + '@workspace:'.length) + snapshot.importers.add(importer) + for (const group of ['dependencies', 'peerDependencies']) { + const dependencies = entry[group] + if (dependencies === null || typeof dependencies !== 'object') { + continue + } + for (const [name, spec] of Object.entries(dependencies)) { + if (typeof spec !== 'string') { + continue + } + snapshot.edges.set(`${importer}\0${group}\0${name}`, { + name, + isLink: spec.startsWith('workspace:') + || spec.startsWith('link:') + || spec.startsWith('portal:') + // Descriptor probing is for regular dependencies only — see + // resolvesToWorkspace above for why peers must not probe. + || (group === 'dependencies' && resolvesToWorkspace(name, spec)), + }) + } + } + } + + return snapshot + } + + if (lockfileName === 'bun.lockb') { + throw new UnsupportedLockfileFormatError( + 'the binary bun.lockb format is not supported;' + + ' regenerate a text lockfile with `bun install --save-text-lockfile`', + ) + } + + throw new UnsupportedLockfileFormatError(`unsupported lockfile ${lockfileName}`) +} + +/** + * Verifies that the regenerated lockfile is a pruned copy of the original + * rather than a (partial) re-resolution. Returns a failure reason, or + * undefined when everything checks out. + */ +function verifyPrunedLockfile ( + original: LockfileSnapshot, + regenerated: LockfileSnapshot, + files: ReadonlyMap, +): string | undefined { + // A changed lockfile format version means the package manager rewrote the + // file wholesale (e.g. a newer pnpm "upgrading" an old lockfile), which is + // a full re-resolution. + if (original.lockfileVersion !== regenerated.lockfileVersion) { + return `the lockfile version changed from ${original.lockfileVersion} to ${regenerated.lockfileVersion}` + } + + // A changed yarn checksum scheme means every checksum was rewritten — + // a wholesale regeneration, not a prune. Only compared when both sides + // record one: yarn 3 omits the cacheKey when a lockfile resolves no + // registry packages, which a prune can legitimately arrive at. + if ( + original.cacheKey !== undefined && regenerated.cacheKey !== undefined + && original.cacheKey !== regenerated.cacheKey + ) { + return `the lockfile cacheKey changed from ${original.cacheKey} to ${regenerated.cacheKey}` + } + + // Any change to the recorded pnpmfile checksum means the resolve ran with + // different pnpm hooks than the user's own install. + if (original.pnpmfileChecksum !== regenerated.pnpmfileChecksum) { + return 'the regenerated lockfile records a different pnpmfile checksum than the original' + } + + // Pruning only removes: every resolution in the regenerated lockfile must + // already exist in the original. A new or changed resolution means the + // lockfile was out of date with the bundled manifests and the package + // manager resolved something fresh from the registry — versions the user + // never installed or tested with. + for (const [key, version] of regenerated.resolutions) { + if (original.resolutions.get(key) !== version) { + return `the regenerated lockfile resolves entries not present in the original ` + + `(is the lockfile out of date with package.json?)` + } + } + + // Every dependency edge that was a workspace link and that still exists + // must still be a link. Catches npm's silent registry substitution (a + // member whose version does not satisfy a range is fetched from the + // registry with exit code 0). + for (const [key, edge] of original.edges) { + if (!edge.isLink) { + continue + } + const after = regenerated.edges.get(key) + if (after !== undefined && !after.isLink) { + return `'${edge.name}' is no longer a workspace link` + } + } + + // Importers may only disappear for members absent from the bundle. Losing + // an importer whose manifest IS bundled would make the remote (frozen) + // install see an importer the lockfile lacks. + for (const importer of original.importers) { + if (regenerated.importers.has(importer)) { + continue + } + const manifestPath = importer === '.' ? 'package.json' : `${importer}/package.json` + if (files.has(manifestPath)) { + return `the regenerated lockfile lost the importer '${importer}' whose manifest is bundled` + } + } +} + +type BackfillResult = + | { manifests: Map } + | { skipReason: string } + +/** + * Synthesizes faux manifests for workspace members that selected manifests + * reference as links (via the `workspace:` protocol, or resolved as links in + * the original lockfile) but that have no manifest among the selected + * entries. Without these the temp-dir resolve would fail (pnpm) or silently + * resolve the member from the registry (npm) — and, crucially, the same + * would happen during the remote install, so the caller must also register + * the returned manifests into the bundle. + */ +async function collectBackfilledManifests ( + workspace: Workspace, + selected: Array<[string, File]>, + original: LockfileSnapshot, +): Promise { + const linkedNames = new Set() + for (const edge of original.edges.values()) { + if (edge.isLink) { + linkedNames.add(edge.name) + } + } + + const manifestEntries = selected.filter( + ([archivePath]) => path.posix.basename(archivePath) === 'package.json', + ) + const presentManifestPaths = new Set(manifestEntries.map(([archivePath]) => archivePath)) + + const backfilled = new Map() + + for (const [, file] of manifestEntries) { + let manifest: any + try { + const content = file.physical + ? await fs.readFile(file.filePath, 'utf8') + : file.content + manifest = JSON.parse(content) + } catch { + continue + } + + for (const group of MANIFEST_DEPENDENCY_GROUPS) { + const entries = manifest?.[group] + if (entries === null || typeof entries !== 'object') { + continue + } + for (const [name, spec] of Object.entries(entries)) { + const member = workspace.memberByName(name) + if (member === undefined || member === workspace.root) { + continue + } + // Optional peers (peerDependenciesMeta.optional) are deliberately NOT + // exempted: pnpm resolves a `workspace:` peer spec regardless of the + // optional flag when auto-install-peers is on (the default), so a + // missing manifest fails the install outright. + const isLink = (typeof spec === 'string' && spec.startsWith('workspace:')) + || linkedNames.has(name) + if (!isLink) { + continue + } + const memberManifestPath = manifestArchivePath(workspace, member) + if (presentManifestPaths.has(memberManifestPath) || backfilled.has(memberManifestPath)) { + continue + } + // Same rule as in shouldPruneLockfile: never feed the 0.0.0 fallback + // version into resolution. + if (member.version === undefined) { + return { skipReason: unknownVersionReason(member) } + } + for (const fauxFile of createFauxPackageFiles(member)) { + backfilled.set( + pathToPosix(path.relative(workspace.root.path, fauxFile.filePath)), + fauxFile, + ) + } + } + } + } + + return { manifests: backfilled } +} + +function buildChildEnv (baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {} + for (const [key, value] of Object.entries(baseEnv)) { + if (STRIPPED_ENV_KEYS.has(key.toLowerCase())) { + continue + } + env[key] = value + } + // The temp project's root package.json may pin a different package manager + // than the one being invoked; relax corepack's mismatch error (this does + // not affect corepack's pinned-version resolution). + env.COREPACK_ENABLE_STRICT = '0' + // A legitimate yarn prune needs no network at all (verified even with a + // cold cache: the fetch step only touches entries that are NEW, which a + // prune never introduces) — but a lockfile that is out of date with a + // manifest would make yarn resolve the missing descriptor against its + // configured registry, and since the project's .yarnrc.yml is not + // materialized (see MATERIALIZED_BASENAMES) that is the PUBLIC registry: + // the request would disclose the (possibly private) package name before + // the subset verification could reject the result. Disabling the network + // makes that case fail fast with yarn's own blocked-request error + // instead. Only Yarn Berry reads this variable, and every Berry + // generation accepts it (unlike YARN_ENABLE_HARDENED_MODE, which is set + // per-run once the yarn generation is known). + env.YARN_ENABLE_NETWORK = '0' + // Yarn honors a .yarnrc.yml (Berry) or .yarnrc (Classic) found in ANY + // ancestor of its working directory — for the prune temp dir that means + // the system temp root and everything above it, none of which this + // process controls (on shared hosts /tmp is world-writable). An ancestor + // rc can redirect the lockfile write (lockfileFilename), re-enable what + // the variables above disable, or worst of all execute an arbitrary + // script via yarnPath/yarn-path — during the version probe already. Two + // independent guards, because one alone is insufficient: + // - YARN_IGNORE_PATH neutralizes yarnPath/yarn-path specifically, and + // is the ONLY mechanism that covers Yarn Classic (which ignores + // YARN_RC_FILENAME and has no env-settable rc path). Verified to + // disable the exploit on 1.22.22, 3.8.7 and 4.18.0. + // - YARN_RC_FILENAME points Berry's rc lookup at a per-invocation + // random name so no ancestor rc is read at all (blocking + // lockfileFilename etc., not just yarnPath). It must be random: a + // fixed name is a compile-time constant an attacker can pre-create + // to re-open the channel. + env.YARN_IGNORE_PATH = '1' + env.YARN_RC_FILENAME = `.checkly-lockfile-prune-no-rc-${randomUUID()}.yml` + // The link step (where lifecycle scripts run) is already skipped by + // --mode=update-lockfile; disabling scripts outright is defense in + // depth, and enableScripts exists in every Berry generation. + env.YARN_ENABLE_SCRIPTS = '0' + return env +} + +function sanitizeDetail (detail: string): string { + // Package manager output can embed registry URLs with userinfo credentials + // (pnpm does not redact them); scrub before surfacing anywhere. + const redacted = detail.replace(/\/\/[^/@\s]+@/g, '//').trim() + if (redacted.length <= MAX_FAILURE_DETAIL_LENGTH) { + return redacted + } + return `${redacted.slice(0, MAX_FAILURE_DETAIL_LENGTH)}…` +} + +// Larger files are not plausible manifests; the cap also keeps a scan of a +// shared temp root from slurping an arbitrarily large unrelated file. +const MAX_ANCESTOR_MANIFEST_BYTES = 4 * 1024 * 1024 + +type WorkspaceAncestor = { dir: string, parseable: boolean } + +async function directoryExists (dir: string): Promise { + try { + await fs.access(dir) + return true + } catch { + return false + } +} + +// Shared between the spawn-ENOENT and lockfile-read-ENOENT branches: both +// must rule out a reaped temp dir before attributing the ENOENT to anything +// more specific. +const TEMP_DIR_VANISHED: PruneBundledLockfileResult = { + status: 'failed', + reason: 'the temp directory disappeared while the command ran', +} + +// A missing package manager binary is a real situation for bun, whose +// detection needs only a committed bun.lock. Pruning was never attempted, +// so it is a notable skip, not a failure whose message would suggest the +// lockfile is broken — and installing the package manager, not disabling +// pruning, is the fix. +function executableMissing (executable: string): PruneBundledLockfileResult { + return { + status: 'skipped', + reason: `${executable} is not installed or not on PATH; install it so the lockfile can be pruned`, + notable: true, + } +} + +/** + * Walks from `startDir` to the filesystem root looking for a package.json + * that declares npm workspaces. The caller treats any hit as "this location + * is not a safe sandbox", so the scan errs toward matching: manifests are + * parsed with the same leniency bun's own package.json parser has (JSONC — + * comments and trailing commas, which strict JSON.parse rejects), and a + * manifest that exists but cannot be parsed even then counts as a hit + * (`parseable: false`), since bun's parser might still accept it. + */ +async function findWorkspaceAncestor (startDir: string): Promise { + for (const dir of lineage(startDir)) { + const manifestPath = path.join(dir, 'package.json') + let raw: string + try { + // Shared temp roots can hold arbitrary files under this name; stat + // first so a FIFO can't hang the read and an oversized file isn't + // slurped. + const stats = await fs.stat(manifestPath) + if (!stats.isFile() || stats.size > MAX_ANCESTOR_MANIFEST_BYTES) { + continue + } + raw = await fs.readFile(manifestPath, 'utf8') + } catch { + // No manifest here — keep walking. + continue + } + try { + const manifest = JSON5.parse(raw) + if (manifest !== null && typeof manifest === 'object' && 'workspaces' in manifest) { + return { dir, parseable: true } + } + } catch { + return { dir, parseable: false } + } + } + return undefined +} + +/** + * Regenerates the bundle's lockfile so it matches the bundle's actual set of + * manifests, by materializing the resolution-relevant bundle entries into a + * temp directory and running the package manager's lockfile-only install. + * + * Returns `skipped` when pruning is unnecessary or unsupported, and `failed` + * when the caller should fall back to the original lockfile. + */ +export async function pruneBundledLockfile ( + options: PruneBundledLockfileOptions, +): Promise { + const { + workspace, + packageManager, + files, + timeoutMs = DEFAULT_TIMEOUT_MS, + env = process.env, + } = options + + const decision = shouldPruneLockfile(workspace, files, env) + if (!decision.prune) { + return { status: 'skipped', reason: decision.reason, notable: decision.notable } + } + + // Every pre-run skip below this point is notable: shouldPruneLockfile has + // already established that the bundle is a partial workspace, so the + // lockfile over-describes the bundle and this setup cannot be helped. + // (The one post-run skip — a byte-identical regeneration — is the + // opposite: pruning ran and proved there was nothing to change.) + + // This capability check must stay ahead of the lockfile read below: an + // unsupported package manager should always skip notably, never surface + // a lockfile read error as a 'failed' warning that implies pruning was + // attempted. + const runnable = packageManager.lockfileOnlyInstallCommand() + if (runnable === undefined) { + return { + status: 'skipped', + reason: `${packageManager.name} has no supported lockfile-only install`, + notable: true, + } + } + + const lockfileName = path.posix.basename(decision.lockfileArchivePath) + + let originalContent: string + try { + originalContent = await fs.readFile(workspace.lockfile.unwrap(), 'utf8') + } catch (err) { + return { status: 'failed', reason: `could not read the lockfile: ${(err as Error).message}` } + } + + let original: LockfileSnapshot + try { + original = parseLockfileSnapshot(originalContent, lockfileName) + } catch (err) { + return { status: 'skipped', reason: (err as Error).message, notable: true } + } + + if (original.excludeLinksFromLockfile) { + // Without link entries in the lockfile, neither the backfill nor the + // link-preservation check can see workspace links. + return { + status: 'skipped', + reason: 'the lockfile is written with excludeLinksFromLockfile', + notable: true, + } + } + + const selected = selectMaterializationEntries(files, decision.lockfileArchivePath) + + if (original.pnpmfileChecksum !== undefined) { + const hasPnpmfile = selected.some(([archivePath]) => isPnpmfilePath(archivePath)) + if (!hasPnpmfile) { + return { + status: 'skipped', + reason: 'the lockfile records a pnpmfile checksum but no pnpmfile is bundled', + notable: true, + } + } + } + + const backfill = await collectBackfilledManifests(workspace, selected, original) + if ('skipReason' in backfill) { + return { status: 'skipped', reason: backfill.skipReason, notable: true } + } + + let tempDir: string | undefined + try { + // Assign before the realpath call so a realpath failure cannot leak the + // freshly created directory. + tempDir = await fs.mkdtemp(path.join(tmpdir(), 'checkly-lockfile-prune-')) + tempDir = await fs.realpath(tempDir) + + // Bun re-roots at an ancestor directory whose package.json declares + // workspaces with a glob matching the working directory — and then + // resolves against THAT root and writes the regenerated lockfile there, + // outside this sandbox, over a real file. This can only happen when the + // system temp dir itself sits inside a workspace (e.g. TMPDIR pointing + // into a repo), so refuse to run rather than risk it. pnpm pins the + // write with --lockfile-dir and anchors at the materialized + // pnpm-workspace.yaml, and npm does not re-root, so only bun needs the + // guard. + if (packageManager.name === 'bun') { + const ancestor = await findWorkspaceAncestor(path.dirname(tempDir)) + if (ancestor !== undefined) { + return { + status: 'skipped', + reason: (ancestor.parseable + ? `the temp directory is inside the npm workspace at '${ancestor.dir}'` + : `an unparseable package.json at '${ancestor.dir}' could not be ruled out as a workspace root`) + + '; point TMPDIR (TEMP/TMP on Windows) outside any workspace to enable pruning', + notable: true, + } + } + } + + const entries: Array<[string, File]> = [ + ...selected, + ...backfill.manifests, + ] + + for (const [archiveRelativePath, file] of entries) { + const target = path.join(tempDir, ...archiveRelativePath.split('/')) + // Defense in depth alongside the '..' filter in + // selectMaterializationEntries: never write outside the temp dir. + // (A plain startsWith('..') would also reject a directory that merely + // begins with two dots, e.g. '..artifacts'.) + const relative = path.relative(tempDir, target) + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + continue + } + await fs.mkdir(path.dirname(target), { recursive: true }) + if (file.physical) { + await fs.copyFile(file.filePath, target) + } else { + await fs.writeFile(target, file.content) + } + } + + // Guard against the selection or the write guards having dropped the + // lockfile — running without one would be a from-scratch registry + // resolution, not a prune. + try { + await fs.access(path.join(tempDir, lockfileName)) + } catch { + return { status: 'failed', reason: 'the lockfile could not be materialized' } + } + + // Yarn Classic must be stopped BEFORE the install is spawned: verified + // on yarn 1.22.22 that it silently ignores --mode=update-lockfile and + // performs a FULL install — fresh registry resolution, node_modules in + // the temp dir, dependency lifecycle scripts — with exit 0, then writes + // a v1 lockfile that fails verification with misleading advice. Classic + // is what a plain `yarn` resolves to when a Berry project pins its + // version via yarnPath (which lives in the unbundled .yarnrc.yml) + // rather than the packageManager field. Only a positive 1.x match + // skips: a failing or unparseable probe falls through to the install, + // whose own error carries the real detail — so a probe hiccup can + // never block a working prune, mirroring the post-hoc PATH probe + // below. Probe and install share the executable, cwd and env, so their + // version resolution cannot diverge. + const childEnv = buildChildEnv(env) + // The probe shares the install's time budget so the yarn path cannot + // block for longer than the documented timeout in total (a stalled + // first-use corepack download would otherwise be paid twice). + // timeoutMs === 0 means "no timeout" to execa, so the whole budget + // dance is skipped in that case. + let installTimeoutMs = timeoutMs + if (packageManager.name === 'yarn') { + // A whole budget below the install floor can never succeed (the yarn + // path spends a probe plus an install), so reject it up front — + // before the probe, whose own outcome under such a budget would be a + // misleading "timed out" rather than this caller-misconfiguration. + if (timeoutMs > 0 && timeoutMs < YARN_PROBE_MIN_INSTALL_BUDGET_MS) { + return { + status: 'failed', + reason: `the prune timeout (${timeoutMs}ms) is below the minimum needed to run yarn`, + } + } + const probeStartedAt = Date.now() + const probe = await execa(runnable.executable, ['--version'], { + cwd: tempDir, + env: childEnv, + extendEnv: false, + timeout: timeoutMs, + reject: false, + }) + if (probe.timedOut) { + // The probe consumed the whole budget; spawning the install with + // the ~zero remainder would only produce a confusing second kill. + return { status: 'failed', reason: `${runnable.executable} timed out after ${timeoutMs}ms` } + } + if (timeoutMs > 0) { + const remaining = timeoutMs - (Date.now() - probeStartedAt) + if (remaining < YARN_PROBE_MIN_INSTALL_BUDGET_MS) { + // The probe (typically a slow first-use corepack toolchain + // download) left too little for the install; a 1 ms install would + // be a misleading second timeout, so say what actually happened. + return { + status: 'failed', + reason: 'provisioning the yarn toolchain used up the prune time budget before the' + + ' lockfile could be regenerated; pre-install yarn or raise the timeout', + } + } + installTimeoutMs = remaining + } + const probeVersion = probe.failed ? '' : probe.stdout?.trim() ?? '' + const major = Number.parseInt(probeVersion, 10) + if (major === 1) { + return { + status: 'skipped', + reason: 'yarn resolves to Yarn Classic (1.x) here, which cannot regenerate' + + ' a Yarn Berry lockfile; set the packageManager field in package.json' + + ' and enable Corepack so a Yarn 2+ binary runs instead', + notable: true, + } + } + // Yarn only REUSES a lockfile written by its own generation — handed + // an older one it re-resolves everything, which the network guard + // blocks (verified: yarn 4.18 re-resolves both v6 and v8 lockfiles). + // A cross-generation mismatch would therefore fail with a message + // about blocked registry requests; skip with the actual remedies + // instead. The parser only accepts versions in the table, so the + // lookup is always defined here. + const requiredMajor = YARN_METADATA_VERSION_TO_MAJOR[original.lockfileVersion ?? ''] + if (major >= 2 && major !== requiredMajor) { + return { + status: 'skipped', + reason: `the lockfile was written by yarn ${requiredMajor} (metadata version` + + ` ${original.lockfileVersion}) but yarn resolves to ${sanitizeDetail(probeVersion)} here;` + + ' run your own install to migrate the lockfile, or pin the matching yarn' + + ' version via the packageManager field in package.json so Corepack provisions it', + notable: true, + } + } + if (major >= 4) { + // Hardened mode revalidates locked entries against the registry — + // yarn 4 enables it automatically on pull-request CI, and with the + // network guard above that would fail every prune there. Only set + // for a CONFIRMED yarn 4+: the setting does not exist before yarn + // 4, which rejects unknown environment settings with a usage error + // (verified on 3.8.7). An unidentified yarn proceeds without it — + // worst case a hardened-mode prune fails closed with a warning. + childEnv.YARN_ENABLE_HARDENED_MODE = '0' + } + } + + debug(`Running ${runnable.unsafeDisplayCommand} in ${tempDir}`) + + const result = await execa(runnable.executable, runnable.args, { + cwd: tempDir, + env: childEnv, + extendEnv: false, + timeout: installTimeoutMs, + reject: false, + }) + + if (result.timedOut) { + return { status: 'failed', reason: `${runnable.executable} timed out after ${installTimeoutMs}ms` } + } + if ((result as any).code === 'ENOENT') { + // A spawn ENOENT can also mean the working directory vanished (a temp + // reaper); only report a missing executable when the temp dir is + // still there. + if (!await directoryExists(tempDir)) { + return TEMP_DIR_VANISHED + } + return executableMissing(runnable.executable) + } + if (result.failed || result.exitCode !== 0) { + // On Windows a missing executable never surfaces as a spawn ENOENT: + // execa spawns through cross-spawn, which wraps an unresolvable + // command in cmd.exe, so the child "runs" and exits non-zero with + // cmd.exe's not-recognized message. Classify after the fact with a + // PATH probe — safe against probe/spawn resolution differences, + // because the command has already failed either way and only the + // reporting is at stake. Executables given as a path are left to the + // spawn's own error detail. + if (path.basename(runnable.executable) === runnable.executable) { + const executablePath = await new PathLookup().lookupPath(runnable.executable) + if (executablePath === undefined) { + return executableMissing(runnable.executable) + } + } + const detail = [result.stderr, result.stdout, (result as any).shortMessage] + .find(value => typeof value === 'string' && value.trim() !== '') ?? 'unknown error' + // Yarn's blocked-request error is the network guard doing its job; + // surfaced verbatim it reads like the user's own configuration is + // broken. Name the two real causes instead — a stale lockfile, or a + // same-generation yarn that still declines to reuse it (e.g. a v8 + // lockfile under a yarn that writes v10). Scan BOTH streams: real + // yarn prints YN0080 on stdout, but the single-stream `detail` above + // prefers a non-empty stderr, so the marker can hide in either one. + const yarnOutput = [result.stdout, result.stderr] + .filter((value): value is string => typeof value === 'string' && value.trim() !== '') + .join('\n') + if (packageManager.name === 'yarn' && /has been blocked/.test(yarnOutput)) { + return { + status: 'failed', + reason: 'yarn needed the network to reuse the lockfile, which pruning forbids' + + ' — the lockfile may be out of date with a package.json, or written by a' + + ' different yarn version than the one that ran (pin it via the' + + ' packageManager field); the request was blocked before any package name' + // Yarn's own output stays attached so the affected descriptor + // is identifiable; echoing it is no new disclosure, the + // request never left the machine. + + ` left the machine: ${sanitizeDetail(yarnOutput)}`, + } + } + return { + status: 'failed', + reason: `${runnable.unsafeDisplayCommand} failed: ${sanitizeDetail(String(detail))}`, + } + } + + let regeneratedContent: string + try { + regeneratedContent = await fs.readFile(path.join(tempDir, lockfileName), 'utf8') + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + // As in the spawn ENOENT branch: distinguish a reaped temp dir from + // a deliberately removed lockfile. + if (!await directoryExists(tempDir)) { + return TEMP_DIR_VANISHED + } + // Some package managers remove rather than write a lockfile in edge + // cases (bun deletes one that would describe no packages at all: + // "No packages! Deleted empty lockfile") — deliberate behavior, not + // a broken lockfile, so don't surface it as a failure whose advice + // says to refresh the lockfile. + return { + status: 'skipped', + reason: 'the regenerated lockfile was not found after the command completed' + + ' (some package managers delete a lockfile that would describe no packages)', + notable: true, + } + } + return { + status: 'failed', + reason: `could not read the regenerated lockfile: ${(err as Error).message}`, + } + } + + // A byte-identical regeneration with backfilled manifests still counts + // as a prune: the manifests must reach the bundle (a lockfile importer + // without a manifest breaks the remote install), and the verification + // below passes trivially for identical content. + if (regeneratedContent === originalContent && backfill.manifests.size === 0) { + return { status: 'skipped', reason: 'the regenerated lockfile is identical to the original' } + } + + let regenerated: LockfileSnapshot + try { + regenerated = parseLockfileSnapshot(regeneratedContent, lockfileName) + } catch (err) { + return { status: 'failed', reason: (err as Error).message } + } + + const problem = verifyPrunedLockfile(original, regenerated, files) + if (problem !== undefined) { + return { status: 'failed', reason: problem } + } + + return { + status: 'pruned', + archivePath: decision.lockfileArchivePath, + content: regeneratedContent, + backfilledManifests: Array.from(backfill.manifests.values()), + } + } catch (err) { + return { status: 'failed', reason: (err as Error).message } + } finally { + if (tempDir !== undefined) { + // Cleanup failures must never override the computed result. + await fs.rm(tempDir, { recursive: true, force: true, maxRetries: 3 }) + .catch(err => debug(`Could not remove temp dir ${tempDir}: ${err}`)) + } + } +} diff --git a/packages/cli/src/services/check-parser/package-files/__tests__/package-manager.spec.ts b/packages/cli/src/services/check-parser/package-files/__tests__/package-manager.spec.ts index 0565fa25f..2795d2236 100644 --- a/packages/cli/src/services/check-parser/package-files/__tests__/package-manager.spec.ts +++ b/packages/cli/src/services/check-parser/package-files/__tests__/package-manager.spec.ts @@ -5,7 +5,9 @@ import path from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { + BunDetector, CNpmDetector, + DenoDetector, detectNearestConfigFiles, detectNearestLockfiles, detectPackageManager, @@ -15,6 +17,8 @@ import { NpmDetector, npmPackageManager, PackageManagerDetector, + PathLookup, + PNpmDetector, YarnDetector, } from '../package-manager.js' @@ -569,3 +573,60 @@ describe('detectNearestConfigFiles', () => { .rejects.toBeInstanceOf(NoConfigFileFoundError) }) }) + +describe('lockfileOnlyInstallCommand', () => { + it('regenerates the lockfile without installing for pnpm, pinning the lockfile location', () => { + const runnable = new PNpmDetector().lockfileOnlyInstallCommand() + expect(runnable?.executable).toEqual('pnpm') + expect(runnable?.args).toEqual([ + 'install', '--lockfile-only', '--ignore-scripts', '--no-frozen-lockfile', '--lockfile-dir', '.', + ]) + }) + + it('regenerates the lockfile without installing for npm', () => { + const runnable = new NpmDetector().lockfileOnlyInstallCommand() + expect(runnable?.executable).toEqual('npm') + expect(runnable?.args).toEqual(['install', '--package-lock-only', '--ignore-scripts', '--no-audit', '--no-fund']) + }) + + it('regenerates the lockfile without installing for bun', () => { + const runnable = new BunDetector().lockfileOnlyInstallCommand() + expect(runnable.executable).toEqual('bun') + expect(runnable.args).toEqual(['install', '--lockfile-only', '--ignore-scripts']) + }) + + it('regenerates the lockfile without installing for yarn', () => { + const runnable = new YarnDetector().lockfileOnlyInstallCommand() + expect(runnable.executable).toEqual('yarn') + expect(runnable.args).toEqual(['install', '--mode=update-lockfile']) + }) + + it('is unsupported for cnpm and deno', () => { + expect(new CNpmDetector().lockfileOnlyInstallCommand()).toBeUndefined() + expect(new DenoDetector().lockfileOnlyInstallCommand()).toBeUndefined() + }) +}) + +describe('PathLookup', () => { + // The lookup must resolve like the spawn's own resolver (cross-spawn → + // which) or the two disagree about whether an executable exists. The + // Windows-specific behaviors depend on the platform's path delimiter, so + // they can only run there — Windows CI covers this. + it.skipIf(process.platform !== 'win32')('strips quoted Path entries and defaults PATHEXT on Windows', () => { + vi.stubEnv('Path', `C:\\Windows;"C:\\Program Files\\nodejs"`) + vi.stubEnv('PATHEXT', undefined) + try { + const lookup = new PathLookup() + expect(lookup.paths).toEqual(['C:\\Windows', 'C:\\Program Files\\nodejs']) + expect(lookup.pathext).toEqual(['.EXE', '.CMD', '.BAT', '.COM']) + } finally { + vi.unstubAllEnvs() + } + }) + + it('resolves an executable that exists on PATH and misses one that does not', async () => { + const lookup = new PathLookup() + expect(await lookup.lookupPath('node')).toBeDefined() + expect(await lookup.lookupPath('checkly-no-such-executable-xyz')).toBeUndefined() + }) +}) diff --git a/packages/cli/src/services/check-parser/package-files/__tests__/pnpmfile.spec.ts b/packages/cli/src/services/check-parser/package-files/__tests__/pnpmfile.spec.ts new file mode 100644 index 000000000..4c2087f26 --- /dev/null +++ b/packages/cli/src/services/check-parser/package-files/__tests__/pnpmfile.spec.ts @@ -0,0 +1,307 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { describe, it, expect, afterAll } from 'vitest' + +import { isPnpmfilePath, loadWorkspacePnpmfiles } from '../pnpmfile.js' + +describe('pnpmfile', () => { + const tempDirs: string[] = [] + + afterAll(async () => { + await Promise.all(tempDirs.map(dir => fs.rm(dir, { recursive: true, force: true, maxRetries: 3 }))) + }) + + const makeRoot = async (files: Record): Promise => { + const dir = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'pnpmfile-'))) + tempDirs.push(dir) + for (const [name, contents] of Object.entries(files)) { + await fs.writeFile(path.join(dir, name), contents) + } + return dir + } + + describe('isPnpmfilePath()', () => { + it('matches the default pnpmfile filenames at any location', () => { + expect(isPnpmfilePath('/ws/.pnpmfile.cjs')).toBe(true) + expect(isPnpmfilePath('/ws/.pnpmfile.mjs')).toBe(true) + expect(isPnpmfilePath('/ws/packages/a/.pnpmfile.cjs')).toBe(true) + expect(isPnpmfilePath('/ws/pnpmfile.cjs')).toBe(false) + expect(isPnpmfilePath('/ws/index.cjs')).toBe(false) + }) + }) + + describe('loadWorkspacePnpmfiles()', () => { + it('returns nothing when no pnpmfile exists', async () => { + const root = await makeRoot({}) + expect(await loadWorkspacePnpmfiles(root)).toEqual([]) + }) + + it('marks a self-contained pnpmfile as bundleable', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': 'module.exports = { hooks: { readPackage: pkg => pkg } }\n', + }) + expect(await loadWorkspacePnpmfiles(root)).toEqual([ + { path: path.join(root, '.pnpmfile.cjs') }, + ]) + }) + + it('allows safe Node.js builtin imports', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': [ + `const path = require('path')`, + `const url = require('node:url')`, + `module.exports = { hooks: {} }`, + ].join('\n'), + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeUndefined() + }) + + it('rejects a pnpmfile that requires an npm package', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const semver = require('semver')\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('semver') + }) + + it('rejects a pnpmfile that requires a local file', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const helper = require('./tools/helper.cjs')\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeDefined() + }) + + it('rejects a pnpmfile that loads unsafe builtins like fs', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const fs = require('node:fs')\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('node:fs') + }) + + it('rejects a pnpmfile that uses createRequire via node:module', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const { createRequire } = require('node:module')\n` + + `const load = createRequire(__filename)\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeDefined() + }) + + it('rejects a pnpmfile that uses require.resolve', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `require.resolve('some-pkg')\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('require') + }) + + it('rejects a pnpmfile that evaluates code via Function', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const env = Function('return process')().env\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('Function') + }) + + it('does not treat new.target as a hazard', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `function Hooks () { if (new.target === undefined) { return } }\n` + + `module.exports = { hooks: {} }\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeUndefined() + }) + + it('rejects a pnpmfile that reaches process via globalThis or global', async () => { + const globalThisRoot = await makeRoot({ + '.pnpmfile.cjs': `const mirror = globalThis.process.env.NPM_MIRROR\nmodule.exports = {}\n`, + }) + expect((await loadWorkspacePnpmfiles(globalThisRoot))[0].skipReason).toContain('globalThis') + + const globalRoot = await makeRoot({ + '.pnpmfile.cjs': `const ci = global.process.env.CI\nmodule.exports = {}\n`, + }) + expect((await loadWorkspacePnpmfiles(globalRoot))[0].skipReason).toContain('global') + }) + + it('rejects a pnpmfile that uses module.require', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const semver = module.require('semver')\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('module') + }) + + it('allows plain module.exports', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `module.exports = { hooks: {} }\nmodule.exports.extra = 1\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeUndefined() + }) + + it('rejects a pnpmfile that references __dirname or process', async () => { + const dirnameRoot = await makeRoot({ + '.pnpmfile.cjs': `const p = __dirname + '/x.json'\nmodule.exports = {}\n`, + }) + expect((await loadWorkspacePnpmfiles(dirnameRoot))[0].skipReason).toContain('__dirname') + + const processRoot = await makeRoot({ + '.pnpmfile.cjs': `const mirror = process.env.NPM_MIRROR\nmodule.exports = {}\n`, + }) + expect((await loadWorkspacePnpmfiles(processRoot))[0].skipReason).toContain('process') + }) + + it('does not treat property names as hazardous references', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': `const obj = { process: 1 }\nconst x = obj.process\nmodule.exports = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeUndefined() + }) + + it('rejects a pnpmfile with a dynamic require target', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': 'const hooks = require(`./hooks/${1}.cjs`)\nmodule.exports = {}\n', + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toBeDefined() + }) + + it('rejects an unparseable pnpmfile', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': 'module.exports = {', + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('parse') + }) + + it('handles an ESM .pnpmfile.mjs', async () => { + const root = await makeRoot({ + '.pnpmfile.mjs': `import path from 'node:path'\nexport const hooks = {}\n`, + }) + expect(await loadWorkspacePnpmfiles(root)).toEqual([ + { path: path.join(root, '.pnpmfile.mjs') }, + ]) + }) + + it('rejects an ESM pnpmfile that references import.meta', async () => { + const root = await makeRoot({ + '.pnpmfile.mjs': `const dir = import.meta.dirname\nexport const hooks = {}\n`, + }) + const [info] = await loadWorkspacePnpmfiles(root) + expect(info.skipReason).toContain('import.meta') + }) + + it('skips both files when both default filenames exist', async () => { + const root = await makeRoot({ + '.pnpmfile.cjs': 'module.exports = {}\n', + '.pnpmfile.mjs': 'export const hooks = {}\n', + }) + const infos = await loadWorkspacePnpmfiles(root) + expect(infos).toHaveLength(2) + for (const info of infos) { + expect(info.skipReason).toContain('depends on the pnpm version') + } + }) + + it('reports a custom pnpmfile path as non-bundleable and ignores default files', async () => { + const root = await makeRoot({ + '.npmrc': 'pnpmfile=./tools/hooks.cjs\n', + '.pnpmfile.cjs': 'module.exports = {}\n', + }) + const infos = await loadWorkspacePnpmfiles(root) + expect(infos).toHaveLength(1) + expect(infos[0].path).toEqual(path.join(root, 'tools/hooks.cjs')) + expect(infos[0].skipReason).toContain('custom path') + }) + + it('treats a pnpmfile setting naming the default file like the default', async () => { + const root = await makeRoot({ + '.npmrc': 'pnpmfile=.pnpmfile.cjs\n', + '.pnpmfile.cjs': 'module.exports = {}\n', + }) + expect(await loadWorkspacePnpmfiles(root)).toEqual([ + { path: path.join(root, '.pnpmfile.cjs') }, + ]) + }) + + it('detects the pnpmfile setting in pnpm-workspace.yaml', async () => { + const root = await makeRoot({ + 'pnpm-workspace.yaml': 'packages:\n - packages/*\npnpmfile: ./tools/hooks.cjs\n', + '.pnpmfile.cjs': 'module.exports = {}\n', + }) + const configFilePath = path.join(root, 'pnpm-workspace.yaml') + const infos = await loadWorkspacePnpmfiles(root, configFilePath) + expect(infos).toHaveLength(1) + expect(infos[0].path).toEqual(path.join(root, 'tools/hooks.cjs')) + expect(infos[0].skipReason).toContain('custom path') + }) + + it('reports each entry of an array-valued pnpmfile setting separately', async () => { + const root = await makeRoot({ + 'pnpm-workspace.yaml': 'pnpmfile:\n - ./tools/a.cjs\n - ./tools/b.cjs\n', + }) + const configFilePath = path.join(root, 'pnpm-workspace.yaml') + const infos = await loadWorkspacePnpmfiles(root, configFilePath) + expect(infos.map(info => info.path)).toEqual([ + path.join(root, 'tools/a.cjs'), + path.join(root, 'tools/b.cjs'), + ]) + for (const info of infos) { + expect(info.skipReason).toContain('custom path') + } + }) + + it('reports no pnpmfiles when the lockfile records no pnpmfile checksum', async () => { + const root = await makeRoot({ + 'pnpm-lock.yaml': `lockfileVersion: '9.0'\n`, + '.pnpmfile.cjs': 'module.exports = { hooks: { readPackage: pkg => pkg } }\n', + }) + expect(await loadWorkspacePnpmfiles(root, undefined, path.join(root, 'pnpm-lock.yaml'))).toEqual([]) + }) + + it('reports pnpmfiles when the lockfile records a pnpmfile checksum', async () => { + const root = await makeRoot({ + 'pnpm-lock.yaml': `lockfileVersion: '9.0'\n\npnpmfileChecksum: sha256-abc\n`, + '.pnpmfile.cjs': 'module.exports = { hooks: { readPackage: pkg => pkg } }\n', + }) + expect(await loadWorkspacePnpmfiles(root, undefined, path.join(root, 'pnpm-lock.yaml'))).toEqual([ + { path: path.join(root, '.pnpmfile.cjs') }, + ]) + }) + + it('reports inspection errors as a skip reason naming the failing file', async () => { + const root = await makeRoot({}) + // A directory where a pnpmfile is expected produces a read error that + // is not ENOENT. + await fs.mkdir(path.join(root, '.pnpmfile.cjs')) + const infos = await loadWorkspacePnpmfiles(root) + expect(infos).toHaveLength(1) + expect(infos[0].path).toEqual(path.join(root, '.pnpmfile.cjs')) + expect(infos[0].skipReason).toContain(`failed to inspect '.pnpmfile.cjs'`) + }) + + it('bundles nothing when ignore-pnpmfile is set in .npmrc', async () => { + const root = await makeRoot({ + '.npmrc': 'ignore-pnpmfile=true\n', + '.pnpmfile.cjs': 'module.exports = {}\n', + }) + expect(await loadWorkspacePnpmfiles(root)).toEqual([]) + }) + + it('bundles nothing when ignorePnpmfile is set in pnpm-workspace.yaml', async () => { + const root = await makeRoot({ + 'pnpm-workspace.yaml': 'packages:\n - packages/*\nignorePnpmfile: true\n', + '.pnpmfile.cjs': 'module.exports = {}\n', + }) + const configFilePath = path.join(root, 'pnpm-workspace.yaml') + expect(await loadWorkspacePnpmfiles(root, configFilePath)).toEqual([]) + }) + }) +}) diff --git a/packages/cli/src/services/check-parser/package-files/package-manager.ts b/packages/cli/src/services/check-parser/package-files/package-manager.ts index 22cdfe9ac..2a5328b39 100644 --- a/packages/cli/src/services/check-parser/package-files/package-manager.ts +++ b/packages/cli/src/services/check-parser/package-files/package-manager.ts @@ -8,6 +8,7 @@ import { shellQuote } from '../../../services/shell.js' import { PackageJsonFile } from './package-json-file.js' import { JsonSourceFile } from './json-source-file.js' import { OptionalWorkspaceFile, Package, Workspace, WorkspaceOptions } from './workspace.js' +import { loadWorkspacePnpmfiles } from './pnpmfile.js' import { Err, Ok } from './result.js' import { LockfilePackageQuery, @@ -43,6 +44,25 @@ export interface PackageManager { installCommand (): Runnable addCommand (options: AddCommandOptions): Runnable execCommand (args: string[]): Runnable + /** + * Command that regenerates the lockfile from the manifests on disk without + * installing anything (resolution only). Undefined when the package + * manager has no such mode the CLI supports. + * + * The command must read and write the lockfile in the directory it is + * spawned in. Package managers that support a lockfile-location setting + * must pin it to the working directory explicitly: the setting can come + * from config layers the caller cannot inspect (e.g. the user-level + * ~/.npmrc), and an inherited value would redirect the subprocess's + * lockfile write outside the working directory — potentially over a + * real lockfile. When the package manager offers no way to pin the + * location (bun re-roots at any ancestor package.json whose workspaces + * glob matches the working directory, with no counteracting flag), the + * CALLER must verify before spawning that no such ancestor exists — the + * lockfile pruner refuses to run when its temp dir sits inside a + * workspace. + */ + lockfileOnlyInstallCommand (): Runnable | undefined lookupWorkspace (dir: string): Promise /** * Resolves the version of a single package as recorded in the package @@ -125,6 +145,19 @@ export abstract class PackageManagerDetector { abstract execCommand (args: string[]): Runnable abstract lookupWorkspace (dir: string): Promise + /** + * Default: no supported lockfile-only resolution mode, so callers skip + * lockfile pruning. Package managers with such a mode override this. + * + * Of the remaining detectors on this default, Yarn Classic simply has no + * lockfile-only mode (the yarn override covers Berry only — a Classic + * lockfile is rejected when it is parsed), and cnpm/deno have no verified + * mode either. + */ + lockfileOnlyInstallCommand (): Runnable | undefined { + return undefined + } + /** * Default: lockfile parsing is unsupported, so callers fall back. Package * managers that can parse their lockfile override this. @@ -186,6 +219,12 @@ export class NpmDetector extends PackageManagerDetector implements PackageManage return new Runnable('npm', ['install']) } + lockfileOnlyInstallCommand (): Runnable { + // npm has no lockfile-location setting; package-lock.json always lives + // next to the package.json npm operates on, so there is nothing to pin. + return new Runnable('npm', ['install', '--package-lock-only', '--ignore-scripts', '--no-audit', '--no-fund']) + } + addCommand (options: AddCommandOptions): Runnable { return new Runnable('npm', [ 'install', @@ -314,6 +353,20 @@ export class PNpmDetector extends PackageManagerDetector implements PackageManag return new Runnable('pnpm', ['install']) } + lockfileOnlyInstallCommand (): Runnable { + // --no-frozen-lockfile is load-bearing: pnpm auto-enables frozen mode + // when CI=true, and a lockfile-only regeneration is by definition not a + // frozen install. --lockfile-dir is equally load-bearing: as a CLI flag + // it outranks a lockfile-dir setting from every config layer (project, + // user and global npmrc, pnpm-workspace.yaml, env), so no inherited + // setting can move the lockfile write elsewhere; '.' resolves against + // the subprocess working directory. Accepted by pnpm 8-11 (pnpm 11 + // dropped the npmrc setting but kept the flag). + return new Runnable('pnpm', [ + 'install', '--lockfile-only', '--ignore-scripts', '--no-frozen-lockfile', '--lockfile-dir', '.', + ]) + } + addCommand (options: AddCommandOptions): Runnable { return new Runnable('pnpm', [ 'add', @@ -352,6 +405,7 @@ export class PNpmDetector extends PackageManagerDetector implements PackageManag type PnpmProjectOutput = { name: string path: string + version?: string } const output: PnpmProjectOutput[] = JSON.parse(result.stdout) @@ -382,13 +436,15 @@ export class PNpmDetector extends PackageManagerDetector implements PackageManag const rootPackage = new Package({ name: root.name, path: root.path, + version: root.version, workspaces: dependencies.map(dep => dep.path), }) - const packages = dependencies.map(({ name, path }) => { + const packages = dependencies.map(({ name, path, version }) => { return new Package({ name, path, + version, }) }) @@ -445,6 +501,28 @@ export class YarnDetector extends PackageManagerDetector implements PackageManag return new Runnable('yarn', ['install']) } + lockfileOnlyInstallCommand (): Runnable { + // Yarn Berry only — a Yarn Classic lockfile is rejected when it is + // parsed, and a Classic BINARY must never run this command: verified + // on 1.22.22 that Classic silently ignores --mode=update-lockfile and + // performs a full install, scripts included, so the pruner refuses + // with a version probe before spawning it. Verified on yarn 4.18.0 and + // 3.8.7: this regenerates yarn.lock fully offline, even with a cold + // cache — the resolution step reuses locked entries and prunes the + // unreachable ones, the fetch step only touches entries that are new + // (a prune introduces none), and the link step is skipped entirely, so + // no install scripts can run. CI detection and enableImmutableInstalls + // do not apply to this mode. With a lockfile present in the working + // directory Berry roots there — it does not re-root at ancestor + // workspace globs the way bun does. Yarn 3's lockfileFilename setting + // could redirect the write, but it would have to live in the unbundled + // .yarnrc.yml; the pruner's regenerated-lockfile-missing skip and its + // child-env network guard bound that case. The remaining config + // hazard, hardened mode revalidating locked entries against the + // registry, is disabled via the child environment as well. + return new Runnable('yarn', ['install', '--mode=update-lockfile']) + } + addCommand (options: AddCommandOptions): Runnable { return new Runnable('yarn', [ 'add', @@ -621,6 +699,29 @@ export class BunDetector extends PackageManagerDetector implements PackageManage return await lookupNearestPackageJsonWorkspace(this, dir) } + lockfileOnlyInstallCommand (): Runnable { + // Verified on bun 1.3.11: this regenerates bun.lock offline from the + // recorded resolutions, pruning entries no longer reachable from the + // manifests on disk, without touching node_modules. One exception: when + // a workspace member's name collides with a registry dependency + // consumed elsewhere, bun rejects its own lockfile (InvalidPackageKey) + // and silently re-resolves from the network with exit 0 — the pruner's + // tuple-subset and importer-preservation checks bound that fail-closed. + // Bun has no lockfile-location setting to pin (unlike pnpm's + // --lockfile-dir) — but it re-roots at any ancestor package.json whose + // workspaces glob matches the working directory and then reads AND + // writes the lockfile at that root, with no flag to prevent it. Callers + // must guard against that themselves; the lockfile pruner refuses to + // run when its temp dir sits inside a workspace. CI=true does not imply + // a frozen lockfile (unlike pnpm), so no unfreeze flag is needed. The + // one freeze vector — bunfig `[install] frozenLockfile = true` — has no + // CLI override at all and simply fails the command, which the pruner + // reports and falls back on. Only the text lockfile (bun.lock) can be + // verified after regeneration; the binary bun.lockb is rejected later, + // when the lockfile is parsed. + return new Runnable('bun', ['install', '--lockfile-only', '--ignore-scripts']) + } + async resolvePackageVersionFromLockfile ( lockfilePath: string, query: LockfilePackageQuery, @@ -664,14 +765,24 @@ function* chunks (array: T[], size: number): Generator { export class PathLookup { static win = process.platform.startsWith('win') + // Mirrors the fallback the `which` package (and through it cross-spawn, + // i.e. what execa actually spawns with) applies when PATHEXT is unset — + // without it, a lookup on such a system would only probe the + // extensionless name and miss every .CMD/.EXE shim. + static defaultWinPathext = ['.EXE', '.CMD', '.BAT', '.COM'] + paths: string[] pathext: string[] pathextSet = new Set() constructor () { if (PathLookup.win) { - this.paths = process.env['Path']?.split(path.delimiter) ?? [] - this.pathext = process.env['PATHEXT']?.split(path.delimiter) ?? [] + // Windows allows individually quoted Path entries + // (`C:\Windows;"C:\Program Files\nodejs"`); the spawn's own resolver + // strips the quotes, so this lookup must too or the two disagree. + this.paths = (process.env['Path']?.split(path.delimiter) ?? []) + .map(entry => entry.replace(/^"(.*)"$/, '$1')) + this.pathext = process.env['PATHEXT']?.split(path.delimiter) ?? [...PathLookup.defaultWinPathext] this.pathext.forEach(ext => this.pathextSet.add(ext.toUpperCase())) } else { this.paths = process.env['PATH']?.split(path.delimiter) ?? [] @@ -679,6 +790,12 @@ export class PathLookup { } } + // FIXME(RED-887): the missing `await` below means `foundPath` is a + // Promise — always defined — so this never throws and executable + // detection always "succeeds". Fixing it changes package-manager + // detection outcomes on machines that lack the executable, so it is + // tracked as a follow-up rather than fixed in passing; use lookupPath() + // for a working check. // eslint-disable-next-line require-await async detectPresence (executable: string): Promise { const foundPath = this.lookupPath(executable) @@ -1053,6 +1170,7 @@ export async function fauxWorkspaceFromPackageJson ( const rootPackage = new Package({ name: packageJsonFile.name!, path: packageJsonFile.basePath, + version: packageJsonFile.version, }) return await initWorkspace(packageManager.detector(), { @@ -1104,9 +1222,17 @@ async function initWorkspace ( reason => Err(reason), ) + // Pnpmfiles are only meaningful for pnpm workspaces. Discovering them here + // gives the bundler and the cache hash a single source of truth for which + // pnpmfiles exist and whether they can be bundled. + const pnpmfiles = detector.name === 'pnpm' + ? await loadWorkspacePnpmfiles(options.root.path, configFile.ok(), lockfile.ok()) + : [] + return new Workspace({ ...options, lockfile, configFile, + pnpmfiles, }) } diff --git a/packages/cli/src/services/check-parser/package-files/pnpmfile.ts b/packages/cli/src/services/check-parser/package-files/pnpmfile.ts new file mode 100644 index 000000000..c3b3806bc --- /dev/null +++ b/packages/cli/src/services/check-parser/package-files/pnpmfile.ts @@ -0,0 +1,379 @@ +import fs from 'node:fs/promises' +import path from 'node:path' + +import * as acorn from 'acorn' +import * as walk from 'acorn-walk' +import Debug from 'debug' +import { parse as parseYaml } from 'yaml' + +import { parseNpmrc } from '../../embedded-packages/npmrc.js' + +const debug = Debug('checkly:cli:services:check-parser:pnpmfile') + +/** + * The default pnpmfile filenames pnpm looks for at the workspace root, in + * pnpm's own precedence order: pnpm 11 loads `.pnpmfile.mjs` and falls back + * to `.pnpmfile.cjs` only when the mjs is absent; pnpm 10 only ever loads + * `.pnpmfile.cjs`. + */ +const PNPMFILE_FILENAMES = ['.pnpmfile.mjs', '.pnpmfile.cjs'] + +export function isPnpmfilePath (filePath: string): boolean { + return PNPMFILE_FILENAMES.includes(path.basename(filePath)) +} + +export interface PnpmfileInfo { + /** + * Absolute path to the pnpmfile. + */ + path: string + + /** + * Why the file is not safe to include in a code bundle, when it isn't. + * Only self-contained pnpmfiles (see {@link analyzePnpmfile}) are + * bundleable: the remote install loads the pnpmfile before any + * dependencies are installed and in a different environment, so a + * pnpmfile that depends on anything outside its own bytes would turn + * today's silent lockfile re-resolution into a hard remote install + * failure. `undefined` means the file is bundleable. + */ + skipReason?: string +} + +/** + * Node.js builtins a bundled pnpmfile may load. Deliberately a small + * allowlist of side-effect-free modules: anything that touches the + * filesystem, environment, network or child processes (fs, child_process, + * module, os, http, ...) can make the pnpmfile behave differently — or + * throw — on the remote runner, where only the pnpmfile's own bytes are + * guaranteed to be present. + */ +const SAFE_BUILTIN_MODULES = new Set([ + 'assert', + 'buffer', + 'crypto', + 'events', + 'path', + 'punycode', + 'querystring', + 'string_decoder', + 'url', + 'util', +]) + +function isSafeBuiltinModule (specifier: string): boolean { + const bare = specifier.startsWith('node:') ? specifier.slice('node:'.length) : specifier + return SAFE_BUILTIN_MODULES.has(bare) +} + +/** + * Identifiers whose mere presence makes a pnpmfile environment-dependent: + * dynamic module access (`require` in non-call positions, `require.resolve`), + * filesystem-relative paths (`__dirname`, `__filename`), process state + * (`process.env`, `process.cwd()`) — also reachable via `globalThis.process` + * and friends — or code evaluation (`eval`, `Function('...')`). + */ +const HAZARDOUS_IDENTIFIERS = new Set([ + '__dirname', + '__filename', + 'process', + 'eval', + 'Function', + 'globalThis', + 'global', +]) + +/** + * Best-effort static analysis of whether a pnpmfile is self-contained: + * parseable, loads nothing beyond {@link SAFE_BUILTIN_MODULES}, and + * references none of {@link HAZARDOUS_IDENTIFIERS}. Returns undefined when + * the file is self-contained, or a human-readable reason when it is not + * (or cannot be confidently analyzed). + * + * The analysis assumes a non-adversarial author: it exists to catch a + * user's own accidentally environment-dependent code, not deliberate + * obfuscation. Reflective escape hatches (e.g. reaching the Function + * constructor through a `.constructor` property, or any computed member + * access) are deliberately not chased — a rule broad enough to close them + * would false-positive on ubiquitous pnpmfile patterns like + * `pkg.dependencies[name]` and silently cost users both pnpmfile bundling + * and lockfile pruning. + */ +function analyzePnpmfile (contents: string, filename: string): string | undefined { + const sourceType = filename.endsWith('.mjs') ? 'module' : 'script' + + let program: acorn.Program + try { + program = acorn.parse(contents, { + ecmaVersion: 'latest', + sourceType, + allowReturnOutsideFunction: true, + }) + } catch { + return `could not parse '${filename}'` + } + + const problems = new Set() + + const recordSource = (node: any) => { + if (node && node.type === 'Literal' && typeof node.value === 'string') { + if (!isSafeBuiltinModule(node.value)) { + problems.add(`loads '${node.value}'`) + } + } else { + problems.add('loads a dynamically computed module') + } + } + + walk.ancestor(program, { + CallExpression (node: any) { + if (node.callee.type === 'Identifier' && node.callee.name === 'require') { + recordSource(node.arguments[0]) + } + }, + ImportExpression (node: any) { + recordSource(node.source) + }, + ImportDeclaration (node: any) { + recordSource(node.source) + }, + ExportNamedDeclaration (node: any) { + if (node.source) { + recordSource(node.source) + } + }, + ExportAllDeclaration (node: any) { + recordSource(node.source) + }, + // Note: acorn-walk only visits Identifiers in reference positions — + // non-computed member properties (`obj.process`) and object literal + // keys (`{ process: 1 }`) are never passed to this visitor, so property + // names are not mistaken for references. + Identifier (node: any, _state: any, ancestors: any[]) { + if (HAZARDOUS_IDENTIFIERS.has(node.name)) { + problems.add(`references '${node.name}'`) + return + } + if (node.name === 'require') { + // A plain `require('specifier')` call is handled above; any other + // use (`require.resolve`, passing `require` around) defeats static + // analysis. + const parent = ancestors[ancestors.length - 2] + if (!(parent?.type === 'CallExpression' && parent.callee === node)) { + problems.add(`uses 'require' outside a plain require('...') call`) + } + } + }, + MemberExpression (node: any) { + // `module.exports` is the standard CJS export, but any other member + // access on `module` (`module.require(...)`, `module['require']`) is a + // dynamic-module-access escape hatch. + if (node.object.type === 'Identifier' && node.object.name === 'module') { + if (node.computed || node.property.name !== 'exports') { + problems.add(`accesses 'module' beyond 'module.exports'`) + } + } + }, + MetaProperty (node: any) { + // Only `import.meta` is environment-dependent; `new.target` is plain + // language semantics. + if (node.meta?.name === 'import') { + problems.add(`references 'import.meta'`) + } + }, + }) + + if (problems.size > 0) { + return `'${filename}' is not self-contained: ${Array.from(problems).join('; ')}` + } +} + +async function readTextFileIfPresent (filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf8') + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + return undefined + } + throw err + } +} + +interface PnpmfileSettings { + /** + * Whether pnpmfiles are disabled via the `ignore-pnpmfile` setting. + */ + ignored: boolean + + /** + * Custom pnpmfile paths from the `pnpmfile` setting. When set, pnpm does + * not load the default pnpmfiles at all. + */ + customPaths: string[] +} + +/** + * Reads pnpm's `pnpmfile` and `ignore-pnpmfile` settings, which can be set + * in the workspace root's `.npmrc` or in `pnpm-workspace.yaml`. + */ +async function readPnpmfileSettings ( + rootPath: string, + configFilePath?: string, +): Promise { + const npmrc = await readTextFileIfPresent(path.join(rootPath, '.npmrc')) + if (npmrc !== undefined) { + const config = parseNpmrc(npmrc) + if (config.get('ignore-pnpmfile') === 'true') { + return { ignored: true, customPaths: [] } + } + const value = config.get('pnpmfile') + if (value !== undefined && value !== '') { + return { ignored: false, customPaths: [value] } + } + } + + if (configFilePath !== undefined && path.basename(configFilePath) === 'pnpm-workspace.yaml') { + const contents = await readTextFileIfPresent(configFilePath) + if (contents !== undefined) { + try { + const parsed = parseYaml(contents) + if (parsed?.ignorePnpmfile === true || parsed?.['ignore-pnpmfile'] === true) { + return { ignored: true, customPaths: [] } + } + const value = parsed?.pnpmfile + if (typeof value === 'string' && value !== '') { + return { ignored: false, customPaths: [value] } + } + if (Array.isArray(value)) { + return { + ignored: false, + customPaths: value.filter(entry => typeof entry === 'string' && entry !== ''), + } + } + } catch { + // An unparseable pnpm-workspace.yaml is not this function's problem; + // treat it as not configuring pnpmfiles. + } + } + } + + return { ignored: false, customPaths: [] } +} + +/** + * Discovers the pnpmfile pnpm would use for the workspace and determines + * whether it can be bundled. Mirrors pnpm's own resolution: + * + * - Bundling only matters — and is only safe — when the lockfile records a + * `pnpmfileChecksum`: without one there is nothing for the remote install + * to reproduce, and shipping a pnpmfile alongside a lockfile that does + * not record its checksum would itself make pnpm treat the lockfile as + * out of date. When the lockfile records no checksum, no pnpmfiles are + * reported at all (no bundling, no warnings). + * - When the `pnpmfile` setting points at custom paths, pnpm loads those + * and ignores the default filenames. Custom paths are reported as + * non-bundleable (except a single setting that just names a default + * file at the root, which is treated like the default). + * - Otherwise the default filenames apply. When both `.pnpmfile.mjs` and + * `.pnpmfile.cjs` exist, the effective file depends on the pnpm version + * (pnpm 11 loads only the mjs, pnpm 10 only the cjs), so the situation + * is ambiguous and both are skipped. + * + * Never throws: any inspection error is reported as a skip reason so that + * workspace detection keeps working for commands that never bundle. + */ +export async function loadWorkspacePnpmfiles ( + rootPath: string, + configFilePath?: string, + lockfilePath?: string, +): Promise { + if (lockfilePath !== undefined) { + try { + const lockfileContent = await readTextFileIfPresent(lockfilePath) + if (lockfileContent === undefined || !/^pnpmfileChecksum:/m.test(lockfileContent)) { + debug(`lockfile '%s' records no pnpmfileChecksum; not bundling any pnpmfile`, lockfilePath) + return [] + } + } catch (err) { + return [{ + path: rootPath, + skipReason: `failed to inspect the lockfile for a pnpmfile checksum: ${(err as Error).message}`, + }] + } + } + + let settings: PnpmfileSettings + try { + settings = await readPnpmfileSettings(rootPath, configFilePath) + } catch (err) { + // A failure to read the configuration (e.g. an unreadable .npmrc) makes + // the effective pnpmfile unknowable; use the workspace root as the + // entry's path since no specific pnpmfile can be named. + return [{ + path: rootPath, + skipReason: `failed to inspect the workspace's pnpmfile configuration: ${(err as Error).message}`, + }] + } + + if (settings.ignored) { + // pnpm loads no pnpmfile at all when `ignore-pnpmfile` is set, and the + // setting's config file travels with the bundle, so the remote install + // ignores pnpmfiles the same way. + return [] + } + + let filenames = PNPMFILE_FILENAMES + if (settings.customPaths.length > 0) { + const defaultAtRoot = settings.customPaths.length === 1 + ? PNPMFILE_FILENAMES.find(filename => { + return path.resolve(rootPath, settings.customPaths[0]) === path.join(rootPath, filename) + }) + : undefined + + if (defaultAtRoot === undefined) { + return settings.customPaths.map(setting => ({ + path: path.resolve(rootPath, setting), + skipReason: `the 'pnpmfile' setting points at a custom path (${setting}), ` + + `which is not supported for bundling`, + })) + } + + // The setting just names a default file at the root; treat it like the + // default (pnpm loads exactly that file). + filenames = [defaultAtRoot] + } + + const infos: PnpmfileInfo[] = [] + const present: { filename: string, contents: string }[] = [] + for (const filename of filenames) { + const filePath = path.join(rootPath, filename) + let contents: string | undefined + try { + contents = await readTextFileIfPresent(filePath) + } catch (err) { + infos.push({ + path: filePath, + skipReason: `failed to inspect '${filename}': ${(err as Error).message}`, + }) + continue + } + if (contents !== undefined) { + present.push({ filename, contents }) + } + } + + if (present.length > 1) { + infos.push(...present.map(({ filename }) => ({ + path: path.join(rootPath, filename), + skipReason: `both ${present.map(f => `'${f.filename}'`).join(' and ')} exist, and the one ` + + `pnpm loads depends on the pnpm version`, + }))) + return infos + } + + infos.push(...present.map(({ filename, contents }) => ({ + path: path.join(rootPath, filename), + skipReason: analyzePnpmfile(contents, filename), + }))) + return infos +} diff --git a/packages/cli/src/services/check-parser/package-files/resolver.ts b/packages/cli/src/services/check-parser/package-files/resolver.ts index 43caff363..ab7ef591a 100644 --- a/packages/cli/src/services/check-parser/package-files/resolver.ts +++ b/packages/cli/src/services/check-parser/package-files/resolver.ts @@ -13,12 +13,21 @@ import { JsonSourceFile } from './json-source-file.js' import { JsonTextSourceFile } from './json-text-source-file.js' import { LookupContext } from './lookup.js' import { lineage, LineageOptions } from './walk.js' +import { PnpmfileInfo } from './pnpmfile.js' import { Package, Workspace } from './workspace.js' const debug = Debug('checkly:cli:services:check-parser:resolver') const NPMRC_FILENAME = '.npmrc' +// resolveDependenciesForFilePath runs for every resolved file, and a project +// creates multiple resolver instances (one per parser), so warnings are +// deduplicated by the PnpmfileInfo object identity: every resolver shares +// the Session's Workspace (and therefore its PnpmfileInfo instances), so the +// user sees each warning once per run, while fresh Workspace instances (new +// runs, tests) warn again. +const warnedPnpmfiles = new WeakSet() + /** * Candidate file paths for an `extends` target. TypeScript appends `.json` when * the specifier has no extension and falls back to `/tsconfig.json` when it @@ -247,6 +256,12 @@ type WorkspaceNpmrcLocalDependency = { sourceFile: SourceFile } +type WorkspaceRootPnpmfileLocalDependency = { + kind: 'workspace-root-pnpmfile' + importPath: string + sourceFile: SourceFile +} + type NearestPackageJsonFileLocalDependency = { kind: 'nearest-package-json-file' importPath: string @@ -315,6 +330,7 @@ type LocalDependency = | WorkspaceRootLockfileLocalDependency | WorkspaceRootConfigFileLocalDependency | WorkspaceNpmrcLocalDependency + | WorkspaceRootPnpmfileLocalDependency | NearestPackageJsonFileLocalDependency | NearestTSConfigFileLocalDependency | SupportingTSConfigFileLocalDependency @@ -612,6 +628,35 @@ export class PackageFilesResolver { }) } } + + // Bundle the workspace root's pnpmfiles (pnpm workspaces only; see + // Workspace.pnpmfiles). pnpm records a checksum of the pnpmfile in the + // lockfile (`pnpmfileChecksum`), and installing without the file makes + // pnpm consider the lockfile out of date and silently re-resolve + // everything — so a bundle that carries the lockfile should carry the + // pnpmfile too. Only bundleable (self-contained) pnpmfiles are included; + // like .npmrc, they are also fed into the workspace cache hash so the + // bundled set is always reflected in the cache key. + for (const pnpmfileInfo of this.workspace.pnpmfiles) { + if (pnpmfileInfo.skipReason !== undefined) { + if (!warnedPnpmfiles.has(pnpmfileInfo)) { + warnedPnpmfiles.add(pnpmfileInfo) + process.stderr.write( + `Warning: not bundling pnpmfile '${pnpmfileInfo.path}': ${pnpmfileInfo.skipReason}. ` + + `The remote install may re-resolve dependencies instead of using the lockfile.\n`) + } + continue + } + const pnpmfile = await this.cache.exactSourceFile(pnpmfileInfo.path) + if (pnpmfile !== undefined) { + debug('Found workspace root pnpmfile %s', pnpmfile.meta.filePath) + resolved.local.push({ + kind: 'workspace-root-pnpmfile', + importPath: filePath, + sourceFile: pnpmfile, + }) + } + } } // As above, only add nearest package files if we are not running in diff --git a/packages/cli/src/services/check-parser/package-files/workspace.ts b/packages/cli/src/services/check-parser/package-files/workspace.ts index 91015d748..85868a005 100644 --- a/packages/cli/src/services/check-parser/package-files/workspace.ts +++ b/packages/cli/src/services/check-parser/package-files/workspace.ts @@ -1,6 +1,7 @@ import { glob } from 'glob' import { PackageJsonFile } from './package-json-file.js' +import { PnpmfileInfo } from './pnpmfile.js' import { Result } from './result.js' export interface PackageOptions { @@ -14,6 +15,12 @@ export interface PackageOptions { */ path: string + /** + * The version of the package, if one is declared. Values that are not + * non-empty strings are normalized to undefined by the constructor. + */ + version?: string + /** * Whether the package is a workspace. */ @@ -23,11 +30,17 @@ export interface PackageOptions { export class Package { name: string path: string + version?: string workspaces?: string[] - constructor ({ name, path, workspaces }: PackageOptions) { + constructor ({ name, path, version, workspaces }: PackageOptions) { this.name = name this.path = path + // The version usually originates from a plain JSON.parse of a package.json + // (or from `pnpm list --json` output), so despite the declared type it may + // be any JSON value at runtime. Normalize here so consumers can trust the + // declared type. + this.version = typeof version === 'string' && version !== '' ? version : undefined this.workspaces = workspaces } @@ -37,7 +50,7 @@ export class Package { // eslint-disable-next-line require-await static async loadFromPackageJsonFile (packageJson: PackageJsonFile): Promise { - const { name, workspaces } = packageJson + const { name, version, workspaces } = packageJson if (name === undefined) { return } @@ -45,6 +58,7 @@ export class Package { return new Package({ name, path: packageJson.meta.dirname, + version, workspaces, }) } @@ -66,6 +80,7 @@ export interface WorkspaceOptions { packages: Package[] lockfile: OptionalWorkspaceFile configFile: OptionalWorkspaceFile + pnpmfiles?: PnpmfileInfo[] } export class Workspace { @@ -89,12 +104,21 @@ export class Workspace { */ configFile: OptionalWorkspaceFile + /** + * The pnpmfiles found at the workspace root, when the workspace uses pnpm. + * Entries without a `skipReason` are safe to bundle; see + * {@link PnpmfileInfo}. Empty for workspaces that use another package + * manager. + */ + pnpmfiles: PnpmfileInfo[] + #membersByName = new Map() #membersByPath = new Map() constructor (options: WorkspaceOptions) { this.root = options.root this.packages = options.packages + this.pnpmfiles = options.pnpmfiles ?? [] this.#membersByName = [options.root, ...options.packages].reduce( (map, pkg) => map.set(pkg.name, pkg), new Map(), diff --git a/packages/cli/src/services/check-parser/parser.ts b/packages/cli/src/services/check-parser/parser.ts index d6000366d..dec1906ba 100644 --- a/packages/cli/src/services/check-parser/parser.ts +++ b/packages/cli/src/services/check-parser/parser.ts @@ -16,6 +16,7 @@ import { pathToPosix } from '../util.js' import { Package, Workspace } from './package-files/workspace.js' import { isCoreExtension, isTSExtension } from './package-files/extension.js' import { createFauxPackageFiles } from './faux-package.js' +import { isPnpmfilePath } from './package-files/pnpmfile.js' import { PlaywrightConfigExpander } from './playwright-config-expander.js' const debug = Debug('checkly:cli:services:check-parser:parser') @@ -231,6 +232,15 @@ export class Parser { private determineFileOps (filePath: string): number { const extension = path.extname(filePath) + // Pnpmfiles are pnpm install-time configuration, not check code: they are + // bundled verbatim and must never be parsed. Parsing would treat their + // require/import statements as check dependencies, and e.g. an optional + // `try { require(...) } catch {}` of a gitignored file would fail the + // whole bundle as a missing dependency. + if (isPnpmfilePath(filePath)) { + return 0 + } + if (this.restricted) { if (isLegacySupportedFileExtension(extension)) { return FILEOP_RESOLVE | FILEOP_PARSE diff --git a/packages/cli/src/services/checkly-config-loader.ts b/packages/cli/src/services/checkly-config-loader.ts index 1ff48c7d7..aa352285d 100644 --- a/packages/cli/src/services/checkly-config-loader.ts +++ b/packages/cli/src/services/checkly-config-loader.ts @@ -122,11 +122,29 @@ export type ChecklyConfig = { * transitive dependencies of other private packages — dependencies of * listed packages are not embedded automatically. * - * Only npm and pnpm are supported at this time: packages are resolved - * against the workspace lockfile (`pnpm-lock.yaml` or - * `package-lock.json`) and always verified against its recorded - * integrity hashes. Changing the resolved set of embedded packages - * invalidates the runner's dependency cache. + * Only npm, pnpm, bun and Yarn Berry are supported at this time: + * packages are resolved against the workspace lockfile + * (`pnpm-lock.yaml`, `package-lock.json`, the text `bun.lock` — + * bun's binary `bun.lockb` is not supported — or a Yarn Berry + * `yarn.lock`; Yarn Classic v1 lockfiles are not) and always + * verified against its recorded integrity hashes. Yarn Berry + * lockfiles record no npm tarball integrity, so it is resolved from + * the registry's package metadata instead — one small metadata + * request per embedded package on every deploy, even with a warm + * cache. Downloads read registry credentials from `.npmrc` only; + * bun or yarn users whose credentials live solely in `bunfig.toml` + * or `.yarnrc.yml` must duplicate them into `.npmrc` — referencing + * them through environment variables (`${NPM_TOKEN}`), never as + * plaintext, because `.npmrc` is uploaded with the code bundle. When the bundled lockfile is pruned to the code + * bundle's contents, the embedded set follows it: packages the pruned + * lockfile no longer references — dependencies of workspace members + * that are not part of the bundle — are neither embedded nor + * downloaded, even if an entry matches them. That usually means the + * runner does not need the package at all; if the checks genuinely + * need it, make the depending workspace member part of the bundle + * rather than disabling pruning (`CHECKLY_LOCKFILE_PRUNE=0` restores + * the unfiltered set, as a last resort). Changing the resolved set of + * embedded packages invalidates the runner's dependency cache. */ embed?: string[] } @@ -143,8 +161,10 @@ export type ChecklyConfig = { dependencyCache?: { /** * Optional value mixed into the code bundle's cache hash in addition - * to its usual inputs (lockfile, package.json and .npmrc files, and - * the resolved `bundle.packages.embed` tarball set). + * to its usual inputs — the workspace's dependency-install inputs. + * The exhaustive input list lives with the hash itself; see + * `ComposeCacheHashInput` in + * `services/check-parser/cache-hash.ts`. * Change the value to force runners to reinstall the bundle's * dependencies. Setting it for the first time invalidates the cache * once. Numbers must be safe integers; unset and empty string leave diff --git a/packages/cli/src/services/embedded-packages/__tests__/lockfile-filter.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/lockfile-filter.spec.ts new file mode 100644 index 000000000..fce580fbf --- /dev/null +++ b/packages/cli/src/services/embedded-packages/__tests__/lockfile-filter.spec.ts @@ -0,0 +1,117 @@ +import { describe, it, expect } from 'vitest' + +import { filterTarballsByLockfile } from '../lockfile-filter.js' + +const PNPM_LOCKFILE = ` +lockfileVersion: '9.0' +packages: + '@acme/kept@1.2.3': + resolution: {integrity: sha512-aaa} + other@2.0.0: + resolution: {integrity: sha512-bbb} +` + +const NPM_LOCKFILE = JSON.stringify({ + lockfileVersion: 3, + packages: { + '': { name: 'root' }, + 'node_modules/@acme/kept': { + version: '1.2.3', + resolved: 'https://registry.npmjs.org/@acme/kept/-/kept-1.2.3.tgz', + integrity: 'sha512-aaa', + }, + }, +}) + +// Trailing comma included: bun.lock is JSONC and the filter must accept +// bun's own serialization. +const BUN_LOCKFILE = `{ + "lockfileVersion": 1, + "packages": { + "@acme/kept": ["@acme/kept@1.2.3", "", {}, "sha512-aaa"], + }, +}` + +const YARN_LOCKFILE = ` +__metadata: + version: 10 + cacheKey: 10c0 + +"@acme/kept@npm:1.2.3": + version: 1.2.3 + resolution: "@acme/kept@npm:1.2.3" + checksum: 10c0/aaa + languageName: node + linkType: hard +` + +describe('filterTarballsByLockfile()', () => { + const kept = { + name: '@acme/kept', + version: '1.2.3', + integrity: 'sha512-aaa', + archiveFilename: '@acme+kept@1.2.3.tgz', + } + const dropped = { + name: '@acme/dropped', + version: '4.5.6', + integrity: 'sha512-bbb', + archiveFilename: '@acme+dropped@4.5.6.tgz', + } + + it('splits tarballs by name@version presence in a pnpm lockfile', () => { + const result = filterTarballsByLockfile([kept, dropped], PNPM_LOCKFILE, 'pnpm-lock.yaml') + expect(result.kept).toEqual([kept]) + expect(result.dropped).toEqual([dropped]) + }) + + it('splits tarballs by name@version presence in an npm lockfile', () => { + const result = filterTarballsByLockfile([kept, dropped], NPM_LOCKFILE, 'package-lock.json') + expect(result.kept).toEqual([kept]) + expect(result.dropped).toEqual([dropped]) + }) + + it('splits tarballs by name@version presence in a yarn lockfile', () => { + const result = filterTarballsByLockfile([kept, dropped], YARN_LOCKFILE, 'yarn.lock') + expect(result.kept).toEqual([kept]) + expect(result.dropped).toEqual([dropped]) + }) + + it('splits tarballs by name@version presence in a bun lockfile', () => { + const result = filterTarballsByLockfile([kept, dropped], BUN_LOCKFILE, 'bun.lock') + expect(result.kept).toEqual([kept]) + expect(result.dropped).toEqual([dropped]) + }) + + it('drops a tarball whose name matches but whose version does not', () => { + const otherVersion = { + name: '@acme/kept', + version: '9.9.9', + integrity: 'sha512-ccc', + archiveFilename: '@acme+kept@9.9.9.tgz', + } + const result = filterTarballsByLockfile([otherVersion], PNPM_LOCKFILE, 'pnpm-lock.yaml') + expect(result.kept).toEqual([]) + expect(result.dropped).toEqual([otherVersion]) + }) + + it('can drop every tarball', () => { + const result = filterTarballsByLockfile([dropped], PNPM_LOCKFILE, 'pnpm-lock.yaml') + expect(result.kept).toEqual([]) + expect(result.dropped).toEqual([dropped]) + }) + + it('keeps every tarball when the lockfile cannot be parsed', () => { + // An unparseable lockfile means the filter cannot know what survived; + // shipping the pre-filter superset is the safe previous behavior. + const result = filterTarballsByLockfile([kept, dropped], 'lockfileVersion: unknown\n', 'pnpm-lock.yaml') + expect(result.kept).toEqual([kept, dropped]) + expect(result.dropped).toEqual([]) + }) + + it('keeps every tarball for an unsupported lockfile name', () => { + const result = filterTarballsByLockfile([kept], 'anything', 'yarn.lock') + expect(result.kept).toEqual([kept]) + expect(result.dropped).toEqual([]) + }) +}) diff --git a/packages/cli/src/services/embedded-packages/__tests__/lockfile-packages.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/lockfile-packages.spec.ts index 6a27a43c9..d3c884630 100644 --- a/packages/cli/src/services/embedded-packages/__tests__/lockfile-packages.spec.ts +++ b/packages/cli/src/services/embedded-packages/__tests__/lockfile-packages.spec.ts @@ -7,8 +7,11 @@ import { describe, it, expect } from 'vitest' import { UnsupportedLockfileError, loadLockfilePackages, + parseBunLockfilePackages, + parseLockfilePackagesContent, parseNpmLockfilePackages, parsePnpmLockfilePackages, + parseYarnLockfilePackages, } from '../lockfile-packages.js' describe('parsePnpmLockfilePackages()', () => { @@ -211,6 +214,375 @@ describe('parseNpmLockfilePackages()', () => { }) }) +describe('parseBunLockfilePackages()', () => { + it('parses registry entries, with and without an explicit tarball URL', () => { + // Trailing commas throughout: bun.lock is JSONC, and the parser must + // accept bun's own serialization. + const { registry, excluded } = parseBunLockfilePackages(`{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { "name": "root" }, + }, + "packages": { + "@acme/foo": ["@acme/foo@1.2.3", "", {}, "sha512-aaa"], + "bar": ["bar@2.0.0", "https://nexus.local/repository/npm/bar/-/bar-2.0.0.tgz", {}, "sha512-bbb"], + }, + }`) + expect(registry).toEqual([ + { name: '@acme/foo', version: '1.2.3', integrity: 'sha512-aaa', tarballUrl: undefined }, + { name: 'bar', version: '2.0.0', integrity: 'sha512-bbb', tarballUrl: 'https://nexus.local/repository/npm/bar/-/bar-2.0.0.tgz' }, + ]) + expect(excluded).toEqual([]) + }) + + it('falls back to the derived URL for a non-http tarball element', () => { + const { registry } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + bar: ['bar@2.0.0', 'file:vendor/bar-2.0.0.tgz', {}, 'sha512-bbb'], + }, + })) + expect(registry[0].tarballUrl).toBeUndefined() + }) + + it('excludes workspace packages with a precise reason', () => { + const { registry, excluded } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + '@acme/shared': ['@acme/shared@workspace:packages/shared'], + 'bar': ['bar@2.0.0', '', {}, 'sha512-bbb'], + }, + })) + expect(registry).toHaveLength(1) + expect(excluded).toEqual([ + { + name: '@acme/shared', + reason: `'@acme/shared' is a workspace package, which cannot be embedded as a registry tarball`, + kind: 'workspace', + }, + ]) + }) + + it('excludes git, remote tarball and file dependencies with a reason', () => { + // Non-registry tuples have kind-dependent arities (git entries carry + // their dependency object at index 1); classification must not trust + // any index beyond the name@ref element. + const { registry, excluded } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + isarray: ['isarray@github:juliangruber/isarray#31d8230', {}, 'juliangruber-isarray-31d8230', 'sha512-xxx'], + mstar: ['ms@https://registry.npmjs.org/ms/-/ms-2.1.3.tgz', {}, 'sha512-yyy'], + baz: ['baz@file:vendor/baz', {}], + }, + })) + expect(registry).toEqual([]) + expect(excluded.map(entry => entry.name).sort()).toEqual(['baz', 'isarray', 'ms']) + expect(excluded[0].reason).toContain('git, file or URL dependency') + }) + + it('keeps the package name intact when a git ref itself contains @', () => { + const { excluded } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + 'foo': ['foo@git+ssh://git@github.com/user/foo.git#abc123', {}, 'user-foo-abc123', 'sha512-xxx'], + '@acme/bar': ['@acme/bar@git+ssh://git@github.com/acme/bar.git#def456', {}, 'acme-bar-def456', 'sha512-yyy'], + }, + })) + expect(excluded.map(entry => entry.name).sort()).toEqual(['@acme/bar', 'foo']) + }) + + it('uses the real package name for aliased installs, deduplicating', () => { + // An alias key records the real name@version in its tuple, so an alias + // and a direct dependency on the same package parse into one entry. + const { registry } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + ms: ['ms@2.1.3', '', {}, 'sha512-aaa'], + msa: ['ms@2.1.3', '', {}, 'sha512-aaa'], + }, + })) + expect(registry).toEqual([ + { name: 'ms', version: '2.1.3', integrity: 'sha512-aaa', tarballUrl: undefined }, + ]) + }) + + it('excludes entries without an integrity hash', () => { + const { registry, excluded } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + bar: ['bar@2.0.0', '', {}], + }, + })) + expect(registry).toEqual([]) + expect(excluded[0].reason).toContain('no integrity hash') + }) + + it('skips malformed package values without failing', () => { + const { registry, excluded } = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + 'not-a-tuple': { some: 'object' }, + 'empty-tuple': [], + 'no-separator': ['plainstring'], + 'bar': ['bar@2.0.0', '', {}, 'sha512-bbb'], + }, + })) + expect(registry.map(pkg => pkg.name)).toEqual(['bar']) + expect(excluded).toEqual([]) + }) + + it('rejects unsupported lockfile versions', () => { + expect(() => parseBunLockfilePackages(JSON.stringify({ lockfileVersion: 0 }))) + .toThrow(UnsupportedLockfileError) + }) +}) + +describe('parseYarnLockfilePackages()', () => { + it('parses registry entries, recording the Berry checksum instead of an integrity', () => { + const { registry, excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"@acme/foo@npm:^1.0.0": + version: 1.2.3 + resolution: "@acme/foo@npm:1.2.3" + checksum: 10c0/aaa + languageName: node + linkType: hard + +"bar@npm:2.0.0": + version: 2.0.0 + resolution: "bar@npm:2.0.0" + checksum: 10c0/bbb + languageName: node + linkType: hard +`) + expect(registry).toEqual([ + { name: '@acme/foo', version: '1.2.3', lockfileChecksum: '10c0/aaa' }, + { name: 'bar', version: '2.0.0', lockfileChecksum: '10c0/bbb' }, + ]) + expect(registry.every(entry => entry.integrity === undefined)).toBe(true) + expect(excluded).toEqual([]) + }) + + it('excludes workspace packages with a precise reason', () => { + const { registry, excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"@acme/shared@workspace:*, @acme/shared@workspace:packages/shared": + version: 0.0.0-use.local + resolution: "@acme/shared@workspace:packages/shared" + languageName: unknown + linkType: soft + +"bar@npm:2.0.0": + version: 2.0.0 + resolution: "bar@npm:2.0.0" + checksum: 10c0/bbb + languageName: node + linkType: hard +`) + expect(registry).toHaveLength(1) + expect(excluded).toEqual([ + { + name: '@acme/shared', + reason: `'@acme/shared' is a workspace package, which cannot be embedded as a registry tarball`, + kind: 'workspace', + }, + ]) + }) + + it('distinguishes in-workspace directory links from escaping ones', () => { + const { excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"inside@portal:./tools/inside::locator=root%40workspace%3A.": + version: 0.0.0-use.local + resolution: "inside@portal:./tools/inside::locator=root%40workspace%3A." + languageName: node + linkType: soft + +"outside@link:../elsewhere::locator=root%40workspace%3A.": + version: 0.0.0-use.local + resolution: "outside@link:../elsewhere::locator=root%40workspace%3A." + languageName: node + linkType: soft +`) + expect(excluded).toEqual([ + { + name: 'inside', + reason: `'inside' is a workspace package, which cannot be embedded as a registry tarball`, + kind: 'workspace', + }, + { + name: 'outside', + reason: `'outside' is a local directory link outside the workspace, which cannot be embedded` + + ` as a registry tarball`, + kind: 'unfetchable', + }, + ]) + }) + + it('excludes git, remote tarball, file and patched dependencies with a reason', () => { + const { registry, excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"mimic-response@github:sindresorhus/mimic-response#v3.1.0": + version: 3.1.0 + resolution: "mimic-response@https://github.com/sindresorhus/mimic-response.git#commit=c781ec5" + checksum: 10c0/ccc + languageName: node + linkType: hard + +"resolve@patch:resolve@npm%3A1.22.8#optional!builtin": + version: 1.22.8 + resolution: "resolve@patch:resolve@npm%3A1.22.8#optional!builtin::version=1.22.8&hash=9bd1a5" + checksum: 10c0/ddd + languageName: node + linkType: hard + +"vendored@file:./vendor/vendored-1.0.0.tgz::locator=root%40workspace%3A.": + version: 1.0.0 + resolution: "vendored@file:./vendor/vendored-1.0.0.tgz::locator=root%40workspace%3A." + checksum: 10c0/eee + languageName: node + linkType: hard +`) + expect(registry).toEqual([]) + expect(excluded.map(entry => entry.name).sort()).toEqual(['mimic-response', 'resolve', 'vendored']) + expect(excluded[0].reason).toContain('git, file, URL or patched dependency') + }) + + it('keeps the package name intact when a git ref itself contains @', () => { + const { excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"foo@git+ssh://git@github.com/user/foo.git#abc123": + version: 1.0.0 + resolution: "foo@git+ssh://git@github.com/user/foo.git#commit=abc123" + checksum: 10c0/fff + languageName: node + linkType: hard + +"@acme/bar@git+ssh://git@github.com/acme/bar.git#def456": + version: 1.0.0 + resolution: "@acme/bar@git+ssh://git@github.com/acme/bar.git#commit=def456" + checksum: 10c0/ggg + languageName: node + linkType: hard +`) + expect(excluded.map(entry => entry.name).sort()).toEqual(['@acme/bar', 'foo']) + }) + + it('uses the real package name for aliased installs, deduplicating', () => { + // An alias key records the real name@version locator in `resolution`, + // so an alias and a direct dependency parse into one entry. + const { registry } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"ms@npm:2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/aaa + languageName: node + linkType: hard + +"msalias@npm:ms@2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/aaa + languageName: node + linkType: hard +`) + expect(registry).toEqual([ + { name: 'ms', version: '2.1.3', lockfileChecksum: '10c0/aaa' }, + ]) + }) + + it('keeps build metadata in versions as written', () => { + const { registry } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"built@npm:1.0.0+sha.abc": + version: 1.0.0+sha.abc + resolution: "built@npm:1.0.0+sha.abc" + checksum: 10c0/hhh + languageName: node + linkType: hard +`) + expect(registry).toEqual([ + { name: 'built', version: '1.0.0+sha.abc', lockfileChecksum: '10c0/hhh' }, + ]) + }) + + it('excludes entries without a checksum', () => { + // checksumBehavior: ignore makes yarn omit checksums, leaving nothing + // to pin the content with. + const { registry, excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"bar@npm:2.0.0": + version: 2.0.0 + resolution: "bar@npm:2.0.0" + languageName: node + linkType: hard +`) + expect(registry).toEqual([]) + expect(excluded[0].reason).toContain('no checksum') + }) + + it('skips malformed entries without failing', () => { + const { registry, excluded } = parseYarnLockfilePackages(` +__metadata: + version: 10 + cacheKey: 10c0 + +"no-resolution@npm:1.0.0": + version: 1.0.0 + +"no-separator@npm:1.0.0": + version: 1.0.0 + resolution: "plainstring" + +"bar@npm:2.0.0": + version: 2.0.0 + resolution: "bar@npm:2.0.0" + checksum: 10c0/bbb + languageName: node + linkType: hard +`) + expect(registry.map(pkg => pkg.name)).toEqual(['bar']) + expect(excluded).toEqual([]) + }) + + it('rejects Yarn Classic lockfiles with a migration hint', () => { + expect(() => parseYarnLockfilePackages(`# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +ms@2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138" +`)).toThrow(/Yarn Classic/) + }) +}) + describe('parsePnpmLockfilePackages() workspace links', () => { it('records workspace-linked packages as excluded with a precise reason', () => { const { registry, excluded } = parsePnpmLockfilePackages(` @@ -295,6 +667,14 @@ packages: }, })) expect(npm.registry[0].version).toBe('1.0.0+sha.abcdef') + + const bun = parseBunLockfilePackages(JSON.stringify({ + lockfileVersion: 1, + packages: { + 'meta-pkg': ['meta-pkg@1.0.0+sha.abcdef', '', {}, 'sha512-eee'], + }, + })) + expect(bun.registry[0].version).toBe('1.0.0+sha.abcdef') }) }) @@ -321,3 +701,58 @@ describe('loadLockfilePackages()', () => { } }) }) + +describe('parseLockfilePackagesContent()', () => { + it('dispatches on the lockfile name for every supported format', () => { + const pnpm = parseLockfilePackagesContent(` +lockfileVersion: '9.0' +packages: + foo@1.0.0: + resolution: {integrity: sha512-aaa} +`, 'pnpm-lock.yaml') + expect(pnpm.registry.map(pkg => pkg.name)).toEqual(['foo']) + + const npm = parseLockfilePackagesContent(JSON.stringify({ + lockfileVersion: 3, + packages: { + 'node_modules/bar': { + version: '2.0.0', + resolved: 'https://registry.npmjs.org/bar/-/bar-2.0.0.tgz', + integrity: 'sha512-bbb', + }, + }, + }), 'package-lock.json') + expect(npm.registry.map(pkg => pkg.name)).toEqual(['bar']) + + const bun = parseLockfilePackagesContent(JSON.stringify({ + lockfileVersion: 1, + packages: { + baz: ['baz@3.0.0', '', {}, 'sha512-ccc'], + }, + }), 'bun.lock') + expect(bun.registry.map(pkg => pkg.name)).toEqual(['baz']) + + const yarn = parseLockfilePackagesContent(` +__metadata: + version: 10 + cacheKey: 10c0 + +"qux@npm:4.0.0": + version: 4.0.0 + resolution: "qux@npm:4.0.0" + checksum: 10c0/ddd + languageName: node + linkType: hard +`, 'yarn.lock') + expect(yarn.registry.map(pkg => pkg.name)).toEqual(['qux']) + }) + + it('rejects unsupported lockfile names', () => { + expect(() => parseLockfilePackagesContent('', 'deno.lock')).toThrow(UnsupportedLockfileError) + }) + + it('rejects the binary bun lockfile with a remedy', () => { + expect(() => parseLockfilePackagesContent('', 'bun.lockb')) + .toThrow(/bun install --save-text-lockfile/) + }) +}) diff --git a/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts b/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts index 991342744..252ecb470 100644 --- a/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts +++ b/packages/cli/src/services/embedded-packages/__tests__/materializer.spec.ts @@ -65,6 +65,12 @@ describe('EmbeddedPackagesMaterializer', () => { }) } + // Materializes the full plan, the way the Bundler does at finalize time + // (production passes the subset the pruned lockfile still references). + const materializeAll = async (materializer: EmbeddedPackagesMaterializer) => { + return materializer.materializeTarballs((await materializer.plan()).tarballs) + } + beforeEach(async () => { workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-embed-ws-')) homedir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-embed-home-')) @@ -232,7 +238,7 @@ packages: expect(warnings[0]).toContain('git-dep') expect(warnings[0]).toContain('cannot be embedded') const written = await captureStderr(async () => { - await materializer.materialize() + await materializeAll(materializer) }) // Filtered rather than asserting total silence: the debug package // also writes to stderr when DEBUG is enabled. @@ -320,7 +326,7 @@ packages: const written = await captureStderr(async () => { const { warnings } = await materializer.plan() expect(warnings).toEqual([]) - await materializer.materialize() + await materializeAll(materializer) }) // Filtered rather than asserting total silence: the debug package // also writes to stderr when DEBUG is enabled. @@ -427,9 +433,37 @@ packages: {} }) }) - describe('materialize()', () => { + describe('materializeTarballs()', () => { + it('materializes only the requested subset of the plan', async () => { + const materializer = makeMaterializer(['@acme/foo', 'bar@2.0.0']) + const { tarballs } = await materializer.plan() + const barOnly = tarballs.filter(tarball => tarball.name === 'bar') + const materialized = await materializer.materializeTarballs(barOnly) + expect(materialized.map(t => t.archivePath)).toEqual([ + '.checkly/embedded-packages/bar@2.0.0.tgz', + ]) + // The unrequested tarball must produce no registry traffic at all. + expect(requests.map(r => r.url)).toEqual(['/bar/-/bar-2.0.0.tgz']) + }) + + it('returns nothing for an empty subset without touching the registry', async () => { + const materialized = await makeMaterializer(['bar@2.0.0']).materializeTarballs([]) + expect(materialized).toEqual([]) + expect(requests).toHaveLength(0) + }) + + it('refuses even an empty subset when the plan has issues', async () => { + // The issues backstop is checked before the empty-list short-circuit: + // an invalid configuration must not pass silently just because + // nothing was requested. + const materializer = makeMaterializer(['no-such-package']) + await expect(materializer.materializeTarballs([])).rejects.toThrow(EmbeddedPackageError) + }) + }) + + describe('materializeTarballs() over the full plan', () => { it('downloads tarballs from the registry and verifies them', async () => { - const tarballs = await makeMaterializer(['@acme/foo', 'bar@2.0.0']).materialize() + const tarballs = await materializeAll(makeMaterializer(['@acme/foo', 'bar@2.0.0'])) expect(tarballs.map(t => t.archivePath)).toEqual([ '.checkly/embedded-packages/@acme+foo@1.2.3.tgz', '.checkly/embedded-packages/bar@2.0.0.tgz', @@ -445,23 +479,23 @@ packages: {} }) it('defaults the cache to node_modules/.cache/checkly under the workspace root', async () => { - const tarballs = await makeMaterializer(['bar@2.0.0'], { env: {} }).materialize() + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'], { env: {} })) expect(tarballs[0].filePath.startsWith( path.join(workspaceRoot, 'node_modules', '.cache', 'checkly', 'embedded-packages'), )).toBe(true) }) it('derives the project root from the lockfile path when no workspace root is given', async () => { - const tarballs = await makeMaterializer(['bar@2.0.0'], { env: {}, workspaceRoot: undefined }).materialize() + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'], { env: {}, workspaceRoot: undefined })) expect(tarballs[0].filePath.startsWith(path.join( path.dirname(lockfilePath), 'node_modules', '.cache', 'checkly', 'embedded-packages', ))).toBe(true) }) it('reuses the CLI cache instead of downloading again', async () => { - await makeMaterializer(['bar@2.0.0']).materialize() + await materializeAll(makeMaterializer(['bar@2.0.0'])) expect(requests).toHaveLength(1) - await makeMaterializer(['bar@2.0.0']).materialize() + await materializeAll(makeMaterializer(['bar@2.0.0'])) expect(requests).toHaveLength(1) }) @@ -481,7 +515,7 @@ packages: {} const materializer = makeMaterializer(['bar@2.0.0'], { env: { CHECKLY_CACHE_DIR: cacheDir, npm_config_cache: npmCacheDir }, }) - const tarballs = await materializer.materialize() + const tarballs = await materializeAll(materializer) expect(requests).toHaveLength(0) await expect(fs.readFile(tarballs[0].filePath)).resolves.toEqual(barTarball) }) @@ -492,7 +526,7 @@ packages: {} `//127.0.0.1:${(server.address() as AddressInfo).port}/:_authToken=secret`, ].join('\n')) - await makeMaterializer(['bar@2.0.0']).materialize() + await materializeAll(makeMaterializer(['bar@2.0.0'])) expect(requests[0].authorization).toBe('Bearer secret') }) @@ -509,7 +543,7 @@ packages: res.end(barTarball) }) - await makeMaterializer(['bar@2.0.0']).materialize() + await materializeAll(makeMaterializer(['bar@2.0.0'])) expect(requests[0].url).toBe('/custom/path/bar-2.0.0.tgz') }) @@ -517,7 +551,7 @@ packages: server.removeAllListeners('request') server.on('request', (req, res) => res.end('tampered content')) - await expect(makeMaterializer(['bar@2.0.0']).materialize()) + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) .rejects.toThrow(/does not match the integrity hash recorded in the lockfile/) }) @@ -528,12 +562,12 @@ packages: secured@1.0.0: resolution: {integrity: ${barIntegrity}} `) - await expect(makeMaterializer(['secured']).materialize()) + await expect(materializeAll(makeMaterializer(['secured']))) .rejects.toThrow(/Failed to download embedded package 'secured@1\.0\.0'.*HTTP 401.*credentials/s) }) it('refuses to materialize when the plan has issues', async () => { - await expect(makeMaterializer(['no-such-package']).materialize()) + await expect(materializeAll(makeMaterializer(['no-such-package']))) .rejects.toThrow(EmbeddedPackageError) }) @@ -543,7 +577,7 @@ packages: await fs.writeFile(path.join(workspaceRoot, '.npmrc'), 'registry=http://127.0.0.1:1/\n') await fs.writeFile(path.join(contextDir, '.npmrc'), `registry=${serverUrl}\n`) - const tarballs = await makeMaterializer(['bar@2.0.0'], { contextDir }).materialize() + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'], { contextDir })) expect(tarballs).toHaveLength(1) expect(requests).toHaveLength(1) }) @@ -554,7 +588,7 @@ packages: const materializer = makeMaterializer(['bar@2.0.0'], { env: { CHECKLY_CACHE_DIR: cacheDir, npm_config_registry: serverUrl }, }) - const tarballs = await materializer.materialize() + const tarballs = await materializeAll(materializer) expect(tarballs).toHaveLength(1) expect(requests).toHaveLength(1) }) @@ -562,7 +596,7 @@ packages: it('fails with a clear error for a registry URL without a protocol', async () => { await fs.writeFile(path.join(workspaceRoot, '.npmrc'), 'registry=nexus.local/repository/npm/\n') - await expect(makeMaterializer(['bar@2.0.0']).materialize()) + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) .rejects.toThrow(/is not a valid URL.*registry/s) }) @@ -579,17 +613,205 @@ packages: resolution: {integrity: ${barIntegrity}} `) - const error = await makeMaterializer(['missing-pkg']).materialize().catch(err => err) + const error = await materializeAll(makeMaterializer(['missing-pkg'])).catch(err => err) expect(error).toBeInstanceOf(EmbeddedPackageError) expect(error.message).not.toContain('super-secret') expect(error.message).toContain('missing-pkg') }) - it('memoizes materialization within an instance', async () => { + it('serves a repeated materialization from the CLI cache without re-downloading', async () => { const materializer = makeMaterializer(['bar@2.0.0']) - const [first, second] = await Promise.all([materializer.materialize(), materializer.materialize()]) - expect(first).toBe(second) + const first = await materializeAll(materializer) + const second = await materializeAll(materializer) + expect(second).toEqual(first) expect(requests).toHaveLength(1) }) }) + + describe('materializeTarballs() from a yarn.lock plan', () => { + // yarn.lock entries carry no npm tarball integrity (Berry checksums + // hash yarn's own cache archive), so the materializer must resolve it + // from the registry's per-version metadata before downloading. + const writeYarnLockfile = async () => { + lockfilePath = path.join(workspaceRoot, 'yarn.lock') + await fs.writeFile(lockfilePath, ` +__metadata: + version: 10 + cacheKey: 10c0 + +"@acme/foo@npm:1.2.3": + version: 1.2.3 + resolution: "@acme/foo@npm:1.2.3" + checksum: 10c0/aaa + languageName: node + linkType: hard + +"bar@npm:2.0.0": + version: 2.0.0 + resolution: "bar@npm:2.0.0" + checksum: 10c0/bbb + languageName: node + linkType: hard +`) + } + + const serveMetadata = (routes: Record) => { + server.removeAllListeners('request') + server.on('request', (req, res) => { + requests.push({ + url: req.url!, + authorization: req.headers.authorization, + acceptEncoding: req.headers['accept-encoding'] as string | undefined, + }) + const body = routes[req.url!] + if (body === undefined) { + res.statusCode = 404 + res.end('not found') + } else if (typeof body === 'number') { + // A numeric route value is an HTTP status to return. + res.statusCode = body + res.end('error') + } else if (Buffer.isBuffer(body)) { + res.end(body) + } else { + res.setHeader('content-type', 'application/json') + res.end(JSON.stringify(body)) + } + }) + } + + it('resolves the integrity from registry metadata and verifies the download', async () => { + await writeYarnLockfile() + serveMetadata({ + '/bar/2.0.0': { dist: { integrity: barIntegrity, tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + '/bar/-/bar-2.0.0.tgz': barTarball, + }) + + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'])) + expect(tarballs).toHaveLength(1) + expect(tarballs[0].integrity).toEqual(barIntegrity) + expect(await fs.readFile(tarballs[0].filePath)).toEqual(barTarball) + expect(requests.map(request => request.url)).toEqual(['/bar/2.0.0', '/bar/-/bar-2.0.0.tgz']) + }) + + it('requests scoped metadata with the scope slash unencoded and derives the tarball URL', async () => { + await writeYarnLockfile() + // No dist.tarball in the metadata: the download must fall back to the + // registry-derived URL. + serveMetadata({ + '/@acme/foo/1.2.3': { dist: { integrity: fooIntegrity } }, + '/@acme/foo/-/foo-1.2.3.tgz': fooTarball, + }) + + const tarballs = await materializeAll(makeMaterializer(['@acme/foo'])) + expect(tarballs[0].integrity).toEqual(fooIntegrity) + expect(requests.map(request => request.url)).toEqual(['/@acme/foo/1.2.3', '/@acme/foo/-/foo-1.2.3.tgz']) + }) + + it('falls back to the full packument when the per-version route 404s', async () => { + // Some private registry proxies serve only the full packument, not + // the abbreviated per-version route. + await writeYarnLockfile() + serveMetadata({ + '/bar': { versions: { '2.0.0': { dist: { integrity: barIntegrity, tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } } } }, + '/bar/-/bar-2.0.0.tgz': barTarball, + }) + + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'])) + expect(tarballs[0].integrity).toEqual(barIntegrity) + // The per-version route was tried first (404), then the packument. + expect(requests.map(request => request.url)).toEqual(['/bar/2.0.0', '/bar', '/bar/-/bar-2.0.0.tgz']) + }) + + it('does not fall back to the packument when the per-version route answers without a hash', async () => { + // A per-version response that simply lacks integrity is a real + // answer, not an absent route, so it must not trigger a second fetch. + await writeYarnLockfile() + serveMetadata({ + '/bar/2.0.0': { dist: { tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + '/bar': { versions: { '2.0.0': { dist: { integrity: barIntegrity } } } }, + }) + + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) + .rejects.toThrow(/provides no usable integrity hash/) + expect(requests.map(request => request.url)).toEqual(['/bar/2.0.0']) + }) + + it('falls back to a sha1 shasum when the metadata has no integrity', async () => { + await writeYarnLockfile() + const shasum = createHash('sha1').update(barTarball).digest('hex') + serveMetadata({ + '/bar/2.0.0': { dist: { shasum, tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + '/bar/-/bar-2.0.0.tgz': barTarball, + }) + + const tarballs = await materializeAll(makeMaterializer(['bar@2.0.0'])) + expect(tarballs[0].integrity).toEqual(`sha1-${Buffer.from(shasum, 'hex').toString('base64')}`) + }) + + it('fails with a clear error when the metadata request errors (non-404)', async () => { + // A 404 means "try the other route"; any other status is a hard + // failure that must surface rather than fall through. + await writeYarnLockfile() + serveMetadata({ '/bar/2.0.0': 500 }) + + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) + .rejects.toThrow(/Failed to fetch registry metadata for embedded package 'bar@2.0.0'/) + // The 500 stops the resolution; the packument fallback is not tried. + expect(requests.map(request => request.url)).toEqual(['/bar/2.0.0']) + }) + + it('fails with a clear error when neither metadata route exists', async () => { + await writeYarnLockfile() + serveMetadata({}) + + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) + .rejects.toThrow(/provides no usable integrity hash/) + expect(requests.map(request => request.url)).toEqual(['/bar/2.0.0', '/bar']) + }) + + it('fails with a clear error when the metadata provides no usable hash', async () => { + await writeYarnLockfile() + serveMetadata({ + '/bar/2.0.0': { dist: { tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + }) + + await expect(materializeAll(makeMaterializer(['bar@2.0.0']))) + .rejects.toThrow(/provides no usable integrity hash/) + }) + + it('sends registry credentials with the metadata request', async () => { + await writeYarnLockfile() + const { port } = server.address() as AddressInfo + await fs.writeFile(path.join(workspaceRoot, '.npmrc'), [ + `registry=${serverUrl}`, + `//127.0.0.1:${port}/:_authToken=secret`, + ].join('\n')) + serveMetadata({ + '/bar/2.0.0': { dist: { integrity: barIntegrity, tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + '/bar/-/bar-2.0.0.tgz': barTarball, + }) + + await materializeAll(makeMaterializer(['bar@2.0.0'])) + expect(requests[0]).toMatchObject({ url: '/bar/2.0.0', authorization: 'Bearer secret' }) + }) + + it('still resolves metadata on a warm cache, but skips the download', async () => { + // The caches are keyed by integrity, which for yarn plans is only + // learnable from the registry — so the (small) metadata roundtrip + // happens every run, while the tarball itself is served from cache. + await writeYarnLockfile() + serveMetadata({ + '/bar/2.0.0': { dist: { integrity: barIntegrity, tarball: `${serverUrl}bar/-/bar-2.0.0.tgz` } }, + '/bar/-/bar-2.0.0.tgz': barTarball, + }) + + await materializeAll(makeMaterializer(['bar@2.0.0'])) + const second = await materializeAll(makeMaterializer(['bar@2.0.0'])) + expect(second).toHaveLength(1) + expect(requests.map(request => request.url)).toEqual([ + '/bar/2.0.0', '/bar/-/bar-2.0.0.tgz', '/bar/2.0.0', + ]) + }) + }) }) diff --git a/packages/cli/src/services/embedded-packages/lockfile-filter.ts b/packages/cli/src/services/embedded-packages/lockfile-filter.ts new file mode 100644 index 000000000..a2e34b9f3 --- /dev/null +++ b/packages/cli/src/services/embedded-packages/lockfile-filter.ts @@ -0,0 +1,51 @@ +import Debug from 'debug' + +import { parseLockfilePackagesContent } from './lockfile-packages.js' +import type { PlannedTarball } from './materializer.js' + +const debug = Debug('checkly:cli:services:embedded-packages:lockfile-filter') + +export interface TarballLockfileFilterResult { + kept: PlannedTarball[] + dropped: PlannedTarball[] +} + +/** + * Splits a planned tarball set by `name@version` presence in a lockfile. + * + * Used after the bundled lockfile has been pruned to the code bundle's + * contents: a pruned lockfile's resolutions are verified to be a subset of + * the original's (no new entries, no version changes), so filtering the + * originally planned set by presence is exactly equivalent to re-running + * spec matching against the pruned lockfile. + * + * Fail-safe: if the lockfile content cannot be parsed — unexpected, since + * a pruned lockfile has already passed the pruner's own parse and + * verification — every tarball is kept (the pre-filter superset is the + * previous behavior) and the failure is debug-logged. + */ +export function filterTarballsByLockfile ( + tarballs: PlannedTarball[], + lockfileContent: string, + lockfileName: string, +): TarballLockfileFilterResult { + let present: Set + try { + const packages = parseLockfilePackagesContent(lockfileContent, lockfileName) + present = new Set(packages.registry.map(pkg => `${pkg.name}@${pkg.version}`)) + } catch (err) { + debug(`Could not parse '${lockfileName}' for tarball filtering, keeping all tarballs: ${err}`) + return { kept: [...tarballs], dropped: [] } + } + + const kept: PlannedTarball[] = [] + const dropped: PlannedTarball[] = [] + for (const tarball of tarballs) { + if (present.has(`${tarball.name}@${tarball.version}`)) { + kept.push(tarball) + } else { + dropped.push(tarball) + } + } + return { kept, dropped } +} diff --git a/packages/cli/src/services/embedded-packages/lockfile-packages.ts b/packages/cli/src/services/embedded-packages/lockfile-packages.ts index 6208076dd..2a0475d1a 100644 --- a/packages/cli/src/services/embedded-packages/lockfile-packages.ts +++ b/packages/cli/src/services/embedded-packages/lockfile-packages.ts @@ -12,7 +12,22 @@ import semver from 'semver' export interface LockfileRegistryPackage { name: string version: string - integrity: string + /** + * SRI integrity of the registry tarball. Absent only for `yarn.lock` + * entries: Yarn Berry checksums hash yarn's own cache archive, not the + * registry tarball, so the tarball integrity is resolved from registry + * metadata at materialization time instead. Entries without either hash + * are excluded by the parsers, so `lockfileChecksum` is always present + * when this is absent. + */ + integrity?: string + /** + * The lockfile's own content pin when it is not an SRI tarball hash + * (Yarn Berry's `checksum` value). Used in place of `integrity` where a + * stable content identifier known at plan time is needed (the dependency + * cache hash). + */ + lockfileChecksum?: string /** * The full tarball URL when the lockfile records one (npm's `resolved`, * pnpm's `resolution.tarball`). When absent, the URL is derived from the @@ -55,25 +70,54 @@ export class UnsupportedLockfileError extends Error { /** * Enumerates every package entry in a lockfile, classified into embeddable * registry packages and excluded (git/file/link/integrity-less) entries. - * Supports `pnpm-lock.yaml` (v6/v9) and `package-lock.json` (v2/v3). + * Supports `pnpm-lock.yaml` (v6/v9), `package-lock.json` (v2/v3), + * `bun.lock` (v1) and Yarn Berry's `yarn.lock`. */ export async function loadLockfilePackages (lockfilePath: string): Promise { const basename = path.basename(lockfilePath) const content = await fs.readFile(lockfilePath, 'utf8') + return parseLockfilePackagesContent(content, basename) +} - switch (basename) { +/** + * Content-based variant of {@link loadLockfilePackages} for lockfiles that + * exist only in memory (e.g. a pruned copy of the bundled lockfile). The + * format is dispatched on the lockfile's filename. + */ +export function parseLockfilePackagesContent (content: string, lockfileName: string): LockfilePackages { + switch (lockfileName) { case 'pnpm-lock.yaml': return parsePnpmLockfilePackages(content) case 'package-lock.json': return parseNpmLockfilePackages(content) + case 'bun.lock': + return parseBunLockfilePackages(content) + case 'yarn.lock': + return parseYarnLockfilePackages(content) + case 'bun.lockb': + throw new UnsupportedLockfileError( + `Embedded packages are not supported for bun's binary lockfile (bun.lockb).` + + ` Regenerate a text lockfile with \`bun install --save-text-lockfile\`.`, + ) default: throw new UnsupportedLockfileError( - `Embedded packages are not supported for '${basename}' lockfiles yet.` - + ` Only pnpm (pnpm-lock.yaml) and npm (package-lock.json) are currently supported.`, + `Embedded packages are not supported for '${lockfileName}' lockfiles yet.` + + ` Only pnpm (pnpm-lock.yaml), npm (package-lock.json), bun (bun.lock)` + + ` and Yarn Berry (yarn.lock) are currently supported.`, ) } } +/** + * Whether a directory-link target points outside the workspace the bundle + * carries. Shared by the parsers' link classification: an in-workspace link + * is part of the project (safe for a wildcard to skip silently), while an + * escaping one is not. + */ +function linkTargetEscapesWorkspace (target: string): boolean { + return target === '..' || target.startsWith('../') || path.isAbsolute(target) +} + /** * Strips a pnpm peer-dependency suffix (`(react@18.2.0)`) from a package * key. The v9 `packages` section doesn't use them (they live in @@ -84,6 +128,70 @@ function stripPeerSuffix (key: string): string { return cut === -1 ? key : key.slice(0, cut) } +/** + * Splits a `name@ref` package key at the first `@` past the name. Searching + * from the front (after the scope, when present) keeps the name intact when + * the ref itself contains `@`, as git refs do + * (`foo@git+ssh://git@github.com/...`). Shared by the pnpm and bun parsers, + * whose keys use the same shape. + */ +function splitPackageKey (key: string): { name: string, ref: string } | undefined { + const searchFrom = key.startsWith('@') ? key.indexOf('/') + 1 : 1 + const separator = searchFrom > 0 ? key.indexOf('@', searchFrom) : -1 + if (separator <= 0) { + return undefined + } + return { name: key.slice(0, separator), ref: key.slice(separator + 1) } +} + +/** + * Classifies one `name@ref` entry into a registry package or an unfetchable + * exclusion, appending to `result`. Shared by the pnpm and bun parsers, + * which record the same information in different places (pnpm's + * `resolution.integrity`/`resolution.tarball` vs bun's tuple indices). + */ +function classifyRegistryEntry ( + result: LockfilePackages, + name: string, + ref: string, + integrity: unknown, + tarball: unknown, +): void { + // Validate with semver but keep the ref as written: semver.valid() + // normalizes away build metadata (`1.0.0+sha.abc` → `1.0.0`), which + // would break both version-pin matching and the derived tarball URL. + const version = semver.valid(ref) !== null ? ref : null + if (version === null) { + result.excluded.push({ + name, + reason: `'${name}@${ref}' resolves to a git, file or URL dependency,` + + ` which cannot be embedded as a registry tarball`, + kind: 'unfetchable', + }) + return + } + + if (typeof integrity !== 'string' || integrity === '') { + result.excluded.push({ + name, + version, + reason: `the lockfile records no integrity hash for '${name}@${version}',` + + ` which is required to embed it`, + kind: 'unfetchable', + }) + return + } + + result.registry.push({ + name, + version, + integrity, + // Only absolute http(s) URLs are usable for downloading; anything + // else falls back to the registry-derived URL. + tarballUrl: typeof tarball === 'string' && /^https?:/.test(tarball) ? tarball : undefined, + }) +} + export function parsePnpmLockfilePackages (content: string): LockfilePackages { const data = parseYaml(content) @@ -119,7 +227,7 @@ export function parsePnpmLockfilePackages (content: string): LockfilePackages { // target escapes the workspace is not part of the project the // bundle carries. const target = version.slice('link:'.length) - const escapesWorkspace = target === '..' || target.startsWith('../') || path.isAbsolute(target) + const escapesWorkspace = linkTargetEscapesWorkspace(target) result.excluded.push({ name, reason: escapesWorkspace @@ -143,58 +251,196 @@ export function parsePnpmLockfilePackages (content: string): LockfilePackages { for (const [rawKey, rawEntry] of Object.entries(packages)) { // v6 keys have a leading slash (`/name@1.2.3`), v9 keys do not. const key = stripPeerSuffix(rawKey.startsWith('/') ? rawKey.slice(1) : rawKey) - // The name/ref separator is the first `@` past the name. Searching from - // the front (after the scope, when present) keeps the name intact when - // the ref itself contains `@`, as git refs do - // (`foo@git+ssh://git@github.com/...`). - const searchFrom = key.startsWith('@') ? key.indexOf('/') + 1 : 1 - const separator = searchFrom > 0 ? key.indexOf('@', searchFrom) : -1 - if (separator <= 0) { + const split = splitPackageKey(key) + if (split === undefined) { continue } - const name = key.slice(0, separator) - const ref = key.slice(separator + 1) + const { name, ref } = split if (seen.has(`${name}@${ref}`)) { continue } seen.add(`${name}@${ref}`) - // Validate with semver but keep the ref as written: semver.valid() - // normalizes away build metadata (`1.0.0+sha.abc` → `1.0.0`), which - // would break both version-pin matching and the derived tarball URL. - const version = semver.valid(ref) !== null ? ref : null + const resolution = rawEntry?.resolution + classifyRegistryEntry(result, name, ref, resolution?.integrity, resolution?.tarball) + } + + return result +} + +export function parseBunLockfilePackages (content: string): LockfilePackages { + // bun.lock is JSONC (bun writes trailing commas), hence JSON5. + const data = JSON5.parse(content) + + const lockfileVersion = data?.lockfileVersion + if (lockfileVersion !== 1) { + throw new UnsupportedLockfileError( + `Embedded packages require bun lockfile version 1` + + ` (found '${lockfileVersion ?? 'unknown'}'). Regenerate the lockfile with a supported` + + ` bun version, or update the Checkly CLI if the lockfile is newer.`, + ) + } + + const packages = data?.packages + const result: LockfilePackages = { registry: [], excluded: [] } + if (typeof packages !== 'object' || packages === null) { + return result + } + + const seen = new Set() + for (const tuple of Object.values(packages)) { + // Package values are tuples whose first element is `name@ref`; the rest + // varies by resolution kind (registry entries carry the tarball URL at + // index 1 — '' for the default registry — and the integrity hash at + // index 3). An unrecognizable value is skipped rather than guessed at, + // like unparseable pnpm keys. Aliased installs appear under the alias + // key but record the real package name in the tuple, so the split below + // always yields the real name. + if (!Array.isArray(tuple) || typeof tuple[0] !== 'string') { + continue + } + const split = splitPackageKey(tuple[0]) + if (split === undefined) { + continue + } + const { name, ref } = split + + if (seen.has(`${name}@${ref}`)) { + continue + } + seen.add(`${name}@${ref}`) + + if (ref.startsWith('workspace:')) { + result.excluded.push({ + name, + reason: `'${name}' is a workspace package, which cannot be embedded as a registry tarball`, + kind: 'workspace', + }) + continue + } + + classifyRegistryEntry(result, name, ref, tuple[3], tuple[1]) + } + + return result +} + +export function parseYarnLockfilePackages (content: string): LockfilePackages { + // Yarn Berry lockfiles are YAML; Yarn Classic's v1 format happens to + // YAML-parse too, so Berry is recognized by its __metadata section. + const data = parseYaml(content) + if (data === null || typeof data !== 'object') { + throw new UnsupportedLockfileError( + `Embedded packages could not parse the yarn.lock file.`, + ) + } + const metadata = (data as Record).__metadata + if (metadata === null || typeof metadata !== 'object' || metadata.version === undefined) { + throw new UnsupportedLockfileError( + `Embedded packages are not supported for Yarn Classic (v1) lockfiles.` + + ` Migrate to Yarn Berry, whose lockfile records the resolution data embedding needs.`, + ) + } + + const result: LockfilePackages = { registry: [], excluded: [] } + + const seen = new Set() + for (const [key, entry] of Object.entries(data)) { + if (key === '__metadata') { + continue + } + // Every entry records its resolved locator as `name@protocol:ref` in + // `resolution` — under an aliased or multi-descriptor key this is the + // REAL package locator, so names and refs are always taken from it. An + // unrecognizable entry is skipped rather than guessed at, like + // unparseable pnpm keys. + const resolution = entry?.resolution + if (typeof resolution !== 'string') { + continue + } + const split = splitPackageKey(resolution) + if (split === undefined) { + continue + } + const { name, ref } = split + + if (seen.has(`${name}@${ref}`)) { + continue + } + seen.add(`${name}@${ref}`) + + if (ref.startsWith('workspace:')) { + result.excluded.push({ + name, + reason: `'${name}' is a workspace package, which cannot be embedded as a registry tarball`, + kind: 'workspace', + }) + continue + } + + // Directory links (`portal:`/`link:`) carry their target between the + // protocol and the `::locator=...` suffix. Same distinction as the + // pnpm/npm parsers: an in-workspace link is part of the project itself. + const linkPrefix = ['portal:', 'link:'].find(prefix => ref.startsWith(prefix)) + if (linkPrefix !== undefined) { + const target = ref.slice(linkPrefix.length).split('::')[0] + const escapesWorkspace = linkTargetEscapesWorkspace(target) + result.excluded.push({ + name, + reason: escapesWorkspace + ? `'${name}' is a local directory link outside the workspace, which cannot be embedded` + + ` as a registry tarball` + : `'${name}' is a workspace package, which cannot be embedded as a registry tarball`, + kind: escapesWorkspace ? 'unfetchable' : 'workspace', + }) + continue + } + + // Anything that is not a plain `npm:` locator cannot be a + // registry tarball: patch:, git, https:, file:, exec:, or an npm ref + // whose version is not valid semver (a range or tag yarn left + // unresolved). file: targets an archive the bundle does not carry, so + // unlike directory links it warrants the unfetchable warning; builtin + // and user patches always coexist with the underlying npm: entry, + // which shadows the exclusion for reporting purposes. + const rawVersion = ref.startsWith('npm:') ? ref.slice('npm:'.length) : null + // As in the other parsers: validate with semver but keep the version as + // written, preserving build metadata. + const version = rawVersion !== null && semver.valid(rawVersion) !== null ? rawVersion : null if (version === null) { result.excluded.push({ name, - reason: `'${name}@${ref}' resolves to a git, file or URL dependency,` + reason: `'${name}@${ref}' resolves to a git, file, URL or patched dependency,` + ` which cannot be embedded as a registry tarball`, kind: 'unfetchable', }) continue } - const resolution = rawEntry?.resolution - const integrity = resolution?.integrity - if (typeof integrity !== 'string' || integrity === '') { + // Berry's checksum hashes yarn's own cache archive rather than the + // registry tarball, so it cannot serve as SRI integrity — the + // materializer resolves the tarball integrity from registry metadata + // instead. It IS a stable content pin, recorded for the dependency + // cache hash. Without one (checksumBehavior: ignore) the lockfile pins + // nothing, and the entry is excluded like the other parsers' + // integrity-less entries. + const checksum = entry?.checksum + if (typeof checksum !== 'string' || checksum === '') { result.excluded.push({ name, version, - reason: `the lockfile records no integrity hash for '${name}@${version}',` + reason: `the lockfile records no checksum for '${name}@${version}',` + ` which is required to embed it`, kind: 'unfetchable', }) continue } - const tarball = resolution?.tarball result.registry.push({ name, version, - integrity, - // Only absolute http(s) URLs are usable for downloading; anything - // else falls back to the registry-derived URL. - tarballUrl: typeof tarball === 'string' && /^https?:/.test(tarball) ? tarball : undefined, + lockfileChecksum: checksum, }) } @@ -238,7 +484,7 @@ export function parseNpmLockfilePackages (content: string): LockfilePackages { // part of the project the bundle carries, so a wildcard must not // skip it silently. const target = typeof entry?.resolved === 'string' ? entry.resolved : '' - const escapesWorkspace = target === '..' || target.startsWith('../') || path.isAbsolute(target) + const escapesWorkspace = linkTargetEscapesWorkspace(target) result.excluded.push({ name: key.slice(lastNodeModules + 'node_modules/'.length), reason: escapesWorkspace diff --git a/packages/cli/src/services/embedded-packages/materializer.ts b/packages/cli/src/services/embedded-packages/materializer.ts index 46d0efa7b..82efc8231 100644 --- a/packages/cli/src/services/embedded-packages/materializer.ts +++ b/packages/cli/src/services/embedded-packages/materializer.ts @@ -77,6 +77,11 @@ export interface EmbeddedPackagesPlan { * to be added to the code bundle. */ export interface MaterializedTarball extends PlannedTarball { + /** + * SRI integrity the tarball was verified against — the lockfile's, or for + * yarn.lock plans the one resolved from registry metadata. + */ + integrity: string /** Absolute path of the verified tarball in the CLI cache. */ filePath: string /** Bundle-root-relative archive path (POSIX). */ @@ -137,15 +142,32 @@ function redactUrl (url: string): string { } } +/** + * Wraps an axios error from a registry request in an EmbeddedPackageError, + * appending the HTTP status and, for 401/403, a credentials hint. `message` + * is the action-specific prefix (e.g. "Failed to download …"). + */ +function registryHttpError (err: any, message: string): EmbeddedPackageError { + const status = err?.response?.status + const statusHint = status !== undefined ? ` (HTTP ${status})` : '' + const authHint = status === 401 || status === 403 + ? ` Check that your .npmrc contains valid credentials for this registry.` + : '' + return new EmbeddedPackageError(`${message}${statusHint}.${authHint}`, { cause: err }) +} + /** * Resolves the configured `bundle.packages.embed` specs against the * workspace lockfile (plan) and sources the selected tarballs into the CLI - * cache (materialize), through a chain of CLI cache → npm cacache → + * cache (materializeTarballs), through a chain of CLI cache → npm cacache → * registry download, always verified against the lockfile integrity. * - * Both stages memoize their in-flight promise: multiple Playwright checks - * bundle concurrently, and validation and bundling share one instance per - * parsed project, so the work runs exactly once. + * The plan memoizes its in-flight promise: validation and bundling share + * one instance per parsed project, so the (purely local) resolution runs + * exactly once. Materialization has a single caller — the Bundler, at + * finalize time, after the bundled lockfile has been pruned — which passes + * the subset of the plan the shipped lockfile still references, so + * pruned-away tarballs are never downloaded. */ export class EmbeddedPackagesMaterializer { #options: EmbeddedPackagesMaterializerOptions @@ -154,7 +176,6 @@ export class EmbeddedPackagesMaterializer { #homedir: string #plan?: Promise - #materialized?: Promise constructor (options: EmbeddedPackagesMaterializerOptions) { this.#options = options @@ -173,9 +194,48 @@ export class EmbeddedPackagesMaterializer { return this.#plan } - materialize (): Promise { - this.#materialized ??= this.#materializeAll() - return this.#materialized + /** + * Sources the given subset of the planned tarballs (CLI cache → npm + * cacache → registry download, verified against the lockfile integrity). + * Lets the caller materialize only the tarballs a pruned bundled lockfile + * still references, so pruned-away packages are never downloaded. + */ + async materializeTarballs (tarballs: PlannedTarball[]): Promise { + const { issues } = await this.plan() + + // Commands validate before bundling and exit on fatal diagnostics, so + // this is a defensive backstop for direct/programmatic use. Checked + // before the empty-list short-circuit: an invalid configuration must + // not pass silently just because nothing was requested. + if (issues.length > 0) { + throw new EmbeddedPackageError( + `Cannot embed packages due to configuration issues:\n\n` + + issues.map(issue => ` ${issue.message}`).join('\n'), + ) + } + + if (tarballs.length === 0) { + return [] + } + + // Safe to assert: a missing lockfile is a plan issue, and issues abort + // above. + const npmrcConfig = await loadNpmrcConfig(defaultNpmrcPaths( + this.#projectRoot!, + this.#homedir, + this.#options.contextDir, + ), this.#env) + + const queue = new PQueue({ concurrency: DOWNLOAD_CONCURRENCY }) + return await queue.addAll(tarballs.map(tarball => async (): Promise => { + const { filePath, integrity } = await this.#obtainTarball(tarball, npmrcConfig) + return { + ...tarball, + integrity, + filePath, + archivePath: `${EMBEDDED_PACKAGES_ARCHIVE_DIR}/${tarball.archiveFilename}`, + } + })) } async #createPlan (): Promise { @@ -335,57 +395,33 @@ export class EmbeddedPackagesMaterializer { } } - async #materializeAll (): Promise { - const { tarballs, issues } = await this.plan() - - // Commands validate before bundling and exit on fatal diagnostics, so - // this is a defensive backstop for direct/programmatic use. - if (issues.length > 0) { - throw new EmbeddedPackageError( - `Cannot embed packages due to configuration issues:\n\n` - + issues.map(issue => ` ${issue.message}`).join('\n'), - ) - } - - if (tarballs.length === 0) { - return [] + async #obtainTarball ( + tarball: PlannedTarball, + npmrcConfig: NpmrcConfig, + ): Promise<{ filePath: string, integrity: string }> { + let { integrity, tarballUrl } = tarball + if (integrity === undefined) { + // yarn.lock plans carry no SRI tarball integrity (Berry checksums + // hash yarn's own cache archive); resolve it from the registry's + // per-version metadata before the caches can be consulted. + const dist = await this.#resolveDistFromRegistry(tarball, npmrcConfig) + integrity = dist.integrity + tarballUrl ??= dist.tarballUrl } - // Safe to assert: a missing lockfile is a plan issue, and issues abort - // above. - const npmrcConfig = await loadNpmrcConfig(defaultNpmrcPaths( - this.#projectRoot!, - this.#homedir, - this.#options.contextDir, - ), this.#env) - - const queue = new PQueue({ concurrency: DOWNLOAD_CONCURRENCY }) - const results = await queue.addAll(tarballs.map(tarball => async (): Promise => { - const filePath = await this.#obtainTarball(tarball, npmrcConfig) - return { - ...tarball, - filePath, - archivePath: `${EMBEDDED_PACKAGES_ARCHIVE_DIR}/${tarball.archiveFilename}`, - } - })) - - return results - } - - async #obtainTarball (tarball: PlannedTarball, npmrcConfig: NpmrcConfig): Promise { - const cached = await this.#cache.get(tarball.integrity) + const cached = await this.#cache.get(integrity) if (cached !== undefined) { debug('%s@%s: CLI cache hit', tarball.name, tarball.version) - return cached + return { filePath: cached, integrity } } - const fromNpmCacache = await lookupNpmCacache(tarball.integrity, this.#env, process.platform, this.#homedir) + const fromNpmCacache = await lookupNpmCacache(integrity, this.#env, process.platform, this.#homedir) if (fromNpmCacache !== undefined) { debug('%s@%s: npm cache hit', tarball.name, tarball.version) - return await this.#cache.put(tarball.integrity, fromNpmCacache) + return { filePath: await this.#cache.put(integrity, fromNpmCacache), integrity } } - const url = tarball.tarballUrl ?? this.#deriveTarballUrl(tarball, npmrcConfig) + const url = tarballUrl ?? this.#deriveTarballUrl(tarball, npmrcConfig) if (!URL.canParse(url)) { throw new EmbeddedPackageError( `The tarball URL for embedded package '${tarball.name}@${tarball.version}'` @@ -396,16 +432,116 @@ export class EmbeddedPackagesMaterializer { debug('%s@%s: downloading from %s', tarball.name, tarball.version, redactUrl(url)) const content = await this.#download(tarball, url, npmrcConfig) - if (!verifyIntegrity(content, tarball.integrity)) { + if (!verifyIntegrity(content, integrity)) { + // For yarn.lock plans the integrity came from the registry's own + // metadata, not the lockfile, so name the right source to check. + const source = tarball.integrity === undefined + ? `the integrity hash the registry's metadata reported` + : `the integrity hash recorded in the lockfile` throw new EmbeddedPackageError( `The tarball downloaded for embedded package '${tarball.name}@${tarball.version}'` - + ` from '${redactUrl(url)}' does not match the integrity hash recorded in the lockfile` - + ` ('${tarball.integrity}'). The registry may be serving a different artifact` + + ` from '${redactUrl(url)}' does not match ${source}` + + ` ('${integrity}'). The registry may be serving a different artifact` + ` than the one the lockfile was created against.`, ) } - return await this.#cache.put(tarball.integrity, content) + return { filePath: await this.#cache.put(integrity, content), integrity } + } + + /** + * Resolves the npm tarball integrity (and canonical tarball URL) for a + * package whose lockfile cannot provide one. Tries the abbreviated + * per-version metadata route (`GET //`) first, + * then falls back to the full packument (`GET /`, whose + * `versions[version].dist` carries the same fields) — some private + * registry proxies serve only one of the two. The scope slash stays + * unencoded, matching npm's own use of these routes. The requests use + * the same registry resolution and credentials as the tarball download + * itself, so they add no trust beyond the download; the end-to-end + * content pin still holds because the package manager re-verifies its + * own lockfile checksums against the served content at install time. + */ + async #resolveDistFromRegistry ( + tarball: PlannedTarball, + npmrcConfig: NpmrcConfig, + ): Promise<{ integrity: string, tarballUrl?: string }> { + const registryUrl = resolveRegistryUrl(npmrcConfig, tarball.name, this.#env) + const versionUrl = `${registryUrl}${tarball.name}/${tarball.version}` + const packumentUrl = `${registryUrl}${tarball.name}` + + // Per-version route: dist is at the document root. + const perVersion = await this.#fetchMetadataDist(tarball, npmrcConfig, versionUrl, data => data?.dist) + // Packument fallback (only when the per-version route was absent, not + // when it answered with unusable data): dist is nested per version. + const dist = perVersion ?? await this.#fetchMetadataDist( + tarball, npmrcConfig, packumentUrl, data => data?.versions?.[tarball.version]?.dist, + ) + + // Modern publishes carry an SRI `integrity`; very old ones only a hex + // sha1 `shasum`, which converts to a (weaker but supported) SRI hash. + const integrity = typeof dist?.integrity === 'string' && dist.integrity !== '' + ? dist.integrity as string + : typeof dist?.shasum === 'string' && /^[0-9a-f]{40}$/.test(dist.shasum) + ? `sha1-${Buffer.from(dist.shasum, 'hex').toString('base64')}` + : undefined + if (integrity === undefined) { + throw new EmbeddedPackageError( + `The registry metadata for embedded package '${tarball.name}@${tarball.version}'` + + ` (from '${redactUrl(versionUrl)}') provides no usable integrity hash, so the` + + ` downloaded tarball could not be verified.`, + ) + } + + return { + integrity, + // Same guard as the lockfile-recorded URLs: only absolute http(s) + // URLs are usable for downloading. + tarballUrl: typeof dist?.tarball === 'string' && /^https?:/.test(dist.tarball) + ? dist.tarball as string + : undefined, + } + } + + /** + * Fetches one metadata URL and extracts its `dist` via `select`. Returns + * undefined on a 404 (so the caller can try another route); any other + * failure — auth, network, malformed response — throws, because retrying + * a different route would only mask it. + */ + async #fetchMetadataDist ( + tarball: PlannedTarball, + npmrcConfig: NpmrcConfig, + url: string, + select: (data: any) => any, + ): Promise { + if (!URL.canParse(url)) { + throw new EmbeddedPackageError( + `The registry metadata URL for embedded package '${tarball.name}@${tarball.version}'` + + ` is not a valid URL: '${redactUrl(url)}'. Check the 'registry' configuration` + + ` in your .npmrc (it must be an absolute URL including the protocol).`, + ) + } + const authHeader = resolveAuthHeader(npmrcConfig, url, this.#env) + debug('%s@%s: resolving integrity from %s', tarball.name, tarball.version, redactUrl(url)) + try { + const response = await axios.get(url, assignProxy(url, { + headers: { + ...(authHeader !== undefined ? { authorization: authHeader } : {}), + }, + timeout: DOWNLOAD_TIMEOUT_MS, + })) + return select(response.data) + } catch (err: any) { + if (err?.response?.status === 404) { + return undefined + } + throw registryHttpError( + err, + `Failed to fetch registry metadata for embedded package` + + ` '${tarball.name}@${tarball.version}' from '${redactUrl(url)}'`, + ) + } } #deriveTarballUrl (tarball: PlannedTarball, npmrcConfig: NpmrcConfig): string { @@ -433,15 +569,10 @@ export class EmbeddedPackagesMaterializer { })) return Buffer.from(response.data) } catch (err: any) { - const status = err?.response?.status - const statusHint = status !== undefined ? ` (HTTP ${status})` : '' - const authHint = status === 401 || status === 403 - ? ` Check that your .npmrc contains valid credentials for this registry.` - : '' - throw new EmbeddedPackageError( + throw registryHttpError( + err, `Failed to download embedded package '${tarball.name}@${tarball.version}'` - + ` from '${redactUrl(url)}'${statusHint}.${authHint}`, - { cause: err }, + + ` from '${redactUrl(url)}'`, ) } } diff --git a/packages/cli/src/services/playwright-project-bundler.ts b/packages/cli/src/services/playwright-project-bundler.ts index 23dbae483..a373d1d51 100644 --- a/packages/cli/src/services/playwright-project-bundler.ts +++ b/packages/cli/src/services/playwright-project-bundler.ts @@ -142,23 +142,10 @@ export class PlaywrightProjectBundler { })) } - // Embedded package tarballs live in the CLI cache, whose on-disk - // location (node_modules/.cache, a per-user dir, or CHECKLY_CACHE_DIR) - // never corresponds to the contract path the runner expects, so they - // carry an explicit archive path instead of relying on the strip - // prefix. The materializer memoizes, so concurrent bundles share one - // download run, and the Bundler dedupes registrations by archive path - // across checks. - const materializer = Session.getEmbeddedPackagesMaterializer() - if (materializer !== undefined) { - for (const tarball of await materializer.materialize()) { - files.push({ - filePath: tarball.filePath, - physical: true, - archivePath: tarball.archivePath, - }) - } - } + // Embedded package tarballs are deliberately NOT part of a check's + // files: the Bundler materializes them once during finalize(), after + // the bundled lockfile has been pruned, so tarballs the shipped + // lockfile no longer references are never downloaded. return { browsers: pwConfigParsed.getBrowsers(), @@ -185,12 +172,29 @@ export function getAutoIncludes ( ): string[] { const autoIncludes: string[] = [] - if (packageManager.name === 'pnpm') { - const patchesDir = path.join(basePath, 'patches') - const alreadyIncluded = existingIncludes.some(p => path.resolve(globCwd, p).startsWith(patchesDir)) - if (!alreadyIncluded) { - const patchesPattern = pathToPosix(path.join(path.relative(globCwd, basePath), 'patches', '*.patch')) - autoIncludes.push(patchesPattern) + // The prefix comparison appends a separator so a sibling directory whose + // name merely starts with the patches dir (e.g. `patches-archive`) does + // not suppress the auto-include. + const includesUnder = (dir: string): boolean => existingIncludes.some(p => { + const resolved = path.resolve(globCwd, p) + return resolved === dir || resolved.startsWith(dir + path.sep) + }) + + // Dependency patches live in a conventional per-manager directory that + // the remote install (and the lockfile pruner's temp-dir install) needs + // alongside the manifests: `patches/` for pnpm and bun, `.yarn/patches` + // for Yarn Berry's patch: protocol. A patch kept at a nonconventional + // path makes the pruner fail closed with a warning instead. + const patchesDirByManager: Record = { + pnpm: ['patches'], + bun: ['patches'], + yarn: ['.yarn', 'patches'], + } + const patchesSegments = patchesDirByManager[packageManager.name] + if (patchesSegments !== undefined) { + const patchesDir = path.join(basePath, ...patchesSegments) + if (!includesUnder(patchesDir)) { + autoIncludes.push(pathToPosix(path.join(path.relative(globCwd, basePath), ...patchesSegments, '*.patch'))) } }