diff --git a/config.json b/config.json index 18dfd74cd..1d601b3c2 100644 --- a/config.json +++ b/config.json @@ -109,6 +109,57 @@ "install" ] } + }, + ">=12.0.0": { + "url": "https://registry.npmjs.org/pnpm/-/pnpm-{}.tgz", + "bin": { + "pnpm": "pnpm", + "pnpx": "pnpx" + }, + "nativePackages": { + "win32-x64": { + "package": "@pnpm/exe.win32-x64", + "bin": "pnpm.exe" + }, + "win32-arm64": { + "package": "@pnpm/exe.win32-arm64", + "bin": "pnpm.exe" + }, + "darwin-x64": { + "package": "@pnpm/exe.darwin-x64", + "bin": "pnpm" + }, + "darwin-arm64": { + "package": "@pnpm/exe.darwin-arm64", + "bin": "pnpm" + }, + "linux-x64": { + "package": "@pnpm/exe.linux-x64", + "bin": "pnpm" + }, + "linux-arm64": { + "package": "@pnpm/exe.linux-arm64", + "bin": "pnpm" + }, + "linux-x64-musl": { + "package": "@pnpm/exe.linux-x64-musl", + "bin": "pnpm" + }, + "linux-arm64-musl": { + "package": "@pnpm/exe.linux-arm64-musl", + "bin": "pnpm" + } + }, + "registry": { + "type": "npm", + "package": "pnpm" + }, + "commands": { + "use": [ + "pnpm", + "install" + ] + } } } }, diff --git a/sources/corepackUtils.ts b/sources/corepackUtils.ts index c02571ab6..99d4cfd57 100644 --- a/sources/corepackUtils.ts +++ b/sources/corepackUtils.ts @@ -1,3 +1,5 @@ +import {spawn} from 'child_process'; +import {UsageError} from 'clipanion'; import {createHash} from 'crypto'; import {once} from 'events'; import fs from 'fs'; @@ -205,6 +207,128 @@ async function download(installTarget: string, url: string, algo: string, binPat }; } +function detectLinuxLibcFamily(): `glibc` | `musl` | null { + if (process.platform !== `linux`) + return null; + + // glibc builds expose `glibcVersionRuntime` in the process report; musl + // builds leave it unset. `process.report` may be unavailable, in which case + // we default to glibc. + try { + const report = process.report?.getReport() as any; + if (report == null) + return null; + + return report.header?.glibcVersionRuntime ? `glibc` : `musl`; + } catch { + return null; + } +} + +function getBinNames(bin: BinSpec | BinList): Array { + return Array.isArray(bin) ? bin : Object.keys(bin); +} + +/** + * Whether all the binaries recorded for an install are present on disk. + * + * Installs of package managers distributed as native executables performed by + * older Corepack releases (which were unaware that the executable must be + * fetched separately) record binary paths that don't exist; such installs + * must be discarded and done anew. + */ +async function isNativeInstallIntact(installFolder: string, bin: BinSpec | BinList): Promise { + if (!isValidBinSpec(bin)) + return false; + + try { + await Promise.all(Object.values(bin).map(target => fs.promises.access(path.join(installFolder, target)))); + return true; + } catch { + return false; + } +} + +/** + * Downloads the platform-specific package containing the package manager's + * native executable, then copies said executable over each of the + * placeholders shipped in `tmpFolder`. This replicates what the package + * manager's own install lifecycle script would have done, since Corepack + * never runs lifecycle scripts. + * + * Returns the bin spec to record for the install. + */ +async function installNativeBinaries(installTarget: string, tmpFolder: string, locator: Locator, version: string, spec: PackageManagerSpec): Promise { + let platformKey = `${process.platform}-${process.arch}`; + if (detectLinuxLibcFamily() === `musl`) + platformKey += `-musl`; + + const nativePackage = spec.nativePackages![platformKey]; + if (nativePackage == null) + throw new UsageError(`${locator.name}@${version} does not ship a prebuilt executable for ${platformKey}`); + + // The main package pins the exact version of its platform-specific + // companion packages in its `optionalDependencies`. + let nativeVersion = version; + try { + const manifest = JSON.parse(await fs.promises.readFile(path.join(tmpFolder, `package.json`), `utf8`)); + nativeVersion = manifest?.optionalDependencies?.[nativePackage.package] ?? version; + } catch { + // Fall back to assuming the companion package shares the main package version. + } + + const {tarball, signatures, integrity} = await npmRegistryUtils.fetchTarballURLAndSignature(nativePackage.package, nativeVersion); + + let url = tarball; + if (process.env.COREPACK_NPM_REGISTRY) { + url = url.replace( + npmRegistryUtils.DEFAULT_NPM_REGISTRY_URL, + () => process.env.COREPACK_NPM_REGISTRY!, + ); + } + + debugUtils.log(`Downloading native executable package ${nativePackage.package}@${nativeVersion} from ${url}`); + const {tmpFolder: nativeTmpFolder, hash: actualHash} = await download(installTarget, url, `sha512`); + + try { + if (!shouldSkipIntegrityCheck()) { + npmRegistryUtils.verifySignature({signatures, integrity, packageName: nativePackage.package, version: nativeVersion}); + + const expectedHash = Buffer.from(integrity.slice(`sha512-`.length), `base64`).toString(`hex`); + if (actualHash !== expectedHash) { + throw new Error(`Mismatch hashes. Expected ${expectedHash}, got ${actualHash}`); + } + } + + const nativeBinPath = path.join(nativeTmpFolder, nativePackage.bin); + const ext = process.platform === `win32` ? `.exe` : ``; + + const bin: BinSpec = {}; + for (const binName of getBinNames(spec.bin)) { + const target = `${binName}${ext}`; + const destPath = path.join(tmpFolder, target); + + // The main package ships placeholders (or shell scripts) under the same + // names; get rid of them so the executable can take their place. The + // executable detects the name it was invoked under, which is how the + // aliases keep working. + await fs.promises.rm(destPath, {force: true}); + try { + await fs.promises.link(nativeBinPath, destPath); + } catch { + await fs.promises.copyFile(nativeBinPath, destPath); + } + await fs.promises.chmod(destPath, 0o755); + + bin[binName] = target; + } + + return bin; + } finally { + await fs.promises.rm(nativeTmpFolder, {recursive: true, force: true}); + } +} + export async function installVersion(installTarget: string, locator: Locator, {spec}: {spec: PackageManagerSpec}): Promise { const locatorIsASupportedPackageManager = isSupportedPackageManagerLocator(locator); const locatorReference = locatorIsASupportedPackageManager ? semverParse(locator.reference)! : parseURLReference(locator); @@ -218,13 +342,21 @@ export async function installVersion(installTarget: string, locator: Locator, {s const corepackData = JSON.parse(corepackContent); - debugUtils.log(`Reusing ${locator.name}@${locator.reference} found in ${installFolder}`); + if (locatorIsASupportedPackageManager && spec.nativePackages != null && !await isNativeInstallIntact(installFolder, corepackData.bin)) { + // The install folder was populated by an older Corepack release that + // didn't know this package manager version requires its native + // executable to be fetched separately; discard it and install anew. + debugUtils.log(`Discarding incomplete install of ${locator.name}@${locator.reference} found in ${installFolder}`); + await fs.promises.rm(installFolder, {recursive: true, force: true}); + } else { + debugUtils.log(`Reusing ${locator.name}@${locator.reference} found in ${installFolder}`); - return { - hash: corepackData.hash as string, - location: installFolder, - bin: corepackData.bin, - }; + return { + hash: corepackData.hash as string, + location: installFolder, + bin: corepackData.bin, + }; + } } catch (err) { if (nodeUtils.isNodeError(err) && err.code !== `ENOENT`) { throw err; @@ -308,6 +440,9 @@ export async function installVersion(installTarget: string, locator: Locator, {s if (build[1] && actualHash !== build[1]) throw new Error(`Mismatch hashes. Expected ${build[1]}, got ${actualHash}`); + if (locatorIsASupportedPackageManager && spec.nativePackages != null) + bin = await installNativeBinaries(installTarget, tmpFolder, locator, version, spec); + const serializedHash = `${algo}.${actualHash}`; await fs.promises.writeFile(path.join(tmpFolder, `.corepack`), JSON.stringify({ @@ -409,6 +544,11 @@ export async function runVersion(locator: Locator, installSpec: InstallSpec & {s if (!binPath) throw new Error(`Assertion failed: Unable to locate path for bin '${binName}'`); + if (installSpec.spec.nativePackages != null) { + await runNativeVersion(binPath, args); + return; + } + if (!Module.enableCompileCache) { // Node.js segfaults when using npm@>=9.7.0 and v8-compile-cache // $ docker run -it node:20.3.0-slim corepack npm@9.7.1 --version @@ -447,6 +587,39 @@ export async function runVersion(locator: Locator, installSpec: InstallSpec & {s } } +/** + * Runs a package manager distributed as a native executable, by spawning it + * as a child process (it cannot be loaded into the current Node.js process + * like the JavaScript-based package managers). + */ +async function runNativeVersion(binPath: string, args: Array): Promise { + process.env.COREPACK_ROOT = path.dirname(require.resolve(`corepack/package.json`)); + + const child = spawn(binPath, args, {stdio: `inherit`}); + + // Terminal-generated signals (e.g. Ctrl+C) are delivered to the whole + // foreground process group, so the child receives them on its own; Corepack + // just has to avoid dying from them before the child had a chance to handle + // them. Signals sent to Corepack itself are forwarded to the child. + const onSigint = () => {}; + const forwardSignal = (signal: NodeJS.Signals) => { + child.kill(signal); + }; + + process.on(`SIGINT`, onSigint); + process.on(`SIGTERM`, forwardSignal); + + const [exitCode, signal] = await once(child, `exit`) as [number | null, NodeJS.Signals | null]; + + process.off(`SIGINT`, onSigint); + process.off(`SIGTERM`, forwardSignal); + + if (signal != null) + process.kill(process.pid, signal); + + process.exitCode = exitCode ?? 1; +} + export function shouldSkipIntegrityCheck() { return process.env.COREPACK_INTEGRITY_KEYS === `` || process.env.COREPACK_INTEGRITY_KEYS === `0`; diff --git a/sources/types.ts b/sources/types.ts index 382f2c3e3..5350fe47e 100644 --- a/sources/types.ts +++ b/sources/types.ts @@ -41,6 +41,18 @@ export type RegistrySpec = | NpmRegistrySpec | UrlRegistrySpec; +export interface NativePackageSpec { + /** + * Name of the npm package containing the native executable for one + * specific platform. + */ + package: string; + /** + * Path of the native executable inside the platform-specific package. + */ + bin: string; +} + /** * Defines how the package manager is meant to be downloaded and accessed. */ @@ -49,6 +61,20 @@ export interface PackageManagerSpec { bin: BinSpec | BinList; registry: RegistrySpec; npmRegistry?: NpmRegistrySpec; + /** + * Some package managers are distributed as native executables: the package + * referenced by `url` only ships placeholders for its binaries, and the + * actual platform-specific executable lives in a companion npm package + * (referenced in the `optionalDependencies` of the main package, and put in + * place by a lifecycle script when installed by a package manager). Since + * Corepack never runs lifecycle scripts, it replicates their effect when + * this field is defined: it downloads the companion package for the current + * platform and copies its executable over the placeholders. + * + * Keys are `${process.platform}-${process.arch}`, plus a `-musl` suffix on + * Linux systems using musl libc. + */ + nativePackages?: {[platformKey: string]: NativePackageSpec}; commands?: { use?: Array; }; diff --git a/tests/_registryServer.mjs b/tests/_registryServer.mjs index 61bb77829..c72ffb6da 100644 --- a/tests/_registryServer.mjs +++ b/tests/_registryServer.mjs @@ -61,19 +61,71 @@ function createSimpleTarArchive(fileName, fileContent, mode = 0o644) { ]); } -const mockPackageTarGz = gzipSync(Buffer.concat([ - createSimpleTarArchive(`package/bin/customPkgManager.js`, `#!/usr/bin/env node\nconsole.log("customPkgManager: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/bin/pnpm.js`, `#!/usr/bin/env node\nconsole.log("pnpm: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/bin/yarn.js`, `#!/usr/bin/env node\nconsole.log("yarn: Hello from custom registry");\n`, 0o755), - createSimpleTarArchive(`package/package.json`, JSON.stringify({bin: {yarn: `bin/yarn.js`, pnpm: `bin/pnpm.js`, customPkgManager: `bin/customPkgManager.js`}})), - Buffer.alloc(1024), -])); -const shasum = createHash(`sha1`).update(mockPackageTarGz).digest(`hex`); -const integrity = `sha512-${createHash(`sha512`).update( - process.env.TEST_INTEGRITY === `invalid_integrity` ? - mockPackageTarGz.subarray(1) : - mockPackageTarGz, -).digest(`base64`)}`; +function createPackageArchive(entries) { + const tarGz = gzipSync(Buffer.concat([ + ...entries.map(([fileName, fileContent, mode]) => createSimpleTarArchive(fileName, fileContent, mode)), + Buffer.alloc(1024), + ])); + return { + tarGz, + shasum: createHash(`sha1`).update(tarGz).digest(`hex`), + integrity: `sha512-${createHash(`sha512`).update( + process.env.TEST_INTEGRITY === `invalid_integrity` ? + tarGz.subarray(1) : + tarGz, + ).digest(`base64`)}`, + }; +} + +const defaultPackageArchive = createPackageArchive([ + [`package/bin/customPkgManager.js`, `#!/usr/bin/env node\nconsole.log("customPkgManager: Hello from custom registry");\n`, 0o755], + [`package/bin/pnpm.js`, `#!/usr/bin/env node\nconsole.log("pnpm: Hello from custom registry");\n`, 0o755], + [`package/bin/yarn.js`, `#!/usr/bin/env node\nconsole.log("yarn: Hello from custom registry");\n`, 0o755], + [`package/package.json`, JSON.stringify({bin: {yarn: `bin/yarn.js`, pnpm: `bin/pnpm.js`, customPkgManager: `bin/customPkgManager.js`}})], +]); + +// pnpm v12 is distributed as a native executable: the `pnpm` package only +// ships placeholders, and the real executable lives in a platform-specific +// companion package pinned in its `optionalDependencies`. +let nativePlatformKey = `${process.platform}-${process.arch}`; +if (process.platform === `linux`) { + try { + const report = process.report?.getReport(); + if (report != null && !report.header?.glibcVersionRuntime) { + nativePlatformKey += `-musl`; + } + } catch {} +} +const PNPM_V12_VERSION = `12.9998.9999`; +const pnpmExePackageName = `@pnpm/exe.${nativePlatformKey}`; +const pnpmExeBinName = process.platform === `win32` ? `pnpm.exe` : `pnpm`; + +const pnpmV12Archive = createPackageArchive([ + [`package/pnpm`, `This is a placeholder replaced by the native executable at install time.\n`], + [`package/pnpx`, `#!/bin/sh\nexec pnpm dlx "$@"\n`, 0o755], + [`package/package.json`, JSON.stringify({ + name: `pnpm`, + version: PNPM_V12_VERSION, + bin: {pnpm: `pnpm`, pnpx: `pnpx`}, + optionalDependencies: {[pnpmExePackageName]: PNPM_V12_VERSION}, + })], +]); +// Stands in for the native executable; prints the name it was invoked under +// so tests can check that the aliases are hardlinked onto it. +const pnpmExeArchive = createPackageArchive([ + [`package/${pnpmExeBinName}`, `#!/bin/sh\necho "pnpm v12 native: $(basename "$0") $@"\n`, 0o755], + [`package/package.json`, JSON.stringify({name: pnpmExePackageName, version: PNPM_V12_VERSION})], +]); + +const packageArchives = { + __proto__: null, + [`pnpm@${PNPM_V12_VERSION}`]: pnpmV12Archive, + [`${pnpmExePackageName}@${PNPM_V12_VERSION}`]: pnpmExeArchive, +}; + +function getPackageArchive(packageName, version) { + return packageArchives[`${packageName}@${version}`] ?? defaultPackageArchive; +} const registry = { __proto__: null, @@ -84,8 +136,15 @@ const registry = { customPkgManager: [`1.0.0`], }; +if (process.env.TEST_PNPM_V12 === `1`) { + // `latest` is the last item of each list, so the v12 pre-release must come first. + registry.pnpm.unshift(PNPM_V12_VERSION); + registry[pnpmExePackageName] = [PNPM_V12_VERSION]; +} + function generateSignature(packageName, version) { if (privateKey == null) return undefined; + const {integrity} = getPackageArchive(packageName, version); const sign = createSign(`SHA256`).end(`${packageName}@${version}:${integrity}`); return {integrity, signatures: [{ keyid, @@ -93,6 +152,7 @@ function generateSignature(packageName, version) { }]}; } function generateVersionMetadata(packageName, version) { + const archive = getPackageArchive(packageName, version); return { name: packageName, version, @@ -100,8 +160,8 @@ function generateVersionMetadata(packageName, version) { [packageName]: `./bin/${packageName}.js`, }, dist: { - shasum, - size: mockPackageTarGz.length, + shasum: archive.shasum, + size: archive.tarGz.length, tarball: `https://registry.npmjs.org/${packageName}/-/${packageName}-${version}.tgz`, ...generateSignature(packageName, version), }, @@ -152,7 +212,7 @@ const server = createServer((req, res) => { if (registry[packageName].includes(version)) { res.end( isDownloadingRequest ? - mockPackageTarGz : + getPackageArchive(packageName, version).tarGz : JSON.stringify(generateVersionMetadata(packageName, version)), ); } else { diff --git a/tests/main.test.ts b/tests/main.test.ts index 507db2e43..055ea377d 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -1321,6 +1321,43 @@ it(`should download latest pnpm from custom registry`, async () => { }); }); +it(`should install the native executable of pnpm v12 from its platform-specific package`, async t => { + // The fake native executable served by the custom registry is a shell + // script, which Windows cannot spawn. + if (process.platform === `win32`) t.skip(); + + await xfs.mktempPromise(async cwd => { + process.env.AUTH_TYPE = `COREPACK_NPM_TOKEN`; // See `_registryServer.mjs` + process.env.TEST_INTEGRITY = `valid`; // See `_registryServer.mjs` + process.env.TEST_PNPM_V12 = `1`; // See `_registryServer.mjs` + + await xfs.writeJsonPromise(ppath.join(cwd, `package.json` as Filename), { + packageManager: `pnpm@12.9998.9999`, + }); + + await expect(runCli(cwd, [`pnpm`, `install`], true)).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpm install\n`, + stderr: ``, + }); + + // The aliases are hardlinked onto the same executable, which adapts its + // behavior to the name it was invoked under. + await expect(runCli(cwd, [`pnpx`, `create-foo`], true)).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpx create-foo\n`, + stderr: ``, + }); + + // Should keep working with cache + await expect(runCli(cwd, [`pnpm`, `run`, `build`])).resolves.toMatchObject({ + exitCode: 0, + stdout: `pnpm v12 native: pnpm run build\n`, + stderr: ``, + }); + }); +}); + describe(`should pick up COREPACK_INTEGRITY_KEYS from env`, () => { beforeEach(() => { process.env.AUTH_TYPE = `COREPACK_NPM_TOKEN`; // See `_registryServer.mjs`