From ec0a7eece651f52c2c387bc15085ccc445c1eaf2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 06:04:52 +0000 Subject: [PATCH 1/2] npm registries as a built-in format plugin npm, pnpm, Yarn and Bun against /npm//: hosted, proxy and group repositories of the new format plugin crates/formats/npm, served by the {mount}/{repo} dispatcher like NuGet. - Packuments, abbreviated and full (with publish times and the README of latest), version documents, tarballs, search, ping. - npm publish: the base64 tarball streams out of the JSON document into the store (Upload::Document, unpack_upload), checked against the integrity and version it claims; the deployment policy decides on republishing. - Dist-tags, deprecations and unpublishing as host changes (package_tag, component.deprecation); retention never deletes a tagged version. - Logins: npm's browser login approved on /cli-login/, a password login (yarn npm login), logout and whoami, minting API tokens; Bearer API tokens are accepted on every plugin repository. - Proxies: Bearer or Basic credentials, the upstream's dist-tags, tarballs verified against the published digests, publish times for the full packument (Query::DatedPackages), 502 for an upstream that refuses the proxy or serves a package that does not hold up. - Groups: dist-tags from the first member that has them, reserved scopes, a writable member for publishes and changes, audits forwarded upstream without reserved names. - Contract 0.4.0: request bodies on demand, document uploads, changes, logins, forwards, tags, host errors in the format's shape. - Web UI: a package's dist-tags, deprecations and README, the npm connect snippets, the browser login approval page, and the deployment policy and writable member for every package format. - Docs (docs/npm.md, format-plugins.md), the npm-e2e CI job with the real clients, changelog fragments. Fixes #41 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BaGpNGgkQV54XvgL5D9ay2 --- .github/scripts/ci-changes.sh | 4 +- .github/scripts/npm-e2e.sh | 433 +++++ .github/workflows/ci.yml | 68 +- .github/workflows/release.yml | 2 +- ...ec0a8744759c462eef93e1c56c452933c1caf.json | 25 + ...c27f8f8f7817a4a80262466c823ae099b17dc.json | 112 ++ ...544d7513b699f27a13fa020659e4133ba3dbd.json | 41 + ...dad1e70c1a120f7f5239ac97d35bd4825678.json} | 18 +- ...7ba1b2fa3bc63b4000d442f3f83a502fd4391.json | 16 + ...d4fec0231ea57ad0e75163af743fac416391.json} | 16 +- ...dd5f8fe600f9fa47b8935b8bc5838ed735708.json | 20 + ...09e334388bbea96ed70a74cd5b79a9d9af35d.json | 52 + ...9f0e708d99b38037deaf4cba0cc0987e33d5d.json | 15 + ...2a4a5decf5084236a1daf71f6a0ce7b30f326.json | 17 + ...3e9890c2b2cabe43451b14e7d25e6a5ddf025.json | 12 + ...5572fbfadab0803e83ac7e4d59be3be92f2a.json} | 18 +- ...4b31bfe9772478bf9be129a50a29deb897db6.json | 16 + ...915c317db9013aba72a3c6346faf481099be7.json | 40 + ...fa213063d86de7736a605a16e1a00ba1ce80d.json | 14 + ...4d4c0f310d5f7f8a40eb0b1146a97840d3851.json | 14 + ...ac58d1613fa732f9eba38fd73cdd8232b9401.json | 23 + ...e39794c7735c62e54bc34fa3d0e8981c8dc0c.json | 17 + Cargo.lock | 40 + Cargo.toml | 1 + DEVELOPMENT.md | 39 +- README.md | 22 +- crates/core/Cargo.toml | 2 + crates/core/src/format.rs | 30 + crates/core/src/net/upstream.rs | 76 +- crates/core/src/repo/group.rs | 108 +- crates/core/src/repo/hosted.rs | 10 + crates/core/src/repo/mod.rs | 59 + crates/core/src/repo/proxy.rs | 291 ++- crates/core/src/retention/apply.rs | 7 + crates/core/src/retention/mod.rs | 22 + crates/core/src/retention/worker.rs | 12 +- crates/core/tests/group_read.rs | 1 + crates/core/tests/package_filtering.rs | 1 + crates/core/tests/proxy_read.rs | 19 +- crates/db/src/cli_login_dao.rs | 157 ++ crates/db/src/component_dao.rs | 35 +- crates/db/src/lib.rs | 2 + crates/db/src/tag_dao.rs | 176 ++ crates/formats/README.md | 4 +- crates/formats/npm/Cargo.toml | 30 + crates/formats/npm/plugin.toml | 19 + crates/formats/npm/src/document.rs | 388 ++++ crates/formats/npm/src/integrity.rs | 168 ++ crates/formats/npm/src/lib.rs | 371 ++++ crates/formats/npm/src/manifest.rs | 243 +++ crates/formats/npm/src/name.rs | 243 +++ crates/formats/npm/src/packument.rs | 297 +++ crates/formats/npm/src/protocol.rs | 884 +++++++++ crates/formats/npm/src/proxy.rs | 406 +++++ crates/formats/npm/src/publish.rs | 639 +++++++ crates/formats/npm/src/tarball.rs | 175 ++ crates/formats/npm/src/upstream.rs | 348 ++++ crates/formats/npm/src/version.rs | 97 + crates/formats/npm/tests/protocol.rs | 568 ++++++ crates/formats/nuget/src/protocol.rs | 1 + crates/formats/nuget/src/upstream.rs | 7 +- crates/formats/nuget/tests/protocol.rs | 5 +- crates/formats/nuget/tests/upstream.rs | 16 +- crates/formats/raw/src/listing.rs | 1 + crates/formats/raw/src/sidecar.rs | 1 + crates/plugin-api/src/lib.rs | 86 +- crates/plugin-api/src/testing.rs | 37 +- crates/plugin-api/src/types.rs | 252 ++- crates/plugin/src/format.rs | 130 +- crates/plugin/src/lib.rs | 2 +- crates/plugin/src/manifest.rs | 27 + crates/server/Cargo.toml | 7 +- crates/server/src/admin/repositories.rs | 260 ++- crates/server/src/admin/retention.rs | 8 +- crates/server/src/auth/mod.rs | 255 ++- crates/server/src/browse.rs | 86 +- crates/server/src/cli_auth.rs | 282 +++ crates/server/src/config.rs | 109 ++ crates/server/src/feed.rs | 66 +- crates/server/src/lib.rs | 1 + crates/server/src/nexus_import/nexus.rs | 1 + crates/server/src/plugins.rs | 1277 ++++++++++--- crates/server/src/state.rs | 40 +- crates/server/src/web_ui.rs | 2 +- crates/server/src/workers.rs | 10 +- crates/server/tests/npm.rs | 1609 +++++++++++++++++ crates/server/tests/plugin_contract.rs | 2 + crates/server/tests/raw.rs | 2 +- crates/server/tests/registry.rs | 2 +- crates/server/tests/release_hardening.rs | 2 +- docs/changes/2.0.0/283.fixed.md | 3 +- .../2.0.0/41-package-options.feature.md | 6 + docs/changes/2.0.0/41.feature.md | 19 + docs/changes/2.0.0/41.fixed.md | 6 + docs/changes/2.0.0/41.limitation.md | 6 + docs/format-plugins.md | 168 +- docs/npm.md | 303 ++++ forge.toml.example | 12 + migrations/0046_package_tags_cli_logins.sql | 50 + web/src/api/cliLogin.ts | 53 + web/src/api/packages.ts | 14 +- web/src/api/repositories.ts | 6 +- web/src/components/DependencyGroups.tsx | 10 +- web/src/components/PackageConnect.test.ts | 26 +- web/src/components/PackageConnect.tsx | 94 +- web/src/components/RepositoryForm.test.tsx | 126 +- web/src/components/RepositoryForm.tsx | 153 +- web/src/components/VersionTable.tsx | 23 + web/src/components/ui/badges.tsx | 1 + web/src/routeTree.gen.ts | 21 + web/src/routes.ts | 3 + web/src/routes/CliLogin.test.tsx | 122 ++ web/src/routes/CliLogin.tsx | 150 ++ web/src/routes/PackageDetail.test.tsx | 73 +- web/src/routes/PackageDetail.tsx | 143 +- web/src/styles/forge-ui.css | 1 + web/vite.config.ts | 1 + 117 files changed, 12514 insertions(+), 648 deletions(-) create mode 100755 .github/scripts/npm-e2e.sh create mode 100644 .sqlx/query-04d359be3fd2a58ac5a3cd6207aec0a8744759c462eef93e1c56c452933c1caf.json create mode 100644 .sqlx/query-0c93dd4457876bbd52e975f8ba8c27f8f8f7817a4a80262466c823ae099b17dc.json create mode 100644 .sqlx/query-0eed893d19cd6e528685a748c1f544d7513b699f27a13fa020659e4133ba3dbd.json rename .sqlx/{query-bee91d65724d2a2b1c2a0cee1ca0d93742f92588a3ad61f44c3e0f8988df7d03.json => query-1f25a2f4aeff0bdc1b0de6315b66dad1e70c1a120f7f5239ac97d35bd4825678.json} (83%) create mode 100644 .sqlx/query-1fa761deb76a7db98f5390e998b7ba1b2fa3bc63b4000d442f3f83a502fd4391.json rename .sqlx/{query-0a80f388f927a6fba4aa4a37cc9c6654a76f3251c3d1cae7a09b57484a3e8db2.json => query-36a76654ad6617689529c9bbe6c9d4fec0231ea57ad0e75163af743fac416391.json} (84%) create mode 100644 .sqlx/query-3ef924fcda6ab461396e254b17bdd5f8fe600f9fa47b8935b8bc5838ed735708.json create mode 100644 .sqlx/query-3f2dc4a59d921bd5846f7c26e6909e334388bbea96ed70a74cd5b79a9d9af35d.json create mode 100644 .sqlx/query-4e0afd23c132ed31eb2380f3f809f0e708d99b38037deaf4cba0cc0987e33d5d.json create mode 100644 .sqlx/query-5a2229c387b996ad1f7bcdec9922a4a5decf5084236a1daf71f6a0ce7b30f326.json create mode 100644 .sqlx/query-62f32a8f0ad8b325b7064977c633e9890c2b2cabe43451b14e7d25e6a5ddf025.json rename .sqlx/{query-8aa6239b751b3255b54d4e02e78955e6c0ebc01ffb3b8c866f03984c422b6e93.json => query-721a7e66544901c81a63cfde95b05572fbfadab0803e83ac7e4d59be3be92f2a.json} (83%) create mode 100644 .sqlx/query-7fa52616dbb07a766382574c42d4b31bfe9772478bf9be129a50a29deb897db6.json create mode 100644 .sqlx/query-afb8561aa8a3f4ea3740499bf63915c317db9013aba72a3c6346faf481099be7.json create mode 100644 .sqlx/query-b9abaf48b56de1f008614de9d88fa213063d86de7736a605a16e1a00ba1ce80d.json create mode 100644 .sqlx/query-cf6b219a3de5f5fd0a5d949c1464d4c0f310d5f7f8a40eb0b1146a97840d3851.json create mode 100644 .sqlx/query-d7b159866489cb91555fd92383fac58d1613fa732f9eba38fd73cdd8232b9401.json create mode 100644 .sqlx/query-f6dac1d112d13d0be04e1ebdd06e39794c7735c62e54bc34fa3d0e8981c8dc0c.json create mode 100644 crates/db/src/cli_login_dao.rs create mode 100644 crates/db/src/tag_dao.rs create mode 100644 crates/formats/npm/Cargo.toml create mode 100644 crates/formats/npm/plugin.toml create mode 100644 crates/formats/npm/src/document.rs create mode 100644 crates/formats/npm/src/integrity.rs create mode 100644 crates/formats/npm/src/lib.rs create mode 100644 crates/formats/npm/src/manifest.rs create mode 100644 crates/formats/npm/src/name.rs create mode 100644 crates/formats/npm/src/packument.rs create mode 100644 crates/formats/npm/src/protocol.rs create mode 100644 crates/formats/npm/src/proxy.rs create mode 100644 crates/formats/npm/src/publish.rs create mode 100644 crates/formats/npm/src/tarball.rs create mode 100644 crates/formats/npm/src/upstream.rs create mode 100644 crates/formats/npm/src/version.rs create mode 100644 crates/formats/npm/tests/protocol.rs create mode 100644 crates/server/src/cli_auth.rs create mode 100644 crates/server/tests/npm.rs create mode 100644 docs/changes/2.0.0/41-package-options.feature.md create mode 100644 docs/changes/2.0.0/41.feature.md create mode 100644 docs/changes/2.0.0/41.fixed.md create mode 100644 docs/changes/2.0.0/41.limitation.md create mode 100644 docs/npm.md create mode 100644 migrations/0046_package_tags_cli_logins.sql create mode 100644 web/src/api/cliLogin.ts create mode 100644 web/src/routes/CliLogin.test.tsx create mode 100644 web/src/routes/CliLogin.tsx diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index ff1df576..3e838f6e 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -42,7 +42,7 @@ CI=.github/workflows/ci.yml declare -A CI_JOB_GROUP=( [fmt]=changelog [clippy]=rust [audit]=audit [test]=rust [docker-platform]=image [docker]=image [kubernetes]=kubernetes [web]=web - [oci-e2e]=rust [raw-e2e]=rust [e2e]=rust [web-e2e]=web + [oci-e2e]=rust [raw-e2e]=rust [npm-e2e]=rust [e2e]=rust [web-e2e]=web ) # Rust: the workspace, what its build and tests read from outside the crates @@ -55,7 +55,7 @@ RUST=( web/index.html docker/forge.docker.toml .github/scripts/test-parallel.sh .github/scripts/test-timings-runner.sh .github/scripts/test-timings-summary.sh .github/scripts/oci-e2e.sh - .github/scripts/raw-e2e.sh + .github/scripts/raw-e2e.sh .github/scripts/npm-e2e.sh ) # fmt's second check: the changelog fragments. CHANGELOG=(docs/changes) diff --git a/.github/scripts/npm-e2e.sh b/.github/scripts/npm-e2e.sh new file mode 100755 index 00000000..1eba917a --- /dev/null +++ b/.github/scripts/npm-e2e.sh @@ -0,0 +1,433 @@ +#!/usr/bin/env bash +# End-to-end check of npm repositories with the real clients (used by +# ci.yml): boots forge-server and a mock upstream registry, then +# +# 1. signs in with npm's web login (`npm login`, approved through the API +# of the browser page it opens) and with the legacy login of Yarn +# (`yarn npm login`: npm's own legacy login refuses a username with an +# `@`, which every Forge account has), checks whoami, and logs out; +# 2. publishes a scoped and an unscoped package with npm, one with pnpm, +# and one through the group (to its writable member); a republish is +# refused; +# 3. installs with npm and pnpm from the hosted repository, from the proxy +# of the mock upstream and from the group, checks that the lockfiles +# point at Forge, and that the proxy's full packument carries the +# upstream's publish times (npm and pnpm 11 read them), its abbreviated +# one none; +# 4. moves dist-tags, deprecates a version and unpublishes; +# 5. the group's dependency-confusion guard: the upstream's package in the +# internal scope `@acme` is never served through the group; +# 6. runs npm audit through the group, whose upstream request leaves the +# internal scope's packages out. +# +# Usage: npm-e2e.sh +# Needs on PATH: curl, jq, python3, node, npm, pnpm, tar, openssl. Yarn 2+ is +# YARN_BERRY (a yarn.js), else fetched from the npm registry with npm pack. +# Postgres and S3 come from the environment (defaults match ci.yml): +# FORGE_E2E_DATABASE_URL (postgres://forge:forge@localhost:5432/forge) +# FORGE_E2E_S3_ENDPOINT (http://localhost:7070), access/secret forge-e2e-key +# FORGE_E2E_PORT (8431: forge-server's) +# FORGE_E2E_UPSTREAM_PORT (5021: the mock upstream registry) +set -euo pipefail + +server_bin="$(realpath "$1")" +results="$(mkdir -p "$2" && realpath "$2")" +db_url="${FORGE_E2E_DATABASE_URL:-postgres://forge:forge@localhost:5432/forge}" +s3_endpoint="${FORGE_E2E_S3_ENDPOINT:-http://localhost:7070}" +port="${FORGE_E2E_PORT:-8431}" +upstream_port="${FORGE_E2E_UPSTREAM_PORT:-5021}" +base="http://127.0.0.1:$port" +upstream="http://127.0.0.1:$upstream_port" +admin_user="admin@example.com" +admin_password="e2e-$(openssl rand -hex 12)" + +work=$(mktemp -d) +server_pid="" +upstream_pid="" +cleanup() { + if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; fi + if [ -n "$upstream_pid" ]; then kill "$upstream_pid" 2>/dev/null || true; fi + cp "$work/server.log" "$work/upstream.log" "$results/" 2>/dev/null || true + cp "$work"/*.out "$results/" 2>/dev/null || true + rm -rf "$work" +} +trap cleanup EXIT + +step() { echo; echo "── $* ──"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +# The clients' configuration and caches stay inside the work directory. +export HOME="$work/home" NPM_CONFIG_USERCONFIG="$work/home/.npmrc" +export npm_config_cache="$work/npm-cache" npm_config_update_notifier=false +export npm_config_fund=false npm_config_audit=false +mkdir -p "$HOME" +reg() { echo "$base/npm/$1/"; } +# The versions `npm view` lists, as a JSON array (one version is printed +# bare by some npm releases). +versions() { # + npm view "$1" versions --json --registry "$2" | jq -c 'if type == "array" then . else [.] end' +} + +step "a mock upstream registry" +# Its packages, made with npm pack: e2e-public (1.0.0, and 1.1.0 depending +# on e2e-dep) and a public @acme/ui that claims the internal scope. +mkdir -p "$work/upstream" +pack() { # [dependencies json] + mkdir -p "$1" + jq -n --arg name "$2" --arg version "$3" --argjson deps "${4:-{\}}" \ + '{name: $name, version: $version, description: "e2e", license: "MIT", main: "index.js", + dependencies: $deps}' > "$1/package.json" + echo "module.exports = '$2@$3'" > "$1/index.js" + echo "# $2" > "$1/README.md" + (cd "$1" && npm pack --silent --pack-destination "$work/upstream" >/dev/null) +} +pack "$work/src/up-public-1.0.0" e2e-public 1.0.0 +pack "$work/src/up-public-1.1.0" e2e-public 1.1.0 '{"e2e-dep":"^2.0.0"}' +pack "$work/src/up-dep-2.0.0" e2e-dep 2.0.0 +pack "$work/src/up-acme-ui" @acme/ui 9.9.9 +cat > "$work/upstream.py" <<'EOF' +import base64, glob, hashlib, http.server, json, os, sys, tarfile, urllib.parse + +root, port = sys.argv[1], int(sys.argv[2]) +packages, tarballs = {}, {} +for path in sorted(glob.glob(os.path.join(root, "*.tgz"))): + data = open(path, "rb").read() + with tarfile.open(path) as tar: + manifest = json.load(tar.extractfile("package/package.json")) + name, version = manifest["name"], manifest["version"] + file = f"{name.split('/')[-1]}-{version}.tgz" + tarballs[f"/{name}/-/{file}"] = data + manifest["_id"] = f"{name}@{version}" + manifest["dist"] = { + "tarball": f"http://127.0.0.1:{port}/{name}/-/{file}", + "shasum": hashlib.sha1(data).hexdigest(), + "integrity": "sha512-" + base64.b64encode(hashlib.sha512(data).digest()).decode(), + } + doc = packages.setdefault(name, {"name": name, "versions": {}, "time": {}, "dist-tags": {}}) + doc["versions"][version] = manifest + doc["time"][version] = "2020-01-01T00:00:00.000Z" + doc["dist-tags"]["latest"] = version + +class Registry(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def answer(self, status, body, content_type="application/json"): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + path = urllib.parse.unquote(self.path.split("?")[0]) + auth = f"auth={'Authorization' in self.headers}" + if path in tarballs: + print(f"GET {path} {auth}", flush=True) + self.answer(200, tarballs[path], "application/octet-stream") + elif path.lstrip("/") in packages: + # Which packument the proxy asked for: the abbreviated one + # installs need, or the full one. + abbreviated = "application/vnd.npm.install-v1+json" in self.headers.get("Accept", "") + print(f"GET {path} {auth} packument={'abbreviated' if abbreviated else 'full'}", flush=True) + self.answer(200, json.dumps(packages[path.lstrip("/")]).encode()) + elif path == "/-/v1/search": + self.answer(200, b'{"objects":[],"total":0,"time":"2020-01-01T00:00:00.000Z"}') + else: + self.answer(404, b'{"error":"not found"}') + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + print(f"POST {self.path} {body.decode()}", flush=True) + if self.path != "/-/npm/v1/security/advisories/bulk": + self.answer(404, b'{"error":"not found"}') + return + advisories = {} + if "e2e-public" in json.loads(body): + advisories["e2e-public"] = [{ + "id": 1001, "url": "https://advisories.example.com/1001", + "title": "Prototype pollution in e2e-public", "severity": "high", + "vulnerable_versions": "<1.1.0", "cwe": ["CWE-1321"], + "cvss": {"score": 7.5, "vectorString": None}, + }] + self.answer(200, json.dumps(advisories).encode()) + +http.server.ThreadingHTTPServer(("127.0.0.1", port), Registry).serve_forever() +EOF +python3 -u "$work/upstream.py" "$work/upstream" "$upstream_port" > "$work/upstream.log" 2>&1 & +upstream_pid=$! +for _ in $(seq 1 30); do curl -fsS "$upstream/e2e-public" >/dev/null 2>&1 && break; sleep 0.5; done +curl -fsS "$upstream/e2e-public" >/dev/null || fail "the mock upstream did not start" + +step "start forge-server" +mkdir -p "$work/web" +echo 'Forge' > "$work/web/index.html" +cat > "$work/forge.toml" < "$work/server.log" 2>&1 & +server_pid=$! +for _ in $(seq 1 60); do + if curl -fsS "$base/healthz" >/dev/null 2>&1; then break; fi + sleep 1 +done +curl -fsS "$base/readyz" >/dev/null || { cat "$work/server.log"; fail "server not ready"; } + +step "create repositories" +login=$(curl -fsS -c "$work/jar" -H 'content-type: application/json' \ + -d "{\"username\":\"$admin_user\",\"password\":\"$admin_password\"}" "$base/auth/login") +csrf=$(jq -r .csrf_token <<<"$login") +admin() { # [json body] + curl -fsS -b "$work/jar" -H "x-csrf-token: $csrf" -H 'content-type: application/json' \ + -X "$1" "$base$2" ${3:+-d "$3"} +} +admin POST /admin/repositories '{"name":"npm-hosted","kind":"hosted","format":"npm"}' >/dev/null +admin POST /admin/repositories "{\"name\":\"npm-proxy\",\"kind\":\"proxy\",\"format\":\"npm\", + \"remote_url\":\"$upstream\"}" >/dev/null +admin POST /admin/repositories '{"name":"npm-all","kind":"group","format":"npm", + "reserved_prefixes":["@acme"],"writable_member":"npm-hosted"}' >/dev/null +admin POST /admin/repositories/npm-all/members '{"member":"npm-hosted","priority":10}' >/dev/null +admin POST /admin/repositories/npm-all/members '{"member":"npm-proxy","priority":5}' >/dev/null + +step "web login with npm" +# `npm login` prints the page to open and polls until it is approved; the +# page's API approves it here, as the signed-in admin would in the browser. +web_login() { # + npm login --browser=false --registry "$(reg npm-all)" --userconfig "$1" \ + > "$2" 2>&1 < /dev/null & + local pid=$! url="" + for _ in $(seq 1 30); do + url=$(grep -o "$base/cli-login/[A-Za-z0-9_-]*" "$2" | head -1 || true) + [ -n "$url" ] && break + sleep 0.5 + done + [ -n "$url" ] || { kill "$pid"; fail "npm login printed no login URL: $(cat "$2")"; } + admin GET "/auth/cli-login/${url##*/}" | jq -e '.client == "npm" and .repository == "npm-all"' \ + >/dev/null || fail "the login request does not say who asks" + admin POST "/auth/cli-login/${url##*/}" '{"approve":true}' + wait "$pid" || fail "npm login failed: $(cat "$2")" +} +web_login "$NPM_CONFIG_USERCONFIG" "$work/npm-login.out" +grep -q "Logged in on $(reg npm-all)" "$work/npm-login.out" || fail "npm login: $(cat "$work/npm-login.out")" +[ "$(npm whoami --registry "$(reg npm-all)")" = "$admin_user" ] || fail "npm whoami" +token=$(sed -n 's#^//127.0.0.1:'"$port"'/npm/npm-all/:_authToken=##p' "$NPM_CONFIG_USERCONFIG") +[ -n "$token" ] || fail "npm login stored no token" +# The token is the account's: the other repositories take it too. +for repo in npm-hosted npm-proxy; do + echo "//127.0.0.1:$port/npm/$repo/:_authToken=$token" >> "$NPM_CONFIG_USERCONFIG" +done +[ "$(npm whoami --registry "$(reg npm-hosted)")" = "$admin_user" ] || fail "npm whoami (hosted)" + +step "legacy login with Yarn, and logout" +yarn_js="${YARN_BERRY:-}" +if [ -z "$yarn_js" ]; then + mkdir -p "$work/yarn" + (cd "$work/yarn" && npm pack --silent @yarnpkg/cli-dist@4.18.1 >/dev/null && tar xzf ./*.tgz) + yarn_js="$work/yarn/package/bin/yarn.js" +fi +# Answers a client's prompts on a terminal: answer.py ... -- +cat > "$work/answer.py" <<'EOF' +import os, pty, sys, time +args = sys.argv[1:] +split = args.index("--") +pairs = list(zip(args[:split:2], args[1:split:2])) +pid, fd = pty.fork() +if pid == 0: + os.execvp(args[split + 1], args[split + 1:]) +seen, deadline = b"", time.time() + 60 +while time.time() < deadline: + try: + chunk = os.read(fd, 4096) + except OSError: + break + if not chunk: + break + sys.stdout.buffer.write(chunk) + seen += chunk + if pairs and pairs[0][0].encode() in seen: + seen = b"" + os.write(fd, pairs.pop(0)[1].encode() + b"\r") +else: + os.kill(pid, 9) + sys.exit("timed out") +_, status = os.waitpid(pid, 0) +sys.exit(os.waitstatus_to_exitcode(status)) +EOF +mkdir -p "$work/yarn-project" +( + cd "$work/yarn-project" + echo '{"name":"yarn-project","version":"1.0.0"}' > package.json + touch yarn.lock + printf 'npmRegistryServer: "%s"\nunsafeHttpWhitelist: ["127.0.0.1"]\nenableTelemetry: false\n' \ + "$(reg npm-all)" > .yarnrc.yml + python3 "$work/answer.py" Username "$admin_user" Password "$admin_password" -- \ + node "$yarn_js" npm login > "$work/yarn-login.out" 2>&1 \ + || fail "yarn npm login: $(cat "$work/yarn-login.out")" + node "$yarn_js" npm whoami | grep -q "$admin_user" || fail "yarn npm whoami" +) +grep -q npmAuthToken "$HOME/.yarnrc.yml" || fail "yarn stored no token" +# A wrong password is refused. +code=$(curl -sS -o /dev/null -w '%{http_code}' -X PUT -H 'content-type: application/json' \ + -d "{\"name\":\"$admin_user\",\"password\":\"wrong\"}" \ + "$base/npm/npm-all/-/user/org.couchdb.user:$admin_user") +[ "$code" = 401 ] || fail "a wrong password answered $code" +# `npm logout` revokes the token it signed in with. +web_login "$work/throwaway-npmrc" "$work/npm-login-2.out" +throwaway=$(sed -n 's#^//127.0.0.1:'"$port"'/npm/npm-all/:_authToken=##p' "$work/throwaway-npmrc") +npm logout --registry "$(reg npm-all)" --userconfig "$work/throwaway-npmrc" >/dev/null +code=$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $throwaway" "$base/npm/npm-all/-/whoami") +[ "$code" = 401 ] || fail "a logged-out token answered $code" + +step "the group keeps the internal scope internal" +npm view @acme/ui version --registry "$(reg npm-proxy)" | grep -qx 9.9.9 \ + || fail "the proxy does not see the upstream's @acme/ui" +if npm view @acme/ui --registry "$(reg npm-all)" > "$work/guard.out" 2>&1; then + fail "the group served the upstream's @acme/ui: $(cat "$work/guard.out")" +fi +grep -q E404 "$work/guard.out" || fail "the guard: $(cat "$work/guard.out")" + +step "publish with npm and pnpm" +package() { # [dependencies json] + mkdir -p "$1" + jq -n --arg name "$2" --arg version "$3" --argjson deps "${4:-{\}}" \ + '{name: $name, version: $version, description: "published by the e2e", license: "MIT", + main: "index.js", dependencies: $deps}' > "$1/package.json" + echo "module.exports = '$2@$3'" > "$1/index.js" + printf '# %s\n\nPublished by the npm e2e.\n' "$2" > "$1/README.md" +} +package "$work/src/acme-ui" @acme/ui 1.0.0 +package "$work/src/hosted-1.0.0" e2e-hosted 1.0.0 +package "$work/src/hosted-1.1.0" e2e-hosted 1.1.0 '{"@acme/ui":"^1.0.0"}' +package "$work/src/acme-core" @acme/core 1.0.0 +package "$work/src/pnpm-made" pnpm-made 1.0.0 +(cd "$work/src/acme-ui" && npm publish --registry "$(reg npm-hosted)") >/dev/null +(cd "$work/src/hosted-1.0.0" && npm publish --registry "$(reg npm-hosted)") >/dev/null +(cd "$work/src/hosted-1.1.0" && npm publish --registry "$(reg npm-hosted)") >/dev/null +# Through the group: it lands in the writable member. +(cd "$work/src/acme-core" && npm publish --registry "$(reg npm-all)") >/dev/null +(cd "$work/src/pnpm-made" && pnpm publish --no-git-checks --registry "$(reg npm-hosted)") >/dev/null +npm view @acme/core version --registry "$(reg npm-hosted)" | grep -qx 1.0.0 \ + || fail "a publish through the group did not land in npm-hosted" +if (cd "$work/src/hosted-1.0.0" && npm publish --registry "$(reg npm-hosted)") > "$work/republish.out" 2>&1; then + fail "a republish of e2e-hosted@1.0.0 was accepted" +fi +grep -q "409 Conflict.*e2e-hosted 1.0.0 already exists" "$work/republish.out" \ + || fail "the republish: $(cat "$work/republish.out")" +# The internal @acme/ui is the group's now, the public one still is not. +[ "$(versions @acme/ui "$(reg npm-all)")" = '["1.0.0"]' ] \ + || fail "the group serves another @acme/ui: $(versions @acme/ui "$(reg npm-all)")" + +step "install with npm and pnpm" +project() { # + mkdir -p "$1" + echo '{"name":"consumer","version":"1.0.0","private":true}' > "$1/package.json" + echo "registry=$2" > "$1/.npmrc" +} +resolved_from() { # : every resolved URL is under it + jq -e --arg reg "$2" '[.packages[] | select(.resolved) | .resolved | startswith($reg)] | all' "$1" \ + >/dev/null || fail "$1 resolves elsewhere than $2: $(jq -c '[.packages[].resolved]' "$1")" +} +version_of() { node -p "require('$1/package.json').version"; } +# Hosted. +project "$work/use-hosted" "$(reg npm-hosted)" +(cd "$work/use-hosted" && npm install e2e-hosted pnpm-made) >/dev/null +[ "$(version_of "$work/use-hosted/node_modules/e2e-hosted")" = 1.1.0 ] || fail "npm install from hosted" +[ "$(version_of "$work/use-hosted/node_modules/@acme/ui")" = 1.0.0 ] || fail "a scoped dependency from hosted" +resolved_from "$work/use-hosted/package-lock.json" "$(reg npm-hosted)" +# Proxy: the tarballs come through Forge, the upstream sees each once. +project "$work/use-proxy" "$(reg npm-proxy)" +(cd "$work/use-proxy" && npm install e2e-public) >/dev/null +[ "$(version_of "$work/use-proxy/node_modules/e2e-public")" = 1.1.0 ] || fail "npm install from the proxy" +[ "$(version_of "$work/use-proxy/node_modules/e2e-dep")" = 2.0.0 ] || fail "a dependency from the proxy" +resolved_from "$work/use-proxy/package-lock.json" "$(reg npm-proxy)" +rm -rf "$work/use-proxy/node_modules" "$work/npm-cache" +(cd "$work/use-proxy" && npm ci) >/dev/null +[ "$(grep -c '^GET /e2e-public/-/e2e-public-1.1.0.tgz' "$work/upstream.log")" = 1 ] \ + || fail "the upstream served the tarball more than once: $(cat "$work/upstream.log")" +# Group, with npm and with pnpm. +project "$work/use-group" "$(reg npm-all)" +(cd "$work/use-group" && npm install e2e-public @acme/ui e2e-hosted) >/dev/null +[ "$(version_of "$work/use-group/node_modules/@acme/ui")" = 1.0.0 ] || fail "the group installed another @acme/ui" +resolved_from "$work/use-group/package-lock.json" "$(reg npm-all)" +project "$work/use-pnpm" "$(reg npm-all)" +(cd "$work/use-pnpm" && pnpm add --store-dir "$work/pnpm-store" e2e-public @acme/ui e2e-hosted pnpm-made) >/dev/null +[ "$(version_of "$work/use-pnpm/node_modules/@acme/ui")" = 1.0.0 ] || fail "pnpm installed another @acme/ui" +[ "$(version_of "$work/use-pnpm/node_modules/e2e-public")" = 1.1.0 ] || fail "pnpm add through the group" +grep -q "tarball: $(reg npm-all)" "$work/use-pnpm/pnpm-lock.yaml" 2>/dev/null \ + || grep -q 'e2e-public@1.1.0' "$work/use-pnpm/pnpm-lock.yaml" || fail "no pnpm lockfile" +project "$work/use-pnpm-proxy" "$(reg npm-proxy)" +(cd "$work/use-pnpm-proxy" && pnpm add --store-dir "$work/pnpm-store-2" e2e-public) >/dev/null +[ "$(version_of "$work/use-pnpm-proxy/node_modules/e2e-public")" = 1.1.0 ] || fail "pnpm add from the proxy" +# The proxy's full packument has the upstream's publish times (`npm view +# time`, `npm install --before`, pnpm's minimumReleaseAge); the abbreviated +# one, which installs may ask for instead, has none. +published=$(npm view e2e-public time --json --registry "$(reg npm-proxy)" | jq -r '.["1.1.0"]') +[ "$published" = 2020-01-01T00:00:00.000Z ] || fail "the proxy's publish time of e2e-public 1.1.0: $published" +curl -fsS -D "$work/corgi.headers" -o "$work/corgi.json" -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.npm.install-v1+json' "$base/npm/npm-proxy/e2e-public" +grep -qi '^content-type: application/vnd.npm.install-v1+json' "$work/corgi.headers" \ + || fail "the abbreviated packument: $(cat "$work/corgi.headers")" +jq -e '(has("time") | not) and .versions["1.1.0"].dist.tarball != null' "$work/corgi.json" >/dev/null \ + || fail "the abbreviated packument: $(cat "$work/corgi.json")" +if command -v bun >/dev/null; then + project "$work/use-bun" "$(reg npm-all)" + (cd "$work/use-bun" && bun add --cache-dir "$work/bun-cache" e2e-hosted) >/dev/null + [ "$(version_of "$work/use-bun/node_modules/e2e-hosted")" = 1.1.0 ] || fail "bun add through the group" +fi + +step "dist-tags, deprecation, unpublishing" +hosted=(--registry "$(reg npm-hosted)") +npm dist-tag add e2e-hosted@1.0.0 legacy "${hosted[@]}" >/dev/null +npm dist-tag ls e2e-hosted "${hosted[@]}" > "$work/tags.out" +grep -qx 'latest: 1.1.0' "$work/tags.out" && grep -qx 'legacy: 1.0.0' "$work/tags.out" \ + || fail "dist-tags: $(cat "$work/tags.out")" +(cd "$work/use-hosted" && npm install e2e-hosted@legacy) >/dev/null +[ "$(version_of "$work/use-hosted/node_modules/e2e-hosted")" = 1.0.0 ] || fail "install by dist-tag" +npm dist-tag rm e2e-hosted legacy "${hosted[@]}" >/dev/null +npm dist-tag ls e2e-hosted "${hosted[@]}" | grep -q legacy && fail "dist-tag rm" +npm deprecate e2e-hosted@1.0.0 "use 1.1.0" "${hosted[@]}" >/dev/null +[ "$(npm view e2e-hosted@1.0.0 deprecated "${hosted[@]}")" = "use 1.1.0" ] || fail "npm deprecate" +rm -rf "$work/use-hosted/node_modules" +(cd "$work/use-hosted" && npm install e2e-hosted@1.0.0) > "$work/deprecated.out" 2>&1 +grep -q "deprecated e2e-hosted@1.0.0: use 1.1.0" "$work/deprecated.out" \ + || fail "npm did not warn about the deprecation: $(cat "$work/deprecated.out")" +npm unpublish e2e-hosted@1.1.0 --force "${hosted[@]}" >/dev/null +[ "$(versions e2e-hosted "$(reg npm-hosted)")" = '["1.0.0"]' ] || fail "unpublish a version" +[ "$(npm view e2e-hosted dist-tags.latest "${hosted[@]}")" = 1.0.0 ] || fail "latest after the unpublish" +npm unpublish pnpm-made --force "${hosted[@]}" >/dev/null +if npm view pnpm-made "${hosted[@]}" >/dev/null 2>&1; then fail "unpublish a package"; fi + +step "npm audit through the group" +project "$work/audit" "$(reg npm-all)" +(cd "$work/audit" && npm install e2e-public@1.0.0 @acme/ui) >/dev/null +(cd "$work/audit" && npm audit --json --audit=true > "$work/audit.out") || true +jq -e '.vulnerabilities["e2e-public"].severity == "high"' "$work/audit.out" >/dev/null \ + || fail "npm audit: $(cat "$work/audit.out")" +grep '^POST /-/npm/v1/security/advisories/bulk' "$work/upstream.log" > "$work/audits.out" \ + || fail "no audit reached the upstream" +if grep -q '@acme' "$work/audits.out"; then fail "the audit told the upstream about @acme: $(cat "$work/audits.out")"; fi + +step "the upstream never saw Forge's credentials" +if grep -q 'auth=True' "$work/upstream.log"; then fail "credentials went upstream"; fi + +echo +echo "npm e2e passed" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d014b0a0..89bf60b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,7 +577,8 @@ jobs: - name: Cache cargo registry + dependency builds uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: - # raw-e2e, oci-e2e and web-e2e build the same debug forge-server. + # raw-e2e, npm-e2e, oci-e2e and web-e2e build the same debug + # forge-server. shared-key: forge-server save-if: false - name: Start the S3 gateway @@ -597,6 +598,71 @@ jobs: path: raw-e2e-results retention-days: 7 + # ── npm registries e2e (npm, pnpm and Yarn against a live server) ───────── + # Boots forge-server and a mock upstream registry, and drives npm + # repositories with the real clients: npm's web login and Yarn's legacy + # login, logout, publishes with npm and pnpm (also through a group to its + # writable member), installs from a hosted repository, a proxy and a group, + # dist-tags, deprecation, unpublishing, the group's dependency-confusion + # guard and npm audit (.github/scripts/npm-e2e.sh). + npm-e2e: + name: npm-e2e + runs-on: ubuntu-latest + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} + services: + postgres: + image: mirror.gcr.io/library/postgres:17@sha256:a63c1ea3f6a3a5a1621f97d2aba3b30faecfab4224c1b0000cc85ced57bb1939 + env: + POSTGRES_DB: forge + POSTGRES_USER: forge + POSTGRES_PASSWORD: forge + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U forge -d forge" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + SQLX_OFFLINE: "true" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Rust toolchain (rust-toolchain.toml) + uses: dtolnay/rust-toolchain@d124f86f28ff42855784ff3319f48ac182c248f8 # 1.99.1 + - name: Cache cargo registry + dependency builds + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + # npm-e2e, raw-e2e, oci-e2e and web-e2e build the same debug + # forge-server. + shared-key: forge-server + save-if: false + - name: Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: "11.5.0" + package_json_file: web/package.json + - name: Setup Node 26 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26" + - name: Start the S3 gateway + run: | + docker run -d --name forge-e2e-s3 -p 7070:7070 \ + mirror.gcr.io/versity/versitygw:v1.4.1@sha256:0400cb59f59da0f1cf9f7fd49505191abc348dfadf54509bf1988caaff4eb96f \ + --port :7070 --access forge-e2e-key --secret forge-e2e-secret posix /tmp + - name: cargo build -p forge-server + run: cargo build -p forge-server --bin forge-server + - name: npm e2e with npm, pnpm and Yarn (logins, publish, install, tags, audit) + run: .github/scripts/npm-e2e.sh target/debug/forge-server npm-e2e-results + - name: Upload the server and upstream logs + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: npm-e2e-results + path: npm-e2e-results + retention-days: 7 + # ── E2E / integration tests (forge-e2e crate, testcontainers) ────────────── # The forge-e2e harness boots its OWN Postgres + versitygw (+ Redis, and a # versitygw with its IAM/STS service for the S3 web identity scenarios) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e00fe78..b3621365 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,7 +90,7 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The `name:` of every ci.yml job that runs on the default branch. The # required status checks in DEVELOPMENT.md (merge queue) list the same. - REQUIRED_JOBS: fmt clippy audit test docker kubernetes web oci-e2e raw-e2e e2e web-e2e + REQUIRED_JOBS: fmt clippy audit test docker kubernetes web oci-e2e raw-e2e npm-e2e e2e web-e2e steps: - name: Wait for the commit's CI run and check its jobs shell: bash # -eo pipefail: a failing gh call fails the step diff --git a/.sqlx/query-04d359be3fd2a58ac5a3cd6207aec0a8744759c462eef93e1c56c452933c1caf.json b/.sqlx/query-04d359be3fd2a58ac5a3cd6207aec0a8744759c462eef93e1c56c452933c1caf.json new file mode 100644 index 00000000..cdfbaf48 --- /dev/null +++ b/.sqlx/query-04d359be3fd2a58ac5a3cd6207aec0a8744759c462eef93e1c56c452933c1caf.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS (\n SELECT 1 FROM package_tag\n WHERE repository_id = $1 AND package = $2 AND version = $3\n ) AS \"tagged!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tagged!", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "04d359be3fd2a58ac5a3cd6207aec0a8744759c462eef93e1c56c452933c1caf" +} diff --git a/.sqlx/query-0c93dd4457876bbd52e975f8ba8c27f8f8f7817a4a80262466c823ae099b17dc.json b/.sqlx/query-0c93dd4457876bbd52e975f8ba8c27f8f8f7817a4a80262466c823ae099b17dc.json new file mode 100644 index 00000000..cfa3b7e7 --- /dev/null +++ b/.sqlx/query-0c93dd4457876bbd52e975f8ba8c27f8f8f7817a4a80262466c823ae099b17dc.json @@ -0,0 +1,112 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, client, repository_id, hostname, client_ip, user_id, created_at, expires_at\n FROM cli_login\n WHERE login_hash = $1 AND expires_at > now()\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "cli_login", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "client", + "type_info": "Text", + "origin": { + "Table": { + "table": "cli_login", + "name": "client" + } + } + }, + { + "ordinal": 2, + "name": "repository_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "cli_login", + "name": "repository_id" + } + } + }, + { + "ordinal": 3, + "name": "hostname", + "type_info": "Text", + "origin": { + "Table": { + "table": "cli_login", + "name": "hostname" + } + } + }, + { + "ordinal": 4, + "name": "client_ip", + "type_info": "Text", + "origin": { + "Table": { + "table": "cli_login", + "name": "client_ip" + } + } + }, + { + "ordinal": 5, + "name": "user_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "cli_login", + "name": "user_id" + } + } + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "cli_login", + "name": "created_at" + } + } + }, + { + "ordinal": 7, + "name": "expires_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "cli_login", + "name": "expires_at" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false, + false + ] + }, + "hash": "0c93dd4457876bbd52e975f8ba8c27f8f8f7817a4a80262466c823ae099b17dc" +} diff --git a/.sqlx/query-0eed893d19cd6e528685a748c1f544d7513b699f27a13fa020659e4133ba3dbd.json b/.sqlx/query-0eed893d19cd6e528685a748c1f544d7513b699f27a13fa020659e4133ba3dbd.json new file mode 100644 index 00000000..31ff1625 --- /dev/null +++ b/.sqlx/query-0eed893d19cd6e528685a748c1f544d7513b699f27a13fa020659e4133ba3dbd.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT tag, version FROM package_tag\n WHERE repository_id = $1 AND package = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Text", + "origin": { + "Table": { + "table": "package_tag", + "name": "tag" + } + } + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Text", + "origin": { + "Table": { + "table": "package_tag", + "name": "version" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "0eed893d19cd6e528685a748c1f544d7513b699f27a13fa020659e4133ba3dbd" +} diff --git a/.sqlx/query-bee91d65724d2a2b1c2a0cee1ca0d93742f92588a3ad61f44c3e0f8988df7d03.json b/.sqlx/query-1f25a2f4aeff0bdc1b0de6315b66dad1e70c1a120f7f5239ac97d35bd4825678.json similarity index 83% rename from .sqlx/query-bee91d65724d2a2b1c2a0cee1ca0d93742f92588a3ad61f44c3e0f8988df7d03.json rename to .sqlx/query-1f25a2f4aeff0bdc1b0de6315b66dad1e70c1a120f7f5239ac97d35bd4825678.json index 34d60263..b9a67418 100644 --- a/.sqlx/query-bee91d65724d2a2b1c2a0cee1ca0d93742f92588a3ad61f44c3e0f8988df7d03.json +++ b/.sqlx/query-1f25a2f4aeff0bdc1b0de6315b66dad1e70c1a120f7f5239ac97d35bd4825678.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\"\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4\n WHERE c.repository_id = $1 AND c.name = $2 AND c.version = ANY($3::text[])\n ORDER BY c.id\n ", + "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\", c.deprecation\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4\n WHERE c.repository_id = $1 AND c.name = $2 AND c.version = ANY($3::text[])\n ORDER BY c.id\n ", "describe": { "columns": [ { @@ -112,6 +112,17 @@ "name": "size" } } + }, + { + "ordinal": 10, + "name": "deprecation", + "type_info": "Text", + "origin": { + "Table": { + "table": "component", + "name": "deprecation" + } + } } ], "parameters": { @@ -132,8 +143,9 @@ true, true, false, - false + false, + true ] }, - "hash": "bee91d65724d2a2b1c2a0cee1ca0d93742f92588a3ad61f44c3e0f8988df7d03" + "hash": "1f25a2f4aeff0bdc1b0de6315b66dad1e70c1a120f7f5239ac97d35bd4825678" } diff --git a/.sqlx/query-1fa761deb76a7db98f5390e998b7ba1b2fa3bc63b4000d442f3f83a502fd4391.json b/.sqlx/query-1fa761deb76a7db98f5390e998b7ba1b2fa3bc63b4000d442f3f83a502fd4391.json new file mode 100644 index 00000000..f0d1cfc1 --- /dev/null +++ b/.sqlx/query-1fa761deb76a7db98f5390e998b7ba1b2fa3bc63b4000d442f3f83a502fd4391.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM package_tag WHERE repository_id = $1 AND package = $2 AND tag = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1fa761deb76a7db98f5390e998b7ba1b2fa3bc63b4000d442f3f83a502fd4391" +} diff --git a/.sqlx/query-0a80f388f927a6fba4aa4a37cc9c6654a76f3251c3d1cae7a09b57484a3e8db2.json b/.sqlx/query-36a76654ad6617689529c9bbe6c9d4fec0231ea57ad0e75163af743fac416391.json similarity index 84% rename from .sqlx/query-0a80f388f927a6fba4aa4a37cc9c6654a76f3251c3d1cae7a09b57484a3e8db2.json rename to .sqlx/query-36a76654ad6617689529c9bbe6c9d4fec0231ea57ad0e75163af743fac416391.json index b52937f8..9b660cfa 100644 --- a/.sqlx/query-0a80f388f927a6fba4aa4a37cc9c6654a76f3251c3d1cae7a09b57484a3e8db2.json +++ b/.sqlx/query-36a76654ad6617689529c9bbe6c9d4fec0231ea57ad0e75163af743fac416391.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\"\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4\n WHERE c.repository_id = $1 AND c.name = $2 AND c.version = $3\n ", + "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\", c.deprecation\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4\n WHERE c.repository_id = $1 AND c.name = $2 AND c.version = $3\n ", "describe": { "columns": [ { @@ -112,6 +112,17 @@ "name": "size" } } + }, + { + "ordinal": 10, + "name": "deprecation", + "type_info": "Text", + "origin": { + "Table": { + "table": "component", + "name": "deprecation" + } + } } ], "parameters": { @@ -132,8 +143,9 @@ true, true, true, + true, true ] }, - "hash": "0a80f388f927a6fba4aa4a37cc9c6654a76f3251c3d1cae7a09b57484a3e8db2" + "hash": "36a76654ad6617689529c9bbe6c9d4fec0231ea57ad0e75163af743fac416391" } diff --git a/.sqlx/query-3ef924fcda6ab461396e254b17bdd5f8fe600f9fa47b8935b8bc5838ed735708.json b/.sqlx/query-3ef924fcda6ab461396e254b17bdd5f8fe600f9fa47b8935b8bc5838ed735708.json new file mode 100644 index 00000000..9351d800 --- /dev/null +++ b/.sqlx/query-3ef924fcda6ab461396e254b17bdd5f8fe600f9fa47b8935b8bc5838ed735708.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO cli_login (login_hash, done_hash, client, repository_id, hostname, client_ip, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Int8", + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "3ef924fcda6ab461396e254b17bdd5f8fe600f9fa47b8935b8bc5838ed735708" +} diff --git a/.sqlx/query-3f2dc4a59d921bd5846f7c26e6909e334388bbea96ed70a74cd5b79a9d9af35d.json b/.sqlx/query-3f2dc4a59d921bd5846f7c26e6909e334388bbea96ed70a74cd5b79a9d9af35d.json new file mode 100644 index 00000000..153a9b39 --- /dev/null +++ b/.sqlx/query-3f2dc4a59d921bd5846f7c26e6909e334388bbea96ed70a74cd5b79a9d9af35d.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM cli_login\n WHERE done_hash = $1 AND expires_at > now() AND user_id IS NOT NULL\n RETURNING user_id AS \"user_id!\", repository_id, hostname\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "user_id!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "cli_login", + "name": "user_id" + } + } + }, + { + "ordinal": 1, + "name": "repository_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "cli_login", + "name": "repository_id" + } + } + }, + { + "ordinal": 2, + "name": "hostname", + "type_info": "Text", + "origin": { + "Table": { + "table": "cli_login", + "name": "hostname" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true + ] + }, + "hash": "3f2dc4a59d921bd5846f7c26e6909e334388bbea96ed70a74cd5b79a9d9af35d" +} diff --git a/.sqlx/query-4e0afd23c132ed31eb2380f3f809f0e708d99b38037deaf4cba0cc0987e33d5d.json b/.sqlx/query-4e0afd23c132ed31eb2380f3f809f0e708d99b38037deaf4cba0cc0987e33d5d.json new file mode 100644 index 00000000..499e766a --- /dev/null +++ b/.sqlx/query-4e0afd23c132ed31eb2380f3f809f0e708d99b38037deaf4cba0cc0987e33d5d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE cli_login SET user_id = $2, approved_at = now()\n WHERE login_hash = $1 AND expires_at > now() AND user_id IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "4e0afd23c132ed31eb2380f3f809f0e708d99b38037deaf4cba0cc0987e33d5d" +} diff --git a/.sqlx/query-5a2229c387b996ad1f7bcdec9922a4a5decf5084236a1daf71f6a0ce7b30f326.json b/.sqlx/query-5a2229c387b996ad1f7bcdec9922a4a5decf5084236a1daf71f6a0ce7b30f326.json new file mode 100644 index 00000000..dcde2717 --- /dev/null +++ b/.sqlx/query-5a2229c387b996ad1f7bcdec9922a4a5decf5084236a1daf71f6a0ce7b30f326.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE component SET deprecation = $4, updated_at = now()\n WHERE repository_id = $1 AND name = $2 AND version = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5a2229c387b996ad1f7bcdec9922a4a5decf5084236a1daf71f6a0ce7b30f326" +} diff --git a/.sqlx/query-62f32a8f0ad8b325b7064977c633e9890c2b2cabe43451b14e7d25e6a5ddf025.json b/.sqlx/query-62f32a8f0ad8b325b7064977c633e9890c2b2cabe43451b14e7d25e6a5ddf025.json new file mode 100644 index 00000000..6294d727 --- /dev/null +++ b/.sqlx/query-62f32a8f0ad8b325b7064977c633e9890c2b2cabe43451b14e7d25e6a5ddf025.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM cli_login WHERE expires_at <= now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "62f32a8f0ad8b325b7064977c633e9890c2b2cabe43451b14e7d25e6a5ddf025" +} diff --git a/.sqlx/query-8aa6239b751b3255b54d4e02e78955e6c0ebc01ffb3b8c866f03984c422b6e93.json b/.sqlx/query-721a7e66544901c81a63cfde95b05572fbfadab0803e83ac7e4d59be3be92f2a.json similarity index 83% rename from .sqlx/query-8aa6239b751b3255b54d4e02e78955e6c0ebc01ffb3b8c866f03984c422b6e93.json rename to .sqlx/query-721a7e66544901c81a63cfde95b05572fbfadab0803e83ac7e4d59be3be92f2a.json index 33e790cf..ee58dabe 100644 --- a/.sqlx/query-8aa6239b751b3255b54d4e02e78955e6c0ebc01ffb3b8c866f03984c422b6e93.json +++ b/.sqlx/query-721a7e66544901c81a63cfde95b05572fbfadab0803e83ac7e4d59be3be92f2a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\"\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $3\n WHERE c.repository_id = $1 AND c.name = $2\n ORDER BY c.id\n ", + "query": "\n SELECT\n c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at,\n c.manifest_summary, a.sha512_base64 AS \"package_hash_sha512_base64?\",\n a.blob_sha256 AS \"package_sha256?\", a.size AS \"package_size?\", c.deprecation\n FROM component c\n LEFT JOIN asset a ON a.component_id = c.id AND a.path = $3\n WHERE c.repository_id = $1 AND c.name = $2\n ORDER BY c.id\n ", "describe": { "columns": [ { @@ -112,6 +112,17 @@ "name": "size" } } + }, + { + "ordinal": 10, + "name": "deprecation", + "type_info": "Text", + "origin": { + "Table": { + "table": "component", + "name": "deprecation" + } + } } ], "parameters": { @@ -131,8 +142,9 @@ true, true, false, - false + false, + true ] }, - "hash": "8aa6239b751b3255b54d4e02e78955e6c0ebc01ffb3b8c866f03984c422b6e93" + "hash": "721a7e66544901c81a63cfde95b05572fbfadab0803e83ac7e4d59be3be92f2a" } diff --git a/.sqlx/query-7fa52616dbb07a766382574c42d4b31bfe9772478bf9be129a50a29deb897db6.json b/.sqlx/query-7fa52616dbb07a766382574c42d4b31bfe9772478bf9be129a50a29deb897db6.json new file mode 100644 index 00000000..72ae25cf --- /dev/null +++ b/.sqlx/query-7fa52616dbb07a766382574c42d4b31bfe9772478bf9be129a50a29deb897db6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM package_tag\n WHERE repository_id = $1 AND package = $2 AND ($3::text IS NULL OR version = $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7fa52616dbb07a766382574c42d4b31bfe9772478bf9be129a50a29deb897db6" +} diff --git a/.sqlx/query-afb8561aa8a3f4ea3740499bf63915c317db9013aba72a3c6346faf481099be7.json b/.sqlx/query-afb8561aa8a3f4ea3740499bf63915c317db9013aba72a3c6346faf481099be7.json new file mode 100644 index 00000000..02a68c91 --- /dev/null +++ b/.sqlx/query-afb8561aa8a3f4ea3740499bf63915c317db9013aba72a3c6346faf481099be7.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT package, version FROM package_tag WHERE repository_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "package", + "type_info": "Text", + "origin": { + "Table": { + "table": "package_tag", + "name": "package" + } + } + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Text", + "origin": { + "Table": { + "table": "package_tag", + "name": "version" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "afb8561aa8a3f4ea3740499bf63915c317db9013aba72a3c6346faf481099be7" +} diff --git a/.sqlx/query-b9abaf48b56de1f008614de9d88fa213063d86de7736a605a16e1a00ba1ce80d.json b/.sqlx/query-b9abaf48b56de1f008614de9d88fa213063d86de7736a605a16e1a00ba1ce80d.json new file mode 100644 index 00000000..3d4617d7 --- /dev/null +++ b/.sqlx/query-b9abaf48b56de1f008614de9d88fa213063d86de7736a605a16e1a00ba1ce80d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM cli_login WHERE login_hash = $1 AND user_id IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "b9abaf48b56de1f008614de9d88fa213063d86de7736a605a16e1a00ba1ce80d" +} diff --git a/.sqlx/query-cf6b219a3de5f5fd0a5d949c1464d4c0f310d5f7f8a40eb0b1146a97840d3851.json b/.sqlx/query-cf6b219a3de5f5fd0a5d949c1464d4c0f310d5f7f8a40eb0b1146a97840d3851.json new file mode 100644 index 00000000..5b92ccbd --- /dev/null +++ b/.sqlx/query-cf6b219a3de5f5fd0a5d949c1464d4c0f310d5f7f8a40eb0b1146a97840d3851.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM package_tag WHERE repository_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cf6b219a3de5f5fd0a5d949c1464d4c0f310d5f7f8a40eb0b1146a97840d3851" +} diff --git a/.sqlx/query-d7b159866489cb91555fd92383fac58d1613fa732f9eba38fd73cdd8232b9401.json b/.sqlx/query-d7b159866489cb91555fd92383fac58d1613fa732f9eba38fd73cdd8232b9401.json new file mode 100644 index 00000000..c8686297 --- /dev/null +++ b/.sqlx/query-d7b159866489cb91555fd92383fac58d1613fa732f9eba38fd73cdd8232b9401.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS (\n SELECT 1 FROM cli_login WHERE done_hash = $1 AND expires_at > now()\n ) AS \"pending!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pending!", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d7b159866489cb91555fd92383fac58d1613fa732f9eba38fd73cdd8232b9401" +} diff --git a/.sqlx/query-f6dac1d112d13d0be04e1ebdd06e39794c7735c62e54bc34fa3d0e8981c8dc0c.json b/.sqlx/query-f6dac1d112d13d0be04e1ebdd06e39794c7735c62e54bc34fa3d0e8981c8dc0c.json new file mode 100644 index 00000000..461eadb4 --- /dev/null +++ b/.sqlx/query-f6dac1d112d13d0be04e1ebdd06e39794c7735c62e54bc34fa3d0e8981c8dc0c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO package_tag (repository_id, package, tag, version)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (repository_id, package, tag)\n DO UPDATE SET version = EXCLUDED.version, updated_at = now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f6dac1d112d13d0be04e1ebdd06e39794c7735c62e54bc34fa3d0e8981c8dc0c" +} diff --git a/Cargo.lock b/Cargo.lock index 173721a4..ae82185b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1544,6 +1544,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.13" @@ -1629,6 +1639,7 @@ dependencies = [ "rsa", "serde", "serde_json", + "sha1 0.11.0", "sha2 0.11.1", "sqlx", "subtle", @@ -1723,6 +1734,24 @@ dependencies = [ "zip", ] +[[package]] +name = "forge-format-npm" +version = "1.0.0" +dependencies = [ + "base64 0.23.1", + "chrono", + "flate2", + "forge-plugin-api", + "hex", + "semver", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "tar", + "tempfile", +] + [[package]] name = "forge-format-nuget" version = "1.0.0" @@ -1815,6 +1844,7 @@ dependencies = [ "forge-core", "forge-crypto", "forge-db", + "forge-format-npm", "forge-format-nuget", "forge-format-raw", "forge-oci", @@ -4581,6 +4611,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", +] + [[package]] name = "tempfile" version = "3.27.0" diff --git a/Cargo.toml b/Cargo.toml index 4a377565..598703d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/server", "crates/e2e", "crates/formats/nuget", + "crates/formats/npm", "crates/formats/raw", "xtask", ] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index b40a2af1..d4928264 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -119,7 +119,7 @@ cargo xtask up && cargo xtask run # terminal 1: infra + API on :8421 cargo xtask web-dev # terminal 2: Vite on :8420 ``` -The Vite dev server proxies `/admin`, `/browse`, `/v2`, `/v3`, `/raw`, `/auth` +The Vite dev server proxies `/admin`, `/browse`, `/v2`, `/v3`, `/npm`, `/raw`, `/auth` to the Rust origin (default `http://127.0.0.1:8421`, override with `FORGE_DEV_API_TARGET`) so the same relative API URLs work in dev — this is a dev-only static route, not a production API proxy. @@ -252,16 +252,22 @@ groups. See [docs/raw.md](docs/raw.md). Every package format is a format plugin, a crate in [`crates/formats/`](./crates/formats) linked into the server (see -[docs/format-plugins.md](./docs/format-plugins.md)): NuGet and raw. A new -format goes in `crates/formats/` too, never in a `crates/format-` +[docs/format-plugins.md](./docs/format-plugins.md)): NuGet, npm and raw. A +new format goes in `crates/formats/` too, never in a `crates/format-` crate or the server. Docker / OCI (`crates/format-oci`) is not there yet and is to move to `crates/formats/docker`. A repository of a plugin format is created with `"format": ""` (hosted, proxy or group; a plugin format has proxies when it implements the upstream side) and served under `/repository//` unless the plugin names another mount (NuGet's is -`/v3`). Helm charts are pushed to docker repositories, see +`/v3`, npm's `/npm`). Helm charts are pushed to docker repositories, see [docs/docker.md](docs/docker.md#helm-charts). +npm repositories (`"format":"npm"`) are served under `/npm/`: `npm publish +--registry http://localhost:8421/npm//` with an API token in +`.npmrc` (`//localhost:8421/npm//:_authToken=`), or after +`npm login --registry …`, which opens the web UI's approval page. See +[docs/npm.md](docs/npm.md). + ## Configuration Deployment configuration is in [`forge.toml`](./forge.toml) (loaded by @@ -630,6 +636,21 @@ cargo build -p forge-server .github/scripts/raw-e2e.sh target/debug/forge-server target/raw-e2e ``` +npm repositories have theirs, CI's `npm-e2e` job: the real clients against a +live server and a mock upstream registry (npm's browser login, approved +through the page's API, and Yarn's password login; logout; publishes with +npm and pnpm, also through a group; installs with npm, pnpm and, when it is +installed, Bun from a hosted repository, a proxy and a group; dist-tags, +deprecation, unpublishing; the group's dependency-confusion guard and +`npm audit`). It needs node with npm, pnpm, curl, jq, python3 and openssl +(Yarn is fetched with `npm pack`, or given as `YARN_BERRY`), a Postgres and +an S3 endpoint: + +```bash +cargo build -p forge-server +.github/scripts/npm-e2e.sh target/debug/forge-server target/npm-e2e +``` + The container image has a smoke test, which CI's `docker` job runs on the image of each platform it built (#239). It starts the image the way the Kubernetes manifests and the Helm chart run it: read-only root filesystem, `/tmp` a tmpfs, user @@ -830,7 +851,7 @@ backups, upgrades) follow [docs/deployment.md](docs/deployment.md). ## CI and the merge queue CI (`.github/workflows/ci.yml`) has the jobs `fmt`, `clippy`, `audit`, -`test`, `docker`, `kubernetes`, `web`, `oci-e2e` and `raw-e2e`, and the slow +`test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e` and `npm-e2e`, and the slow `e2e` and `web-e2e`, which pull requests skip: those two run in the merge queue and on every push to `main`. It runs on pull requests that target `main` only; a stacked one waits (see [Stacked pull @@ -861,7 +882,7 @@ It sorts the change into groups of paths: | Group | Paths | Jobs | |---|---|---| -| Rust | `crates/`, `xtask/`, `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, `.cargo/` (but `audit.toml`), `migrations/`, `.sqlx/`, `fixtures/`; the files the code or its tests read from outside the crates: `web/index.html`, `docker/forge.docker.toml`; and the scripts the jobs run: `test-parallel.sh`, `test-timings-runner.sh`, `test-timings-summary.sh`, `oci-e2e.sh`, `raw-e2e.sh` | `fmt`, `clippy`, `test`, `oci-e2e`, `raw-e2e`, `e2e`, `web-e2e` | +| Rust | `crates/`, `xtask/`, `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, `.cargo/` (but `audit.toml`), `migrations/`, `.sqlx/`, `fixtures/`; the files the code or its tests read from outside the crates: `web/index.html`, `docker/forge.docker.toml`; and the scripts the jobs run: `test-parallel.sh`, `test-timings-runner.sh`, `test-timings-summary.sh`, `oci-e2e.sh`, `raw-e2e.sh`, `npm-e2e.sh` | `fmt`, `clippy`, `test`, `oci-e2e`, `raw-e2e`, `npm-e2e`, `e2e`, `web-e2e` | | Changelog | `docs/changes/` | `fmt` | | Release | what xtask's tests read from outside `xtask/`: the version lines `cargo xtask release` sets (`web/package.json`, `deploy/kubernetes/base/kustomization.yaml`, `deploy/helm/forge/Chart.yaml`) and `release-channels.sh` | `fmt` (which runs xtask's tests) | | Audit | `Cargo.lock`, `.cargo/audit.toml` | `audit` | @@ -871,7 +892,7 @@ It sorts the change into groups of paths: A change to `ci-changes.sh` counts for every group. A change to `ci.yml` counts for the groups whose jobs it changes (#365): `fmt` is in Changelog; -`clippy`, `test`, `oci-e2e`, `raw-e2e` and `e2e` in Rust; `audit` +`clippy`, `test`, `oci-e2e`, `raw-e2e`, `npm-e2e` and `e2e` in Rust; `audit` in Audit; `web` and `web-e2e` in Web; `docker` and `docker-platform` in Image and `kubernetes` in Kubernetes. A change outside the jobs (the triggers, the settings, the `changes` job) or to a job missing from `CI_JOB_GROUP` in `ci-changes.sh` @@ -917,8 +938,8 @@ To turn it on, edit the ruleset (or branch protection rule) of `main`: 1. Enable **Require merge queue**. Keep its **status check timeout** above the run time of `e2e` and `web-e2e` (60 minutes by default). 2. Under **Require status checks to pass**, require `fmt`, `clippy`, `audit`, - `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, `e2e` and - `web-e2e` from GitHub Actions: every job of `ci.yml` but + `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, `npm-e2e`, + `e2e` and `web-e2e` from GitHub Actions: every job of `ci.yml` but `changes`, the jobs the release workflow requires of the tagged commit (`REQUIRED_JOBS` in `.github/workflows/release.yml`). A job left out here can fail on a pull request that merges anyway, and the release then diff --git a/README.md b/README.md index 13cffcb7..43aa0dc8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Forge -A self-hosted **NuGet package registry, Docker / OCI registry and raw file -repository** written in Rust. It speaks the NuGet **V3** protocol and the **OCI Distribution Spec** +A self-hosted **NuGet package registry, npm registry, Docker / OCI registry +and raw file repository** written in Rust. It speaks the NuGet **V3** +protocol, the **npm registry** protocol (npm, pnpm, Yarn, Bun) and the **OCI Distribution Spec** v1.1 (Docker, Podman, containerd, crane, Helm, cosign), and supports the three repository types you expect from an enterprise registry (more package formats come as **format plugins**, see below): @@ -15,7 +16,11 @@ formats come as **format plugins**, see below): priority; tolerates a failing member and, by default, keeps public feeds from shadowing internal package ids (dependency-confusion guard). -The same three types exist for container images and other OCI artifacts, +The same three types exist for npm packages: publish with `npm publish`, +log in with `npm login` (approved in the browser), proxies of +`registry.npmjs.org` or any registry, and groups that keep your scopes +internal and pass `npm audit` on — see **[docs/npm.md](./docs/npm.md)**; +for container images and other OCI artifacts, Helm charts included (`helm push`, `helm install oci://…`): hosted image repositories, pull-through proxies of Docker Hub, GHCR, Quay or any OCI registry, and groups — see **[docs/docker.md](./docs/docker.md)** — and for plain files @@ -53,8 +58,8 @@ A Cargo workspace of small crates plus the web SPA: | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | | `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | | `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | -| `formats/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `raw`. | -| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. | +| `formats/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `npm`, `raw`. | +| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, npm at `/npm`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | @@ -65,7 +70,7 @@ server. Docker / OCI (`crates/format-oci`) is not there yet and is to move to `crates/formats/docker`. **Single-origin serving.** `forge-server` mounts each format plugin's feed -(NuGet at `/v3`, others at `/repository`), the Docker / OCI registry (`/v2`), +(NuGet at `/v3`, npm at `/npm`, others at `/repository`), the Docker / OCI registry (`/v2`), raw file repositories (`/raw`), the admin/browse APIs (`/admin`, `/browse`, `/auth`) and then serves the built SPA from `web/dist/` via a `ServeDir` SPA fallback. The browser talks to the API on @@ -94,6 +99,7 @@ role/db; use `--db docker` if you don't have one — see below.) - **UI:** http://localhost:8420 - **NuGet feed:** `http://localhost:8421/v3/{repo}/index.json` +- **npm registry:** `http://localhost:8421/npm/{repo}/` Add demo data (anonymous-readable `nuget` repo + a pushed sample package) in a second terminal: @@ -204,6 +210,10 @@ curl -u admin@example.com: http://localhost:8421/raw/binaries/tools/1.0/t - **[Docker / OCI registry](./docs/docker.md)** — image repositories (hosted, proxy, group), naming and host-based routing, `docker login`, retention and garbage collection. +- **[npm registries](./docs/npm.md)** — npm packages (hosted, proxy, group) + for npm, pnpm, Yarn and Bun: `.npmrc`, logins, publishing, dist-tags, + deprecations, unpublishing, proxies of `registry.npmjs.org`, internal + scopes, audits and retention. - **[Raw file repositories](./docs/raw.md)** — files under arbitrary paths (hosted, proxy, group): uploads with curl, ranges and checksum sidecars, listings, the deployment policy, proxies of release downloads (the pdfium diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4671f1d8..dfafbd2c 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -35,6 +35,8 @@ argon2 = { workspace = true } sha2 = { workspace = true } hmac = { workspace = true } hex = { workspace = true } +# The SHA-1 an upstream publishes for a package (npm's `shasum`). +sha1 = "0.11" subtle = { workspace = true } rand = { workspace = true } base64 = { workspace = true } diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index cb736d2a..c60765a7 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -130,6 +130,36 @@ pub trait Format: Send + Sync { } } + /// Whether a proxy of this format with a password and no username sends + /// the password as an `Authorization: Bearer` token (npm's registry + /// tokens) rather than no credentials at all. + fn upstream_bearer_token(&self) -> bool { + false + } + + /// The README a manifest carries, for the web UI's package page. + fn readme(&self, manifest: &[u8]) -> Option { + let _ = manifest; + None + } + + /// A group's reserved prefix as stored, from what an administrator typed + /// (trimmed, a trailing `*` dropped); refused when it cannot start an + /// id. The default takes letters, digits, `.`, `_` and `-`, lowercased. + fn reserved_prefix(&self, prefix: &str) -> Result { + let valid = !prefix.is_empty() + && prefix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); + if valid { + Ok(prefix.to_ascii_lowercase()) + } else { + Err(FormatError::InvalidPackage(format!( + "{prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" + ))) + } + } + /// One step of the upstream side of a proxy repository /// ([`forge_plugin_api::FormatPlugin::upstream`]): what to fetch from /// `remote` for `op`, or what came of it. The proxy read path diff --git a/crates/core/src/net/upstream.rs b/crates/core/src/net/upstream.rs index 2453f4e9..ce9096d3 100644 --- a/crates/core/src/net/upstream.rs +++ b/crates/core/src/net/upstream.rs @@ -164,6 +164,20 @@ impl UpstreamError { } } +impl UpstreamError { + /// Whether the upstream refused the request (a 401 or a 403): the + /// proxy's credentials are missing or wrong, or they may not read it. + /// Asking again will not help until the proxy is fixed. + pub fn is_refusal(&self) -> bool { + matches!( + self, + UpstreamError::Status { status, .. } + if *status == reqwest::StatusCode::UNAUTHORIZED + || *status == reqwest::StatusCode::FORBIDDEN + ) + } +} + /// Whether an upstream's `status` says it cannot serve now: a 5xx, a 429. pub fn status_is_unavailable(status: reqwest::StatusCode) -> bool { status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS @@ -304,6 +318,9 @@ pub struct UpstreamAuth<'a> { pub username: &'a str, /// Basic-auth password. pub password: &'a str, + /// Send `password` as `Authorization: Bearer` instead (a registry token + /// stored without a username, for a format that takes tokens so). + pub bearer: bool, } /// Scheme + host + port triple used to scope credentials, and the requests @@ -471,12 +488,28 @@ impl UpstreamHttp { /// sent only to `url`'s origin: it is dropped once a redirect leaves it /// (registries redirect blobs to a CDN or an object store). pub async fn send( + &self, + method: reqwest::Method, + url: &str, + class: RequestClass, + headers: reqwest::header::HeaderMap, + auth: Option<&UpstreamAuth<'_>>, + ) -> Result { + self.send_with_body(method, url, class, headers, auth, None) + .await + } + + /// [`Self::send`] with a request `body` (an `npm audit` passed to a + /// registry). A redirect keeps it, except a `303`, which is followed + /// with a GET. + pub async fn send_with_body( &self, mut method: reqwest::Method, url: &str, class: RequestClass, mut headers: reqwest::header::HeaderMap, auth: Option<&UpstreamAuth<'_>>, + mut body: Option>, ) -> Result { let timeout = match class { RequestClass::Metadata => self.config.metadata_timeout, @@ -502,9 +535,16 @@ impl UpstreamHttp { .request(method.clone(), &hop) .headers(headers.clone()) .timeout(remaining); + if let Some(body) = &body { + req = req.body(body.clone()); + } if let Some(auth) = auth { if auth.origin.is_some() && auth.origin == Origin::of(&hop) { - req = req.basic_auth(auth.username, Some(auth.password)); + req = if auth.bearer { + req.bearer_auth(auth.password) + } else { + req.basic_auth(auth.username, Some(auth.password)) + }; } else { tracing::debug!(url = %hop, "upstream resource is off-origin; not sending proxy credentials"); } @@ -530,9 +570,10 @@ impl UpstreamHttp { } hops += 1; // A 303 is answered with a GET; the other redirects keep the - // method (the registry sends only GET and HEAD). + // method and the body. if resp.status() == reqwest::StatusCode::SEE_OTHER && method != reqwest::Method::HEAD { method = reqwest::Method::GET; + body = None; } tracing::debug!(from = %hop, to = %next, "following upstream redirect"); hop = next.into(); @@ -587,7 +628,7 @@ impl UpstreamHttp { url: &str, auth: Option<&UpstreamAuth<'_>>, ) -> Result, UpstreamError> { - let Some(spool) = self.get_metadata_body(url, auth).await? else { + let Some(spool) = self.get_metadata_body(url, auth, None).await? else { return Ok(None); }; let url_owned = url.to_string(); @@ -609,7 +650,17 @@ impl UpstreamHttp { url: &str, auth: Option<&UpstreamAuth<'_>>, ) -> Result>, UpstreamError> { - let Some(spool) = self.get_metadata_body(url, auth).await? else { + self.get_bytes_accepting(url, auth, None).await + } + + /// [`Self::get_bytes`] with an `Accept` header. + pub async fn get_bytes_accepting( + &self, + url: &str, + auth: Option<&UpstreamAuth<'_>>, + accept: Option<&str>, + ) -> Result>, UpstreamError> { + let Some(spool) = self.get_metadata_body(url, auth, accept).await? else { return Ok(None); }; let bytes = tokio::task::spawn_blocking(move || -> std::io::Result> { @@ -628,8 +679,21 @@ impl UpstreamHttp { &self, url: &str, auth: Option<&UpstreamAuth<'_>>, + accept: Option<&str>, ) -> Result, UpstreamError> { - let resp = self.get(url, RequestClass::Metadata, auth).await?; + let mut headers = reqwest::header::HeaderMap::new(); + if let Some(accept) = accept.and_then(|a| reqwest::header::HeaderValue::from_str(a).ok()) { + headers.insert(reqwest::header::ACCEPT, accept); + } + let resp = self + .send( + reqwest::Method::GET, + url, + RequestClass::Metadata, + headers, + auth, + ) + .await?; if resp.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } @@ -744,7 +808,7 @@ impl SpooledBody { /// Inflate a gzip-encoded body into a new spool, failing as soon as the /// decoded size exceeds `limit`. Blocking: call from the blocking pool. - fn gunzip( + pub fn gunzip( self, url: &str, limit: u64, diff --git a/crates/core/src/repo/group.rs b/crates/core/src/repo/group.rs index 71ccdc0c..6a88e826 100644 --- a/crates/core/src/repo/group.rs +++ b/crates/core/src/repo/group.rs @@ -676,6 +676,63 @@ impl GroupRepo { .collect()) } + /// The leaf repositories (hosted and proxy) that serve `package_id` + /// through group `repo` for this caller, highest priority first: the + /// members [`RepoReadOps`] reads it from, after every group's guards + /// (see [`Self::serving_members_for_id`]). + pub async fn serving_leaves( + &self, + repo: &str, + package_id: &str, + ) -> Result, RepoError> { + let graph = self.group_graph(repo).await?; + Ok(self + .serving_members_for_id(&graph, package_id) + .await? + .into_iter() + .map(|member| member.repo) + .collect()) + } + + /// The members' registrations of `id` merged, deduplicated by version, + /// the highest-priority member's entry winning; `dated`: their + /// [`RepoReadOps::dated_registration`]s. + async fn merged_registration( + &self, + repo: &str, + id: &str, + dated: bool, + ) -> Result, RepoError> { + let graph = self.group_graph(repo).await?; + let members = self.serving_members_for_id(&graph, id).await?; + + // Merge, dedup by version (id is fixed here), highest-priority wins. + let mut seen: HashSet = HashSet::new(); + let mut merged: Vec = Vec::new(); + let mut failures = MemberFailures::default(); + for m in &members { + let entries = if dated { + m.ops.dated_registration(&m.repo.name, id).await + } else { + m.ops.registration(&m.repo.name, id).await + }; + match entries { + Ok(entries) => { + failures.ok(); + for entry in entries { + if seen.insert(entry.version.clone()) { + merged.push(entry); + } + } + } + Err(e) => failures.failed(&m.repo.name, e), + } + } + failures.into_merged_result(merged.is_empty())?; + merged.sort_by(|a, b| self.compare_versions(&a.version, &b.version)); + Ok(merged) + } + /// The members allowed to serve `package_id`, in priority order: the /// visible members that every group on their way lets through /// ([`GroupGraph::serving_leaves`]). A group lets only its owners serve an @@ -887,6 +944,27 @@ impl GroupRepo { #[async_trait] impl RepoReadOps for GroupRepo { + /// The tags of the first serving member, by priority, that has any: a + /// tag names a version of the member that set it. A member that fails + /// is skipped, as for versions. + async fn tags( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + let graph = self.group_graph(repo).await?; + for m in self.serving_members_for_id(&graph, id).await? { + match m.ops.tags(&m.repo.name, id).await { + Ok(tags) if !tags.is_empty() => return Ok(tags), + Ok(_) => {} + Err(e) => { + tracing::warn!(group = %repo, member = %m.repo.name, error = %e, "group member tags failed; skipped"); + } + } + } + Ok(std::collections::BTreeMap::new()) + } + async fn list_versions(&self, repo: &str, id: &str) -> Result, RepoError> { let graph = self.group_graph(repo).await?; let members = self.serving_members_for_id(&graph, id).await?; @@ -960,29 +1038,15 @@ impl RepoReadOps for GroupRepo { repo: &str, id: &str, ) -> Result, RepoError> { - let graph = self.group_graph(repo).await?; - let members = self.serving_members_for_id(&graph, id).await?; + self.merged_registration(repo, id, false).await + } - // Merge, dedup by version (id is fixed here), highest-priority wins. - let mut seen: HashSet = HashSet::new(); - let mut merged: Vec = Vec::new(); - let mut failures = MemberFailures::default(); - for m in &members { - match m.ops.registration(&m.repo.name, id).await { - Ok(entries) => { - failures.ok(); - for entry in entries { - if seen.insert(entry.version.clone()) { - merged.push(entry); - } - } - } - Err(e) => failures.failed(&m.repo.name, e), - } - } - failures.into_merged_result(merged.is_empty())?; - merged.sort_by(|a, b| self.compare_versions(&a.version, &b.version)); - Ok(merged) + async fn dated_registration( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + self.merged_registration(repo, id, true).await } async fn registration_versions(&self, repo: &str, id: &str) -> Result, RepoError> { diff --git a/crates/core/src/repo/hosted.rs b/crates/core/src/repo/hosted.rs index b9087162..5e508166 100644 --- a/crates/core/src/repo/hosted.rs +++ b/crates/core/src/repo/hosted.rs @@ -274,6 +274,7 @@ impl HostedRepo { nuspec_bytes, manifest_summary, published: Some(row.created_at), + deprecation: row.deprecation, }) } @@ -350,6 +351,15 @@ impl HostedRepo { #[async_trait] impl RepoReadOps for HostedRepo { + async fn tags( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + let repo_id = self.repo_id(repo).await?; + Ok(forge_db::tag_dao::tags(&self.pool, repo_id, id).await?) + } + async fn list_versions(&self, repo: &str, id: &str) -> Result, RepoError> { let repo_id = self.repo_id(repo).await?; let comps = component_dao::list_versions(&self.pool, repo_id, id).await?; diff --git a/crates/core/src/repo/mod.rs b/crates/core/src/repo/mod.rs index aad448b3..4c9f7deb 100644 --- a/crates/core/src/repo/mod.rs +++ b/crates/core/src/repo/mod.rs @@ -71,6 +71,14 @@ pub enum RepoError { /// retry later → 503. #[error("upstream unavailable: {0}")] Upstream(String), + /// A proxy's upstream answered, but not usably, and nothing cached + /// answers: it refused the proxy (a 401 or a 403: its credentials are + /// missing or wrong) or served a package the proxy does not take (too + /// large, unreadable, not the one asked for or not matching its + /// published digest). Not an outage: retrying does not help until the + /// upstream or the proxy changes → 502. + #[error("bad upstream: {0}")] + BadUpstream(String), /// The repository's format is not installed on this server (a format /// plugin missing from the configuration) → 503: nothing of it is served /// until the format is back, and it is never read as another format. @@ -103,6 +111,7 @@ impl From for RepoError { RepoError::Storage(reason) => RepoError::Storage(reason.clone()), RepoError::Busy(reason) => RepoError::Busy(reason.clone()), RepoError::Upstream(reason) => RepoError::Upstream(reason.clone()), + RepoError::BadUpstream(reason) => RepoError::BadUpstream(reason.clone()), RepoError::FormatUnavailable(format) => RepoError::FormatUnavailable(format.clone()), }) } @@ -315,6 +324,9 @@ pub struct RegistrationEntry { /// When this version was published into the repository (`None` when /// unknown, e.g. a proxied version not cached yet). pub published: Option>, + /// Why the version is deprecated, if it was deprecated in this + /// repository (`component.deprecation`). + pub deprecation: Option, } /// A search hit: one package id with all of its (filtered) versions. @@ -543,6 +555,19 @@ pub trait RepoReadOps: Send + Sync { async fn registration(&self, repo: &str, id: &str) -> Result, RepoError>; + /// [`Self::registration`] with every version's publish time, even where + /// that costs more: a proxy asks its upstream with + /// [`forge_plugin_api::UpstreamOp::Dating`] (npm: the full packument, + /// not the abbreviated one installs read). The default is + /// [`Self::registration`], whose times hosted repositories know. + async fn dated_registration( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + self.registration(repo, id).await + } + /// The versions [`Self::registration`] lists for `id` (including /// unlisted), ascending by version precedence, without their metadata: /// what a registration index needs when it does not inline its leaves. @@ -596,6 +621,18 @@ pub trait RepoReadOps: Send + Sync { /// `skip`/`take` paging. async fn search(&self, repo: &str, params: &SearchParams) -> Result; + /// The tags of package `id` in `repo`, tag → normalized version + /// (`package_tag`): a hosted repository's own, a proxy's upstream's, a + /// group's from its first serving member that has any. None by default. + async fn tags( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + let _ = (repo, id); + Ok(std::collections::BTreeMap::new()) + } + /// Search with an authorization predicate applied **before** paging and /// total-count calculation. /// @@ -763,6 +800,17 @@ impl RepoReadOps for PackageFilteredReadOps { self.inner.registration(repo, id).await } + async fn dated_registration( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + if !self.allows(id) { + return Ok(Vec::new()); + } + self.inner.dated_registration(repo, id).await + } + async fn registration_entry( &self, repo: &str, @@ -794,6 +842,17 @@ impl RepoReadOps for PackageFilteredReadOps { self.inner.registration_entries(repo, id, versions).await } + async fn tags( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + if !self.allows(id) { + return Ok(std::collections::BTreeMap::new()); + } + self.inner.tags(repo, id).await + } + async fn search(&self, repo: &str, params: &SearchParams) -> Result { self.inner .search_with_package_filter(repo, params, &self.filter) diff --git a/crates/core/src/repo/proxy.rs b/crates/core/src/repo/proxy.rs index c15c3953..5e0539e0 100644 --- a/crates/core/src/repo/proxy.rs +++ b/crates/core/src/repo/proxy.rs @@ -243,21 +243,22 @@ enum CachedFetch { /// Map an upstream transport/size/parse failure onto the read-path error: /// an unavailable upstream ([`UpstreamError::is_unavailable`]) is -/// [`RepoError::Upstream`], any other failure the proxy's own. +/// [`RepoError::Upstream`], a refusing one ([`UpstreamError::is_refusal`]) +/// [`RepoError::BadUpstream`], any other failure the proxy's own. fn upstream_err(e: UpstreamError) -> RepoError { - if e.is_unavailable() { - RepoError::Upstream(e.to_string()) - } else { - RepoError::Storage(e.to_string()) + match Trouble::of(&e) { + Trouble::Unavailable => RepoError::Upstream(e.to_string()), + Trouble::Refused => RepoError::BadUpstream(e.to_string()), + Trouble::None => RepoError::Storage(e.to_string()), } } /// Why a fetch of [`ProxyRepo::fetch`] failed: the reason the format is told, -/// and whether the upstream was unavailable ([`UpstreamError::is_unavailable`]). +/// and what it says about the upstream ([`Trouble`]). #[derive(Clone)] struct FetchFailure { reason: String, - unavailable: bool, + trouble: Trouble, } impl FetchFailure { @@ -265,7 +266,31 @@ impl FetchFailure { fn local(reason: String) -> Self { Self { reason, - unavailable: false, + trouble: Trouble::None, + } + } +} + +/// What a failed fetch says about the upstream, worst last. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +enum Trouble { + /// Nothing: the fetch went well, or failed on this side. + #[default] + None, + /// It refused the proxy (a 401, a 403): [`RepoError::BadUpstream`]. + Refused, + /// It cannot serve now: [`RepoError::Upstream`]. + Unavailable, +} + +impl Trouble { + fn of(e: &UpstreamError) -> Self { + if e.is_unavailable() { + Trouble::Unavailable + } else if e.is_refusal() { + Trouble::Refused + } else { + Trouble::None } } } @@ -275,6 +300,7 @@ pub struct Credentials { origin: Option, username: String, password: forge_crypto::PlaintextSecret, + bearer: bool, } impl Credentials { @@ -285,9 +311,18 @@ impl Credentials { origin: Origin::of(url), username, password, + bearer: false, } } + /// Send the password as an `Authorization: Bearer` token when there is + /// no username and `enabled` (a format whose upstreams take tokens so: + /// npm's `_authToken`). + pub fn bearer_without_username(mut self, enabled: bool) -> Self { + self.bearer = enabled && self.username.is_empty(); + self + } + /// The credentials, scoped to the origin of the `remote_url` they were /// stored for. pub fn auth(&self) -> UpstreamAuth<'_> { @@ -295,6 +330,7 @@ impl Credentials { origin: self.origin.clone(), username: &self.username, password: self.password.expose_secret(), + bearer: self.bearer, } } @@ -306,6 +342,7 @@ impl Credentials { origin: Some(origin), username: &self.username, password: self.password.expose_secret(), + bearer: self.bearer, } } } @@ -344,6 +381,17 @@ async fn upstream_password( } } +/// The upstream secret of `repo` alone, with the `remote_url` it belongs +/// to: what a proxy sends as a Bearer token when it has no username (npm +/// registries take a token). Decrypted like [`upstream_credentials`]. +pub async fn upstream_secret( + pool: &PgPool, + envelope: Option<&Arc>, + repo: &Repository, +) -> Result)>, RepoError> { + upstream_password(pool, envelope, repo).await +} + /// The Basic credentials for `repo`'s upstream, if both a username and a /// password are configured. They are attached only to the origin of the /// `remote_url` the password was read (and, encrypted, bound) with, which @@ -363,9 +411,33 @@ pub async fn upstream_credentials( origin: remote_url.as_deref().and_then(Origin::of), username, password, + bearer: false, })) } +/// The credentials a proxy sends its upstream: Basic with a username +/// ([`upstream_credentials`]), or, for a format whose upstreams take tokens +/// so (`bearer_token`, [`Format::upstream_bearer_token`]), a password stored +/// without one as a Bearer token. +pub async fn upstream_auth( + pool: &PgPool, + envelope: Option<&Arc>, + repo: &Repository, + bearer_token: bool, +) -> Result, RepoError> { + if repo.upstream_username.is_none() && bearer_token { + return Ok(upstream_password(pool, envelope, repo) + .await? + .map(|(password, remote_url)| { + let url = remote_url + .or_else(|| repo.remote_url.clone()) + .unwrap_or_default(); + Credentials::for_url(&url, String::new(), password).bearer_without_username(true) + })); + } + upstream_credentials(pool, envelope, repo).await +} + /// Key of an in-flight first-time `.nupkg` fetch: `(repository id, lowercase /// id, version)`. type InflightKey = (i64, String, String); @@ -480,7 +552,13 @@ impl ProxyRepo { /// The Basic credentials for `repo`'s upstream (see /// [`upstream_credentials`]). async fn credentials(&self, repo: &Repository) -> Result, RepoError> { - upstream_credentials(&self.pool, self.envelope.as_ref(), repo).await + upstream_auth( + &self.pool, + self.envelope.as_ref(), + repo, + self.format.upstream_bearer_token(), + ) + .await } /// Run one upstream conversation: ask the format what to fetch for `op`, @@ -515,9 +593,10 @@ impl ProxyRepo { .unwrap_or_default(), ); let mut fetched: Vec = Vec::new(); - // A fetch found the upstream unavailable: the format's failed answer - // is then the upstream's outage, not a fault of the proxy. - let mut unavailable = false; + // A fetch found the upstream unavailable, or refusing the proxy: the + // format's failed answer is then the upstream's outage or refusal, + // not a fault of the proxy. + let mut trouble = Trouble::None; for _ in 0..MAX_UPSTREAM_ROUNDS { let format = self.format.clone(); let (asked, round) = (remote.clone(), op.clone()); @@ -551,9 +630,27 @@ impl ProxyRepo { ); return Ok((api::UpstreamAnswer::Failed(reason), origins)); } - api::UpstreamStep::Done(api::UpstreamAnswer::Failed(reason)) if unavailable => { + api::UpstreamStep::Done(api::UpstreamAnswer::Located(location)) + if !origins.permits(&location.url) => + { + tracing::warn!(format = %self.format.id(), repo = %repo.name, url = %location.url, "the format answered a package URL on an origin neither the remote_url nor a document fetched from the upstream names; not downloaded"); + let reason = format!( + "upstream package {}: not downloaded, its origin is neither the \ + remote_url's nor named by a document fetched from the upstream", + location.url + ); + return Ok((api::UpstreamAnswer::Failed(reason), origins)); + } + api::UpstreamStep::Done(api::UpstreamAnswer::Failed(reason)) + if trouble == Trouble::Unavailable => + { return Err(RepoError::Upstream(reason)); } + api::UpstreamStep::Done(api::UpstreamAnswer::Failed(reason)) + if trouble == Trouble::Refused => + { + return Err(RepoError::BadUpstream(reason)); + } api::UpstreamStep::Done(answer) => return Ok((answer, origins)), api::UpstreamStep::Fetch(wanted) => wanted, }; @@ -566,7 +663,7 @@ impl ProxyRepo { } // Each fetch is judged by the origins known when the round // starts; what its document names counts from the next round. - let results: Vec<(api::FetchResult, Vec, bool)> = + let results: Vec<(api::FetchResult, Vec, Trouble)> = futures_util::stream::iter(wanted) .map(|fetch| { let permitted = origins.permits(&fetch.url); @@ -574,7 +671,7 @@ impl ProxyRepo { let (outcome, named, down) = if permitted { self.fetch(repo, &fetch).await } else { - (self.refuse(repo, &fetch.url), Vec::new(), false) + (self.refuse(repo, &fetch.url), Vec::new(), Trouble::None) }; (api::FetchResult { fetch, outcome }, named, down) } @@ -585,7 +682,7 @@ impl ProxyRepo { for (result, named, down) in results { origins.extend(named); fetched.push(result); - unavailable |= down; + trouble = trouble.max(down); } } Err(RepoError::Storage(format!( @@ -611,18 +708,24 @@ impl ProxyRepo { /// A 404 is cached only for an immutable document: a metadata 404 (a /// package the upstream does not know yet) is what the not-found /// markers track, and they are cleared as soon as the package turns up. - /// The flag says whether a failed fetch found the upstream unavailable - /// ([`UpstreamError::is_unavailable`]). + /// The [`Trouble`] says whether a failed fetch found the upstream + /// unavailable ([`UpstreamError::is_unavailable`]) or refusing the proxy + /// ([`UpstreamError::is_refusal`]). async fn fetch( &self, repo: &Repository, fetch: &api::Fetch, - ) -> (api::Fetched, Vec, bool) { + ) -> (api::Fetched, Vec, Trouble) { let ttl = match fetch.freshness { api::Freshness::Metadata => Self::metadata_ttl(repo).to_std().unwrap_or_default(), api::Freshness::Immutable => IMMUTABLE_FETCH_TTL, }; - let key = format!("{}/{}", Self::upstream_scope(repo), fetch.url); + // What the fetch accepts is part of what it is (npm's abbreviated + // and full packuments share a URL). + let key = match &fetch.accept { + Some(accept) => format!("{}/{}\n{accept}", Self::upstream_scope(repo), fetch.url), + None => format!("{}/{}", Self::upstream_scope(repo), fetch.url), + }; let cached: Result> = self .cache .get_or_try_insert_local(NS_UPSTREAM_FETCH, &key, ttl, || async { @@ -631,7 +734,11 @@ impl ProxyRepo { .await .map_err(|e| Some(FetchFailure::local(e.to_string())))?; let auth = creds.as_ref().map(Credentials::auth); - match self.http.get_bytes(&fetch.url, auth.as_ref()).await { + match self + .http + .get_bytes_accepting(&fetch.url, auth.as_ref(), fetch.accept.as_deref()) + .await + { Ok(Some(bytes)) => { // A document may be megabytes: read off the async // workers. @@ -653,7 +760,7 @@ impl ProxyRepo { } Ok(None) => Err(None), Err(e) => Err(Some(FetchFailure { - unavailable: e.is_unavailable(), + trouble: Trouble::of(&e), reason: e.to_string(), })), } @@ -662,19 +769,21 @@ impl ProxyRepo { match cached { Ok(CachedFetch::Found { body, named }) => { match base64::engine::general_purpose::STANDARD.decode(body) { - Ok(bytes) => (api::Fetched::Found(bytes), named, false), + Ok(bytes) => (api::Fetched::Found(bytes), named, Trouble::None), Err(e) => ( api::Fetched::Failed(format!("cached {}: {e}", fetch.url)), Vec::new(), - false, + Trouble::None, ), } } - Ok(CachedFetch::NotFound) | Err(None) => (api::Fetched::NotFound, Vec::new(), false), + Ok(CachedFetch::NotFound) | Err(None) => { + (api::Fetched::NotFound, Vec::new(), Trouble::None) + } Err(Some(failure)) => ( api::Fetched::Failed(failure.reason), Vec::new(), - failure.unavailable, + failure.trouble, ), } } @@ -1309,7 +1418,7 @@ impl ProxyRepo { // Bytes rejected for their content recently: fail fast instead of // downloading them again on every client retry. if let Some(reason) = self.recent_rejection(repo, lower_id, version).await { - return Err(RepoError::Storage(format!( + return Err(RepoError::BadUpstream(format!( "upstream package {lower_id} {version} was rejected recently: {reason}" ))); } @@ -1319,7 +1428,7 @@ impl ProxyRepo { // NuGet, `{flat container}{id}/{version}/{id}.{version}.nupkg`), on // an origin the upstream points at: [`Self::converse`] answers any // other as an upstream failure. - let url = match self + let (url, digests) = match self .converse_resolved( repo, api::UpstreamOp::Package(api::PackageRef { @@ -1329,7 +1438,12 @@ impl ProxyRepo { ) .await? { - api::UpstreamAnswer::Package(url) => url, + api::UpstreamAnswer::Package(url) => (url, None), + // The upstream publishes the package's digests: what is + // downloaded must match them. + api::UpstreamAnswer::Located(location) => { + (location.url, Some((location.sha512, location.sha1))) + } api::UpstreamAnswer::Failed(reason) => return Err(RepoError::Storage(reason)), _ => { return Err(RepoError::Storage( @@ -1381,9 +1495,14 @@ impl ProxyRepo { return Ok(None); } if !resp.status().is_success() { - let reason = format!("upstream nupkg fetch {url} returned {}", resp.status()); + let reason = format!("upstream package fetch {url} returned {}", resp.status()); return Err(if status_is_unavailable(resp.status()) { RepoError::Upstream(reason) + } else if matches!( + resp.status(), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN + ) { + RepoError::BadUpstream(reason) } else { RepoError::Storage(reason) }); @@ -1424,6 +1543,25 @@ impl ProxyRepo { } }; + if let Some((Some(sha512), _)) = &digests { + if *sha512 != staged.sha512_base64 { + return Err(self + .reject( + repo, + lower_id, + version, + format!("upstream package {url} does not match the SHA-512 its upstream publishes"), + ) + .await); + } + } + // Without a SHA-512, the SHA-1 the upstream publishes (npm's + // `shasum`), computed while the package is parsed. + let sha1 = match &digests { + Some((None, Some(sha1))) => Some(sha1.to_ascii_lowercase()), + _ => None, + }; + // Parse from the temp file on the blocking pool to obtain the canonical // identity and the manifest, and summarize the manifest for // registration and search (stored with the component, like a push). @@ -1431,6 +1569,25 @@ impl ProxyRepo { let format = self.format.clone(); let path = staged.read_path(); tokio::task::spawn_blocking(move || { + if let Some(expected) = sha1 { + use sha1::Digest; + use std::io::Read; + let mut hasher = sha1::Sha1::new(); + let mut file = std::fs::File::open(&path)?; + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + if hex::encode(hasher.finalize()) != expected { + return Ok(Err(crate::domain::FormatError::InvalidPackage( + "the package does not match the SHA-1 its upstream publishes".into(), + ))); + } + } let mut file = std::fs::File::open(&path)?; Ok::<_, std::io::Error>(format.parse_upload(&mut file).map(|parsed| { let summary = format.manifest_summary(&parsed.nuspec_bytes); @@ -1684,7 +1841,7 @@ impl ProxyRepo { }) .await; } - RepoError::Storage(reason) + RepoError::BadUpstream(reason) } /// Resolve the locally-stored `.nupkg` location for a cached version. @@ -2043,12 +2200,14 @@ impl ProxyRepo { /// The registration entries of `id`: every listable version, or only /// those in `only` (normalized). The upstream's metadata is asked for the /// versions returned only, so a registration page or leaf reads just the - /// upstream documents covering its own versions. + /// upstream documents covering its own versions; `dated` asks it for + /// their publish times too ([`api::UpstreamOp::Dating`]). async fn registration_of( &self, repo: &str, id: &str, only: Option<&[String]>, + dated: bool, ) -> Result, RepoError> { let repo_row = self.proxy_repo(repo).await?; let Some(lower) = self.upstream_id(id) else { @@ -2072,7 +2231,7 @@ impl ProxyRepo { let wanted: Vec = visible.iter().map(|r| r.version.clone()).collect(); let upstream = self - .upstream_metadata(&repo_row, &lower, &wanted, false) + .upstream_metadata(&repo_row, &lower, &wanted, dated) .await; // Persist the upstream listing state so the flat index and search @@ -2186,6 +2345,8 @@ impl ProxyRepo { // Only the upstream knows the publish time (the cache time // would be misleading). published: leaf.and_then(|l| rfc3339(l.published.as_deref())), + // A proxy's deprecations are its upstream's, in the summary. + deprecation: None, }); } Ok(out) @@ -2252,6 +2413,56 @@ const SEARCH_ENRICH_CONCURRENCY: usize = 8; #[async_trait] impl RepoReadOps for ProxyRepo { + /// The upstream's tags ([`api::UpstreamOp::Tags`], read through the + /// fetch cache for the metadata TTL), kept in `package_tag`: what was + /// last read is served while the proxy is offline or the upstream + /// fails. + async fn tags( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + let repo_row = self.proxy_repo(repo).await?; + let Some(lower) = self.upstream_id(id) else { + return Ok(std::collections::BTreeMap::new()); + }; + // A package the upstream does not know has no tags either. + if self.negative_cached(&repo_row, &lower, "*").await? { + return Ok(std::collections::BTreeMap::new()); + } + if repo_row.online { + match self + .converse_resolved(&repo_row, api::UpstreamOp::Tags(lower.clone())) + .await + { + Ok(api::UpstreamAnswer::Tags(Some(tags))) => { + let tags: std::collections::BTreeMap = tags + .into_iter() + .filter_map(|tag| { + let version = self.format.normalize_version(&tag.version).ok()?; + Some((tag.name, version)) + }) + .collect(); + let mut tx = self.pool.begin().await?; + forge_db::tag_dao::replace_tags(&mut tx, repo_row.id, &lower, &tags).await?; + tx.commit().await?; + return Ok(tags); + } + Ok(api::UpstreamAnswer::Tags(None)) => return Ok(std::collections::BTreeMap::new()), + Ok(api::UpstreamAnswer::Failed(reason)) => { + tracing::warn!(repo = %repo_row.name, id = %lower, %reason, "upstream tags unavailable; serving the last read"); + } + Ok(_) => { + tracing::warn!(repo = %repo_row.name, id = %lower, "upstream protocol answered tags with something else"); + } + Err(e) => { + tracing::warn!(repo = %repo_row.name, id = %lower, error = %e, "upstream tags unavailable; serving the last read"); + } + } + } + Ok(forge_db::tag_dao::tags(&self.pool, repo_row.id, &lower).await?) + } + async fn list_versions(&self, repo: &str, id: &str) -> Result, RepoError> { let repo_row = self.proxy_repo(repo).await?; let Some(lower) = self.upstream_id(id) else { @@ -2395,7 +2606,15 @@ impl RepoReadOps for ProxyRepo { repo: &str, id: &str, ) -> Result, RepoError> { - self.registration_of(repo, id, None).await + self.registration_of(repo, id, None, false).await + } + + async fn dated_registration( + &self, + repo: &str, + id: &str, + ) -> Result, RepoError> { + self.registration_of(repo, id, None, true).await } async fn registration_versions(&self, repo: &str, id: &str) -> Result, RepoError> { @@ -2425,7 +2644,7 @@ impl RepoReadOps for ProxyRepo { id: &str, versions: &[String], ) -> Result, RepoError> { - self.registration_of(repo, id, Some(versions)).await + self.registration_of(repo, id, Some(versions), false).await } async fn registration_entry( @@ -2435,7 +2654,7 @@ impl RepoReadOps for ProxyRepo { version: &str, ) -> Result, RepoError> { Ok(self - .registration_of(repo, id, Some(&[version.to_string()])) + .registration_of(repo, id, Some(&[version.to_string()]), false) .await? .into_iter() .next()) diff --git a/crates/core/src/retention/apply.rs b/crates/core/src/retention/apply.rs index 7ff050b7..14deef77 100644 --- a/crates/core/src/retention/apply.rs +++ b/crates/core/src/retention/apply.rs @@ -350,6 +350,13 @@ async fn apply_hosted( summary.skipped += 1; return Ok(()); } + // A package tag (npm's `latest`, `next`, …) names it now: never touched, + // whatever the evaluation said. + if forge_db::tag_dao::is_tagged(&mut **tx, ctx.repository_id, &row.name, &row.version).await? { + audit_skipped(tx, ctx, d, "component").await?; + summary.skipped += 1; + return Ok(()); + } match d.action { Action::Unlist => { diff --git a/crates/core/src/retention/mod.rs b/crates/core/src/retention/mod.rs index d1388067..1cc1a8a7 100644 --- a/crates/core/src/retention/mod.rs +++ b/crates/core/src/retention/mod.rs @@ -29,3 +29,25 @@ pub use model::{ Scope, POLICY_VERSION, }; pub use worker::{FormatResolver, RetentionConfig, RetentionWorker, WorkerError}; + +/// Leave out of `decisions` the versions retention never touches: every +/// version a package tag names (npm's dist-tags, `latest` included). +/// Returns how many decisions were dropped. The apply path checks again, +/// with the version locked. +pub async fn spare_protected( + pool: &sqlx::PgPool, + repository_id: i64, + decisions: &mut Vec, +) -> sqlx::Result { + if decisions.is_empty() { + return Ok(0); + } + let tagged: std::collections::HashSet<(String, String)> = + forge_db::tag_dao::tagged_versions(pool, repository_id) + .await? + .into_iter() + .collect(); + let before = decisions.len(); + decisions.retain(|d| !tagged.contains(&(d.package_name.clone(), d.version.clone()))); + Ok(before - decisions.len()) +} diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index a53428c8..3475b138 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -331,12 +331,22 @@ impl RetentionWorker { }; // A panic there (e.g. a panicking `Format`) fails the run like a panic // anywhere else in `process`. - let (hosted_decisions, proxy_decisions) = evaluation.await.map_err(|e| { + let (mut hosted_decisions, proxy_decisions) = evaluation.await.map_err(|e| { WorkerError::Panicked(match e.try_into_panic() { Ok(panic) => panic_message(&*panic), Err(e) => e.to_string(), // cancelled: the runtime is shutting down }) })?; + // A version a package tag names is never touched. + let spared = crate::retention::spare_protected( + &self.pool, + repo.repository_id, + &mut hosted_decisions, + ) + .await?; + if spared > 0 { + tracing::info!(repo = %repo.name, spared, "retention spared tagged versions"); + } // Affected package names (for cache invalidation after the run). let mut affected: BTreeSet = BTreeSet::new(); diff --git a/crates/core/tests/group_read.rs b/crates/core/tests/group_read.rs index 0a8f1e57..1da8e95e 100644 --- a/crates/core/tests/group_read.rs +++ b/crates/core/tests/group_read.rs @@ -1119,6 +1119,7 @@ impl RepoReadOps for FakeProxy { nuspec_bytes: None, manifest_summary: None, published: None, + deprecation: None, }) .collect()) } diff --git a/crates/core/tests/package_filtering.rs b/crates/core/tests/package_filtering.rs index c5340cb2..0cead992 100644 --- a/crates/core/tests/package_filtering.rs +++ b/crates/core/tests/package_filtering.rs @@ -68,6 +68,7 @@ impl RepoReadOps for FixtureReads { nuspec_bytes: None, manifest_summary: None, published: None, + deprecation: None, }]) } diff --git a/crates/core/tests/proxy_read.rs b/crates/core/tests/proxy_read.rs index 6bc924b4..85f8a371 100644 --- a/crates/core/tests/proxy_read.rs +++ b/crates/core/tests/proxy_read.rs @@ -891,8 +891,9 @@ async fn credentialed_upstream_gets_basic_auth_on_every_request() { assert_eq!(mock.nupkg_hits(), 1); } -/// Without credentials the same upstream is a hard error, never a silent -/// empty listing that clients would treat as "package does not exist". +/// Without credentials the same upstream is a hard error (a bad upstream: +/// the server answers 502), never a silent empty listing that clients would +/// treat as "package does not exist". #[tokio::test] async fn unauthorized_upstream_is_an_error_not_an_empty_listing() { let pool = common::pool().await; @@ -902,14 +903,14 @@ async fn unauthorized_upstream_is_an_error_not_an_empty_listing() { let px = proxy_repo(&pool, cas, Cache::l1_only(128)); assert!(matches!( px.list_versions(&repo, "Sample.Lib").await, - Err(RepoError::Storage(_)) + Err(RepoError::BadUpstream(_)) )); } /// #283: an upstream that does not answer (nothing listens on its port), or /// answers 5xx, while nothing is cached is unavailable (the server answers /// 503 with `Retry-After`), not a failure of the proxy (500). An upstream -/// refusing the proxy's credentials stays one (above). +/// refusing the proxy's credentials is a bad upstream (above). #[tokio::test] async fn an_unavailable_upstream_is_an_upstream_error() { let pool = common::pool().await; @@ -3351,10 +3352,12 @@ async fn a_format_cannot_reach_an_origin_its_upstream_does_not_name() { scripted_proxy(&pool, cas, move |nuget, remote, op, fetched| match op { // The listing reads the internal service first, then lists // what the upstream lists, as NuGet does. - UpstreamOp::Versions(_) if fetched.is_empty() => Ok(UpstreamStep::Fetch(vec![Fetch { - url: secret.clone(), - freshness: Freshness::Metadata, - }])), + UpstreamOp::Versions(_) if fetched.is_empty() => { + Ok(UpstreamStep::Fetch(vec![Fetch::new( + secret.clone(), + Freshness::Metadata, + )])) + } UpstreamOp::Versions(_) => { *seen.lock().unwrap() = fetched .iter() diff --git a/crates/db/src/cli_login_dao.rs b/crates/db/src/cli_login_dao.rs new file mode 100644 index 00000000..c5a201da --- /dev/null +++ b/crates/db/src/cli_login_dao.rs @@ -0,0 +1,157 @@ +//! Command-line logins approved in the browser (migration 0046): a client +//! such as `npm login` is given a login URL to open and a done URL to poll. + +use chrono::{DateTime, Utc}; +use sqlx::PgPool; + +/// A login as its approval page shows it. +#[derive(Debug, Clone)] +pub struct CliLogin { + /// The client (the repository's format, `npm`). + pub client: String, + pub id: i64, + pub repository_id: i64, + pub hostname: Option, + pub client_ip: Option, + pub user_id: Option, + pub created_at: DateTime, + pub expires_at: DateTime, +} + +/// Start a login of `client` for repository `repository_id`, valid until +/// `expires_at`. +#[allow(clippy::too_many_arguments)] +pub async fn insert( + pool: &PgPool, + login_hash: &str, + done_hash: &str, + client: &str, + repository_id: i64, + hostname: Option<&str>, + client_ip: Option<&str>, + expires_at: DateTime, +) -> sqlx::Result<()> { + sqlx::query!( + r#" + INSERT INTO cli_login (login_hash, done_hash, client, repository_id, hostname, client_ip, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + "#, + login_hash, + done_hash, + client, + repository_id, + hostname, + client_ip, + expires_at, + ) + .execute(pool) + .await?; + Ok(()) +} + +/// The live (not expired) login whose login URL secret hashes to +/// `login_hash`. +pub async fn find(pool: &PgPool, login_hash: &str) -> sqlx::Result> { + sqlx::query_as!( + CliLogin, + r#" + SELECT id, client, repository_id, hostname, client_ip, user_id, created_at, expires_at + FROM cli_login + WHERE login_hash = $1 AND expires_at > now() + "#, + login_hash, + ) + .fetch_optional(pool) + .await +} + +/// Approve the live, not yet approved login `login_hash` for `user_id`. +/// Returns whether it was approved now. +pub async fn approve(pool: &PgPool, login_hash: &str, user_id: i64) -> sqlx::Result { + let res = sqlx::query!( + r#" + UPDATE cli_login SET user_id = $2, approved_at = now() + WHERE login_hash = $1 AND expires_at > now() AND user_id IS NULL + "#, + login_hash, + user_id, + ) + .execute(pool) + .await?; + Ok(res.rows_affected() > 0) +} + +/// Deny (delete) the live login `login_hash`. Returns whether it existed. +pub async fn deny(pool: &PgPool, login_hash: &str) -> sqlx::Result { + let res = sqlx::query!( + r#"DELETE FROM cli_login WHERE login_hash = $1 AND user_id IS NULL"#, + login_hash, + ) + .execute(pool) + .await?; + Ok(res.rows_affected() > 0) +} + +/// What polling a login's done URL finds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CliLoginState { + /// No live login has this done URL (unknown, expired, denied or + /// collected already). + Unknown, + /// Waiting for the browser. + Pending, + /// Approved by `user_id`, for `hostname`: the row is gone now, the + /// caller mints the token. + Approved { + user_id: i64, + repository_id: i64, + hostname: Option, + }, +} + +/// Poll the login whose done URL secret hashes to `done_hash`. An +/// approved login is taken (deleted) by the first poll that finds it. +pub async fn poll(pool: &PgPool, done_hash: &str) -> sqlx::Result { + if let Some(row) = sqlx::query!( + r#" + DELETE FROM cli_login + WHERE done_hash = $1 AND expires_at > now() AND user_id IS NOT NULL + RETURNING user_id AS "user_id!", repository_id, hostname + "#, + done_hash, + ) + .fetch_optional(pool) + .await? + { + return Ok(CliLoginState::Approved { + user_id: row.user_id, + repository_id: row.repository_id, + hostname: row.hostname, + }); + } + let pending = sqlx::query_scalar!( + r#" + SELECT EXISTS ( + SELECT 1 FROM cli_login WHERE done_hash = $1 AND expires_at > now() + ) AS "pending!" + "#, + done_hash, + ) + .fetch_one(pool) + .await?; + Ok(if pending { + CliLoginState::Pending + } else { + CliLoginState::Unknown + }) +} + +/// Delete expired logins (the auth reaper). Returns how many. +pub async fn delete_expired(pool: &PgPool) -> sqlx::Result { + Ok( + sqlx::query!(r#"DELETE FROM cli_login WHERE expires_at <= now()"#) + .execute(pool) + .await? + .rows_affected(), + ) +} diff --git a/crates/db/src/component_dao.rs b/crates/db/src/component_dao.rs index 1cd9fe7b..180dfa7f 100644 --- a/crates/db/src/component_dao.rs +++ b/crates/db/src/component_dao.rs @@ -388,6 +388,8 @@ pub struct RegistrationRow { pub package_sha256: Option, /// Byte size of the package asset. pub package_size: Option, + /// Why the version is deprecated, if it is (migration 0046). + pub deprecation: Option, } /// Every version of `name` with its registration data, in one query (the @@ -404,7 +406,7 @@ pub async fn list_registration_rows( SELECT c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at, c.manifest_summary, a.sha512_base64 AS "package_hash_sha512_base64?", - a.blob_sha256 AS "package_sha256?", a.size AS "package_size?" + a.blob_sha256 AS "package_sha256?", a.size AS "package_size?", c.deprecation FROM component c LEFT JOIN asset a ON a.component_id = c.id AND a.path = $3 WHERE c.repository_id = $1 AND c.name = $2 @@ -464,7 +466,7 @@ pub async fn list_registration_rows_for_versions( SELECT c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at, c.manifest_summary, a.sha512_base64 AS "package_hash_sha512_base64?", - a.blob_sha256 AS "package_sha256?", a.size AS "package_size?" + a.blob_sha256 AS "package_sha256?", a.size AS "package_size?", c.deprecation FROM component c LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4 WHERE c.repository_id = $1 AND c.name = $2 AND c.version = ANY($3::text[]) @@ -493,7 +495,7 @@ pub async fn find_registration_row( SELECT c.id, c.original_id, c.version, c.version_original, c.listed, c.created_at, c.manifest_summary, a.sha512_base64 AS "package_hash_sha512_base64?", - a.blob_sha256 AS "package_sha256?", a.size AS "package_size?" + a.blob_sha256 AS "package_sha256?", a.size AS "package_size?", c.deprecation FROM component c LEFT JOIN asset a ON a.component_id = c.id AND a.path = $4 WHERE c.repository_id = $1 AND c.name = $2 AND c.version = $3 @@ -611,3 +613,30 @@ pub async fn set_imported_times( } Ok(()) } + +/// Deprecate version `version` of `name` with `message`, or (`None`) no +/// longer. Returns whether the version exists. +pub async fn set_deprecation<'e, E>( + executor: E, + repository_id: i64, + name: &str, + version: &str, + message: Option<&str>, +) -> sqlx::Result +where + E: sqlx::PgExecutor<'e>, +{ + let res = sqlx::query!( + r#" + UPDATE component SET deprecation = $4, updated_at = now() + WHERE repository_id = $1 AND name = $2 AND version = $3 + "#, + repository_id, + name, + version, + message, + ) + .execute(executor) + .await?; + Ok(res.rows_affected() > 0) +} diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs index 06ba83ec..3759228b 100644 --- a/crates/db/src/lib.rs +++ b/crates/db/src/lib.rs @@ -11,6 +11,7 @@ pub mod asset_dao; pub mod auth_realm_dao; pub mod authz_dao; pub mod blob_dao; +pub mod cli_login_dao; pub mod component_dao; pub mod download_dao; pub mod email_oauth_flow_dao; @@ -31,6 +32,7 @@ pub mod repository_dao; pub mod retention_dao; pub mod secret_dao; pub mod session_dao; +pub mod tag_dao; pub mod token_dao; pub mod trusted_publish_audit_dao; pub mod user_dao; diff --git a/crates/db/src/tag_dao.rs b/crates/db/src/tag_dao.rs new file mode 100644 index 00000000..ef219ab9 --- /dev/null +++ b/crates/db/src/tag_dao.rs @@ -0,0 +1,176 @@ +//! Package tags (migration 0046): names for versions of a package, such as +//! npm's dist-tags (`latest`, `next`). A hosted repository's are set by its +//! clients (a format plugin's tag changes); a proxy's are its upstream's +//! as last read ([`replace_tags`]). + +use std::collections::BTreeMap; + +use sqlx::{PgExecutor, PgPool, Postgres, Transaction}; + +/// The tags of `package` in repository `repository_id`, tag → version. +pub async fn tags<'e, E>( + executor: E, + repository_id: i64, + package: &str, +) -> sqlx::Result> +where + E: PgExecutor<'e>, +{ + let rows = sqlx::query!( + r#" + SELECT tag, version FROM package_tag + WHERE repository_id = $1 AND package = $2 + "#, + repository_id, + package, + ) + .fetch_all(executor) + .await?; + Ok(rows.into_iter().map(|r| (r.tag, r.version)).collect()) +} + +/// Point `tag` of `package` at `version`. +pub async fn set_tag<'e, E>( + executor: E, + repository_id: i64, + package: &str, + tag: &str, + version: &str, +) -> sqlx::Result<()> +where + E: PgExecutor<'e>, +{ + sqlx::query!( + r#" + INSERT INTO package_tag (repository_id, package, tag, version) + VALUES ($1, $2, $3, $4) + ON CONFLICT (repository_id, package, tag) + DO UPDATE SET version = EXCLUDED.version, updated_at = now() + "#, + repository_id, + package, + tag, + version, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// Remove `tag` of `package`. Returns whether it existed. +pub async fn delete_tag<'e, E>( + executor: E, + repository_id: i64, + package: &str, + tag: &str, +) -> sqlx::Result +where + E: PgExecutor<'e>, +{ + let res = sqlx::query!( + r#"DELETE FROM package_tag WHERE repository_id = $1 AND package = $2 AND tag = $3"#, + repository_id, + package, + tag, + ) + .execute(executor) + .await?; + Ok(res.rows_affected() > 0) +} + +/// Remove every tag of `package`, or only those naming `version`. +pub async fn delete_tags<'e, E>( + executor: E, + repository_id: i64, + package: &str, + version: Option<&str>, +) -> sqlx::Result +where + E: PgExecutor<'e>, +{ + let res = sqlx::query!( + r#" + DELETE FROM package_tag + WHERE repository_id = $1 AND package = $2 AND ($3::text IS NULL OR version = $3) + "#, + repository_id, + package, + version, + ) + .execute(executor) + .await?; + Ok(res.rows_affected()) +} + +/// Every `(package, version)` a tag of repository `repository_id` +/// names: what retention never deletes. +pub async fn tagged_versions( + pool: &PgPool, + repository_id: i64, +) -> sqlx::Result> { + let rows = sqlx::query!( + r#"SELECT DISTINCT package, version FROM package_tag WHERE repository_id = $1"#, + repository_id, + ) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().map(|r| (r.package, r.version)).collect()) +} + +/// Whether a tag of `package` names `version`. +pub async fn is_tagged<'e, E>( + executor: E, + repository_id: i64, + package: &str, + version: &str, +) -> sqlx::Result +where + E: PgExecutor<'e>, +{ + sqlx::query_scalar!( + r#" + SELECT EXISTS ( + SELECT 1 FROM package_tag + WHERE repository_id = $1 AND package = $2 AND version = $3 + ) AS "tagged!" + "#, + repository_id, + package, + version, + ) + .fetch_one(executor) + .await +} + +/// Make `tags` the tags of `package` (a proxy's, as its upstream gave them): +/// the others are removed. Returns whether anything changed. +pub async fn replace_tags( + tx: &mut Transaction<'_, Postgres>, + repository_id: i64, + package: &str, + tags: &BTreeMap, +) -> sqlx::Result { + let current = self::tags(&mut **tx, repository_id, package).await?; + if ¤t == tags { + return Ok(false); + } + delete_tags(&mut **tx, repository_id, package, None).await?; + for (tag, version) in tags { + set_tag(&mut **tx, repository_id, package, tag, version).await?; + } + Ok(true) +} + +/// Remove every tag of repository `repository_id` (a repointed proxy's). +pub async fn delete_repository_tags<'e, E>(executor: E, repository_id: i64) -> sqlx::Result +where + E: PgExecutor<'e>, +{ + Ok(sqlx::query!( + r#"DELETE FROM package_tag WHERE repository_id = $1"#, + repository_id + ) + .execute(executor) + .await? + .rows_affected()) +} diff --git a/crates/formats/README.md b/crates/formats/README.md index f545f4d9..36da7bff 100644 --- a/crates/formats/README.md +++ b/crates/formats/README.md @@ -8,6 +8,7 @@ the design, the contract and how to write one. | Plugin | Format id | What it serves | | --- | --- | --- | | [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | +| [`npm`](npm) | `npm` | npm registries at `/npm` for npm, pnpm, Yarn and Bun: publish, install, dist-tags, deprecations, unpublishing, search, audits and logins; hosted, proxy and group repositories ([docs/npm.md](../../docs/npm.md)) | | [`raw`](raw) | `raw` | Raw files at `/raw`: files under paths, uploads with `PUT` or a form, ranges, checksum sidecars, listings; hosted, proxy and group repositories ([docs/raw.md](../../docs/raw.md)) | A plugin directory holds: @@ -17,7 +18,8 @@ A plugin directory holds: version, the `forge:format` contract version it targets (`api`), and optionally its mount, repository kinds (`proxy` when it implements `upstream`), its `storage` (`packages`, or `files` for a files format such - as raw) and `[assets]` names; + as raw), how a proxy sends a token (`upstream_token`) and `[assets]` + names; - `src/`: `impl FormatPlugin`, and `pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!();`. diff --git a/crates/formats/npm/Cargo.toml b/crates/formats/npm/Cargo.toml new file mode 100644 index 00000000..ee80b6e7 --- /dev/null +++ b/crates/formats/npm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "forge-format-npm" +# The plugin's own version: keep it equal to `version` in plugin.toml. +version = "1.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "npm registries for Forge, as a format plugin" + +[dependencies] +forge-plugin-api = { path = "../../plugin-api" } + +serde = { workspace = true } +serde_json = { workspace = true } +chrono = { workspace = true } +sha2 = { workspace = true } +# `dist.shasum`, the SHA-1 every npm client still checks. +sha1 = "0.11" +hex = { workspace = true } +base64 = { workspace = true } +# Tarballs: gzip, then tar (read only, never unpacked to disk). +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +tar = { version = "0.4", default-features = false } +# npm versions are SemVer 2.0.0 (node-semver), ordered by its precedence. +semver = "1" + +[dev-dependencies] +forge-plugin-api = { path = "../../plugin-api", features = ["testing"] } +tempfile = { workspace = true } diff --git a/crates/formats/npm/plugin.toml b/crates/formats/npm/plugin.toml new file mode 100644 index 00000000..5b0c427d --- /dev/null +++ b/crates/formats/npm/plugin.toml @@ -0,0 +1,19 @@ +# A Forge format plugin: npm registries (see crates/formats/README.md). +[plugin] +id = "npm" +name = "npm" +version = "1.0.0" +api = "0.4.0" +description = "npm registries: publish, install and log in with npm, pnpm, Yarn and Bun; dist-tags, deprecations and audits; proxies of registry.npmjs.org and other registries." +mount = "/npm" +# Proxies mirror another registry through the plugin's upstream side. +kinds = ["hosted", "proxy", "group"] +# A proxy given only a password (a registry token) sends it as a Bearer +# token, as npm clients do with `_authToken`. +upstream_token = "bearer" + +[assets] +package = "package.tgz" +package_type = "application/octet-stream" +manifest = "package.json" +manifest_type = "application/json" diff --git a/crates/formats/npm/src/document.rs b/crates/formats/npm/src/document.rs new file mode 100644 index 00000000..e439e7f6 --- /dev/null +++ b/crates/formats/npm/src/document.rs @@ -0,0 +1,388 @@ +//! The documents npm clients `PUT` to a package: a publish (a new version +//! with its tarball attached) and the packument updates of `npm deprecate`, +//! `npm unpublish ` and `npm owner`. + +use std::collections::BTreeMap; +use std::io::{Read, Seek, SeekFrom}; + +use serde_json::{Map, Value}; + +use crate::integrity::Digests; +use crate::name::PackageName; +use crate::{tarball, version}; + +/// Whether `tag` may name a dist-tag: a short word of letters, digits, +/// `.`, `_` and `-` that no client could read as a version or a range +/// (`1.x`, `v2`, `x`, `^1`), as npm refuses those. +pub fn valid_tag(tag: &str) -> bool { + let mut chars = tag.chars(); + let Some(first) = chars.next() else { + return false; + }; + let second = chars.next(); + tag.len() <= 64 + && tag + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) + && !first.is_ascii_digit() + && !(matches!(first, 'v' | 'V') && second.is_some_and(|c| c.is_ascii_digit())) + && !matches!(tag, "x" | "X") +} + +/// What a publish document says. +#[derive(Debug, Clone, PartialEq)] +pub struct PublishDocument { + pub name: PackageName, + /// The published version, normalized. + pub version: String, + /// The version as the document writes it. + pub version_original: String, + /// The version object. + pub manifest: Map, + /// The dist-tags the document points at the version. + pub dist_tags: Vec, + /// The attachment's name and its declared `length`. + pub attachment: String, + pub length: Option, + /// The document's top-level README. + pub readme: Option, +} + +/// What a packument update (a `PUT` without a tarball) says of the +/// package's versions. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct UpdateDocument { + /// Every version the document lists (normalized), with its + /// `deprecated` message (`None`: not deprecated). + pub versions: BTreeMap>, + /// The dist-tags (versions normalized). + pub dist_tags: BTreeMap, + /// Whether the document has dist-tags at all (a document without them + /// changes none). + pub dist_tags_given: bool, +} + +fn object<'a>(doc: &'a Value, key: &str) -> Option<&'a Map> { + doc.get(key).and_then(Value::as_object) +} + +/// Check that a document's `name` (and `_id`, when present) is `name`. +fn check_name(doc: &Value, name: &PackageName) -> Result<(), String> { + let expected = name.to_string(); + match doc.get("name").and_then(Value::as_str) { + Some(n) if n == expected => {} + Some(n) => { + return Err(format!( + "the document names package {n:?}, not {expected:?}" + )) + } + None => return Err("the document has no name".into()), + } + if let Some(id) = doc.get("_id").and_then(Value::as_str) { + if id != expected { + return Err(format!("the document's _id {id:?} is not {expected:?}")); + } + } + Ok(()) +} + +/// Read a publish document (its attachment's data extracted already) of +/// package `name`, which the URL named. +pub fn read_publish(doc: &Value, name: &PackageName) -> Result { + if !doc.is_object() { + return Err("the document is not a JSON object".into()); + } + check_name(doc, name)?; + let versions = object(doc, "versions").ok_or("the document has no versions")?; + let [(version_original, manifest)] = versions.iter().collect::>()[..] else { + return Err("a publish carries exactly one version".into()); + }; + let manifest = manifest + .as_object() + .ok_or("the version is not a JSON object")?; + let version = version::normalize(version_original) + .map_err(|e| format!("invalid version {version_original:?}: {e}"))?; + match manifest.get("version").and_then(Value::as_str) { + Some(v) if version::normalize(v).as_deref() == Ok(version.as_str()) => {} + _ => { + return Err(format!( + "the manifest of {version_original:?} names another version" + )) + } + } + match manifest.get("name").and_then(Value::as_str) { + Some(n) if n == name.to_string() => {} + _ => return Err("the version's manifest names another package".into()), + } + let attachments = object(doc, "_attachments").ok_or("the document attaches no tarball")?; + let [(attachment, meta)] = attachments.iter().collect::>()[..] else { + return Err("a publish attaches exactly one tarball".into()); + }; + let length = meta.get("length").and_then(Value::as_u64); + let dist_tags = object(doc, "dist-tags") + .map(|tags| { + tags.iter() + .filter(|(tag, _)| valid_tag(tag)) + .filter(|(_, v)| { + v.as_str() + .is_some_and(|v| version::normalize(v).as_deref() == Ok(version.as_str())) + }) + .map(|(t, _)| t.clone()) + .collect() + }) + .unwrap_or_default(); + Ok(PublishDocument { + name: name.clone(), + version, + version_original: version_original.clone(), + manifest: manifest.clone(), + dist_tags, + attachment: attachment.clone(), + length, + readme: doc + .get("readme") + .and_then(Value::as_str) + .map(str::to_string), + }) +} + +/// Read a packument update of package `name`. +pub fn read_update(doc: &Value, name: &PackageName) -> Result { + if !doc.is_object() { + return Err("the document is not a JSON object".into()); + } + check_name(doc, name)?; + let mut out = UpdateDocument::default(); + for (raw, manifest) in object(doc, "versions").into_iter().flatten() { + let version = + version::normalize(raw).map_err(|e| format!("invalid version {raw:?}: {e}"))?; + let deprecated = manifest + .get("deprecated") + .and_then(Value::as_str) + .filter(|m| !m.is_empty()) + .map(str::to_string); + out.versions.insert(version, deprecated); + } + out.dist_tags_given = doc.get("dist-tags").is_some_and(Value::is_object); + for (tag, v) in object(doc, "dist-tags").into_iter().flatten() { + if !valid_tag(tag) { + continue; + } + if let Some(v) = v.as_str().and_then(|v| version::normalize(v).ok()) { + out.dist_tags.insert(tag.clone(), v); + } + } + Ok(out) +} + +/// The version a publish stores: its manifest, completed from the tarball. +#[derive(Debug, Clone, PartialEq)] +pub struct Release { + /// The manifest stored (README included): the version object, with the + /// `dist` the registry computes from the tarball, `_id`, `_npmUser` and + /// `hasInstallScript`. + pub manifest: Map, + pub digests: Digests, +} + +/// Check the staged `tarball` against the publish `doc` and build the +/// manifest to store. `publisher` is the account publishing it. +pub fn release( + doc: &PublishDocument, + tarball: &mut R, + publisher: Option<&str>, + max_unpacked: u64, +) -> Result { + let digests = Digests::of(&mut *tarball).map_err(|e| format!("read the tarball: {e}"))?; + if let Some(length) = doc.length { + if length != digests.size { + return Err(format!( + "the attachment's length is {length}, but its data is {} bytes", + digests.size + )); + } + } + let dist = doc.manifest.get("dist").and_then(Value::as_object); + let claimed = |key: &str| dist.and_then(|d| d.get(key)).and_then(Value::as_str); + digests.verify(claimed("integrity"), claimed("shasum"))?; + tarball + .seek(SeekFrom::Start(0)) + .map_err(|e| format!("read the tarball: {e}"))?; + let tarball = tarball::read(&mut *tarball, max_unpacked)?; + if tarball.name != doc.name.to_string() { + return Err(format!( + "the tarball's package.json names package {:?}, not {:?}", + tarball.name, + doc.name.to_string() + )); + } + if version::normalize(&tarball.version).as_deref() != Ok(doc.version.as_str()) { + return Err(format!( + "the tarball's package.json is version {:?}, not {:?}", + tarball.version, doc.version + )); + } + + let mut manifest = doc.manifest.clone(); + manifest.insert("name".into(), Value::String(doc.name.to_string())); + manifest.insert("version".into(), Value::String(doc.version.clone())); + manifest.insert( + "_id".into(), + Value::String(format!("{}@{}", doc.name, doc.version)), + ); + let mut dist = dist.cloned().unwrap_or_default(); + dist.remove("tarball"); + dist.insert("integrity".into(), Value::String(digests.integrity())); + dist.insert("shasum".into(), Value::String(digests.sha1_hex.clone())); + dist.insert("fileCount".into(), Value::from(tarball.file_count)); + dist.insert("unpackedSize".into(), Value::from(tarball.unpacked_size)); + manifest.insert("dist".into(), Value::Object(dist)); + if crate::manifest::has_install_script(&manifest) { + manifest.insert("hasInstallScript".into(), Value::Bool(true)); + } + if !manifest.contains_key("readme") { + if let Some(readme) = &doc.readme { + manifest.insert("readme".into(), Value::String(readme.clone())); + } + } + if let Some(publisher) = publisher { + let user = serde_json::json!({ "name": publisher }); + manifest.insert("_npmUser".into(), user.clone()); + if !manifest.get("maintainers").is_some_and(Value::is_array) { + manifest.insert("maintainers".into(), Value::Array(vec![user])); + } + } + Ok(Release { manifest, digests }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn name() -> PackageName { + PackageName::parse("@acme/a").unwrap() + } + + fn doc(tarball: &[u8]) -> Value { + let d = Digests::of(tarball).unwrap(); + json!({ + "_id": "@acme/a", "name": "@acme/a", + "dist-tags": {"latest": "1.0.0", "beta": "1.0.0", "other": "0.1.0"}, + "versions": {"1.0.0": { + "name": "@acme/a", "version": "1.0.0", "scripts": {"install": "node-gyp rebuild"}, + "dist": {"integrity": d.integrity(), "shasum": d.sha1_hex, "tarball": "http://x"} + }}, + "_attachments": {"@acme/a-1.0.0.tgz": {"content_type": "application/octet-stream", "data": "", "length": tarball.len()}}, + "readme": "# A" + }) + } + + fn staged(bytes: &[u8]) -> std::io::Cursor> { + std::io::Cursor::new(bytes.to_vec()) + } + + #[test] + fn reads_and_releases_a_publish() { + let tgz = tarball::build(&[( + "package/package.json", + br#"{"name":"@acme/a","version":"1.0.0"}"#, + )]); + let parsed = read_publish(&doc(&tgz), &name()).unwrap(); + assert_eq!(parsed.version, "1.0.0"); + assert_eq!(parsed.attachment, "@acme/a-1.0.0.tgz"); + assert_eq!(parsed.dist_tags, ["beta", "latest"]); + let mut file = staged(&tgz); + let release = release(&parsed, &mut file, Some("alice@example.com"), 1 << 20).unwrap(); + let m = &release.manifest; + assert_eq!(m["_id"], "@acme/a@1.0.0"); + assert_eq!(m["readme"], "# A"); + assert_eq!(m["hasInstallScript"], true); + assert_eq!(m["_npmUser"], json!({"name": "alice@example.com"})); + assert_eq!(m["dist"]["integrity"], release.digests.integrity()); + assert_eq!(m["dist"]["fileCount"], 1); + assert!(m["dist"].get("tarball").is_none()); + } + + #[test] + fn refuses_a_tarball_that_is_not_the_version() { + let tgz = tarball::build(&[( + "package/package.json", + br#"{"name":"@acme/a","version":"1.0.0"}"#, + )]); + let other = tarball::build(&[( + "package/package.json", + br#"{"name":"@acme/a","version":"2.0.0"}"#, + )]); + let parsed = read_publish(&doc(&tgz), &name()).unwrap(); + // Bytes other than the integrity claims. + let mut file = staged(&other); + let err = release(&parsed, &mut file, None, 1 << 20).unwrap_err(); + assert!( + err.contains("does not match") || err.contains("length"), + "{err}" + ); + // A tarball of another version, with matching digests. + let parsed = read_publish(&doc(&other), &name()).unwrap(); + let mut file = staged(&other); + let err = release(&parsed, &mut file, None, 1 << 20).unwrap_err(); + assert!(err.contains("version"), "{err}"); + } + + #[test] + fn refuses_documents_of_another_shape() { + let tgz = tarball::build(&[( + "package/package.json", + br#"{"name":"@acme/a","version":"1.0.0"}"#, + )]); + let mut d = doc(&tgz); + d["name"] = json!("@acme/b"); + assert!(read_publish(&d, &name()).is_err()); + let mut d = doc(&tgz); + d["versions"]["1.1.0"] = json!({"name": "@acme/a", "version": "1.1.0"}); + assert!(read_publish(&d, &name()).is_err()); + let mut d = doc(&tgz); + d["versions"]["1.0.0"]["version"] = json!("1.0.1"); + assert!(read_publish(&d, &name()).is_err()); + let mut d = doc(&tgz); + d["_attachments"] = json!({}); + assert!(read_publish(&d, &name()).is_err()); + } + + #[test] + fn dist_tags_are_words() { + for ok in ["latest", "next", "beta-2", "rc.1", "v-next", "legacy_1"] { + assert!(valid_tag(ok), "{ok}"); + } + for bad in [ + "", + "1.x", + "1", + "v2", + "x", + "X", + "^1", + "~1", + ">=1", + "a b", + "a/b", + &"a".repeat(65), + ] { + assert!(!valid_tag(bad), "{bad}"); + } + } + + #[test] + fn reads_an_update() { + let d = json!({ + "name": "@acme/a", "_rev": "3-x", + "versions": {"1.0.0": {"deprecated": "use 2"}, "2.0.0": {"deprecated": ""}}, + "dist-tags": {"latest": "2.0.0"} + }); + let u = read_update(&d, &name()).unwrap(); + assert_eq!(u.versions["1.0.0"].as_deref(), Some("use 2")); + assert_eq!(u.versions["2.0.0"], None); + assert_eq!(u.dist_tags["latest"], "2.0.0"); + assert!(read_update(&json!({"name": "x"}), &name()).is_err()); + } +} diff --git a/crates/formats/npm/src/integrity.rs b/crates/formats/npm/src/integrity.rs new file mode 100644 index 00000000..eb50b6fb --- /dev/null +++ b/crates/formats/npm/src/integrity.rs @@ -0,0 +1,168 @@ +//! Tarball digests: `dist.integrity` (Subresource Integrity, as `ssri` reads +//! it) and `dist.shasum` (the hex SHA-1). + +use std::io::Read; + +use base64::Engine as _; +use sha1::Digest as _; + +/// The digests of a tarball. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Digests { + /// Lowercase hex SHA-1: `dist.shasum`. + pub sha1_hex: String, + /// Base64 SHA-256. + pub sha256_base64: String, + /// Base64 SHA-384. + pub sha384_base64: String, + /// Base64 SHA-512. + pub sha512_base64: String, + /// The number of bytes read. + pub size: u64, +} + +impl Digests { + /// Digest everything `reader` yields. + pub fn of(mut reader: impl Read) -> std::io::Result { + let mut sha1 = sha1::Sha1::new(); + let mut sha256 = sha2::Sha256::new(); + let mut sha384 = sha2::Sha384::new(); + let mut sha512 = sha2::Sha512::new(); + let mut buf = vec![0u8; 64 * 1024]; + let mut size = 0u64; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + let chunk = &buf[..n]; + sha1.update(chunk); + sha256.update(chunk); + sha384.update(chunk); + sha512.update(chunk); + size += n as u64; + } + let b64 = |bytes: &[u8]| base64::engine::general_purpose::STANDARD.encode(bytes); + Ok(Digests { + sha1_hex: hex::encode(sha1.finalize()), + sha256_base64: b64(&sha256.finalize()), + sha384_base64: b64(&sha384.finalize()), + sha512_base64: b64(&sha512.finalize()), + size, + }) + } + + /// The `dist.integrity` Forge serves: `sha512-`. + pub fn integrity(&self) -> String { + format!("sha512-{}", self.sha512_base64) + } + + /// The base64 digest of algorithm `alg`, if it is one of these. + fn of_algorithm(&self, alg: &str) -> Option<&str> { + match alg { + "sha1" => None, + "sha256" => Some(&self.sha256_base64), + "sha384" => Some(&self.sha384_base64), + "sha512" => Some(&self.sha512_base64), + _ => None, + } + } + + /// Check the tarball against what its metadata claims: `integrity` (an + /// SRI string) and `shasum` (hex SHA-1), each when given. At least one + /// must be given. Like `ssri`, an SRI string is checked with its + /// strongest algorithm, any of whose hashes may match. + pub fn verify(&self, integrity: Option<&str>, shasum: Option<&str>) -> Result<(), String> { + let integrity = integrity.map(str::trim).filter(|s| !s.is_empty()); + let shasum = shasum.map(str::trim).filter(|s| !s.is_empty()); + if integrity.is_none() && shasum.is_none() { + return Err("the version has neither dist.integrity nor dist.shasum".into()); + } + if let Some(integrity) = integrity { + self.verify_sri(integrity)?; + } + if let Some(shasum) = shasum { + if !shasum.eq_ignore_ascii_case(&self.sha1_hex) { + return Err(format!( + "dist.shasum {shasum} does not match the tarball (sha1 {})", + self.sha1_hex + )); + } + } + Ok(()) + } + + fn verify_sri(&self, integrity: &str) -> Result<(), String> { + const STRENGTH: [&str; 4] = ["sha512", "sha384", "sha256", "sha1"]; + let hashes: Vec<(&str, &str)> = integrity + .split_ascii_whitespace() + .filter_map(|entry| { + let (alg, rest) = entry.split_once('-')?; + // Options after `?` (SRI's extension point) are ignored. + let digest = rest.split('?').next().unwrap_or(rest); + Some((alg, digest)) + }) + .filter(|(alg, _)| STRENGTH.contains(alg)) + .collect(); + let Some(strongest) = STRENGTH + .iter() + .find(|alg| hashes.iter().any(|(a, _)| a == *alg)) + else { + return Err(format!( + "dist.integrity {integrity:?} names no supported algorithm" + )); + }; + let matches = hashes + .iter() + .filter(|(alg, _)| alg == strongest) + .any(|(_, digest)| { + match self.of_algorithm(strongest) { + Some(ours) => *digest == ours, + // SHA-1 integrity: base64 of the raw digest. + None => hex::decode(&self.sha1_hex) + .map(|raw| base64::engine::general_purpose::STANDARD.encode(raw)) + .is_ok_and(|ours| *digest == ours), + } + }); + if matches { + Ok(()) + } else { + Err(format!( + "dist.integrity does not match the tarball ({})", + self.integrity() + )) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn verifies_integrity_and_shasum() { + let d = Digests::of(&b"hello"[..]).unwrap(); + assert_eq!(d.size, 5); + assert_eq!(d.sha1_hex, "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"); + let sri = d.integrity(); + assert!(sri.starts_with("sha512-")); + d.verify(Some(&sri), Some(&d.sha1_hex)).unwrap(); + d.verify(Some(&sri), None).unwrap(); + d.verify(None, Some("AAF4C61DDCC5E8A2DABEDE0F3B482CD9AEA9434D")) + .unwrap(); + // The strongest algorithm decides; a weaker one may be wrong. + d.verify(Some(&format!("sha1-AAAA {sri}")), None).unwrap(); + assert!(d + .verify(Some(&format!("{} sha1-x", "sha512-AAAA")), None) + .is_err()); + assert!(d.verify(Some("sha512-AAAA"), None).is_err()); + assert!(d.verify(Some("md5-AAAA"), None).is_err()); + assert!(d.verify(None, Some("00")).is_err()); + assert!(d.verify(None, None).is_err()); + // SHA-1 integrity strings, as old packages carry. + d.verify(Some("sha1-qvTGHdzF6KLavt4PO0gs2a6pQ00="), None) + .unwrap(); + // Options are ignored. + d.verify(Some(&format!("{sri}?foo")), None).unwrap(); + } +} diff --git a/crates/formats/npm/src/lib.rs b/crates/formats/npm/src/lib.rs new file mode 100644 index 00000000..62df8d30 --- /dev/null +++ b/crates/formats/npm/src/lib.rs @@ -0,0 +1,371 @@ +//! npm registries for Forge, as a built-in format plugin: what npm, pnpm, +//! Yarn and Bun read and write under `/npm//`. +//! +//! [`Npm`] implements [`FormatPlugin`]: the rules of package names and +//! versions, reading a tarball, and the registry protocol +//! ([`protocol`]): packuments (full and abbreviated), versions, tarballs, +//! search, publishing (the tarball comes inside a JSON document, +//! [`publish`]), dist-tags, deprecations, unpublishing, the logins, and +//! audits passed to an upstream. Proxies read another registry through +//! [`proxy`]. Forge keeps the rest: storage, authentication, the tags and +//! deprecations themselves, hosted/proxy/group semantics and retention. +//! +//! The pure parts: [`name`] and [`version`] rules, [`tarball`] reading, +//! [`integrity`] checks, the [`document`]s clients `PUT`, version +//! [`manifest`] summaries, [`packument`] rendering and the parsing of +//! [`upstream`] packuments. + +pub mod document; +pub mod integrity; +pub mod manifest; +pub mod name; +pub mod packument; +pub mod protocol; +pub mod proxy; +pub mod publish; +pub mod tarball; +pub mod upstream; +pub mod version; + +use std::cmp::Ordering; +use std::io::{Read, SeekFrom, Write}; + +use forge_plugin_api::{ + Action, AuthOutcome, Change, Dependency, DependencyGroup, FetchResult, FormatError, + FormatPlugin, Loaded, PackageRef, PackageSource, ParsedPackage, Remote, Repository, Request, + Response, Unpacked, Uploaded, UpstreamOp, UpstreamStep, +}; +use serde_json::Value; + +pub use name::PackageName; + +/// The plugin's `plugin.toml`, as the server installs it. +pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); + +/// What the plugin reads of a request, beyond the package the repository's +/// upload limit bounds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NpmLimits { + /// The largest publish document, its tarball's data aside, and the + /// largest other document a client sends (an unpublish, an audit). + pub max_document_bytes: u64, + /// The most a tarball may unpack to. + pub max_unpacked_bytes: u64, +} + +impl Default for NpmLimits { + fn default() -> Self { + NpmLimits { + max_document_bytes: 32 * 1024 * 1024, + max_unpacked_bytes: tarball::DEFAULT_MAX_UNPACKED_BYTES, + } + } +} + +/// The npm format plugin. +/// +/// Package ids are names as npm writes them (`left-pad`, `@acme/ui`), case +/// and all: the registry keeps names that differ only in case apart +/// (`JSONStream` is not `jsonstream`). +#[derive(Debug, Clone, Default)] +pub struct Npm { + limits: NpmLimits, +} + +impl Npm { + pub fn with_limits(limits: NpmLimits) -> Self { + Npm { limits } + } + + pub fn limits(&self) -> NpmLimits { + self.limits + } +} + +fn invalid(e: impl std::fmt::Display) -> FormatError { + FormatError::InvalidPackage(e.to_string()) +} + +fn io(e: std::io::Error) -> FormatError { + FormatError::InvalidPackage(format!("read the package: {e}")) +} + +/// A manifest's dependencies as the host records them: a group per kind. +fn dependency_groups(manifest: &serde_json::Map) -> Vec { + manifest::dependencies(manifest) + .into_iter() + .map(|(kind, deps)| DependencyGroup { + target: Some(kind), + dependencies: deps + .into_iter() + .map(|(id, range)| Dependency { + id, + range: Some(range), + }) + .collect(), + }) + .collect() +} + +impl FormatPlugin for Npm { + fn canonical_id(&self, id: &str) -> String { + id.to_string() + } + + fn validate_id(&self, id: &str) -> Result<(), FormatError> { + PackageName::parse(id) + .map(|_| ()) + .map_err(|e| FormatError::InvalidId(e.to_string())) + } + + fn normalize_version(&self, version: &str) -> Result { + version::normalize(version).map_err(FormatError::InvalidVersion) + } + + fn compare_versions(&self, a: &str, b: &str) -> Ordering { + version::compare(a, b) + } + + /// A tarball on its own (what a proxy downloads): its `package.json` is + /// its manifest, with the `dist` the registry computes. + fn parse_package(&self, package: &mut dyn PackageSource) -> Result { + let digests = integrity::Digests::of(&mut *package).map_err(io)?; + package.seek(SeekFrom::Start(0)).map_err(io)?; + let tarball = + tarball::read(&mut *package, self.limits.max_unpacked_bytes).map_err(invalid)?; + let name = PackageName::parse(&tarball.name).map_err(invalid)?; + let version = version::normalize(&tarball.version).map_err(FormatError::InvalidVersion)?; + let mut manifest = tarball.package_json.clone(); + manifest.insert("version".into(), Value::String(version.clone())); + manifest.insert("_id".into(), Value::String(format!("{name}@{version}"))); + manifest.insert( + "dist".into(), + serde_json::json!({ + "integrity": digests.integrity(), + "shasum": digests.sha1_hex, + "fileCount": tarball.file_count, + "unpackedSize": tarball.unpacked_size, + }), + ); + Ok(ParsedPackage { + id: name.to_string(), + original_id: name.to_string(), + version, + version_original: tarball.version, + listed: true, + dependencies: dependency_groups(&manifest), + manifest: serde_json::to_vec(&manifest).map_err(invalid)?, + }) + } + + fn manifest_summary(&self, manifest: &[u8]) -> Option { + match serde_json::from_slice(manifest).ok()? { + Value::Object(m) => serde_json::to_string(&manifest::summary(&m)).ok(), + _ => None, + } + } + + fn readme(&self, manifest: &[u8]) -> Option { + let manifest: serde_json::Map = serde_json::from_slice(manifest).ok()?; + manifest + .get("readme") + .and_then(Value::as_str) + .filter(|readme| !readme.trim().is_empty()) + .map(str::to_string) + } + + /// A scope (`@acme`, stored as `@acme/` so that it reserves the scope's + /// packages, not every scope starting like it) or a name prefix. + fn reserved_prefix(&self, prefix: &str) -> Result { + let word = |s: &str| { + s.bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + }; + // `@scope`, `@scope/` or `@scope/name-start`; or `name-start`. + let valid = match prefix.strip_prefix('@') { + Some(scoped) => { + let (scope, rest) = scoped.split_once('/').unwrap_or((scoped, "")); + !scope.is_empty() && word(scope) && word(rest) + } + None => !prefix.is_empty() && word(prefix), + }; + if !valid { + return Err(FormatError::InvalidId(format!( + "{prefix:?} is not a package name prefix (a scope such as @acme, or letters, \ + digits, '.', '_', '-')" + ))); + } + let mut prefix = prefix.to_ascii_lowercase(); + if prefix.starts_with('@') && !prefix.contains('/') { + prefix.push('/'); + } + Ok(prefix) + } + + fn handle(&self, repo: &Repository, req: &Request) -> Action { + protocol::handle(self, repo, req) + } + + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + protocol::render(self, repo, req, data) + } + + fn auth_response(&self, repo: &Repository, req: &Request, outcome: &AuthOutcome) -> Response { + protocol::auth_response(repo, req, outcome) + } + + fn error_response(&self, status: u16, message: &str) -> Option { + Some(protocol::error(status, message)) + } + + /// The publish document's tarball, decoded from its base64 as the body + /// streams by ([`publish::Extractor`]); the document is what remains. + fn unpack_upload( + &self, + body: &mut dyn Read, + package: &mut dyn Write, + ) -> Result { + let max_document = usize::try_from(self.limits.max_document_bytes).unwrap_or(usize::MAX); + let mut extractor = publish::Extractor::new(max_document); + let mut buf = vec![0u8; 64 * 1024]; + let mut out = Vec::new(); + loop { + let n = body.read(&mut buf).map_err(io)?; + if n == 0 { + break; + } + out.clear(); + extractor.feed(&buf[..n], &mut out).map_err(invalid)?; + package.write_all(&out).map_err(io)?; + } + let extracted = extractor.finish().map_err(invalid)?; + Ok(match extracted.attachment { + Some(_) => Unpacked::Package { + document: extracted.document, + }, + None => Unpacked::Document(extracted.document), + }) + } + + /// The version a publish stores: the version object its document + /// describes, checked against the tarball ([`document::release`]). + fn parse_upload( + &self, + upload: &Uploaded<'_>, + package: &mut dyn PackageSource, + ) -> Result { + let doc = publish_document(upload.document)?; + let release = document::release( + &doc, + package, + upload.publisher, + self.limits.max_unpacked_bytes, + ) + .map_err(invalid)?; + Ok(ParsedPackage { + id: doc.name.to_string(), + original_id: doc.name.to_string(), + version: doc.version.clone(), + version_original: doc.version_original.clone(), + listed: true, + dependencies: dependency_groups(&release.manifest), + manifest: serde_json::to_vec(&release.manifest).map_err(invalid)?, + }) + } + + /// The dist-tags the publish document points at the new version. + fn published(&self, document: &[u8], package: &PackageRef) -> Vec { + let Ok(doc) = publish_document(document) else { + return Vec::new(); + }; + doc.dist_tags + .into_iter() + .map(|tag| Change::Tag { + id: package.id.clone(), + tag, + version: package.version.clone(), + }) + .collect() + } + + fn upstream( + &self, + remote: &Remote, + op: &UpstreamOp, + fetched: &[FetchResult], + ) -> Result { + Ok(proxy::upstream(remote, op, fetched)) + } +} + +/// The publish document of an upload: its name must be one a new package +/// may take. +fn publish_document(document: &[u8]) -> Result { + let doc: Value = serde_json::from_slice(document) + .map_err(|e| invalid(format!("the document is not valid JSON: {e}")))?; + let raw = doc + .get("name") + .and_then(Value::as_str) + .ok_or_else(|| invalid("the document has no name"))?; + let name = PackageName::parse_new(raw).map_err(invalid)?; + document::read_publish(&doc, &name).map_err(invalid) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn parses_a_tarball_on_its_own() { + let tgz = tarball::build(&[( + "package/package.json", + br##"{"name":"@acme/a","version":"v1.2.3","readme":"# A","dependencies":{"b":"^1"}}"##, + )]); + let npm = Npm::default(); + let parsed = npm.parse_package(&mut Cursor::new(tgz)).unwrap(); + assert_eq!(parsed.id, "@acme/a"); + assert_eq!(parsed.version, "1.2.3"); + assert_eq!(parsed.version_original, "v1.2.3"); + assert_eq!( + parsed.dependencies[0].target.as_deref(), + Some("dependencies") + ); + let summary: Value = + serde_json::from_str(&npm.manifest_summary(&parsed.manifest).unwrap()).unwrap(); + assert!(summary.get("readme").is_none()); + assert_eq!( + summary["dependency_groups"][0]["dependencies"][0]["id"], + "b" + ); + assert!(summary["dist"]["integrity"] + .as_str() + .unwrap() + .starts_with("sha512-")); + assert_eq!(npm.readme(&parsed.manifest).as_deref(), Some("# A")); + } + + #[test] + fn ids_keep_their_case() { + let npm = Npm::default(); + assert_eq!(npm.canonical_id("JSONStream"), "JSONStream"); + assert!(npm.validate_id("@acme/a").is_ok()); + assert!(npm.validate_id("../etc").is_err()); + assert_eq!(npm.compare_versions("1.10.0", "1.9.0"), Ordering::Greater); + } + + #[test] + fn reserved_prefixes_are_scopes_or_name_prefixes() { + let npm = Npm::default(); + assert_eq!(npm.reserved_prefix("@Acme").unwrap(), "@acme/"); + assert_eq!(npm.reserved_prefix("@acme/ui-").unwrap(), "@acme/ui-"); + assert_eq!(npm.reserved_prefix("acme-").unwrap(), "acme-"); + assert!(npm.reserved_prefix("a@b").is_err()); + assert!(npm.reserved_prefix("").is_err()); + // Nothing is named so: these would reserve nothing. + assert!(npm.reserved_prefix("@").is_err()); + assert!(npm.reserved_prefix("@/").is_err()); + assert!(npm.reserved_prefix("acme/ui").is_err()); + assert!(npm.reserved_prefix("@acme/ui/").is_err()); + } +} diff --git a/crates/formats/npm/src/manifest.rs b/crates/formats/npm/src/manifest.rs new file mode 100644 index 00000000..bfec01b1 --- /dev/null +++ b/crates/formats/npm/src/manifest.rs @@ -0,0 +1,243 @@ +//! Version manifests: the object a packument lists under `versions`. +//! +//! Forge stores two forms of a version's manifest: +//! +//! * the **manifest** as published (the version object of the publish +//! document, or of the upstream packument), README included, as the +//! version's `package.json` asset; +//! * its **summary** ([`summary`]), the manifest without its README, as the +//! version's manifest summary, with the dependencies as the web UI's +//! package page reads them ([`DEPENDENCY_GROUPS`]). Every packument is +//! rendered from the summaries alone, one query for every version. +//! +//! A version deprecated with `npm deprecate` keeps its summary: Forge +//! stores the message beside it, and it wins over the summary's own +//! `deprecated` (an upstream's). + +use serde_json::{json, Map, Value}; + +/// The key of a summary holding the version's dependencies in the shape the +/// web UI's package page reads (`[{"target_framework", "dependencies": +/// [{"id", "range"}]}]`, a group per dependency kind). Packuments leave it +/// out. +pub const DEPENDENCY_GROUPS: &str = "dependency_groups"; + +/// The kinds of dependencies that install with a version, in the order the +/// web UI lists them. +pub const DEPENDENCY_KINDS: &[&str] = &["dependencies", "optionalDependencies", "peerDependencies"]; + +/// A manifest's dependencies, `(kind, [(name, range)])` for each kind of +/// [`DEPENDENCY_KINDS`] it has. +pub fn dependencies(manifest: &Map) -> Vec<(String, Vec<(String, String)>)> { + DEPENDENCY_KINDS + .iter() + .filter_map(|kind| { + let deps: Vec<(String, String)> = manifest + .get(*kind)? + .as_object()? + .iter() + .map(|(name, range)| (name.clone(), range.as_str().unwrap_or_default().to_string())) + .collect(); + (!deps.is_empty()).then(|| (kind.to_string(), deps)) + }) + .collect() +} + +/// The fields of a version an install needs: what the abbreviated +/// metadata (`application/vnd.npm.install-v1+json`) carries of it. +pub const ABBREVIATED_FIELDS: &[&str] = &[ + "name", + "version", + "deprecated", + "dependencies", + "optionalDependencies", + "devDependencies", + "bundleDependencies", + "bundledDependencies", + "peerDependencies", + "peerDependenciesMeta", + "bin", + "directories", + "dist", + "engines", + "_hasShrinkwrap", + "hasInstallScript", + "cpu", + "os", + "libc", +]; + +/// Fields a summary drops: the README, which the packument carries once, +/// at the top. +const UNSUMMARIZED: &[&str] = &["readme", "readmeFilename"]; + +/// The largest summary kept whole. A larger one keeps the fields an +/// install needs and the small display fields only. +pub const MAX_SUMMARY_BYTES: usize = 256 * 1024; + +/// Display fields kept in a cut-down summary, besides [`ABBREVIATED_FIELDS`]. +const DISPLAY_FIELDS: &[&str] = &[ + "_id", + "description", + "license", + "homepage", + "repository", + "bugs", + "author", + "keywords", + "maintainers", + "_npmUser", + "gitHead", + "type", + "main", +]; + +/// The summary of a version manifest: see the module documentation. +pub fn summary(manifest: &Map) -> Map { + let mut out: Map = manifest + .iter() + .filter(|(key, _)| { + !UNSUMMARIZED.contains(&key.as_str()) && key.as_str() != DEPENDENCY_GROUPS + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + if serde_json::to_vec(&out).map_or(0, |v| v.len()) > MAX_SUMMARY_BYTES { + out.retain(|key, _| { + ABBREVIATED_FIELDS.contains(&key.as_str()) || DISPLAY_FIELDS.contains(&key.as_str()) + }); + if let Some(Value::String(description)) = out.get_mut("description") { + truncate(description, 1024); + } + for key in DISPLAY_FIELDS { + if out + .get(*key) + .is_some_and(|v| serde_json::to_vec(v).map_or(0, |v| v.len()) > 16 * 1024) + { + out.remove(*key); + } + } + } + let groups: Vec = dependencies(&out) + .into_iter() + .map(|(kind, deps)| { + json!({ + "target_framework": kind, + "dependencies": deps + .into_iter() + .map(|(id, range)| json!({"id": id, "range": range})) + .collect::>(), + }) + }) + .collect(); + if !groups.is_empty() { + out.insert(DEPENDENCY_GROUPS.into(), Value::Array(groups)); + } + out +} + +/// A summary as a packument lists it: without what Forge keeps for itself +/// ([`DEPENDENCY_GROUPS`]). +pub fn served(summary: &Map) -> Map { + let mut out = summary.clone(); + out.remove(DEPENDENCY_GROUPS); + out +} + +/// Cut `s` to at most `max` bytes, at a character boundary. +pub fn truncate(s: &mut String, max: usize) { + if s.len() > max { + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + s.truncate(end); + } +} + +/// The abbreviated form of a summary. +pub fn abbreviated(summary: &Map) -> Map { + ABBREVIATED_FIELDS + .iter() + .filter_map(|key| summary.get(*key).map(|v| (key.to_string(), v.clone()))) + .collect() +} + +/// Whether a manifest runs an install script: what the registry reports as +/// `hasInstallScript`, so that installers know before they unpack it. +pub fn has_install_script(manifest: &Map) -> bool { + let scripts = manifest.get("scripts").and_then(Value::as_object); + ["preinstall", "install", "postinstall"] + .iter() + .any(|s| scripts.is_some_and(|scripts| scripts.contains_key(*s))) + || manifest.get("gypfile").and_then(Value::as_bool) == Some(true) +} + +/// The `deprecated` message of a summary, if the version is deprecated (an +/// empty message is none). +pub fn deprecation(summary: &Map) -> Option<&str> { + summary + .get("deprecated") + .and_then(Value::as_str) + .filter(|m| !m.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn object(v: Value) -> Map { + v.as_object().unwrap().clone() + } + + #[test] + fn summaries_drop_the_readme() { + let m = object(json!({ + "name": "a", "version": "1.0.0", "readme": "# A", "readmeFilename": "README.md", + "dependencies": {"b": "^1"}, "scripts": {"test": "x"} + })); + let s = summary(&m); + assert!(!s.contains_key("readme") && !s.contains_key("readmeFilename")); + assert_eq!(s["scripts"], json!({"test": "x"})); + let a = abbreviated(&s); + assert_eq!( + a, + object(json!({"name": "a", "version": "1.0.0", "dependencies": {"b": "^1"}})) + ); + } + + #[test] + fn a_huge_manifest_keeps_what_installs_need() { + let m = object(json!({ + "name": "a", "version": "1.0.0", + "description": "x".repeat(400_000), + "jsii": {"huge": "y".repeat(400_000)}, + "dependencies": {"b": "^1"}, + "repository": {"url": "z".repeat(20_000)}, + })); + let s = summary(&m); + assert!(serde_json::to_vec(&s).unwrap().len() < MAX_SUMMARY_BYTES); + assert_eq!(s["dependencies"], json!({"b": "^1"})); + assert_eq!(s["description"].as_str().unwrap().len(), 1024); + assert!(!s.contains_key("jsii") && !s.contains_key("repository")); + } + + #[test] + fn install_scripts_and_deprecation() { + assert!(has_install_script(&object( + json!({"scripts": {"postinstall": "x"}}) + ))); + assert!(has_install_script(&object(json!({"gypfile": true})))); + assert!(!has_install_script(&object( + json!({"scripts": {"test": "x"}}) + ))); + assert_eq!( + deprecation(&object(json!({"deprecated": "use b"}))), + Some("use b") + ); + assert_eq!(deprecation(&object(json!({"deprecated": ""}))), None); + let mut s = "héllo".to_string(); + truncate(&mut s, 2); + assert_eq!(s, "h"); + } +} diff --git a/crates/formats/npm/src/name.rs b/crates/formats/npm/src/name.rs new file mode 100644 index 00000000..80ef3083 --- /dev/null +++ b/crates/formats/npm/src/name.rs @@ -0,0 +1,243 @@ +//! npm package names, as `validate-npm-package-name` reads them. +//! +//! A name is `name` or `@scope/name`. Both parts are URL-safe as they are +//! (what `encodeURIComponent` leaves alone), so a name is pasted into a URL +//! path without escaping, save the `/` of a scoped name, which the packument +//! URL escapes as `%2f`. +//! +//! The registry has two rules: +//! +//! * a name a **new** package may take ([`PackageName::parse_new`]): what a +//! hosted repository accepts from a publish. Lowercase, at most 214 +//! characters, and no `~'!()*` in the name part. +//! * a name an **existing** package may have ([`PackageName::parse`]): what a +//! proxy reads from its upstream, which still serves packages published +//! before the stricter rule (`JSONStream`). +//! +//! Neither part may start with `.` or `_`, so no part is ever `.` or `..`. + +use std::fmt; + +/// The longest name a new package may take. +pub const MAX_NEW_NAME_LEN: usize = 214; + +/// The longest name read at all (the host's bound on ids). +pub const MAX_NAME_LEN: usize = 512; + +/// A valid npm package name. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PackageName { + scope: Option, + name: String, +} + +/// Why a name is not one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvalidName(pub String); + +impl fmt::Display for InvalidName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for InvalidName {} + +/// Whether `c` is a character `encodeURIComponent` leaves as it is. +fn uri_safe(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '!' | '~' | '*' | '\'' | '(' | ')') +} + +fn check_part(name: &str, part: &str, what: &str) -> Result<(), InvalidName> { + if part.is_empty() { + return Err(InvalidName(format!("{name:?}: the {what} is empty"))); + } + if part.starts_with('.') || part.starts_with('_') { + return Err(InvalidName(format!( + "{name:?}: the {what} may not start with '.' or '_'" + ))); + } + if !part.chars().all(uri_safe) { + return Err(InvalidName(format!( + "{name:?}: the {what} may only contain URL-safe characters (letters, digits, '-', '_', '.', '!', '~', '*', ''', '(' and ')')" + ))); + } + Ok(()) +} + +impl PackageName { + /// A name an existing package may have. + pub fn parse(raw: &str) -> Result { + if raw.len() > MAX_NAME_LEN { + return Err(InvalidName(format!( + "the name is longer than {MAX_NAME_LEN} characters" + ))); + } + if raw.trim() != raw { + return Err(InvalidName(format!( + "{raw:?}: the name has leading or trailing spaces" + ))); + } + if raw.eq_ignore_ascii_case("node_modules") || raw.eq_ignore_ascii_case("favicon.ico") { + return Err(InvalidName(format!("{raw:?} is not a package name"))); + } + match raw.strip_prefix('@') { + Some(scoped) => { + let Some((scope, name)) = scoped.split_once('/') else { + return Err(InvalidName(format!( + "{raw:?}: a scoped name is @scope/name" + ))); + }; + check_part(raw, scope, "scope")?; + check_part(raw, name, "name")?; + Ok(PackageName { + scope: Some(scope.to_string()), + name: name.to_string(), + }) + } + None => { + check_part(raw, raw, "name")?; + Ok(PackageName { + scope: None, + name: raw.to_string(), + }) + } + } + } + + /// A name a new package may take (a publish to a hosted repository). + pub fn parse_new(raw: &str) -> Result { + let name = Self::parse(raw)?; + if raw.len() > MAX_NEW_NAME_LEN { + return Err(InvalidName(format!( + "{raw:?}: a new package's name is at most {MAX_NEW_NAME_LEN} characters" + ))); + } + if raw.chars().any(|c| c.is_ascii_uppercase()) { + return Err(InvalidName(format!( + "{raw:?}: a new package's name may not contain capital letters" + ))); + } + if name.name.contains(['~', '\'', '!', '(', ')', '*']) { + return Err(InvalidName(format!( + "{raw:?}: a new package's name may not contain ~'!()*" + ))); + } + Ok(name) + } + + /// The scope, without its `@` (`None` for an unscoped name). + pub fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + /// The name without its scope: what a tarball's file name starts with. + pub fn unscoped(&self) -> &str { + &self.name + } + + /// The name as it is written in a URL path segment, `@scope%2fname` for + /// a scoped name (npm's `escapedName`). + pub fn escaped(&self) -> String { + match &self.scope { + Some(scope) => format!("@{scope}%2f{}", self.name), + None => self.name.clone(), + } + } + + /// The file name of a version's tarball, `-.tgz`, as the + /// npm registry names it. + pub fn tarball_file(&self, version: &str) -> String { + format!("{}-{version}.tgz", self.name) + } + + /// The version a tarball file name `file` names, if it is one of this + /// package's (`-.tgz`). + pub fn version_of_tarball<'a>(&self, file: &'a str) -> Option<&'a str> { + let version = file + .strip_suffix(".tgz")? + .strip_prefix(self.name.as_str())? + .strip_prefix('-')?; + (!version.is_empty()).then_some(version) + } +} + +impl fmt::Display for PackageName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.scope { + Some(scope) => write!(f, "@{scope}/{}", self.name), + None => f.write_str(&self.name), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_plain_and_scoped_names() { + let plain = PackageName::parse("left-pad").unwrap(); + assert_eq!(plain.scope(), None); + assert_eq!(plain.to_string(), "left-pad"); + assert_eq!(plain.escaped(), "left-pad"); + + let scoped = PackageName::parse("@envisia/ui-kit").unwrap(); + assert_eq!(scoped.scope(), Some("envisia")); + assert_eq!(scoped.unscoped(), "ui-kit"); + assert_eq!(scoped.to_string(), "@envisia/ui-kit"); + assert_eq!(scoped.escaped(), "@envisia%2fui-kit"); + assert_eq!(scoped.tarball_file("1.0.0"), "ui-kit-1.0.0.tgz"); + assert_eq!( + scoped.version_of_tarball("ui-kit-1.0.0-rc.1.tgz"), + Some("1.0.0-rc.1") + ); + assert_eq!(scoped.version_of_tarball("other-1.0.0.tgz"), None); + assert_eq!(scoped.version_of_tarball("ui-kit-.tgz"), None); + assert_eq!(scoped.version_of_tarball("ui-kit-1.0.0.tar"), None); + } + + #[test] + fn old_names_stay_readable() { + assert!(PackageName::parse("JSONStream").is_ok()); + assert!(PackageName::parse_new("JSONStream").is_err()); + assert!(PackageName::parse("highlight.js").is_ok()); + assert!(PackageName::parse("lodash.merge").is_ok()); + assert!(PackageName::parse("ok!").is_ok()); + assert!(PackageName::parse_new("ok!").is_err()); + assert!(PackageName::parse_new(&"a".repeat(214)).is_ok()); + assert!(PackageName::parse_new(&"a".repeat(215)).is_err()); + assert!(PackageName::parse(&"a".repeat(215)).is_ok()); + } + + #[test] + fn refuses_what_is_no_name() { + for bad in [ + "", + " x", + "x ", + ".hidden", + "_private", + "..", + ".", + "a/b", + "@scope", + "@scope/", + "@/x", + "@sc/ope/x", + "@.scope/x", + "@scope/.x", + "@scope/_x", + "a b", + "a%2fb", + "a?b", + "a#b", + "a\\b", + "ünicode", + "node_modules", + "favicon.ico", + ] { + assert!(PackageName::parse(bad).is_err(), "{bad:?}"); + } + } +} diff --git a/crates/formats/npm/src/packument.rs b/crates/formats/npm/src/packument.rs new file mode 100644 index 00000000..2ed9eb0d --- /dev/null +++ b/crates/formats/npm/src/packument.rs @@ -0,0 +1,297 @@ +//! Packuments: the package documents an npm client reads. +//! +//! A [`Packument`] is what a repository knows of a package (its versions' +//! summaries, its dist-tags, the latest README); [`Packument::render`] +//! writes it as the full document (`application/json`) or as the +//! abbreviated metadata installs ask for +//! (`application/vnd.npm.install-v1+json`). Every `dist.tarball` points at +//! the repository the packument is served from, never upstream. + +use std::collections::BTreeMap; + +use chrono::{DateTime, SecondsFormat, Utc}; +use serde_json::{json, Map, Value}; + +use crate::manifest; +use crate::name::PackageName; +use crate::version; + +/// The media type of abbreviated metadata. +pub const ABBREVIATED_MEDIA_TYPE: &str = "application/vnd.npm.install-v1+json"; + +/// The dist-tag every package has, unless it has no version. +pub const LATEST: &str = "latest"; + +/// One version of a packument. +#[derive(Debug, Clone, PartialEq)] +pub struct PackumentVersion { + /// The version, normalized. + pub version: String, + /// Its manifest summary ([`manifest::summary`]). + pub summary: Map, + /// When it was published, if known. + pub published: Option>, +} + +/// What a repository knows of a package. +#[derive(Debug, Clone, PartialEq)] +pub struct Packument { + pub name: PackageName, + /// In any order. + pub versions: Vec, + pub dist_tags: BTreeMap, + /// The README of the latest version. + pub readme: Option, +} + +/// The URL of a version's tarball in the repository at `base` (no trailing +/// slash): `{base}/{name}/-/{name without scope}-{version}.tgz`, as the npm +/// registry writes it. +pub fn tarball_url(base: &str, name: &PackageName, version: &str) -> String { + format!("{base}/{name}/-/{}", name.tarball_file(version)) +} + +/// The time format of a packument's `time`: RFC 3339 in UTC with +/// milliseconds. +pub fn timestamp(t: DateTime) -> String { + t.to_rfc3339_opts(SecondsFormat::Millis, true) +} + +/// The version `latest` defaults to: the highest stable version, else the +/// highest. +pub fn default_latest<'a>(versions: impl Iterator) -> Option<&'a str> { + let mut all: Vec<&str> = versions.collect(); + all.sort_by(|a, b| version::compare(a, b)); + all.iter() + .rev() + .find(|v| !version::is_prerelease(v)) + .or(all.last()) + .copied() +} + +/// `tags` without the ones naming a version not in `versions`, and with a +/// `latest` whenever there is a version: the default one if `latest` was +/// missing or named a version that is gone. +pub fn heal_dist_tags( + tags: &BTreeMap, + versions: &[&str], +) -> BTreeMap { + let mut out: BTreeMap = tags + .iter() + .filter(|(_, v)| versions.contains(&v.as_str())) + .map(|(t, v)| (t.clone(), v.clone())) + .collect(); + if !out.contains_key(LATEST) { + if let Some(latest) = default_latest(versions.iter().copied()) { + out.insert(LATEST.to_string(), latest.to_string()); + } + } + out +} + +impl Packument { + /// The versions in ascending precedence. + fn sorted(&self) -> Vec<&PackumentVersion> { + let mut versions: Vec<&PackumentVersion> = self.versions.iter().collect(); + versions.sort_by(|a, b| version::compare(&a.version, &b.version)); + versions + } + + /// The dist-tags as served: see [`heal_dist_tags`]. + pub fn served_dist_tags(&self) -> BTreeMap { + let versions: Vec<&str> = self.versions.iter().map(|v| v.version.as_str()).collect(); + heal_dist_tags(&self.dist_tags, &versions) + } + + /// When the package last changed: its newest version's publish time. + pub fn modified(&self) -> Option> { + self.versions.iter().filter_map(|v| v.published).max() + } + + /// A revision for clients that send one back (`npm unpublish`): it is + /// not checked. + pub fn rev(&self) -> String { + let stamp = self.modified().map_or(0, |t| t.timestamp_millis()); + format!("{}-{stamp:x}", self.versions.len()) + } + + /// The summary of a version with its `dist.tarball` pointing at `base`. + fn served_summary(&self, base: &str, v: &PackumentVersion) -> Map { + let mut summary = manifest::served(&v.summary); + let url = tarball_url(base, &self.name, &v.version); + match summary.get_mut("dist").and_then(Value::as_object_mut) { + Some(dist) => { + dist.insert("tarball".into(), Value::String(url)); + } + None => { + summary.insert("dist".into(), json!({ "tarball": url })); + } + } + summary.insert("name".into(), Value::String(self.name.to_string())); + summary.insert("version".into(), Value::String(v.version.clone())); + summary + } + + /// One version's object (`GET //`), with its + /// tarball at `base`. + pub fn render_version(&self, base: &str, version: &str) -> Option { + let v = self.versions.iter().find(|v| v.version == version)?; + Some(Value::Object(self.served_summary(base, v))) + } + + /// The document, full or `abbreviated`, with tarballs at `base`. + pub fn render(&self, base: &str, abbreviated: bool) -> Value { + let versions = self.sorted(); + let tags = self.served_dist_tags(); + let modified = self.modified(); + let mut rendered = Map::new(); + for v in &versions { + let summary = self.served_summary(base, v); + let summary = if abbreviated { + manifest::abbreviated(&summary) + } else { + summary + }; + rendered.insert(v.version.clone(), Value::Object(summary)); + } + let name = self.name.to_string(); + if abbreviated { + let mut doc = Map::new(); + doc.insert("name".into(), Value::String(name)); + if let Some(modified) = modified { + doc.insert("modified".into(), Value::String(timestamp(modified))); + } + doc.insert("dist-tags".into(), json!(tags)); + doc.insert("versions".into(), Value::Object(rendered)); + return Value::Object(doc); + } + let mut time = Map::new(); + if let Some(created) = versions.iter().filter_map(|v| v.published).min() { + time.insert("created".into(), Value::String(timestamp(created))); + } + if let Some(modified) = modified { + time.insert("modified".into(), Value::String(timestamp(modified))); + } + for v in &versions { + if let Some(published) = v.published { + time.insert(v.version.clone(), Value::String(timestamp(published))); + } + } + let mut doc = Map::new(); + doc.insert("_id".into(), Value::String(name.clone())); + doc.insert("_rev".into(), Value::String(self.rev())); + doc.insert("name".into(), Value::String(name)); + // The display fields of the package are its latest version's. + let latest = tags + .get(LATEST) + .and_then(|l| versions.iter().find(|v| &v.version == l)); + if let Some(latest) = latest { + for key in [ + "description", + "keywords", + "license", + "homepage", + "repository", + "bugs", + "author", + "maintainers", + ] { + if let Some(value) = latest.summary.get(key) { + doc.insert(key.into(), value.clone()); + } + } + } + doc.insert("dist-tags".into(), json!(tags)); + doc.insert("versions".into(), Value::Object(rendered)); + doc.insert("time".into(), Value::Object(time)); + if let Some(readme) = &self.readme { + doc.insert("readme".into(), Value::String(readme.clone())); + } + Value::Object(doc) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn version(v: &str, day: u32) -> PackumentVersion { + PackumentVersion { + version: v.to_string(), + summary: json!({ + "name": "@acme/a", "version": v, "description": format!("v{v}"), + "dependencies": {"b": "^1"}, "scripts": {"test": "x"}, + "dist": {"integrity": "sha512-x", "shasum": "ab", "tarball": "https://upstream/x.tgz"} + }) + .as_object() + .unwrap() + .clone(), + published: Some(Utc.with_ymd_and_hms(2026, 1, day, 0, 0, 0).unwrap()), + } + } + + fn packument() -> Packument { + Packument { + name: PackageName::parse("@acme/a").unwrap(), + versions: vec![ + version("1.10.0", 3), + version("1.2.0", 1), + version("2.0.0-rc.1", 4), + ], + dist_tags: BTreeMap::from([ + ("next".to_string(), "2.0.0-rc.1".to_string()), + ("gone".to_string(), "0.1.0".to_string()), + ]), + readme: Some("# A".into()), + } + } + + #[test] + fn renders_the_abbreviated_metadata() { + let doc = packument().render("https://forge/npm/r", true); + assert_eq!(doc["name"], "@acme/a"); + assert_eq!(doc["modified"], "2026-01-04T00:00:00.000Z"); + assert_eq!( + doc["dist-tags"], + json!({"latest": "1.10.0", "next": "2.0.0-rc.1"}) + ); + let v = &doc["versions"]["1.2.0"]; + assert_eq!( + v["dist"]["tarball"], + "https://forge/npm/r/@acme/a/-/a-1.2.0.tgz" + ); + assert_eq!(v["dist"]["integrity"], "sha512-x"); + assert_eq!(v["dependencies"], json!({"b": "^1"})); + assert!(v.get("scripts").is_none() && v.get("description").is_none()); + assert!(doc.get("time").is_none() && doc.get("readme").is_none()); + let keys: Vec<&String> = doc["versions"].as_object().unwrap().keys().collect(); + assert_eq!(keys, ["1.10.0", "1.2.0", "2.0.0-rc.1"]); + } + + #[test] + fn renders_the_full_document() { + let doc = packument().render("https://forge/npm/r", false); + assert_eq!(doc["_id"], "@acme/a"); + assert_eq!(doc["description"], "v1.10.0"); + assert_eq!(doc["readme"], "# A"); + assert_eq!(doc["time"]["created"], "2026-01-01T00:00:00.000Z"); + assert_eq!(doc["time"]["1.10.0"], "2026-01-03T00:00:00.000Z"); + assert_eq!(doc["versions"]["1.10.0"]["scripts"], json!({"test": "x"})); + assert!(doc["_rev"].as_str().unwrap().starts_with("3-")); + } + + #[test] + fn heals_dist_tags() { + let tags = BTreeMap::from([("latest".to_string(), "9.9.9".to_string())]); + assert_eq!( + heal_dist_tags(&tags, &["1.0.0", "2.0.0-rc.1"]), + BTreeMap::from([("latest".to_string(), "1.0.0".to_string())]) + ); + assert_eq!( + heal_dist_tags(&BTreeMap::new(), &["2.0.0-rc.1"]), + BTreeMap::from([("latest".to_string(), "2.0.0-rc.1".to_string())]) + ); + assert!(heal_dist_tags(&tags, &[]).is_empty()); + } +} diff --git a/crates/formats/npm/src/protocol.rs b/crates/formats/npm/src/protocol.rs new file mode 100644 index 00000000..48756c21 --- /dev/null +++ b/crates/formats/npm/src/protocol.rs @@ -0,0 +1,884 @@ +//! The registry protocol: what npm clients request under a repository, +//! `/npm//…`, and what they get. +//! +//! A package is named in one path segment, `left-pad` or `@scope%2fname` +//! (npm's `escapedName`, `%2f` or `%2F`), or in two, `@scope/name` (how +//! tarball URLs and some clients write it). Segments are percent-decoded +//! one by one after the path is split, so an encoded `/` never adds a +//! segment, and a name is only ever what [`PackageName::parse`] accepts. +//! +//! Reads load the package's versions and tags and render the packument, +//! full or abbreviated (`Accept: application/vnd.npm.install-v1+json`, what +//! installs ask for). Writes are a publish (the tarball inside the JSON +//! document, [`Upload::Document`]), and documents that change versions: +//! deprecations, unpublishing the versions a document leaves out, dist-tags. +//! Logins and `whoami` are the host's ([`Action::Auth`]); audits go to the +//! upstream ([`Action::Forward`]). + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use forge_plugin_api::{ + Action, Answer, AuthOutcome, AuthRequest, BodyRequest, BodyUse, Change, ChangeSet, Forward, + Loaded, PackageInfo, PackageRef, PublishRequest, Query, Repository, Request, Response, + SearchQuery, Upload, +}; +use serde_json::{json, Map, Value}; + +use crate::document; +use crate::manifest; +use crate::packument::{Packument, PackumentVersion, ABBREVIATED_MEDIA_TYPE, LATEST}; +use crate::{version, Npm, PackageName}; + +/// The endpoints of `npm audit` passed on to the upstream, under +/// `/-/npm/v1/security/`. +pub const AUDITS: &[&str] = &["advisories/bulk", "audits/quick", "audits"]; + +/// The largest body of a login, a web login's start or `npm dist-tag add`. +const SMALL_BODY: u64 = 64 * 1024; + +/// How long a client waits before it polls a web login again, in seconds. +const POLL_SECONDS: &str = "2"; + +/// What a request is. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Route { + /// `GET /-/ping`. + Ping, + /// `GET /-/whoami`. + Whoami, + /// `GET /-/v1/search`. + Search, + /// `POST /-/v1/login`: start a web login. + WebLogin, + /// `GET /-/v1/login/done/`: poll a web login. + WebLoginDone(String), + /// `PUT /-/user/org.couchdb.user:[/-rev/]`: a legacy login + /// (`npm login --auth-type=legacy`, `yarn npm login`). + UserLogin(String), + /// `DELETE /-/user/token/`: `npm logout`. + Logout(String), + /// `POST /-/npm/v1/security/…`: `npm audit`. + Audit(String), + /// `GET /-/package//dist-tags`. + DistTags(PackageName), + /// `PUT`/`DELETE /-/package//dist-tags/`. + DistTag(PackageName, String), + /// `GET`/`PUT /`: the packument, a publish or an update. + Package(PackageName), + /// `PUT`/`DELETE //-rev/`: an update, or unpublishing + /// the whole package. + PackageRev(PackageName), + /// `GET //`. + Version(PackageName, String), + /// `GET //-/.tgz`. + Tarball(PackageName, String), + /// `DELETE //-/.tgz/-rev/`. + TarballRev(PackageName, String), +} + +/// Why a path is no route. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RouteError { + NotFound, + BadRequest(String), +} + +/// Percent-decode one path segment. A malformed escape, or one decoding to +/// a control character or invalid UTF-8, is refused. +pub fn decode(segment: &str) -> Result { + let bad = || RouteError::BadRequest(format!("bad percent-encoding in {segment:?}")); + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + let hex = bytes.get(i + 1..i + 3).ok_or_else(bad)?; + let hex = std::str::from_utf8(hex).map_err(|_| bad())?; + out.push(u8::from_str_radix(hex, 16).map_err(|_| bad())?); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + let decoded = String::from_utf8(out).map_err(|_| bad())?; + if decoded.chars().any(char::is_control) { + return Err(bad()); + } + Ok(decoded) +} + +/// The package named at the start of `segments`, and the segments after it. +fn package<'a>(segments: &'a [&'a str]) -> Result<(PackageName, &'a [&'a str]), RouteError> { + let first = segments.first().ok_or(RouteError::NotFound)?; + let first = decode(first)?; + let (raw, rest) = if first.starts_with('@') && !first.contains('/') { + let second = segments.get(1).ok_or(RouteError::NotFound)?; + (format!("{first}/{}", decode(second)?), &segments[2..]) + } else { + (first, &segments[1..]) + }; + let name = PackageName::parse(&raw).map_err(|_| RouteError::NotFound)?; + Ok((name, rest)) +} + +/// The route of a request path below the repository (`/left-pad`, raw, +/// not decoded). +pub fn parse(path: &str) -> Result { + let rest = path.strip_prefix('/').unwrap_or(path); + let segments: Vec<&str> = rest.split('/').collect(); + if segments.iter().any(|s| s.is_empty()) { + return Err(RouteError::NotFound); + } + match segments.as_slice() { + ["-", "ping"] => Ok(Route::Ping), + ["-", "whoami"] => Ok(Route::Whoami), + ["-", "v1", "search"] => Ok(Route::Search), + ["-", "v1", "login"] => Ok(Route::WebLogin), + ["-", "v1", "login", "done", secret] => Ok(Route::WebLoginDone(decode(secret)?)), + ["-", "user", "token", token] => Ok(Route::Logout(decode(token)?)), + ["-", "user", user] | ["-", "user", user, "-rev", _] => { + let user = decode(user)?; + let name = user + .strip_prefix("org.couchdb.user:") + .ok_or(RouteError::NotFound)?; + Ok(Route::UserLogin(name.to_string())) + } + ["-", "npm", "v1", "security", rest @ ..] if !rest.is_empty() => { + Ok(Route::Audit(rest.join("/"))) + } + ["-", "package", rest @ ..] => { + let (name, rest) = package(rest)?; + match rest { + ["dist-tags"] => Ok(Route::DistTags(name)), + ["dist-tags", tag] => Ok(Route::DistTag(name, decode(tag)?)), + _ => Err(RouteError::NotFound), + } + } + ["-", ..] => Err(RouteError::NotFound), + _ => { + let (name, rest) = package(&segments)?; + match rest { + [] => Ok(Route::Package(name)), + ["-rev", _] => Ok(Route::PackageRev(name)), + ["-", file] => Ok(Route::Tarball(name, decode(file)?)), + ["-", file, "-rev", _] => Ok(Route::TarballRev(name, decode(file)?)), + [version] => Ok(Route::Version(name, decode(version)?)), + _ => Err(RouteError::NotFound), + } + } + } +} + +fn json_response(status: u16, body: &Value) -> Response { + Response::new( + status, + "application/json", + serde_json::to_vec(body).unwrap_or_default(), + ) +} + +/// An error as npm clients read it: `{"error": "…"}`. +pub fn error(status: u16, message: impl Into) -> Response { + json_response(status, &json!({ "error": message.into() })) +} + +fn ok(status: u16) -> Response { + json_response(status, &json!({ "ok": true, "success": true })) +} + +fn respond(response: Response) -> Action { + Action::Respond(response) +} + +/// Whether the request asks for abbreviated metadata (and does not refuse +/// it with `q=0`). +pub fn wants_abbreviated(req: &Request) -> bool { + req.header("accept").is_some_and(|accept| { + accept.split(',').any(|part| { + let mut params = part.split(';'); + let media = params.next().unwrap_or("").trim(); + let refused = params.any(|p| { + p.trim() + .strip_prefix("q=") + .and_then(|q| q.trim().parse::().ok()) + .is_some_and(|q| q == 0.0) + }); + media.eq_ignore_ascii_case(ABBREVIATED_MEDIA_TYPE) && !refused + }) + }) +} + +fn reads(name: &PackageName) -> Action { + let id = name.to_string(); + Action::Load(vec![Query::Packages(id.clone()), Query::Tags(id)]) +} + +/// The full packument's: with the publish times (`time`) it shows, which +/// cost a proxy the upstream's full packument too. +fn dated_reads(name: &PackageName) -> Action { + let id = name.to_string(); + Action::Load(vec![Query::DatedPackages(id.clone()), Query::Tags(id)]) +} + +/// The search of `GET /-/v1/search?text=&size=&from=`. +fn search_query(req: &Request) -> SearchQuery { + let number = |key: &str, default: u32| { + req.query_param(key) + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) + }; + SearchQuery { + text: req.query_param("text").unwrap_or_default(), + skip: number("from", 0), + take: number("size", 20).clamp(1, 250), + prerelease: true, + semver2: true, + upstream: true, + } +} + +/// What a request is: see the module documentation. +pub fn handle(npm: &Npm, repo: &Repository, req: &Request) -> Action { + let route = match parse(&req.path) { + Ok(route) => route, + Err(RouteError::NotFound) => return respond(error(404, "not found")), + Err(RouteError::BadRequest(m)) => return respond(error(400, m)), + }; + let document_limit = npm.limits().max_document_bytes; + let body = req.body.as_deref(); + match (req.method.as_str(), route) { + ("GET" | "HEAD", Route::Ping) => respond(json_response(200, &json!({}))), + ("GET", Route::Whoami) => Action::Auth(AuthRequest::Whoami), + ("GET", Route::Search) => Action::Load(vec![Query::Search(search_query(req))]), + ("POST", Route::WebLogin) => match body { + None => Action::ReadBody(BodyRequest { + limit: SMALL_BODY, + used_for: BodyUse::Login, + }), + Some(body) => { + let hostname = serde_json::from_slice::(body) + .ok() + .and_then(|v| v.get("hostname")?.as_str().map(str::to_string)) + .filter(|h| !h.is_empty() && h.len() <= 255); + Action::Auth(AuthRequest::StartWebLogin { hostname }) + } + }, + ("GET", Route::WebLoginDone(secret)) => Action::Auth(AuthRequest::PollWebLogin { secret }), + ("PUT", Route::UserLogin(user)) => match body { + None => Action::ReadBody(BodyRequest { + limit: SMALL_BODY, + used_for: BodyUse::Login, + }), + Some(body) => { + let doc: Value = serde_json::from_slice(body).unwrap_or(Value::Null); + let username = doc + .get("name") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or(user); + match doc.get("password").and_then(Value::as_str) { + Some(password) if !password.is_empty() => Action::Auth(AuthRequest::Password { + username, + password: password.to_string(), + }), + _ => respond(error(400, "the login has no password")), + } + } + }, + ("DELETE", Route::Logout(token)) => Action::Auth(AuthRequest::Logout { token }), + ("POST", Route::Audit(kind)) => { + if !AUDITS.contains(&kind.as_str()) { + return respond(error(404, "not found")); + } + match body { + None => Action::ReadBody(BodyRequest { + limit: document_limit, + used_for: BodyUse::Read, + }), + Some(body) => audit(repo, req, &kind, body), + } + } + ("GET", Route::DistTags(name)) => reads(&name), + ("PUT", Route::DistTag(name, tag)) => { + if !document::valid_tag(&tag) { + return respond(error( + 400, + format!("{tag:?} is not a dist-tag: use letters, digits, '.', '_' and '-', not starting with a digit"), + )); + } + let Some(body) = body else { + return Action::ReadBody(BodyRequest { + limit: SMALL_BODY, + used_for: BodyUse::Change, + }); + }; + let raw = match serde_json::from_slice::(body) { + Ok(Value::String(v)) => v, + _ => String::from_utf8_lossy(body).trim().to_string(), + }; + match version::normalize(&raw) { + Ok(version) => Action::Change(ChangeSet { + changes: vec![Change::Tag { + id: name.to_string(), + tag, + version, + }], + done: ok(201), + }), + Err(_) => respond(error(400, format!("{raw:?} is not a version"))), + } + } + ("DELETE", Route::DistTag(name, tag)) => { + if tag == LATEST { + return respond(error( + 400, + "the latest tag cannot be removed; point it at another version", + )); + } + Action::Change(ChangeSet { + changes: vec![Change::Untag { + id: name.to_string(), + tag, + }], + done: ok(200), + }) + } + ("GET" | "HEAD", Route::Package(name)) if wants_abbreviated(req) => reads(&name), + ("GET" | "HEAD", Route::Package(name)) => dated_reads(&name), + ("PUT", Route::Package(name)) => match body { + // A publish, unless the document carries no tarball: then it is + // an update, and the host asks again with it. + None => Action::Publish(PublishRequest { + upload: Upload::Document, + id: Some(name.to_string()), + done: ok(201), + }), + Some(_) => reads(&name), + }, + ("PUT", Route::PackageRev(name)) => match body { + None => Action::ReadBody(BodyRequest { + limit: document_limit, + used_for: BodyUse::Change, + }), + Some(_) => reads(&name), + }, + ("DELETE", Route::PackageRev(name)) => reads(&name), + ("GET" | "HEAD", Route::Version(name, _)) => reads(&name), + ("GET" | "HEAD", Route::Tarball(name, file)) => match tarball_version(&name, &file) { + Some(version) => Action::Download(PackageRef { + id: name.to_string(), + version, + }), + None => respond(error(404, format!("tarball {file} of {name} not found"))), + }, + ("DELETE", Route::TarballRev(name, file)) => match tarball_version(&name, &file) { + // npm unpublishes a version with a packument update first, and + // takes the 404 of this second request for done. + Some(version) => Action::Change(ChangeSet { + changes: vec![Change::Delete(PackageRef { + id: name.to_string(), + version, + })], + done: ok(200), + }), + None => respond(error(404, format!("tarball {file} of {name} not found"))), + }, + _ => respond(error(405, "method not allowed")), + } +} + +/// The version a tarball's file name (`name-1.0.0.tgz`) names. +fn tarball_version(name: &PackageName, file: &str) -> Option { + name.version_of_tarball(file) + .and_then(|v| version::normalize(v).ok()) +} + +/// An audit, to the upstream: a bulk request through a group without the +/// packages under its reserved prefixes (an internal scope's names stay +/// inside; the upstream knows no advisories of them anyway). +fn audit(repo: &Repository, req: &Request, kind: &str, body: &[u8]) -> Action { + let mut body = body.to_vec(); + if kind == "advisories/bulk" && !repo.reserved_prefixes.is_empty() { + if let Some(filtered) = without_reserved(&body, &repo.reserved_prefixes) { + body = filtered; + } + } + let query = if req.query.is_empty() { + String::new() + } else { + format!("?{}", req.query) + }; + Action::Forward(Forward { + path: format!("/-/npm/v1/security/{kind}{query}"), + content_type: "application/json".into(), + body, + }) +} + +/// A bulk advisory request (`{name: [versions]}`) without the packages +/// under `prefixes`. +pub fn without_reserved(body: &[u8], prefixes: &[String]) -> Option> { + let Ok(Value::Object(mut packages)) = serde_json::from_slice::(body) else { + return None; + }; + packages.retain(|name, _| { + let lower = name.to_ascii_lowercase(); + !prefixes.iter().any(|p| lower.starts_with(p.as_str())) + }); + serde_json::to_vec(&Value::Object(packages)).ok() +} + +/// The versions of `id` loaded, if they were. +fn packages<'a>(data: &'a [Loaded], id: &str) -> Option<&'a [PackageInfo]> { + data.iter() + .find_map(|loaded| match (&loaded.query, &loaded.answer) { + (Query::Packages(q) | Query::DatedPackages(q), Answer::Packages(infos)) if q == id => { + Some(infos.as_slice()) + } + _ => None, + }) +} + +/// The tags of `id` loaded (none when they were not). +fn tags(data: &[Loaded], id: &str) -> BTreeMap { + data.iter() + .find_map(|loaded| match (&loaded.query, &loaded.answer) { + (Query::Tags(q), Answer::Tags(tags)) if q == id => Some( + tags.iter() + .map(|t| (t.name.clone(), t.version.clone())) + .collect(), + ), + _ => None, + }) + .unwrap_or_default() +} + +/// The manifest of `target` loaded: `None` when it was not asked for yet. +fn loaded_manifest<'a>(data: &'a [Loaded], target: &PackageRef) -> Option> { + data.iter() + .find_map(|loaded| match (&loaded.query, &loaded.answer) { + (Query::Manifest(q), Answer::Manifest(m)) if q == target => Some(m.as_deref()), + _ => None, + }) +} + +/// A version's summary as the packument lists it: its deprecation is the +/// one Forge stores, when there is one. +fn summary_of(info: &PackageInfo) -> Map { + let mut summary = info + .summary + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()) + .and_then(|v| match v { + Value::Object(m) => Some(m), + _ => None, + }) + .unwrap_or_default(); + if let Some(message) = &info.deprecated { + summary.insert("deprecated".into(), Value::String(message.clone())); + } + summary +} + +fn published(info: &PackageInfo) -> Option> { + info.published + .as_deref() + .and_then(|t| DateTime::parse_from_rfc3339(t).ok()) + .map(|t| t.with_timezone(&Utc)) +} + +/// What the repository serves of `name`: its loaded versions and tags. +fn packument_of( + name: &PackageName, + infos: &[PackageInfo], + tags: BTreeMap, +) -> Packument { + Packument { + name: name.clone(), + versions: infos + .iter() + .map(|info| PackumentVersion { + version: info.version.clone(), + summary: summary_of(info), + published: published(info), + }) + .collect(), + dist_tags: tags, + readme: None, + } +} + +/// The response to a request [`handle`] asked data for. +pub fn render(npm: &Npm, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + let Ok(route) = parse(&req.path) else { + return Action::Respond(error(404, "not found")); + }; + let name = match &route { + Route::Search => return Action::Respond(search(data)), + Route::DistTags(name) + | Route::Package(name) + | Route::PackageRev(name) + | Route::Version(name, _) => name, + _ => return Action::Respond(error(500, "nothing to render")), + }; + let id = name.to_string(); + let infos = packages(data, &id).unwrap_or_default(); + let not_found = || Action::Respond(error(404, format!("package {name} not found"))); + if infos.is_empty() { + return not_found(); + } + let mut packument = packument_of(name, infos, tags(data, &id)); + match (req.method.as_str(), &route) { + (_, Route::DistTags(_)) => { + Action::Respond(json_response(200, &json!(packument.served_dist_tags()))) + } + ("GET" | "HEAD", Route::Package(_)) => { + let abbreviated = wants_abbreviated(req); + // The full document carries the latest version's README. + if !abbreviated { + if let Some(latest) = packument.served_dist_tags().get(LATEST) { + let target = PackageRef { + id: id.clone(), + version: latest.clone(), + }; + match loaded_manifest(data, &target) { + None => return Action::Load(vec![Query::Manifest(target)]), + Some(manifest) => { + packument.readme = manifest + .and_then(|m| forge_plugin_api::FormatPlugin::readme(npm, m)) + } + } + } + } + let doc = packument.render(&repo.base_url, abbreviated); + let content_type = if abbreviated { + ABBREVIATED_MEDIA_TYPE + } else { + "application/json" + }; + Action::Respond( + Response::new( + 200, + content_type, + serde_json::to_vec(&doc).unwrap_or_default(), + ) + .with_header("vary", "Accept"), + ) + } + ("GET" | "HEAD", Route::Version(_, spec)) => { + let version = match packument.served_dist_tags().get(spec.as_str()) { + Some(v) => v.clone(), + None => match version::normalize(spec) { + Ok(v) => v, + Err(_) => { + return Action::Respond(error( + 404, + format!("version {spec:?} of {name} not found"), + )) + } + }, + }; + match packument.render_version(&repo.base_url, &version) { + Some(doc) => Action::Respond(json_response(200, &doc)), + None => { + Action::Respond(error(404, format!("version {version} of {name} not found"))) + } + } + } + ("PUT", Route::Package(_) | Route::PackageRev(_)) => update(req, name, infos, data), + ("DELETE", Route::PackageRev(_)) => Action::Change(ChangeSet { + changes: infos + .iter() + .map(|info| { + Change::Delete(PackageRef { + id: id.clone(), + version: info.version.clone(), + }) + }) + .collect(), + done: ok(200), + }), + _ => Action::Respond(error(405, "method not allowed")), + } +} + +/// A packument update (a `PUT` without a tarball): `npm deprecate` +/// (versions' `deprecated`), `npm unpublish ` (versions left out) +/// and dist-tag changes, as changes to make. +fn update(req: &Request, name: &PackageName, infos: &[PackageInfo], data: &[Loaded]) -> Action { + let bad = |m: String| Action::Respond(error(400, m)); + let Some(body) = req.body.as_deref() else { + return bad("the document is missing".into()); + }; + let doc: Value = match serde_json::from_slice(body) { + Ok(doc) => doc, + Err(e) => return bad(format!("the document is not valid JSON: {e}")), + }; + let update = match document::read_update(&doc, name) { + Ok(update) => update, + Err(e) => return bad(e), + }; + let id = name.to_string(); + let target = |version: &str| PackageRef { + id: id.clone(), + version: version.to_string(), + }; + let removed: Vec<&PackageInfo> = infos + .iter() + .filter(|info| !update.versions.contains_key(&info.version)) + .collect(); + if removed.len() == infos.len() { + // npm unpublishes a package's last version with `DELETE /-rev`. + return bad( + "the document leaves the package no version: unpublish the package instead".into(), + ); + } + // Leaving versions out removes them: the delete permission's. + let mut changes: Vec = removed + .iter() + .map(|info| Change::Delete(target(&info.version))) + .collect(); + for info in infos { + let Some(wanted) = update.versions.get(&info.version) else { + continue; + }; + let summary = summary_of(info); + if manifest::deprecation(&summary) != wanted.as_deref() { + changes.push(Change::Deprecate { + target: target(&info.version), + message: wanted.clone(), + }); + } + } + if update.dist_tags_given { + let current = tags(data, &id); + for (tag, version) in &update.dist_tags { + let kept = infos + .iter() + .any(|info| &info.version == version && update.versions.contains_key(version)); + if kept && current.get(tag) != Some(version) { + changes.push(Change::Tag { + id: id.clone(), + tag: tag.clone(), + version: version.clone(), + }); + } + } + for (tag, version) in ¤t { + // Deleting a version drops its tags already. + let deleted = removed.iter().any(|info| &info.version == version); + if !deleted && !update.dist_tags.contains_key(tag) { + changes.push(Change::Untag { + id: id.clone(), + tag: tag.clone(), + }); + } + } + } + if changes.is_empty() { + return Action::Respond(ok(200)); + } + Action::Change(ChangeSet { + changes, + done: ok(200), + }) +} + +/// `GET /-/v1/search`: the results as npm prints them. +fn search(data: &[Loaded]) -> Response { + let Some(result) = data.iter().find_map(|loaded| match &loaded.answer { + Answer::Search(result) => Some(result), + _ => None, + }) else { + return error(500, "no search results"); + }; + let objects: Vec = result + .hits + .iter() + .map(|hit| { + let summary: Map = hit + .summary + .as_deref() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or_default(); + let scope = PackageName::parse(&hit.original_id) + .ok() + .and_then(|n| n.scope().map(str::to_string)) + .unwrap_or_else(|| "unscoped".into()); + let mut package = Map::new(); + package.insert("name".into(), json!(hit.original_id)); + package.insert("scope".into(), json!(scope)); + package.insert("version".into(), json!(hit.latest_version)); + for key in ["description", "keywords", "date", "links", "author"] { + if let Some(value) = summary.get(key) { + package.insert(key.into(), value.clone()); + } + } + // Search results name people by `username` (a packument by + // `name`), and clients expect the list. + let person = |p: &Value| { + let username = p + .get("username") + .or_else(|| p.get("name")) + .and_then(Value::as_str)?; + let mut out = json!({ "username": username }); + if let Some(email) = p.get("email").and_then(Value::as_str) { + out["email"] = json!(email); + } + Some(out) + }; + let maintainers: Vec = summary + .get("maintainers") + .and_then(Value::as_array) + .map(|m| m.iter().filter_map(person).collect()) + .unwrap_or_default(); + package.insert("maintainers".into(), Value::Array(maintainers)); + if let Some(publisher) = summary + .get("publisher") + .or_else(|| summary.get("_npmUser")) + .and_then(person) + { + package.insert("publisher".into(), publisher); + } + json!({ + "package": package, + "score": {"final": 1.0, "detail": {"quality": 1.0, "popularity": 1.0, "maintenance": 1.0}}, + "searchScore": 1.0, + }) + }) + .collect(); + json_response( + 200, + &json!({ + "objects": objects, + "total": result.total, + "time": Utc::now().to_rfc2822(), + }), + ) +} + +/// The response to a login, a logout or `whoami` the host performed. +pub fn auth_response(repo: &Repository, req: &Request, outcome: &AuthOutcome) -> Response { + match outcome { + AuthOutcome::Token { token, username } => match parse(&req.path) { + Ok(Route::UserLogin(_)) => json_response( + 201, + &json!({ + "ok": true, + "id": format!("org.couchdb.user:{username}"), + "rev": "_we_dont_use_revs_any_more", + "username": username, + "token": token, + }), + ), + _ => json_response(200, &json!({ "token": token })), + }, + AuthOutcome::WebLoginStarted { login_url, secret } => json_response( + 200, + &json!({ + "loginUrl": login_url, + "doneUrl": format!("{}/-/v1/login/done/{secret}", repo.base_url), + }), + ), + AuthOutcome::WebLoginPending => { + json_response(202, &json!({})).with_header("retry-after", POLL_SECONDS) + } + AuthOutcome::WebLoginUnknown => error(404, "no such login, or it expired"), + AuthOutcome::LoggedOut => json_response(200, &json!({ "ok": true })), + AuthOutcome::Identity { username } => json_response(200, &json!({ "username": username })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn name(raw: &str) -> PackageName { + PackageName::parse(raw).unwrap() + } + + #[test] + fn parses_routes() { + assert_eq!(parse("/-/ping"), Ok(Route::Ping)); + assert_eq!(parse("/left-pad"), Ok(Route::Package(name("left-pad")))); + assert_eq!(parse("/@acme%2fui"), Ok(Route::Package(name("@acme/ui")))); + assert_eq!(parse("/@acme%2Fui"), Ok(Route::Package(name("@acme/ui")))); + assert_eq!(parse("/@acme/ui"), Ok(Route::Package(name("@acme/ui")))); + assert_eq!( + parse("/@acme/ui/-/ui-1.0.0.tgz"), + Ok(Route::Tarball(name("@acme/ui"), "ui-1.0.0.tgz".into())) + ); + assert_eq!( + parse("/left-pad/-/left-pad-1.0.0.tgz/-rev/3-x"), + Ok(Route::TarballRev( + name("left-pad"), + "left-pad-1.0.0.tgz".into() + )) + ); + assert_eq!( + parse("/left-pad/-rev/3-x"), + Ok(Route::PackageRev(name("left-pad"))) + ); + assert_eq!( + parse("/left-pad/latest"), + Ok(Route::Version(name("left-pad"), "latest".into())) + ); + assert_eq!( + parse("/-/package/@acme%2fui/dist-tags/next"), + Ok(Route::DistTag(name("@acme/ui"), "next".into())) + ); + assert_eq!( + parse("/-/user/org.couchdb.user:alice@example.com"), + Ok(Route::UserLogin("alice@example.com".into())) + ); + assert_eq!(parse("/-/user/token/abc"), Ok(Route::Logout("abc".into()))); + assert_eq!( + parse("/-/npm/v1/security/advisories/bulk"), + Ok(Route::Audit("advisories/bulk".into())) + ); + assert_eq!( + parse("/-/v1/login/done/s3cret"), + Ok(Route::WebLoginDone("s3cret".into())) + ); + } + + #[test] + fn refuses_paths_that_are_no_package() { + for bad in [ + "/", + "/..%2f..%2fadmin", + "/%2e%2e", + "/a%2fb", + "/-/nope", + "/a//b", + "/@acme", + ] { + assert!(parse(bad).is_err(), "{bad}"); + } + assert!(matches!(parse("/%zz"), Err(RouteError::BadRequest(_)))); + } + + #[test] + fn abbreviated_metadata_is_asked_for() { + let req = |accept: &str| Request::new("GET", "/x").with_header("accept", accept); + assert!(wants_abbreviated(&req( + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*" + ))); + assert!(!wants_abbreviated(&req("application/json"))); + assert!(!wants_abbreviated(&req( + "application/vnd.npm.install-v1+json;q=0" + ))); + assert!(!wants_abbreviated(&Request::new("GET", "/x"))); + } + + #[test] + fn bulk_audits_keep_internal_names_inside() { + let body = br#"{"@acme/ui":["1.0.0"],"left-pad":["1.3.0"],"@ACME/core":["2.0.0"]}"#; + let filtered = without_reserved(body, &["@acme/".to_string()]).unwrap(); + let v: Value = serde_json::from_slice(&filtered).unwrap(); + assert_eq!(v, json!({"left-pad": ["1.3.0"]})); + assert!(without_reserved(b"[1]", &[]).is_none()); + } +} diff --git a/crates/formats/npm/src/proxy.rs b/crates/formats/npm/src/proxy.rs new file mode 100644 index 00000000..5bfbf502 --- /dev/null +++ b/crates/formats/npm/src/proxy.rs @@ -0,0 +1,406 @@ +//! The upstream side of npm proxies ([`FormatPlugin::upstream`]): reading +//! another registry's packuments, tarball locations and search. +//! +//! A package's listing, its tags, its tarballs and its versions' summaries +//! come from the abbreviated packument (`application/vnd.npm.install-v1+json`, +//! what installs read: much smaller than the full document). A version's +//! manifest (with the README of `latest`) and the publish times the package +//! delay counts from come from the full one. Both are fetched for the +//! repository's metadata TTL. A tarball is downloaded from where the +//! packument says, and the host checks it against the packument's +//! `dist.integrity` and `dist.shasum`. +//! +//! [`FormatPlugin::upstream`]: forge_plugin_api::FormatPlugin::upstream + +use forge_plugin_api::{ + Fetch, FetchResult, Fetched, Freshness, PackageLocation, PackageMetadata, PackageRef, Remote, + SearchQuery, Tag, UpstreamAnswer, UpstreamHit, UpstreamOp, UpstreamSearch, UpstreamStep, + VersionMetadata, +}; +use serde_json::{Map, Value}; + +use crate::packument::{timestamp, LATEST}; +use crate::upstream::{self, UpstreamPackument}; +use crate::{version, PackageName}; + +/// What a proxy asks for when it wants abbreviated metadata (as npm does). +pub const ABBREVIATED_ACCEPT: &str = + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*"; + +/// What it asks for when it wants the full document. +pub const FULL_ACCEPT: &str = "application/json"; + +fn done(answer: UpstreamAnswer) -> UpstreamStep { + UpstreamStep::Done(answer) +} + +fn failed(reason: impl Into) -> UpstreamStep { + done(UpstreamAnswer::Failed(reason.into())) +} + +/// The upstream's packument of `id`: `{remote}/{escaped name}`. +fn packument_fetch(remote: &Remote, id: &str, full: bool) -> Option { + let name = PackageName::parse(id).ok()?; + let url = format!("{}/{}", remote.url.trim_end_matches('/'), name.escaped()); + let accept = if full { + FULL_ACCEPT + } else { + ABBREVIATED_ACCEPT + }; + Some(Fetch::new(url, Freshness::Metadata).accepting(accept)) +} + +/// How `fetch` went, if it was sent. +fn outcome<'a>(fetched: &'a [FetchResult], fetch: &Fetch) -> Option<&'a Fetched> { + fetched + .iter() + .find(|f| &f.fetch == fetch) + .map(|f| &f.outcome) +} + +/// Answer `op` from the packument of `id` (full or abbreviated): fetch it +/// first, then hand it to `answer` (`None` when the upstream does not know +/// the package). +fn with_packument( + remote: &Remote, + id: &str, + full: bool, + fetched: &[FetchResult], + answer: impl FnOnce(Option<&UpstreamPackument>) -> UpstreamStep, +) -> UpstreamStep { + let Some(fetch) = packument_fetch(remote, id, full) else { + return failed(format!("{id:?} is not a package name")); + }; + match outcome(fetched, &fetch) { + None => UpstreamStep::Fetch(vec![fetch]), + Some(Fetched::NotFound) => answer(None), + Some(Fetched::Failed(reason)) => failed(reason.clone()), + Some(Fetched::Found(body)) => match upstream::read(&body[..]) { + Ok(doc) if doc.name.is_empty() || doc.name == id => answer(Some(&doc)), + Ok(doc) => failed(format!( + "the upstream answered package {:?} for {id:?}", + doc.name + )), + Err(e) => failed(format!("the upstream's packument of {id}: {e}")), + }, + } +} + +/// The SHA-512 of an SRI `integrity` (`sha512-`), if it has one. +fn sha512_of(integrity: &str) -> Option { + integrity + .split_whitespace() + .find_map(|sri| sri.strip_prefix("sha512-")) + .map(|b64| b64.split('?').next().unwrap_or(b64).to_string()) +} + +/// `s` percent-encoded for a query string. +fn encode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} + +/// One round of the upstream conversation for `op`. +pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> UpstreamStep { + match op { + // A registry needs no index: packuments are at fixed URLs. + UpstreamOp::Index => done(UpstreamAnswer::Index("{}".into())), + UpstreamOp::Versions(id) => with_packument(remote, id, false, fetched, |doc| { + done(UpstreamAnswer::Versions(doc.map(|doc| { + doc.versions().into_iter().map(|v| v.version).collect() + }))) + }), + UpstreamOp::Tags(id) => with_packument(remote, id, false, fetched, |doc| { + done(UpstreamAnswer::Tags(doc.map(|doc| { + doc.normalized_dist_tags() + .into_iter() + .map(|(name, version)| Tag { name, version }) + .collect() + }))) + }), + UpstreamOp::Package(target) => with_packument(remote, &target.id, false, fetched, |doc| { + let Some(v) = doc.and_then(|doc| doc.version(&target.version)) else { + return failed(format!( + "the upstream has no version {} of {}", + target.version, target.id + )); + }; + match v.tarball() { + Some(url) if url.starts_with("https://") || url.starts_with("http://") => { + done(UpstreamAnswer::Located(PackageLocation { + url: url.to_string(), + sha512: v.integrity().and_then(sha512_of), + sha1: v.shasum().map(str::to_ascii_lowercase), + })) + } + _ => failed(format!( + "the upstream names no tarball of {} {}", + target.id, target.version + )), + } + }), + UpstreamOp::Manifest(target) => with_packument(remote, &target.id, true, fetched, |doc| { + done(UpstreamAnswer::Manifest( + doc.and_then(|doc| manifest_of(doc, target)), + )) + }), + UpstreamOp::Metadata(set) => with_packument(remote, &set.id, false, fetched, |doc| { + done(UpstreamAnswer::Metadata(metadata( + doc, + &set.versions, + false, + ))) + }), + UpstreamOp::Dating(set) => with_packument(remote, &set.id, true, fetched, |doc| { + done(UpstreamAnswer::Metadata(metadata(doc, &set.versions, true))) + }), + UpstreamOp::Search(query) => search(remote, query, fetched), + // A registry holds packages: it has no files to fetch. + UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), + } +} + +/// A version's manifest, the README of `latest` with it, when the upstream +/// has exactly that version of that package. +fn manifest_of(doc: &UpstreamPackument, target: &PackageRef) -> Option> { + let v = doc.version(&target.version)?; + let mut manifest: Map = v.manifest.clone(); + if manifest.get("name").and_then(Value::as_str) != Some(target.id.as_str()) { + return None; + } + if version::normalize(manifest.get("version")?.as_str()?).ok()? != target.version { + return None; + } + let latest = doc.normalized_dist_tags().get(LATEST).cloned(); + if latest.as_deref() == Some(target.version.as_str()) && !manifest.contains_key("readme") { + if let Some(readme) = &doc.readme { + manifest.insert("readme".into(), Value::String(readme.clone())); + } + } + serde_json::to_vec(&manifest).ok() +} + +/// The metadata of `wanted` versions: their summaries, and with `dating` +/// when the upstream published them. +fn metadata(doc: Option<&UpstreamPackument>, wanted: &[String], dating: bool) -> PackageMetadata { + let Some(doc) = doc else { + return PackageMetadata::default(); + }; + PackageMetadata { + original_id: (!doc.name.is_empty()).then(|| doc.name.clone()), + versions: doc + .versions() + .into_iter() + .filter(|v| wanted.contains(&v.version)) + .map(|v| { + let time = if dating { + v.published.map(timestamp) + } else { + None + }; + VersionMetadata { + original_id: (!doc.name.is_empty()).then(|| doc.name.clone()), + listed: true, + published: time.clone(), + created: time, + summary: serde_json::to_string(&v.summary()).ok(), + version: v.version, + } + }) + .collect(), + } +} + +/// A page of the upstream's `/-/v1/search`. +fn search(remote: &Remote, query: &SearchQuery, fetched: &[FetchResult]) -> UpstreamStep { + let take = match query.take { + 0 => 20, + n => n.min(250), + }; + let url = format!( + "{}/-/v1/search?text={}&size={take}&from={}", + remote.url.trim_end_matches('/'), + encode(&query.text), + query.skip + ); + let fetch = Fetch::new(url, Freshness::Metadata).accepting(FULL_ACCEPT); + let body = match outcome(fetched, &fetch) { + None => return UpstreamStep::Fetch(vec![fetch]), + Some(Fetched::NotFound) => return done(UpstreamAnswer::Search(None)), + Some(Fetched::Failed(reason)) => return failed(reason.clone()), + Some(Fetched::Found(body)) => body, + }; + let Ok(page) = serde_json::from_slice::(body) else { + return failed("the upstream's search answer is not JSON"); + }; + let hits = page + .get("objects") + .and_then(Value::as_array) + .map(|objects| { + objects + .iter() + .filter_map(|object| { + let package = object.get("package")?.as_object()?; + let id = package.get("name")?.as_str()?; + PackageName::parse(id).ok()?; + let latest = version::normalize(package.get("version")?.as_str()?).ok()?; + Some(UpstreamHit { + id: id.to_string(), + versions: vec![latest], + summary: serde_json::to_string(package).ok(), + total_downloads: None, + }) + }) + .collect() + }) + .unwrap_or_default(); + done(UpstreamAnswer::Search(Some(UpstreamSearch { + total: page.get("total").and_then(Value::as_u64).unwrap_or(0), + hits, + }))) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::testing::converse; + use forge_plugin_api::FormatPlugin; + use serde_json::json; + + fn remote() -> Remote { + Remote { + url: "https://registry.example.com/".into(), + index: Some("{}".into()), + } + } + + fn doc() -> Value { + json!({ + "name": "@acme/ui", + "dist-tags": {"latest": "1.1.0", "next": "2.0.0-rc.1"}, + "time": {"1.0.0": "2020-01-01T00:00:00.000Z", "1.1.0": "2021-01-01T00:00:00.000Z"}, + "readme": "# UI", + "versions": { + "1.0.0": {"name": "@acme/ui", "version": "1.0.0", "dist": {"tarball": "https://cdn.example.com/ui-1.0.0.tgz", "integrity": "sha512-AAAA", "shasum": "ABCD"}}, + "1.1.0": {"name": "@acme/ui", "version": "1.1.0", "dependencies": {"b": "^1"}, "dist": {"tarball": "https://cdn.example.com/ui-1.1.0.tgz"}}, + "2.0.0-rc.1": {"name": "@acme/ui", "version": "2.0.0-rc.1", "dist": {}} + } + }) + } + + fn found(fetch: &Fetch) -> Fetched { + assert_eq!(fetch.url, "https://registry.example.com/@acme%2fui"); + Fetched::Found(serde_json::to_vec(&doc()).unwrap()) + } + + fn run(op: UpstreamOp) -> (UpstreamAnswer, Vec) { + let conversed = converse(&crate::Npm::default(), &remote(), &op, found).unwrap(); + (conversed.answer, conversed.fetches) + } + + #[test] + fn lists_versions_and_tags_from_the_abbreviated_packument() { + let (answer, fetches) = run(UpstreamOp::Versions("@acme/ui".into())); + assert_eq!( + answer, + UpstreamAnswer::Versions(Some(vec![ + "1.0.0".into(), + "1.1.0".into(), + "2.0.0-rc.1".into() + ])) + ); + assert_eq!(fetches[0].accept.as_deref(), Some(ABBREVIATED_ACCEPT)); + let (answer, _) = run(UpstreamOp::Tags("@acme/ui".into())); + let UpstreamAnswer::Tags(Some(tags)) = answer else { + panic!("{answer:?}") + }; + assert!(tags.contains(&Tag { + name: "next".into(), + version: "2.0.0-rc.1".into() + })); + } + + #[test] + fn locates_tarballs_with_their_digests() { + let (answer, _) = run(UpstreamOp::Package(PackageRef { + id: "@acme/ui".into(), + version: "1.0.0".into(), + })); + assert_eq!( + answer, + UpstreamAnswer::Located(PackageLocation { + url: "https://cdn.example.com/ui-1.0.0.tgz".into(), + sha512: Some("AAAA".into()), + sha1: Some("abcd".into()), + }) + ); + let (answer, _) = run(UpstreamOp::Package(PackageRef { + id: "@acme/ui".into(), + version: "2.0.0-rc.1".into(), + })); + assert!(matches!(answer, UpstreamAnswer::Failed(_)), "{answer:?}"); + } + + #[test] + fn manifests_carry_the_readme_of_latest_only() { + let (answer, fetches) = run(UpstreamOp::Manifest(PackageRef { + id: "@acme/ui".into(), + version: "1.1.0".into(), + })); + assert_eq!(fetches[0].accept.as_deref(), Some(FULL_ACCEPT)); + let UpstreamAnswer::Manifest(Some(manifest)) = answer else { + panic!("{answer:?}") + }; + assert_eq!( + crate::Npm::default().readme(&manifest).as_deref(), + Some("# UI") + ); + let (answer, _) = run(UpstreamOp::Manifest(PackageRef { + id: "@acme/ui".into(), + version: "1.0.0".into(), + })); + let UpstreamAnswer::Manifest(Some(manifest)) = answer else { + panic!("{answer:?}") + }; + assert_eq!(crate::Npm::default().readme(&manifest), None); + } + + #[test] + fn dates_versions_for_the_package_delay() { + let (answer, _) = run(UpstreamOp::Dating(forge_plugin_api::PackageVersions { + id: "@acme/ui".into(), + versions: vec!["1.1.0".into()], + })); + let UpstreamAnswer::Metadata(metadata) = answer else { + panic!("{answer:?}") + }; + assert_eq!(metadata.versions.len(), 1); + assert_eq!( + metadata.versions[0].created.as_deref(), + Some("2021-01-01T00:00:00.000Z") + ); + assert!(metadata.versions[0] + .summary + .as_deref() + .unwrap() + .contains("dependency_groups")); + } + + #[test] + fn an_unknown_package_is_none() { + let conversed = converse( + &crate::Npm::default(), + &remote(), + &UpstreamOp::Versions("left-pad".into()), + |_| Fetched::NotFound, + ) + .unwrap(); + assert_eq!(conversed.answer, UpstreamAnswer::Versions(None)); + } +} diff --git a/crates/formats/npm/src/publish.rs b/crates/formats/npm/src/publish.rs new file mode 100644 index 00000000..c4b7652a --- /dev/null +++ b/crates/formats/npm/src/publish.rs @@ -0,0 +1,639 @@ +//! The body of `npm publish`: a packument document carrying the tarball +//! base64-encoded in `_attachments..data`. +//! +//! [`Extractor`] reads the body as it arrives, chunk by chunk, and splits it +//! in two without ever holding the tarball in memory: the decoded tarball, +//! handed to the caller as it is decoded (to be staged on disk), and the +//! document with the attachment's `data` replaced by `""`, small enough to +//! parse. It is a push parser (no I/O of its own), so it runs on the async +//! task that reads the request body. +//! +//! The document is copied token by token, without the whitespace between +//! tokens. The extractor delimits strings, objects and arrays itself and +//! leaves numbers and literals to the JSON parser the caller runs on the +//! document afterwards, which also catches anything else malformed. + +use base64::alphabet; +use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}; +use base64::Engine as _; + +/// The deepest nesting accepted. +const MAX_DEPTH: usize = 64; + +/// Base64 decoded in blocks of this many characters (a multiple of 4). +const DECODE_BLOCK: usize = 64 * 1024; + +/// Keys longer than this are not decoded: no key the extractor looks for is. +const MAX_MATCHED_KEY: usize = 64; + +/// npm's base64 (Node's `Buffer#toString('base64')`), padded or not. +const BASE64: GeneralPurpose = GeneralPurpose::new( + &alphabet::STANDARD, + GeneralPurposeConfig::new().with_decode_padding_mode(DecodePaddingMode::Indifferent), +); + +/// Why a publish body is refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtractError { + /// The body is not a JSON document of the expected shape. + Malformed(String), + /// The document, without the tarball, is larger than allowed. + DocumentTooLarge(usize), +} + +impl std::fmt::Display for ExtractError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExtractError::Malformed(m) => write!(f, "malformed publish document: {m}"), + ExtractError::DocumentTooLarge(limit) => write!( + f, + "the publish document is larger than {limit} bytes (without its tarball)" + ), + } + } +} + +impl std::error::Error for ExtractError {} + +/// What the extractor made of a body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Extracted { + /// The document, its attachment's `data` replaced by `""`. + pub document: Vec, + /// The attachment whose data was extracted: its name (the key under + /// `_attachments`) and the number of bytes decoded. + pub attachment: Option<(String, u64)>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Container { + Object, + Array, +} + +#[derive(Debug)] +struct Frame { + container: Container, + /// The key of the member being read (objects), decoded when short. + key: Option, + /// Whether no member or element has been read yet. + empty: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + Key, + Value, + Data, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + /// A value is expected. + Value, + /// A key (or the end of an empty object) is expected. + Key, + /// The `:` after a key is expected. + Colon, + /// A `,` or the end of the container is expected (or the end of the + /// body, at the top level). + After, + /// Inside a string; `escaped` after a backslash, `unicode` counting the + /// hex digits of a `\u` escape still to come. + Str { + role: Role, + escaped: bool, + unicode: u8, + }, + /// Inside a number or a literal (`true`, `false`, `null`). + Literal, + /// The top-level value is complete. + Done, +} + +/// Splits a publish body into its document and its tarball; see the module +/// documentation. +pub struct Extractor { + max_document: usize, + document: Vec, + stack: Vec, + state: State, + key: Vec, + /// Base64 characters of the data not decoded yet. + pending: Vec, + /// The `\u` escape being read inside the data. + unicode: Vec, + attachment: Option<(String, u64)>, + offset: u64, +} + +impl Extractor { + /// An extractor refusing a document over `max_document` bytes. + pub fn new(max_document: usize) -> Self { + Extractor { + max_document, + document: Vec::new(), + stack: Vec::new(), + state: State::Value, + key: Vec::new(), + pending: Vec::new(), + unicode: Vec::new(), + attachment: None, + offset: 0, + } + } + + /// Whether the extractor is inside the attachment's data, or past it. + pub fn found_attachment(&self) -> bool { + self.attachment.is_some() + } + + fn malformed(&self, what: &str) -> ExtractError { + ExtractError::Malformed(format!("{what} at byte {}", self.offset)) + } + + fn emit(&mut self, bytes: &[u8]) -> Result<(), ExtractError> { + if self.document.len() + bytes.len() > self.max_document { + return Err(ExtractError::DocumentTooLarge(self.max_document)); + } + self.document.extend_from_slice(bytes); + Ok(()) + } + + /// Whether a string value starting now is an attachment's data: + /// `_attachments..data` at the top level. + fn at_data(&self) -> bool { + matches!( + self.stack.as_slice(), + [root, attachments, attachment] + if root.container == Container::Object + && root.key.as_deref() == Some("_attachments") + && attachments.container == Container::Object + && attachment.container == Container::Object + && attachment.key.as_deref() == Some("data") + ) + } + + /// The state after a value ends. + fn value_done(&mut self) -> State { + if self.stack.is_empty() { + State::Done + } else { + State::After + } + } + + /// Decode the complete base64 blocks of the data (all of it at its + /// end), appending the bytes to `out`. + fn decode(&mut self, out: &mut Vec, last: bool) -> Result<(), ExtractError> { + let take = if last { + self.pending.len() + } else { + self.pending.len() - self.pending.len() % 4 + }; + if take == 0 { + return Ok(()); + } + let start = out.len(); + BASE64 + .decode_vec(&self.pending[..take], out) + .map_err(|e| self.malformed(&format!("invalid base64 in the attachment ({e})")))?; + if let Some((_, len)) = &mut self.attachment { + *len += (out.len() - start) as u64; + } + self.pending.drain(..take); + Ok(()) + } + + /// Read `chunk`, appending the tarball bytes it decodes to `out`. + pub fn feed(&mut self, chunk: &[u8], out: &mut Vec) -> Result<(), ExtractError> { + let mut i = 0; + while i < chunk.len() { + let b = chunk[i]; + match self.state { + State::Str { + role, + escaped, + unicode, + } => { + if role == Role::Data { + // Copy a run of plain base64 characters at once. + if !escaped && unicode == 0 && b != b'"' && b != b'\\' { + let run = chunk[i..] + .iter() + .position(|&c| c == b'"' || c == b'\\') + .unwrap_or(chunk.len() - i); + self.pending.extend_from_slice(&chunk[i..i + run]); + if self.pending.len() >= DECODE_BLOCK { + self.decode(out, false)?; + } + i += run; + self.offset += run as u64; + continue; + } + self.data_byte(b, escaped, unicode, out)?; + } else { + self.string_byte(b, role, escaped, unicode)?; + } + } + State::Literal => { + if matches!(b, b',' | b'}' | b']' | b' ' | b'\t' | b'\r' | b'\n') { + self.state = self.value_done(); + // Re-read this byte in the new state. + continue; + } + self.emit(&[b])?; + } + _ if matches!(b, b' ' | b'\t' | b'\r' | b'\n') => {} + State::Done => return Err(self.malformed("data after the document")), + State::Value => self.value_start(b)?, + State::Key => match b { + b'"' => { + self.emit(b"\"")?; + self.key.clear(); + self.state = State::Str { + role: Role::Key, + escaped: false, + unicode: 0, + }; + } + b'}' if self.stack.last().is_some_and(|f| f.empty) => { + self.close(b, Container::Object)? + } + _ => return Err(self.malformed("expected a key")), + }, + State::Colon => { + if b != b':' { + return Err(self.malformed("expected ':'")); + } + self.emit(b":")?; + self.state = State::Value; + } + State::After => match (b, self.stack.last().map(|f| f.container)) { + (b',', Some(container)) => { + self.emit(b",")?; + self.state = match container { + Container::Object => State::Key, + Container::Array => State::Value, + }; + } + (b'}', Some(Container::Object)) => self.close(b, Container::Object)?, + (b']', Some(Container::Array)) => self.close(b, Container::Array)?, + _ => return Err(self.malformed("expected ',' or the end of a container")), + }, + } + i += 1; + self.offset += 1; + } + Ok(()) + } + + fn value_start(&mut self, b: u8) -> Result<(), ExtractError> { + if b == b']' { + // Only an empty array may end where a value is expected. + return match self.stack.last() { + Some(frame) if frame.container == Container::Array && frame.empty => { + self.close(b, Container::Array) + } + _ => Err(self.malformed("expected a value")), + }; + } + if let Some(frame) = self.stack.last_mut() { + frame.empty = false; + } + match b { + b'{' | b'[' => { + if self.stack.len() >= MAX_DEPTH { + return Err(self.malformed("nested too deeply")); + } + self.emit(&[b])?; + let container = if b == b'{' { + Container::Object + } else { + Container::Array + }; + self.stack.push(Frame { + container, + key: None, + empty: true, + }); + self.state = match container { + Container::Object => State::Key, + Container::Array => State::Value, + }; + } + b'"' => { + let role = if self.at_data() { + if self.attachment.is_some() { + return Err(self.malformed("a publish carries one tarball")); + } + let name = self.stack[1].key.clone().unwrap_or_default(); + self.attachment = Some((name, 0)); + Role::Data + } else { + Role::Value + }; + self.emit(b"\"")?; + self.state = State::Str { + role, + escaped: false, + unicode: 0, + }; + } + b'-' | b'0'..=b'9' | b't' | b'f' | b'n' => { + self.emit(&[b])?; + self.state = State::Literal; + } + _ => return Err(self.malformed("expected a value")), + } + Ok(()) + } + + fn close(&mut self, b: u8, container: Container) -> Result<(), ExtractError> { + match self.stack.pop() { + Some(frame) if frame.container == container => {} + _ => return Err(self.malformed("unbalanced brackets")), + } + self.emit(&[b])?; + self.state = self.value_done(); + Ok(()) + } + + /// A byte of a key or of a string value: copied as it is. + fn string_byte( + &mut self, + b: u8, + role: Role, + escaped: bool, + unicode: u8, + ) -> Result<(), ExtractError> { + if b < 0x20 { + return Err(self.malformed("a control character in a string")); + } + self.emit(&[b])?; + if role == Role::Key && self.key.len() <= MAX_MATCHED_KEY { + self.key.push(b); + } + let (escaped, unicode) = match (escaped, unicode, b) { + (false, _, b'"') if unicode == 0 => { + if role == Role::Key { + let key = self.decoded_key(); + if let Some(frame) = self.stack.last_mut() { + frame.key = key; + } + self.state = State::Colon; + } else { + self.state = self.value_done(); + } + return Ok(()); + } + (false, 0, b'\\') => (true, 0), + (true, _, b'u') => (false, 4), + (true, _, _) => (false, 0), + (false, n, _) if n > 0 => { + if !b.is_ascii_hexdigit() { + return Err(self.malformed("a bad \\u escape")); + } + (false, n - 1) + } + (false, _, _) => (false, 0), + }; + self.state = State::Str { + role, + escaped, + unicode, + }; + Ok(()) + } + + /// The key just read, decoded, unless it is too long to be one the + /// extractor looks for. + fn decoded_key(&self) -> Option { + if self.key.len() > MAX_MATCHED_KEY { + return None; + } + // `self.key` ends with the closing quote. + let mut quoted = Vec::with_capacity(self.key.len() + 1); + quoted.push(b'"'); + quoted.extend_from_slice(&self.key); + serde_json::from_slice::("ed).ok() + } + + /// A byte of the attachment's data that is not a plain base64 character. + fn data_byte( + &mut self, + b: u8, + escaped: bool, + unicode: u8, + out: &mut Vec, + ) -> Result<(), ExtractError> { + if unicode > 0 { + if !b.is_ascii_hexdigit() { + return Err(self.malformed("a bad \\u escape")); + } + self.unicode.push(b); + if unicode == 1 { + let code = std::str::from_utf8(&self.unicode).ok().and_then(|hex| { + u8::from_str_radix(hex, 16) + .ok() + .filter(|_| hex.starts_with("00")) + }); + self.unicode.clear(); + match code { + Some(c) if c.is_ascii_alphanumeric() || matches!(c, b'+' | b'/' | b'=') => { + self.pending.push(c) + } + Some(b'\n' | b'\r') => {} + _ => return Err(self.malformed("a non-base64 character in the attachment")), + } + } + self.state = State::Str { + role: Role::Data, + escaped: false, + unicode: unicode - 1, + }; + return Ok(()); + } + if escaped { + let unicode = match b { + b'/' => { + self.pending.push(b'/'); + 0 + } + // Line breaks some encoders wrap base64 with. + b'n' | b'r' => 0, + b'u' => 4, + _ => return Err(self.malformed("a non-base64 escape in the attachment")), + }; + self.state = State::Str { + role: Role::Data, + escaped: false, + unicode, + }; + return Ok(()); + } + match b { + b'\\' => { + self.state = State::Str { + role: Role::Data, + escaped: true, + unicode: 0, + }; + } + b'"' => { + self.decode(out, true)?; + self.emit(b"\"")?; + self.state = self.value_done(); + } + _ => unreachable!("plain characters are copied in runs"), + } + Ok(()) + } + + /// The body is complete: the document, and the attachment found. + pub fn finish(self) -> Result { + match self.state { + State::Done => {} + // A top-level number ends with the body. + State::Literal if self.stack.is_empty() => {} + _ => return Err(self.malformed("the document ends early")), + } + Ok(Extracted { + document: self.document, + attachment: self.attachment, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + /// The document, the attachment's data and the attachment's name and + /// length. + type Run = (Value, Vec, Option<(String, u64)>); + + /// Run the extractor over `body` in chunks of `size`. + fn run(body: &[u8], size: usize) -> Result { + let mut x = Extractor::new(1024 * 1024); + let mut out = Vec::new(); + for chunk in body.chunks(size) { + x.feed(chunk, &mut out)?; + } + let extracted = x.finish()?; + let doc = serde_json::from_slice(&extracted.document).expect("valid document"); + Ok((doc, out, extracted.attachment)) + } + + fn body(tarball: &[u8]) -> Vec { + let data = base64::engine::general_purpose::STANDARD.encode(tarball); + serde_json::to_vec_pretty(&json!({ + "_id": "@acme/a", + "name": "@acme/a", + "dist-tags": {"latest": "1.0.0"}, + "versions": {"1.0.0": {"name": "@acme/a", "version": "1.0.0", "data": "kept", "n": [1, 2.5, -3e2, true, false, null, {}, []]}}, + "access": null, + "_attachments": {"@acme/a-1.0.0.tgz": {"content_type": "application/octet-stream", "data": data, "length": tarball.len()}}, + "readme": "say \"hi\" \\ \u{e9} \u{1F600}" + })) + .unwrap() + } + + #[test] + fn splits_the_document_from_the_tarball() { + let tarball: Vec = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect(); + let body = body(&tarball); + for size in [1, 3, 7, 4096, body.len()] { + let (doc, out, attachment) = run(&body, size).unwrap(); + assert_eq!(out, tarball, "chunk size {size}"); + assert_eq!( + attachment, + Some(("@acme/a-1.0.0.tgz".to_string(), tarball.len() as u64)) + ); + assert_eq!(doc["_attachments"]["@acme/a-1.0.0.tgz"]["data"], ""); + assert_eq!(doc["_attachments"]["@acme/a-1.0.0.tgz"]["length"], 200_000); + assert_eq!(doc["versions"]["1.0.0"]["data"], "kept"); + assert_eq!( + doc["versions"]["1.0.0"]["n"], + json!([1, 2.5, -3e2, true, false, null, {}, []]) + ); + assert_eq!(doc["readme"], "say \"hi\" \\ \u{e9} \u{1F600}"); + } + } + + #[test] + fn reads_escaped_and_wrapped_base64() { + let tarball = b"\xff\xfe\xfd hello world, this is a tarball".to_vec(); + let data = base64::engine::general_purpose::STANDARD.encode(&tarball); + let wrapped: String = data + .chars() + .enumerate() + .flat_map(|(i, c)| { + let mut s = String::new(); + if i > 0 && i % 8 == 0 { + s.push_str("\\n"); + } + if c == '/' { + s.push_str("\\/"); + } else if c == 'A' { + s.push_str("\\u0041"); + } else { + s.push(c); + } + s.chars().collect::>() + }) + .collect(); + let body = format!(r#"{{"_attachments":{{"x.tgz":{{"data":"{wrapped}"}}}}}}"#); + let (_, out, _) = run(body.as_bytes(), 5).unwrap(); + assert_eq!(out, tarball); + } + + #[test] + fn a_document_without_an_attachment() { + let (doc, out, attachment) = + run(br#"{"name":"a","versions":{},"_attachments":{}}"#, 2).unwrap(); + assert!(out.is_empty()); + assert_eq!(attachment, None); + assert_eq!(doc["name"], "a"); + // `data` elsewhere is a string like any other. + let (doc, out, attachment) = run( + br#"{"data":"QUJD","x":{"_attachments":{"a":{"data":"QUJD"}}}}"#, + 2, + ) + .unwrap(); + assert!(out.is_empty() && attachment.is_none()); + assert_eq!(doc["data"], "QUJD"); + } + + #[test] + fn refuses_what_it_cannot_read() { + for bad in [ + &br#"{"a":1"#[..], + br#"{"a" 1}"#, + br#"{"a":1,}"#, + br#"{"a":1]"#, + br#"[1,2}"#, + br#"{"a":1} {}"#, + br#"{a:1}"#, + br#"{"_attachments":{"x":{"data":"QUJD!"}}}"#, + br#"{"_attachments":{"x":{"data":"QU\tJD"}}}"#, + br#"{"_attachments":{"x":{"data":"QUJD"},"y":{"data":"QUJD"}}}"#, + b"{\"a\":\"line\nbreak\"}", + ] { + let result = run(bad, 3); + assert!( + matches!(result, Err(ExtractError::Malformed(_))), + "{:?}: {result:?}", + String::from_utf8_lossy(bad) + ); + } + let deep = format!("{}{}", "[".repeat(100), "]".repeat(100)); + assert!(run(deep.as_bytes(), 10).is_err()); + let mut x = Extractor::new(10); + assert_eq!( + x.feed(br#"{"description":"far too long"}"#, &mut Vec::new()), + Err(ExtractError::DocumentTooLarge(10)) + ); + } +} diff --git a/crates/formats/npm/src/tarball.rs b/crates/formats/npm/src/tarball.rs new file mode 100644 index 00000000..c2fcd74e --- /dev/null +++ b/crates/formats/npm/src/tarball.rs @@ -0,0 +1,175 @@ +//! Reading a package tarball the way npm installs it. +//! +//! npm (pacote) unpacks a tarball with `strip: 1`: the first component of +//! every path is dropped, whatever it is called (`package/` by convention), +//! so the package's `package.json` is any entry at depth one. When several +//! entries map to it, the last one wins on disk, so the last one is read. +//! The whole archive is read, bounded in what it unpacks to. + +use std::io::Read; + +use serde_json::{Map, Value}; + +/// The most a tarball may unpack to by default: 1 GiB. +pub const DEFAULT_MAX_UNPACKED_BYTES: u64 = 1024 * 1024 * 1024; + +/// The largest `package.json` read. +pub const MAX_PACKAGE_JSON_BYTES: u64 = 8 * 1024 * 1024; + +/// What a tarball holds, as far as the registry cares. +#[derive(Debug, Clone, PartialEq)] +pub struct Tarball { + /// The `package.json` npm installs, as an object. + pub package_json: Map, + /// Its `name`. + pub name: String, + /// Its `version`, as written. + pub version: String, + /// Regular files in the archive (`dist.fileCount`). + pub file_count: u64, + /// Their unpacked size (`dist.unpackedSize`). + pub unpacked_size: u64, +} + +/// The path an entry named `raw` unpacks to with `strip: 1`, as components. +fn stripped(raw: &[u8]) -> Vec<&[u8]> { + let mut parts = raw + .split(|b| *b == b'/' || *b == b'\\') + .filter(|part| !part.is_empty() && *part != b"."); + parts.next(); + parts.collect() +} + +/// Read the gzip-compressed tarball in `tgz`, refusing one that unpacks to +/// more than `max_unpacked` bytes or holds no `package.json` npm would read. +pub fn read(tgz: impl Read, max_unpacked: u64) -> Result { + let gz = flate2::read::GzDecoder::new(tgz); + let mut archive = tar::Archive::new(gz); + let entries = archive + .entries() + .map_err(|e| format!("not a gzip-compressed tar archive: {e}"))?; + let mut package_json: Option> = None; + let mut file_count = 0u64; + let mut unpacked = 0u64; + for entry in entries { + let mut entry = entry.map_err(|e| format!("unreadable tar entry: {e}"))?; + let size = entry.header().entry_size().unwrap_or(0); + unpacked = unpacked.saturating_add(size); + if unpacked > max_unpacked { + return Err(format!( + "the tarball unpacks to more than {max_unpacked} bytes" + )); + } + if !entry.header().entry_type().is_file() { + continue; + } + file_count += 1; + let path = entry.path_bytes().into_owned(); + if stripped(&path) == [b"package.json".as_slice()] { + if size > MAX_PACKAGE_JSON_BYTES { + return Err(format!( + "package.json is larger than {MAX_PACKAGE_JSON_BYTES} bytes" + )); + } + let mut data = Vec::with_capacity(size as usize); + (&mut entry) + .take(MAX_PACKAGE_JSON_BYTES + 1) + .read_to_end(&mut data) + .map_err(|e| format!("unreadable package.json: {e}"))?; + package_json = Some(data); + } + // The rest of the entry's data is skipped by the archive reader, + // which still inflates it: what the size bound above is for. + } + let data = package_json.ok_or("the tarball holds no package/package.json")?; + let data = data.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&data); + let package_json: Map = match serde_json::from_slice(data) { + Ok(Value::Object(map)) => map, + Ok(_) => return Err("package.json is not a JSON object".into()), + Err(e) => return Err(format!("package.json is not valid JSON: {e}")), + }; + let field = |key: &str| { + package_json + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("package.json has no {key:?}")) + }; + let name = field("name")?; + let version = field("version")?; + Ok(Tarball { + package_json, + name, + version, + file_count, + unpacked_size: unpacked, + }) +} + +/// Build a gzip-compressed tarball of `entries` (path, contents), for tests. +#[doc(hidden)] +pub fn build(entries: &[(&str, &[u8])]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + for (path, data) in entries { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + builder + .append_data(&mut header, path, *data) + .expect("append tar entry"); + } + let tar = builder.into_inner().expect("finish tar"); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gz, &tar).expect("gzip"); + gz.finish().expect("finish gzip") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_package_json_npm_installs() { + let tgz = build(&[ + ("package/package.json", br#"{"name":"a","version":"1.0.0"}"#), + ("package/index.js", b"module.exports = 1"), + ("package/lib/package.json", br#"{"name":"nested"}"#), + ]); + let t = read(&tgz[..], DEFAULT_MAX_UNPACKED_BYTES).unwrap(); + assert_eq!((t.name.as_str(), t.version.as_str()), ("a", "1.0.0")); + assert_eq!(t.file_count, 3); + + // Any top-level directory; the last entry wins. + let tgz = build(&[ + ( + "node-uuid/package.json", + br#"{"name":"first","version":"1.0.0"}"#, + ), + ( + "./package/package.json", + "\u{feff}{\"name\":\"last\",\"version\":\"2.0.0\"}".as_bytes(), + ), + ]); + let t = read(&tgz[..], DEFAULT_MAX_UNPACKED_BYTES).unwrap(); + assert_eq!((t.name.as_str(), t.version.as_str()), ("last", "2.0.0")); + } + + #[test] + fn refuses_what_npm_would_not_install() { + let none = build(&[("package.json", br#"{"name":"a","version":"1.0.0"}"#)]); + assert!(read(&none[..], DEFAULT_MAX_UNPACKED_BYTES).is_err()); + let not_json = build(&[("package/package.json", b"name: a")]); + assert!(read(¬_json[..], DEFAULT_MAX_UNPACKED_BYTES).is_err()); + let no_version = build(&[("package/package.json", br#"{"name":"a"}"#)]); + assert!(read(&no_version[..], DEFAULT_MAX_UNPACKED_BYTES).is_err()); + assert!(read(&b"not gzip"[..], DEFAULT_MAX_UNPACKED_BYTES).is_err()); + let big = build(&[ + ("package/package.json", br#"{"name":"a","version":"1.0.0"}"#), + ("package/blob", &[0u8; 4096]), + ]); + let err = read(&big[..], 1024).unwrap_err(); + assert!(err.contains("unpacks to more than"), "{err}"); + } +} diff --git a/crates/formats/npm/src/upstream.rs b/crates/formats/npm/src/upstream.rs new file mode 100644 index 00000000..1b92f446 --- /dev/null +++ b/crates/formats/npm/src/upstream.rs @@ -0,0 +1,348 @@ +//! Packuments a proxy reads from its upstream registry. +//! +//! An upstream packument may be large (tens of MiB for a package with +//! thousands of versions, most of it READMEs and the `users` who starred +//! it). [`read`] reads it as a stream and keeps only what a proxy serves: +//! the name, the dist-tags, the publish times, the version objects without +//! their READMEs, and the top-level README (bounded). The result is what a +//! proxy stores and serves from until it asks the upstream again. + +use std::collections::BTreeMap; +use std::fmt; +use std::io::Read; + +use chrono::{DateTime, Utc}; +use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::{Map, Value}; + +use crate::manifest; +use crate::version; + +/// The longest top-level README kept. +pub const MAX_README_BYTES: usize = 512 * 1024; + +/// What a proxy keeps of an upstream packument. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct UpstreamPackument { + #[serde(default)] + pub name: String, + #[serde(rename = "dist-tags", default)] + pub dist_tags: BTreeMap, + /// `created`, `modified` and a publish time per version, as written. + #[serde(default)] + pub time: BTreeMap, + /// The version objects by version as the upstream writes it, without + /// their READMEs. + #[serde(default)] + pub versions: BTreeMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readme: Option, +} + +/// One version of an upstream packument, normalized. +#[derive(Debug, Clone, PartialEq)] +pub struct UpstreamVersion<'a> { + /// The version, normalized. + pub version: String, + /// The version as the upstream writes it (its key in `versions`). + pub original: &'a str, + pub manifest: &'a Map, + /// When it was published, if the upstream says. + pub published: Option>, +} + +impl UpstreamVersion<'_> { + /// Where the upstream serves the tarball. + pub fn tarball(&self) -> Option<&str> { + self.dist() + .and_then(|d| d.get("tarball")) + .and_then(Value::as_str) + } + + /// `dist.integrity`. + pub fn integrity(&self) -> Option<&str> { + self.dist() + .and_then(|d| d.get("integrity")) + .and_then(Value::as_str) + } + + /// `dist.shasum`. + pub fn shasum(&self) -> Option<&str> { + self.dist() + .and_then(|d| d.get("shasum")) + .and_then(Value::as_str) + } + + fn dist(&self) -> Option<&Map> { + self.manifest.get("dist").and_then(Value::as_object) + } + + /// The version's manifest summary. + pub fn summary(&self) -> Map { + manifest::summary(self.manifest) + } +} + +/// A publish time as the registry writes it. +pub fn parse_time(raw: &str) -> Option> { + DateTime::parse_from_rfc3339(raw) + .ok() + .map(|t| t.with_timezone(&Utc)) +} + +impl UpstreamPackument { + /// The versions that normalize, each once (a key that is already + /// normalized wins over another spelling of the same version), in + /// ascending precedence. + pub fn versions(&self) -> Vec> { + let mut by_version: BTreeMap> = BTreeMap::new(); + for (original, manifest) in &self.versions { + let Ok(normalized) = version::normalize(original) else { + continue; + }; + let entry = UpstreamVersion { + version: normalized.clone(), + original, + manifest, + published: self.time.get(original).and_then(|t| parse_time(t)), + }; + match by_version.get(&normalized) { + Some(existing) if existing.original == normalized => {} + _ => { + by_version.insert(normalized, entry); + } + } + } + let mut out: Vec> = by_version.into_values().collect(); + out.sort_by(|a, b| version::compare(&a.version, &b.version)); + out + } + + /// The version `version` (normalized), if the upstream has it. + pub fn version(&self, version: &str) -> Option> { + self.versions().into_iter().find(|v| v.version == version) + } + + /// The dist-tags, with normalized versions. + pub fn normalized_dist_tags(&self) -> BTreeMap { + self.dist_tags + .iter() + .filter_map(|(tag, v)| Some((tag.clone(), version::normalize(v).ok()?))) + .collect() + } +} + +/// Read an upstream packument from `reader`; see the module documentation. +pub fn read(reader: impl Read) -> Result { + let mut de = serde_json::Deserializer::from_reader(std::io::BufReader::new(reader)); + let doc = de + .deserialize_map(TopVisitor) + .map_err(|e| format!("not a packument: {e}"))?; + de.end().map_err(|e| format!("not a packument: {e}"))?; + Ok(doc) +} + +struct TopVisitor; + +/// The string values of a map, other values left out. +fn strings(map: BTreeMap) -> BTreeMap { + map.into_iter() + .filter_map(|(k, v)| match v { + Value::String(s) => Some((k, s)), + _ => None, + }) + .collect() +} + +impl<'de> Visitor<'de> for TopVisitor { + type Value = UpstreamPackument; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a packument object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = UpstreamPackument::default(); + while let Some(key) = map.next_key::()? { + match key.as_str() { + "name" => { + out.name = map + .next_value::()? + .as_str() + .unwrap_or_default() + .to_string() + } + "dist-tags" => { + out.dist_tags = strings( + map.next_value::>>()? + .unwrap_or_default(), + ) + } + "time" => { + out.time = strings( + map.next_value::>>()? + .unwrap_or_default(), + ) + } + "versions" => out.versions = map.next_value_seed(VersionsSeed)?, + "readme" => { + out.readme = map.next_value::()?.as_str().map(|r| { + let mut r = r.to_string(); + manifest::truncate(&mut r, MAX_README_BYTES); + r + }) + } + _ => { + map.next_value::()?; + } + } + } + Ok(out) + } +} + +struct VersionsSeed; + +impl<'de> DeserializeSeed<'de> for VersionsSeed { + type Value = BTreeMap>; + + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(VersionsVisitor) + } +} + +struct VersionsVisitor; + +impl<'de> Visitor<'de> for VersionsVisitor { + type Value = BTreeMap>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("the versions object") + } + + fn visit_unit(self) -> Result { + Ok(BTreeMap::new()) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if let Some(manifest) = map.next_value_seed(ManifestSeed)? { + out.insert(key, manifest); + } + } + Ok(out) + } +} + +struct ManifestSeed; + +impl<'de> DeserializeSeed<'de> for ManifestSeed { + type Value = Option>; + + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(ManifestVisitor) + } +} + +/// A version object without its README; anything else is no version. +struct ManifestVisitor; + +macro_rules! not_a_manifest { + ($($name:ident: $ty:ty),*) => { + $(fn $name(self, _: $ty) -> Result { Ok(None) })* + }; +} + +impl<'de> Visitor<'de> for ManifestVisitor { + type Value = Option>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a version object") + } + + not_a_manifest!(visit_bool: bool, visit_i64: i64, visit_u64: u64, visit_f64: f64, visit_str: &str); + + fn visit_unit(self) -> Result { + Ok(None) + } + + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element::()?.is_some() {} + Ok(None) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = Map::new(); + while let Some(key) = map.next_key::()? { + if key == "readme" || key == "readmeFilename" { + map.next_value::()?; + } else { + out.insert(key, map.next_value::()?); + } + } + Ok(Some(out)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn keeps_what_a_proxy_serves() { + let body = json!({ + "_id": "left-pad", "_rev": "12-abc", "name": "left-pad", + "dist-tags": {"latest": "1.3.0", "bogus": 1}, + "versions": { + "1.3.0": {"name": "left-pad", "version": "1.3.0", "readme": "# huge", "dist": { + "tarball": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-x", "shasum": "ab"}}, + "v1.2.0": {"name": "left-pad", "version": "1.2.0"}, + "1.2.0": {"name": "left-pad", "version": "1.2.0", "canonical": true}, + "not-a-version": {"name": "left-pad"}, + "1.1.0": null + }, + "time": {"created": "2016-03-23T20:00:00.000Z", "1.3.0": "2018-04-09T01:57:31.231Z", "x": 5}, + "users": {"someone": true}, + "readme": "# left-pad", + "_attachments": {} + }); + let doc = read(serde_json::to_vec(&body).unwrap().as_slice()).unwrap(); + assert_eq!(doc.name, "left-pad"); + assert_eq!( + doc.dist_tags, + BTreeMap::from([("latest".into(), "1.3.0".into())]) + ); + assert_eq!(doc.readme.as_deref(), Some("# left-pad")); + assert!(!doc.time.contains_key("x")); + assert!(doc.versions["1.3.0"].get("readme").is_none()); + let versions = doc.versions(); + let names: Vec<&str> = versions.iter().map(|v| v.version.as_str()).collect(); + assert_eq!(names, ["1.2.0", "1.3.0"]); + assert_eq!(versions[0].original, "1.2.0"); + assert_eq!(versions[0].manifest["canonical"], true); + let latest = doc.version("1.3.0").unwrap(); + assert_eq!( + latest.tarball(), + Some("https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz") + ); + assert_eq!(latest.integrity(), Some("sha512-x")); + assert_eq!(latest.published, parse_time("2018-04-09T01:57:31.231Z")); + // What a proxy stores reads back the same. + let stored = serde_json::to_vec(&doc).unwrap(); + assert_eq!( + serde_json::from_slice::(&stored).unwrap(), + doc + ); + } + + #[test] + fn refuses_what_is_no_packument() { + assert!(read(&b"[1,2]"[..]).is_err()); + assert!(read(&b"{\"name\": \"x\"} trailing"[..]).is_err()); + assert!(read(&b""[..]).is_err()); + } +} diff --git a/crates/formats/npm/src/version.rs b/crates/formats/npm/src/version.rs new file mode 100644 index 00000000..74be273b --- /dev/null +++ b/crates/formats/npm/src/version.rs @@ -0,0 +1,97 @@ +//! npm versions: SemVer 2.0.0, as node-semver cleans them. +//! +//! npm stores a version as `semver.clean` makes it: surrounding spaces and +//! one leading `v` or `=` dropped, and build metadata left out +//! (`1.2.3+build.5` is `1.2.3`). Versions order by SemVer precedence. + +use std::cmp::Ordering; + +/// The longest version read (the host's bound on versions). +pub const MAX_VERSION_LEN: usize = 256; + +/// The normalized form of `raw`, or why it is no version. +pub fn normalize(raw: &str) -> Result { + if raw.len() > MAX_VERSION_LEN { + return Err(format!("longer than {MAX_VERSION_LEN} characters")); + } + let trimmed = raw.trim(); + let trimmed = trimmed + .strip_prefix('=') + .or_else(|| trimmed.strip_prefix('v')) + .unwrap_or(trimmed) + .trim_start(); + let parsed = semver::Version::parse(trimmed).map_err(|e| e.to_string())?; + Ok(format(&parsed)) +} + +/// `major.minor.patch[-pre]`, without build metadata. +fn format(v: &semver::Version) -> String { + if v.pre.is_empty() { + format!("{}.{}.{}", v.major, v.minor, v.patch) + } else { + format!("{}.{}.{}-{}", v.major, v.minor, v.patch, v.pre) + } +} + +/// Order two versions by SemVer precedence; a version that does not parse +/// sorts before every one that does, and among themselves by text. +pub fn compare(a: &str, b: &str) -> Ordering { + match (semver::Version::parse(a), semver::Version::parse(b)) { + (Ok(a), Ok(b)) => a.cmp_precedence(&b), + (Ok(_), Err(_)) => Ordering::Greater, + (Err(_), Ok(_)) => Ordering::Less, + (Err(_), Err(_)) => a.cmp(b), + } +} + +/// Whether a normalized version is a prerelease (`1.0.0-rc.1`). +pub fn is_prerelease(version: &str) -> bool { + version.contains('-') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalizes_like_semver_clean() { + assert_eq!(normalize("1.2.3").unwrap(), "1.2.3"); + assert_eq!(normalize(" v1.2.3 ").unwrap(), "1.2.3"); + assert_eq!(normalize("=1.2.3").unwrap(), "1.2.3"); + assert_eq!(normalize("1.2.3+build.5").unwrap(), "1.2.3"); + assert_eq!(normalize("1.2.3-rc.1+x").unwrap(), "1.2.3-rc.1"); + assert!(normalize("1.2").is_err()); + assert!(normalize("01.2.3").is_err()); + assert!(normalize("latest").is_err()); + assert!(normalize("").is_err()); + assert!(normalize(&"1".repeat(300)).is_err()); + } + + #[test] + fn orders_by_precedence() { + let mut versions = vec![ + "1.10.0", + "1.2.0", + "1.2.0-beta.11", + "1.2.0-beta.2", + "1.2.0-alpha", + "0.9.9", + "2.0.0-rc.1", + ]; + versions.sort_by(|a, b| compare(a, b)); + assert_eq!( + versions, + [ + "0.9.9", + "1.2.0-alpha", + "1.2.0-beta.2", + "1.2.0-beta.11", + "1.2.0", + "1.10.0", + "2.0.0-rc.1" + ] + ); + assert!(is_prerelease("2.0.0-rc.1")); + assert!(!is_prerelease("2.0.0")); + } +} diff --git a/crates/formats/npm/tests/protocol.rs b/crates/formats/npm/tests/protocol.rs new file mode 100644 index 00000000..9b5d4568 --- /dev/null +++ b/crates/formats/npm/tests/protocol.rs @@ -0,0 +1,568 @@ +//! The npm registry protocol as the plugin speaks it, driven through +//! `forge_plugin_api::testing::serve` against an in-memory repository: the +//! packuments (abbreviated for installs, full with publish times and the +//! README), the host actions it asks for (downloads, publishes, changes, +//! logins, forwards), the bodies it reads first, and its errors. What the +//! host does with those actions is tested with the server +//! (`crates/server/tests/npm.rs`). + +use std::collections::HashMap; + +use forge_format_npm::Npm; +use forge_plugin_api::testing::{serve, serve_with_body, Served}; +use forge_plugin_api::{ + Action, Answer, AuthOutcome, AuthRequest, BodyUse, Change, FormatPlugin, PackageInfo, + PackageRef, Query, Repository, RepositoryKind, Request, SearchHit, SearchResult, Tag, Upload, + VersionInfo, +}; +use serde_json::{json, Value}; + +const BASE: &str = "https://forge.example.com/npm/main"; +const ABBREVIATED: &str = + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*"; + +fn repo(kind: RepositoryKind) -> Repository { + Repository { + name: "main".into(), + kind, + base_url: BASE.into(), + inline: false, + reserved_prefixes: Vec::new(), + } +} + +fn hosted() -> Repository { + repo(RepositoryKind::Hosted) +} + +/// An in-memory repository answering the plugin's queries as the host does. +#[derive(Default, Clone)] +struct MockRepo { + packages: HashMap>, + tags: HashMap>, + manifests: HashMap<(String, String), Vec>, + hits: Vec, +} + +impl MockRepo { + /// `name` with `versions` (ascending, with their publish times) and + /// `tags`; every version's manifest has a README. + fn with(mut self, name: &str, versions: &[(&str, &str)], tags: &[(&str, &str)]) -> Self { + let infos = versions + .iter() + .map(|(version, published)| { + let manifest = json!({ + "name": name, + "version": version, + "description": format!("{name} {version}"), + "readme": format!("# {name} {version}"), + "dependencies": {"left-pad": "^1.0.0"}, + "devDependencies": {"jest": "^29.0.0"}, + "scripts": {"test": "jest"}, + "dist": {"integrity": format!("sha512-{version}"), "shasum": "ab12"}, + }); + let bytes = serde_json::to_vec(&manifest).unwrap(); + self.manifests + .insert((name.to_string(), version.to_string()), bytes.clone()); + PackageInfo { + original_id: name.into(), + version: (*version).into(), + version_original: (*version).into(), + listed: true, + published: Some((*published).into()), + sha256: None, + sha512: None, + size: Some(1234), + summary: Npm::default().manifest_summary(&bytes), + deprecated: None, + } + }) + .collect(); + self.packages.insert(name.into(), infos); + self.tags.insert( + name.into(), + tags.iter() + .map(|(tag, version)| Tag { + name: (*tag).into(), + version: (*version).into(), + }) + .collect(), + ); + self + } + + fn answer(&self, query: &Query) -> Answer { + match query { + Query::Packages(id) | Query::DatedPackages(id) => { + Answer::Packages(self.packages.get(id).cloned().unwrap_or_default()) + } + Query::Versions(id) => Answer::Versions( + self.packages + .get(id) + .into_iter() + .flatten() + .map(|p| VersionInfo { + original_id: p.original_id.clone(), + version: p.version.clone(), + version_original: p.version_original.clone(), + listed: p.listed, + }) + .collect(), + ), + Query::Tags(id) => Answer::Tags(self.tags.get(id).cloned().unwrap_or_default()), + Query::Manifest(target) => Answer::Manifest( + self.manifests + .get(&(target.id.clone(), target.version.clone())) + .cloned(), + ), + Query::Search(_) => Answer::Search(SearchResult { + total: self.hits.len() as u32, + hits: self.hits.clone(), + }), + other => panic!("unexpected query {other:?}"), + } + } + + fn serve(&self, req: &Request) -> Served { + serve(&Npm::default(), &hosted(), req, |q| self.answer(q)) + } + + fn serve_with_body(&self, req: &Request, body: &[u8]) -> Served { + serve_with_body(&Npm::default(), &hosted(), req, body, |q| self.answer(q)) + } +} + +fn get(path: &str) -> Request { + Request::new("GET", path) +} + +fn json_of(served: &Served) -> Value { + serde_json::from_slice(&served.response().body).expect("a JSON body") +} + +fn sample() -> MockRepo { + MockRepo::default().with( + "@acme/ui", + &[ + ("1.0.0", "2026-01-01T00:00:00.000+00:00"), + ("1.1.0-beta.1", "2026-02-01T00:00:00.000+00:00"), + ("2.0.0", "2026-03-01T00:00:00.000+00:00"), + ], + &[("latest", "2.0.0"), ("next", "1.1.0-beta.1")], + ) +} + +#[test] +fn ping_answers_without_loading_anything() { + let served = MockRepo::default().serve(&get("/-/ping")); + assert_eq!(served.response().status, 200); + assert_eq!(json_of(&served), json!({})); + assert!(served.queries.is_empty()); +} + +#[test] +fn installs_read_the_abbreviated_packument() { + let served = sample().serve(&get("/@acme%2fui").with_header("accept", ABBREVIATED)); + // No publish times, no manifest: what an install needs only. + assert_eq!( + served.queries, + [ + Query::Packages("@acme/ui".into()), + Query::Tags("@acme/ui".into()) + ] + ); + assert_eq!( + served.header("content-type"), + Some("application/vnd.npm.install-v1+json") + ); + assert_eq!(served.header("vary"), Some("Accept")); + let doc = json_of(&served); + assert_eq!(doc["name"], "@acme/ui"); + assert_eq!(doc["modified"], "2026-03-01T00:00:00.000Z"); + assert_eq!( + doc["dist-tags"], + json!({"latest": "2.0.0", "next": "1.1.0-beta.1"}) + ); + let v = &doc["versions"]["2.0.0"]; + assert_eq!( + v["dist"]["tarball"], + format!("{BASE}/@acme/ui/-/ui-2.0.0.tgz") + ); + assert_eq!(v["dist"]["integrity"], "sha512-2.0.0"); + assert_eq!(v["dependencies"], json!({"left-pad": "^1.0.0"})); + assert_eq!(v["devDependencies"], json!({"jest": "^29.0.0"})); + // Fields installs do not read are left out. + assert!( + v.get("description").is_none() && v.get("scripts").is_none(), + "{v}" + ); + assert!(doc.get("time").is_none() && doc.get("readme").is_none()); +} + +#[test] +fn the_full_packument_has_publish_times_and_the_readme_of_latest() { + let served = sample().serve(&get("/@acme%2Fui").with_header("accept", "application/json")); + assert_eq!( + served.queries, + [ + Query::DatedPackages("@acme/ui".into()), + Query::Tags("@acme/ui".into()), + Query::Manifest(PackageRef { + id: "@acme/ui".into(), + version: "2.0.0".into(), + }), + ] + ); + assert_eq!(served.header("content-type"), Some("application/json")); + let doc = json_of(&served); + assert_eq!(doc["readme"], "# @acme/ui 2.0.0"); + assert_eq!(doc["time"]["created"], "2026-01-01T00:00:00.000Z"); + assert_eq!(doc["time"]["2.0.0"], "2026-03-01T00:00:00.000Z"); + let v = &doc["versions"]["1.0.0"]; + assert_eq!(v["description"], "@acme/ui 1.0.0"); + assert!( + v.get("readme").is_none(), + "only the package has a README: {v}" + ); + // An Accept that refuses the abbreviated document gets the full one. + let refused = sample().serve(&get("/@acme%2fui").with_header( + "accept", + "application/vnd.npm.install-v1+json; q=0, application/json", + )); + assert!(json_of(&refused).get("time").is_some()); +} + +#[test] +fn tags_point_at_the_latest_version_when_none_is_set() { + let repo = MockRepo::default().with( + "plain", + &[ + ("0.9.0", "2026-01-01T00:00:00.000Z"), + ("1.0.0", "2026-01-02T00:00:00.000Z"), + ], + &[], + ); + let served = repo.serve(&get("/-/package/plain/dist-tags")); + assert_eq!(json_of(&served), json!({"latest": "1.0.0"})); +} + +#[test] +fn versions_resolve_by_tag_or_by_version() { + let repo = sample(); + let by_tag = repo.serve(&get("/@acme%2fui/next")); + assert_eq!(json_of(&by_tag)["version"], "1.1.0-beta.1"); + let by_version = repo.serve(&get("/@acme%2fui/1.0.0")); + assert_eq!(json_of(&by_version)["version"], "1.0.0"); + let missing = repo.serve(&get("/@acme%2fui/9.9.9")); + assert_eq!(missing.response().status, 404); +} + +#[test] +fn an_unknown_package_is_a_404_npm_reads() { + let served = MockRepo::default().serve(&get("/nope")); + assert_eq!(served.response().status, 404); + assert!(json_of(&served)["error"].as_str().unwrap().contains("nope")); + // So is a path that names no package, before anything is loaded. + let served = MockRepo::default().serve(&get("/a/b/c/d")); + assert_eq!(served.response().status, 404); + assert!(served.queries.is_empty()); +} + +#[test] +fn host_errors_are_rendered_as_npm_reads_them() { + let response = Npm::default() + .error_response(409, "package left-pad 1.0.0 already exists") + .unwrap(); + assert_eq!(response.status, 409); + let body: Value = serde_json::from_slice(&response.body).unwrap(); + assert_eq!( + body, + json!({"error": "package left-pad 1.0.0 already exists"}) + ); +} + +#[test] +fn tarballs_are_downloads() { + let served = MockRepo::default().serve(&get("/@acme%2fui/-/ui-2.0.0.tgz")); + assert_eq!( + served.outcome, + Action::Download(PackageRef { + id: "@acme/ui".into(), + version: "2.0.0".into(), + }) + ); + // A file of another package is not one. + let served = MockRepo::default().serve(&get("/@acme%2fui/-/other-2.0.0.tgz")); + assert_eq!(served.response().status, 404); +} + +#[test] +fn a_publish_streams_its_document_to_the_host() { + let served = MockRepo::default().serve(&Request::new("PUT", "/@acme%2fui")); + let Action::Publish(publish) = served.outcome else { + panic!("expected a publish, got {:?}", served.outcome); + }; + assert_eq!(publish.upload, Upload::Document); + assert_eq!(publish.id.as_deref(), Some("@acme/ui")); + assert_eq!(publish.done.status, 201); +} + +#[test] +fn dist_tags_are_set_with_a_body_and_removed_without() { + let repo = sample(); + let put = Request::new("PUT", "/-/package/@acme%2fui/dist-tags/stable"); + let Action::ReadBody(wanted) = repo.serve(&put).outcome else { + panic!("a dist-tag's version comes in the body"); + }; + assert_eq!(wanted.used_for, BodyUse::Change); + let served = repo.serve_with_body(&put, b"\"1.0.0\""); + let Action::Change(set) = served.outcome else { + panic!("expected a change, got {:?}", served.outcome); + }; + assert_eq!( + set.changes, + [Change::Tag { + id: "@acme/ui".into(), + tag: "stable".into(), + version: "1.0.0".into(), + }] + ); + assert_eq!(set.done.status, 201); + + let served = repo.serve(&Request::new( + "DELETE", + "/-/package/@acme%2fui/dist-tags/next", + )); + let Action::Change(set) = served.outcome else { + panic!("expected a change"); + }; + assert_eq!( + set.changes, + [Change::Untag { + id: "@acme/ui".into(), + tag: "next".into(), + }] + ); + // `latest` always names a version. + let served = repo.serve(&Request::new( + "DELETE", + "/-/package/@acme%2fui/dist-tags/latest", + )); + assert_eq!(served.response().status, 400); + // A tag that looks like a version would shadow it. + let served = repo.serve(&Request::new("PUT", "/-/package/@acme%2fui/dist-tags/1.0")); + assert_eq!(served.response().status, 400); +} + +/// The packument `npm deprecate` and `npm unpublish ` send back. +fn updated(versions: &[(&str, Option<&str>)], tags: Value) -> Vec { + let versions: serde_json::Map = versions + .iter() + .map(|(v, deprecated)| { + let mut doc = json!({"name": "@acme/ui", "version": v}); + if let Some(message) = deprecated { + doc["deprecated"] = json!(message); + } + (v.to_string(), doc) + }) + .collect(); + serde_json::to_vec(&json!({ + "_id": "@acme/ui", + "_rev": "3-abc", + "name": "@acme/ui", + "dist-tags": tags, + "versions": versions, + })) + .unwrap() +} + +#[test] +fn a_packument_update_deprecates_unpublishes_and_retags() { + let repo = sample(); + let put = Request::new("PUT", "/@acme%2fui/-rev/3-abc"); + let Action::ReadBody(wanted) = repo.serve(&put).outcome else { + panic!("an update comes in the body"); + }; + assert_eq!(wanted.used_for, BodyUse::Change); + + // 1.1.0-beta.1 is left out (unpublished; its `next` tag goes with it), + // 1.0.0 deprecated, and `stable` added. + let body = updated( + &[("1.0.0", Some("use 2")), ("2.0.0", None)], + json!({"latest": "2.0.0", "stable": "1.0.0"}), + ); + let served = repo.serve_with_body(&put, &body); + let Action::Change(set) = served.outcome else { + panic!("expected changes, got {:?}", served.outcome); + }; + let target = |v: &str| PackageRef { + id: "@acme/ui".into(), + version: v.into(), + }; + assert_eq!( + set.changes, + [ + Change::Delete(target("1.1.0-beta.1")), + Change::Deprecate { + target: target("1.0.0"), + message: Some("use 2".into()), + }, + Change::Tag { + id: "@acme/ui".into(), + tag: "stable".into(), + version: "1.0.0".into(), + }, + ] + ); + + // A document that leaves no version is refused: that is an unpublish + // of the package. + let body = updated(&[], json!({})); + assert_eq!(repo.serve_with_body(&put, &body).response().status, 400); + // Nothing changed: nothing to do. + let body = updated( + &[("1.0.0", None), ("1.1.0-beta.1", None), ("2.0.0", None)], + json!({"latest": "2.0.0", "next": "1.1.0-beta.1"}), + ); + assert_eq!(repo.serve_with_body(&put, &body).response().status, 200); +} + +#[test] +fn unpublishing_a_package_deletes_every_version() { + let served = sample().serve(&Request::new("DELETE", "/@acme%2fui/-rev/3-abc")); + let Action::Change(set) = served.outcome else { + panic!("expected changes, got {:?}", served.outcome); + }; + assert_eq!(set.changes.len(), 3); + assert!(set.changes.iter().all(|c| matches!(c, Change::Delete(_)))); +} + +#[test] +fn logins_are_the_hosts() { + let npm = Npm::default(); + let repo = MockRepo::default(); + // Legacy: `npm login --auth-type=legacy`, `yarn npm login`. + let put = Request::new("PUT", "/-/user/org.couchdb.user:jane@example.com"); + let Action::ReadBody(wanted) = repo.serve(&put).outcome else { + panic!("a login comes in the body"); + }; + assert_eq!(wanted.used_for, BodyUse::Login); + let served = repo.serve_with_body( + &put, + br#"{"name": "jane@example.com", "password": "secret", "type": "user"}"#, + ); + assert_eq!( + served.outcome, + Action::Auth(AuthRequest::Password { + username: "jane@example.com".into(), + password: "secret".into(), + }) + ); + let minted = npm.auth_response( + &hosted(), + &put, + &AuthOutcome::Token { + token: "t0k3n".into(), + username: "jane@example.com".into(), + }, + ); + assert_eq!(minted.status, 201); + let body: Value = serde_json::from_slice(&minted.body).unwrap(); + assert_eq!(body["token"], "t0k3n"); + assert_eq!(body["ok"], true); + + // Web: `npm login` opens the login URL and polls the done URL. + let start = Request::new("POST", "/-/v1/login"); + let served = repo.serve_with_body(&start, br#"{"hostname": "laptop"}"#); + assert_eq!( + served.outcome, + Action::Auth(AuthRequest::StartWebLogin { + hostname: Some("laptop".into()), + }) + ); + let started = npm.auth_response( + &hosted(), + &start, + &AuthOutcome::WebLoginStarted { + login_url: "https://forge.example.com/cli-login/abc".into(), + secret: "def".into(), + }, + ); + let body: Value = serde_json::from_slice(&started.body).unwrap(); + assert_eq!(body["loginUrl"], "https://forge.example.com/cli-login/abc"); + assert_eq!(body["doneUrl"], format!("{BASE}/-/v1/login/done/def")); + let done = get("/-/v1/login/done/def"); + assert_eq!( + repo.serve(&done).outcome, + Action::Auth(AuthRequest::PollWebLogin { + secret: "def".into(), + }) + ); + let pending = npm.auth_response(&hosted(), &done, &AuthOutcome::WebLoginPending); + assert_eq!(pending.status, 202); + assert!(pending + .headers + .iter() + .any(|(k, _)| k.eq_ignore_ascii_case("retry-after"))); + + assert_eq!( + repo.serve(&get("/-/whoami")).outcome, + Action::Auth(AuthRequest::Whoami) + ); + assert_eq!( + repo.serve(&Request::new("DELETE", "/-/user/token/t0k3n")) + .outcome, + Action::Auth(AuthRequest::Logout { + token: "t0k3n".into(), + }) + ); +} + +#[test] +fn search_renders_what_npm_search_prints() { + let mut repo = MockRepo::default(); + let summary = Npm::default().manifest_summary( + br#"{"name": "left-pad", "version": "1.3.0", "description": "String left pad", "keywords": ["pad"]}"#, + ); + repo.hits.push(SearchHit { + original_id: "left-pad".into(), + latest_version: "1.3.0".into(), + versions: Vec::new(), + summary, + total_downloads: None, + }); + let served = repo.serve(&get("/-/v1/search?text=pad&size=5&from=10")); + let [Query::Search(search)] = served.queries.as_slice() else { + panic!("expected one search, got {:?}", served.queries); + }; + assert_eq!( + (search.text.as_str(), search.take, search.skip), + ("pad", 5, 10) + ); + let doc = json_of(&served); + assert_eq!(doc["total"], 1); + let package = &doc["objects"][0]["package"]; + assert_eq!(package["name"], "left-pad"); + assert_eq!(package["version"], "1.3.0"); + assert_eq!(package["description"], "String left pad"); +} + +#[test] +fn audits_go_upstream_without_internal_names() { + let mut group = repo(RepositoryKind::Group); + group.reserved_prefixes = vec!["@acme/".into()]; + let req = Request::new("POST", "/-/npm/v1/security/advisories/bulk"); + let body = br#"{"@acme/ui": ["1.0.0"], "left-pad": ["1.3.0"]}"#; + let served = serve_with_body(&Npm::default(), &group, &req, body, |q| { + panic!("unexpected query {q:?}") + }); + let Action::Forward(forward) = served.outcome else { + panic!("expected a forward, got {:?}", served.outcome); + }; + assert_eq!(forward.path, "/-/npm/v1/security/advisories/bulk"); + let sent: Value = serde_json::from_slice(&forward.body).unwrap(); + assert_eq!(sent, json!({"left-pad": ["1.3.0"]})); + // Other audit endpoints are not forwarded. + let served = MockRepo::default().serve(&Request::new("POST", "/-/npm/v1/security/anything")); + assert_eq!(served.response().status, 404); +} diff --git a/crates/formats/nuget/src/protocol.rs b/crates/formats/nuget/src/protocol.rs index dd821b19..5e4636b6 100644 --- a/crates/formats/nuget/src/protocol.rs +++ b/crates/formats/nuget/src/protocol.rs @@ -249,6 +249,7 @@ pub(crate) fn handle(repo: &Repository, req: &Request) -> Action { Route::Push => Action::Publish(PublishRequest { // The first part is the raw `.nupkg`, whatever its name. upload: Upload::FirstPart, + id: None, // Both create and redeploy answer 201. done: empty(201), }), diff --git a/crates/formats/nuget/src/upstream.rs b/crates/formats/nuget/src/upstream.rs index 696c6e66..23a46a05 100644 --- a/crates/formats/nuget/src/upstream.rs +++ b/crates/formats/nuget/src/upstream.rs @@ -139,6 +139,8 @@ pub(crate) fn step( UpstreamOp::Search(query) => search(&resources, query, fetched), // A feed holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), + // NuGet has no tags. + UpstreamOp::Tags(_) => done(UpstreamAnswer::Tags(None)), }) } @@ -151,10 +153,7 @@ fn failed(reason: impl Into) -> UpstreamStep { } fn get(url: &str, freshness: Freshness) -> Fetch { - Fetch { - url: url.to_string(), - freshness, - } + Fetch::new(url, freshness) } /// How the fetch of `url` went, if it has been fetched. diff --git a/crates/formats/nuget/tests/protocol.rs b/crates/formats/nuget/tests/protocol.rs index 1fca7409..5bddf983 100644 --- a/crates/formats/nuget/tests/protocol.rs +++ b/crates/formats/nuget/tests/protocol.rs @@ -24,6 +24,7 @@ fn repo(kind: RepositoryKind) -> Repository { kind, base_url: format!("{BASE}/v3/main"), inline: false, + reserved_prefixes: Vec::new(), } } @@ -46,7 +47,7 @@ impl MockRepo { Query::Versions(id) => { Answer::Versions(self.versions.get(id).cloned().unwrap_or_default()) } - Query::Packages(id) => { + Query::Packages(id) | Query::DatedPackages(id) => { Answer::Packages(self.packages.get(id).cloned().unwrap_or_default()) } Query::PackagesOf(set) => Answer::Packages( @@ -66,6 +67,7 @@ impl MockRepo { .find(|p| p.version == target.version) .cloned(), ), + Query::Tags(_) => Answer::Tags(Vec::new()), Query::Manifest(target) => Answer::Manifest( self.manifests .get(&(target.id.clone(), target.version.clone())) @@ -143,6 +145,7 @@ fn package(id: &str, v: &str, nuspec: Option<&[u8]>) -> PackageInfo { sha512: None, size: None, summary: nuspec.and_then(|n| Nuget::default().manifest_summary(n)), + deprecated: None, } } diff --git a/crates/formats/nuget/tests/upstream.rs b/crates/formats/nuget/tests/upstream.rs index b0d0c6a3..4b3a720e 100644 --- a/crates/formats/nuget/tests/upstream.rs +++ b/crates/formats/nuget/tests/upstream.rs @@ -79,13 +79,7 @@ fn the_index_is_the_service_indexs_resources() { UpstreamOp::Index, &docs([(FEED.into(), index)]), ); - assert_eq!( - done.fetches, - [Fetch { - url: FEED.into(), - freshness: Freshness::Metadata - }] - ); + assert_eq!(done.fetches, [Fetch::new(FEED, Freshness::Metadata)]); let UpstreamAnswer::Index(index) = done.answer else { panic!("{:?}", done.answer) }; @@ -281,10 +275,10 @@ fn dating_reads_the_catalog_of_unlisted_versions() { ); assert_eq!( done.fetches.last().unwrap(), - &Fetch { - url: "https://up.example/catalog/1.0.0.json".into(), - freshness: Freshness::Immutable, - } + &Fetch::new( + "https://up.example/catalog/1.0.0.json", + Freshness::Immutable + ) ); let UpstreamAnswer::Metadata(metadata) = done.answer else { panic!("{:?}", done.answer) diff --git a/crates/formats/raw/src/listing.rs b/crates/formats/raw/src/listing.rs index f5d22999..f2de5ae2 100644 --- a/crates/formats/raw/src/listing.rs +++ b/crates/formats/raw/src/listing.rs @@ -351,6 +351,7 @@ mod tests { kind: RepositoryKind::Hosted, base_url: "https://forge.example.com/raw/bin".into(), inline: false, + reserved_prefixes: Vec::new(), } } diff --git a/crates/formats/raw/src/sidecar.rs b/crates/formats/raw/src/sidecar.rs index c1d380af..aba6f857 100644 --- a/crates/formats/raw/src/sidecar.rs +++ b/crates/formats/raw/src/sidecar.rs @@ -196,6 +196,7 @@ mod tests { kind: RepositoryKind::Hosted, base_url: "https://forge.example.com/raw/r".into(), inline: false, + reserved_prefixes: Vec::new(), }; assert_eq!( content_disposition(&repo, "a/b.txt"), diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 563dba70..4906311b 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -24,7 +24,7 @@ mod types; pub mod testing; use std::cmp::Ordering; -use std::io::{Read, Seek}; +use std::io::{Read, Seek, Write}; use std::path::Path; pub use types::*; @@ -115,12 +115,90 @@ pub trait FormatPlugin: Send + Sync + 'static { None } + /// The README a manifest carries, shown on the web UI's package page. + /// `None` by default. + fn readme(&self, manifest: &[u8]) -> Option { + let _ = manifest; + None + } + + /// A group's reserved prefix as the host stores it, from what an + /// administrator typed (trimmed, a trailing `*` dropped): refused when it + /// cannot start a package id. The default takes letters, digits, `.`, + /// `_` and `-`, lowercased. + fn reserved_prefix(&self, prefix: &str) -> Result { + let valid = !prefix.is_empty() + && prefix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); + if valid { + Ok(prefix.to_ascii_lowercase()) + } else { + Err(FormatError::InvalidId(format!( + "{prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" + ))) + } + } + + /// An [`Upload::Document`]: read the request `body`, write the package it + /// carries to `package`, and return the rest of the document — or say + /// there is no package in it. Runs off the async workers; the host bounds + /// the package by the repository's upload limit, the plugin the document. + fn unpack_upload( + &self, + body: &mut dyn Read, + package: &mut dyn Write, + ) -> Result { + let _ = (body, package); + Err(FormatError::Internal( + "this format takes no document uploads".into(), + )) + } + + /// [`Self::parse_package`] of a package [`Self::unpack_upload`] wrote, + /// with the document it came in: what the document says about the + /// package (its digests, its manifest) is checked here. The default + /// ignores the document. + fn parse_upload( + &self, + upload: &Uploaded<'_>, + package: &mut dyn PackageSource, + ) -> Result { + let _ = upload; + self.parse_package(package) + } + + /// The changes to make once a document's package is stored as + /// `package` (npm: the dist-tags the document sets). None by default. + fn published(&self, document: &[u8], package: &PackageRef) -> Vec { + let _ = (document, package); + Vec::new() + } + + /// The response to an [`Action::Auth`] that succeeded. + fn auth_response(&self, repo: &Repository, req: &Request, outcome: &AuthOutcome) -> Response { + let _ = (repo, req, outcome); + Response::text(500, "this format has no logins") + } + + /// The body of an error the host answers with (a refusal, a conflict, a + /// missing package), for clients that read errors in a shape of their + /// own: npm prints `{"error": ...}`. The host keeps the status and its + /// own headers (`WWW-Authenticate`, `Retry-After`) and takes the + /// content type and the body. `None` (the default): plain text. + fn error_response(&self, status: u16, message: &str) -> Option { + let _ = (status, message); + None + } + /// What a request is: a response right away, data to load first (then /// [`Self::render`]), or an action only the host can perform. /// - /// The host has authenticated the caller and checked that it may perform - /// the request's action on the repository (GET/HEAD read, PUT/POST - /// write, DELETE delete) before it asks. + /// The host has authenticated the caller before it asks, and checks + /// the answer against what the caller may do (GET/HEAD read, PUT/POST + /// write, DELETE delete) — loads, downloads, forwards and responses + /// need read, a publish or a change the permission it is for, a body + /// what [`BodyUse`] says, and a login nothing. fn handle(&self, repo: &Repository, req: &Request) -> Action; /// What to do once the data [`Self::handle`] (and earlier rounds of diff --git a/crates/plugin-api/src/testing.rs b/crates/plugin-api/src/testing.rs index deaeb877..e5904eb2 100644 --- a/crates/plugin-api/src/testing.rs +++ b/crates/plugin-api/src/testing.rs @@ -37,6 +37,7 @@ pub fn repository(kind: RepositoryKind, mount: &str) -> Repository { kind, base_url: format!("https://forge.example.com{mount}/repo"), inline: false, + reserved_prefixes: Vec::new(), } } @@ -80,11 +81,43 @@ pub fn serve( req: &Request, mut answer: impl FnMut(&Query) -> Answer, ) -> Served { - let mut queries = Vec::new(); - let mut pending = match plugin.handle(repo, req) { + let queries = Vec::new(); + let pending = match plugin.handle(repo, req) { Action::Load(pending) => pending, outcome => return Served { outcome, queries }, }; + serve_loads(plugin, repo, req, &mut answer, pending, queries) +} + +/// [`serve`] for a request with a body ([`Action::ReadBody`]): the plugin is +/// asked again with `body`, as the host does once it read it. +pub fn serve_with_body( + plugin: &dyn FormatPlugin, + repo: &Repository, + req: &Request, + body: &[u8], + answer: impl FnMut(&Query) -> Answer, +) -> Served { + match plugin.handle(repo, req) { + Action::ReadBody(_) => {} + outcome => { + return Served { + outcome, + queries: Vec::new(), + } + } + } + serve(plugin, repo, &req.clone().with_body(body.to_vec()), answer) +} + +fn serve_loads( + plugin: &dyn FormatPlugin, + repo: &Repository, + req: &Request, + answer: &mut impl FnMut(&Query) -> Answer, + mut pending: Vec, + mut queries: Vec, +) -> Served { let mut data: Vec = Vec::new(); for _ in 0..MAX_ROUNDS { if data.len() + pending.len() > MAX_QUERIES { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index f1f7e023..cc49a185 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -92,6 +92,10 @@ pub struct Repository { /// [`Action::ServeFile`] serves them. Always `false` for a package /// format. pub inline: bool, + /// A group's reserved prefixes ([`crate::FormatPlugin::reserved_prefix`] + /// forms): package ids that are never served from its proxy members. + /// Empty for any other repository. + pub reserved_prefixes: Vec, } /// An HTTP request from a package client. @@ -109,6 +113,9 @@ pub struct Request { /// accept-language, content-type, if-match, if-modified-since, /// if-none-match, user-agent. Never credentials or cookies. pub headers: Vec<(String, String)>, + /// The request body, once the plugin asked for it ([`Action::ReadBody`]); + /// `None` before. A `Content-Encoding: gzip` body is inflated. + pub body: Option>, } impl Request { @@ -121,9 +128,16 @@ impl Request { path: path.to_string(), query: query.to_string(), headers: Vec::new(), + body: None, } } + /// The request with its body read ([`Action::ReadBody`]). + pub fn with_body(mut self, body: impl Into>) -> Self { + self.body = Some(body.into()); + self + } + /// Add a header (`name` lowercase). pub fn with_header(mut self, name: &str, value: &str) -> Self { self.headers @@ -416,6 +430,11 @@ pub enum Query { Versions(String), /// Every version of a package with its metadata, ascending. Packages(String), + /// [`Query::Packages`] with every version's publish time + /// ([`PackageInfo::published`]), even where that costs more: a proxy + /// asks its upstream with [`UpstreamOp::Dating`] instead of + /// [`UpstreamOp::Metadata`]. + DatedPackages(String), /// One version with its metadata. Package(PackageRef), /// Some versions of a package with their metadata, ascending; versions @@ -425,6 +444,11 @@ pub enum Query { Manifest(PackageRef), /// The packages matching a search. Search(SearchQuery), + /// A package's tags ([`Tag`]). A hosted repository's are the ones set + /// with [`Change::Tag`], a proxy's its upstream's + /// ([`UpstreamOp::Tags`]), a group's those of its first member, by + /// priority, that serves the package and has tags for it. + Tags(String), /// The file stored at a path. A proxy answers with its cached copy, /// re-validated with its upstream once it is due; a group with its first /// member, by priority, that has the path. @@ -470,6 +494,15 @@ impl Query { } } +/// A name for one version of a package (npm's dist-tags: `latest`, `next`). +/// A tag names one version; a version may have several tags. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tag { + pub name: String, + /// Normalized. + pub version: String, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VersionInfo { /// The id as the package writes it. @@ -498,6 +531,9 @@ pub struct PackageInfo { /// /// [`FormatPlugin::manifest_summary`]: crate::FormatPlugin::manifest_summary pub summary: Option, + /// Why the version should not be used any more, when it was deprecated + /// ([`Change::Deprecate`]). + pub deprecated: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -523,11 +559,14 @@ pub struct SearchResult { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Answer { Versions(Vec), - /// The answer to [`Query::Packages`] and to [`Query::PackagesOf`]. + /// The answer to [`Query::Packages`], [`Query::DatedPackages`] and + /// [`Query::PackagesOf`]. Packages(Vec), Package(Option), Manifest(Option>), Search(SearchResult), + /// The answer to [`Query::Tags`], by name. + Tags(Vec), /// The answer to [`Query::File`] and to [`Query::FetchFile`]. File(Option), /// `None`: the directory does not exist (a directory other than the root @@ -557,6 +596,16 @@ pub enum Upload { /// The package is the first part of a `multipart/form-data` body, /// whatever its name. FirstPart, + /// The body is a document carrying the package (npm's publish: a JSON + /// document with the tarball in base64): the host streams it through + /// [`FormatPlugin::unpack_upload`], parses the package with + /// [`FormatPlugin::parse_upload`] and, once it is stored, applies + /// [`FormatPlugin::published`]'s changes. + /// + /// [`FormatPlugin::unpack_upload`]: crate::FormatPlugin::unpack_upload + /// [`FormatPlugin::parse_upload`]: crate::FormatPlugin::parse_upload + /// [`FormatPlugin::published`]: crate::FormatPlugin::published + Document, } /// A publish, and the response when it succeeds. When it fails the host @@ -565,9 +614,164 @@ pub enum Upload { #[derive(Debug, Clone, PartialEq, Eq)] pub struct PublishRequest { pub upload: Upload, + /// The canonical id the package must have (the one its URL names): + /// another is refused with 400. `None`: any. + pub id: Option, + pub done: Response, +} + +/// What [`FormatPlugin::unpack_upload`] found in an [`Upload::Document`]. +/// +/// [`FormatPlugin::unpack_upload`]: crate::FormatPlugin::unpack_upload +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Unpacked { + /// A package, written out; `document` is the rest of the body, handed + /// to [`FormatPlugin::parse_upload`] and [`FormatPlugin::published`]. + /// + /// [`FormatPlugin::parse_upload`]: crate::FormatPlugin::parse_upload + /// [`FormatPlugin::published`]: crate::FormatPlugin::published + Package { document: Vec }, + /// No package: the body is a document of another kind (npm's + /// deprecations and dist-tag edits are the same request). The host + /// drops what was staged and asks [`FormatPlugin::handle`] again, with + /// the body as [`Request::body`]. + /// + /// [`FormatPlugin::handle`]: crate::FormatPlugin::handle + Document(Vec), +} + +/// What a request's body is for ([`BodyRequest`]): what the caller needs +/// for the host to read it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BodyUse { + /// A login's ([`AuthRequest::Password`], [`AuthRequest::StartWebLogin`]): + /// read for any caller, at most 64 KiB, and what the plugin answers then + /// must be [`Action::Auth`]. + Login, + /// A query sent as a body (npm's audits): the caller needs read. + Read, + /// A change: the caller needs the permission the request's method takes + /// (PUT/POST write, DELETE delete). + Change, +} + +/// A request whose body the plugin needs to decide what it is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BodyRequest { + /// The largest body accepted: a larger one is answered 413. The host + /// caps it at 64 MiB. + pub limit: u64, + pub used_for: BodyUse, +} + +/// A package [`FormatPlugin::unpack_upload`] wrote, with what came with it. +/// +/// [`FormatPlugin::unpack_upload`]: crate::FormatPlugin::unpack_upload +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Uploaded<'a> { + /// The rest of the document ([`Unpacked::Package`]). + pub document: &'a [u8], + /// The username of the account publishing, when it is a user's. + pub publisher: Option<&'a str>, +} + +/// One change to a repository's packages ([`ChangeSet`]). Ids are +/// canonical, versions normalized. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Change { + /// Hide a version from searches and "latest" (delete permission). + Unlist(PackageRef), + /// Show an unlisted version again (write permission). + Relist(PackageRef), + /// Remove a version, its bytes and its tags (delete permission). + Delete(PackageRef), + /// Mark a version deprecated with `message`, or (`None`) no longer + /// (write permission). + Deprecate { + target: PackageRef, + message: Option, + }, + /// Point tag `tag` of package `id` at `version`, which must exist + /// (write permission). + Tag { + id: String, + tag: String, + version: String, + }, + /// Remove tag `tag` of package `id` (write permission). + Untag { id: String, tag: String }, +} + +/// Changes the host makes in order, and the response when every one +/// succeeded. The first that fails answers instead (403, 404 for a version +/// the caller cannot see, 400), and those before it stay made. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChangeSet { + pub changes: Vec, pub done: Response, } +/// What only the host can do about who the caller is: log a client in, +/// out, or tell it who it is. Its outcome comes back through +/// [`FormatPlugin::auth_response`]; refusals the host answers itself (401, +/// 429, 503). +/// +/// [`FormatPlugin::auth_response`]: crate::FormatPlugin::auth_response +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AuthRequest { + /// Sign in with a password; the outcome is a new API token + /// ([`AuthOutcome::Token`]). Any caller may ask. + Password { username: String, password: String }, + /// Start a login the user approves in the browser + /// ([`AuthOutcome::WebLoginStarted`]). Any caller may ask; 404 when the + /// server serves no web UI. + StartWebLogin { hostname: Option }, + /// Whether the web login `secret` was approved: [`AuthOutcome::Token`] + /// once, [`AuthOutcome::WebLoginPending`] before, + /// [`AuthOutcome::WebLoginUnknown`] when refused, expired or collected. + PollWebLogin { secret: String }, + /// Revoke the caller's own API token `token` ([`AuthOutcome::LoggedOut`]). + Logout { token: String }, + /// Who the caller is ([`AuthOutcome::Identity`]). + Whoami, +} + +/// What came of an [`AuthRequest`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AuthOutcome { + /// A new API token, for `username`. + Token { + token: String, + username: String, + }, + /// The page the user approves the login on, and the login's secret + /// (what the client polls with). + WebLoginStarted { + login_url: String, + secret: String, + }, + WebLoginPending, + WebLoginUnknown, + LoggedOut, + Identity { + username: String, + }, +} + +/// A request the host sends on to the upstream of the repository (a proxy) +/// or of a group's first proxy member the caller may read: npm's audits. +/// The upstream's status, content type and body are the response. A hosted +/// repository has no upstream: 404. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Forward { + /// The path below the upstream's `remote_url`, starting with `/` + /// (with a query, if any). + pub path: String, + pub content_type: String, + /// Sent as a POST. + pub body: Vec, +} + /// An unlist, relist or delete, and the response when it succeeds (the host /// answers 404 for an unknown version). #[derive(Debug, Clone, PartialEq, Eq)] @@ -600,6 +804,15 @@ pub enum Action { Relist(ChangeRequest), /// Remove a version and its bytes (delete permission). Delete(ChangeRequest), + /// Make these changes ([`ChangeSet`]): never for a GET or HEAD. + Change(ChangeSet), + /// Read the request body, then call `handle` again with it + /// ([`Request::body`]). + ReadBody(BodyRequest), + /// Log in, out, or say who the caller is ([`AuthRequest`]). + Auth(AuthRequest), + /// Send the request on to the upstream ([`Forward`]; read permission). + Forward(Forward), /// Serve the file at this path (read requests), found as /// [`Query::FetchFile`] finds it, or 404. The host streams it with its /// validators (`ETag`, its SHA-256; `Last-Modified`), answers conditional @@ -647,6 +860,8 @@ pub enum UpstreamOp { Dating(PackageVersions), /// A page of the upstream's search results. Search(SearchQuery), + /// The tags the upstream gives a package (canonical id). + Tags(String), /// The URL of the file at a path (files formats). The host downloads it, /// following redirects (its credentials go to the remote's origin only), /// caches it as the file, and re-validates it with a conditional request @@ -684,6 +899,25 @@ pub enum Freshness { pub struct Fetch { pub url: String, pub freshness: Freshness, + /// The `Accept` header to send, if any (npm's abbreviated metadata). + pub accept: Option, +} + +impl Fetch { + /// A fetch of `url` without an `Accept` header. + pub fn new(url: impl Into, freshness: Freshness) -> Self { + Fetch { + url: url.into(), + freshness, + accept: None, + } + } + + /// The fetch with an `Accept` header. + pub fn accepting(mut self, accept: &str) -> Self { + self.accept = Some(accept.to_string()); + self + } } /// How a [`Fetch`] went. @@ -747,6 +981,17 @@ pub struct UpstreamHit { pub total_downloads: Option, } +/// Where an upstream keeps a version's package, and the digests it +/// publishes for it: the host refuses a package that does not match them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageLocation { + pub url: String, + /// SHA-512, base64. + pub sha512: Option, + /// SHA-1, lowercase hex. + pub sha1: Option, +} + /// One page of an upstream's search results. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UpstreamSearch { @@ -774,6 +1019,11 @@ pub enum UpstreamAnswer { Metadata(PackageMetadata), /// `None`: the upstream offers no search. Search(Option), + /// [`UpstreamAnswer::Package`] with the digests the upstream publishes. + Located(PackageLocation), + /// The answer to [`UpstreamOp::Tags`]; `None`: the upstream does not know + /// the package. + Tags(Option>), /// The file's URL, on an origin a [`Fetch`] may reach; `None`: the /// upstream cannot have the path. File(Option), diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index 60178b6a..41670a2b 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -5,10 +5,12 @@ use std::path::Path; use std::sync::Arc; use forge_core::domain::{Dependency, DependencyGroup, FormatError, ParsedUpload}; +use std::sync::OnceLock; + use forge_core::format::{Format, ReadSeek}; use forge_plugin_api::{self as api, FormatPlugin}; -use crate::manifest::{Assets, Storage}; +use crate::manifest::{Assets, Storage, UpstreamToken}; use crate::Plugin; /// The longest version the host takes from a format plugin, in bytes: a @@ -31,11 +33,13 @@ const MAX_ID_LENGTH: usize = 512; /// A format plugin as a [`Format`]: what the hosted write path, the read /// ops, the proxy read path and retention work with. +#[derive(Clone)] pub struct PluginFormat { id: &'static str, plugin: Arc, assets: Assets, storage: Storage, + upstream_token: UpstreamToken, } impl PluginFormat { @@ -46,8 +50,120 @@ impl PluginFormat { plugin: plugin.implementation().clone(), assets: plugin.manifest().assets.clone(), storage: plugin.manifest().storage, + upstream_token: plugin.manifest().upstream_token, } } + + /// The format of one publish through the plugin's protocol: the package + /// is parsed with the document it came in, when there is one + /// ([`FormatPlugin::parse_upload`]; the host sets + /// [`UploadFormat::document`] once the body is read, before the staged + /// package is parsed), must have the canonical id `expected_id`, when + /// one is given, and its identity is recorded in + /// [`UploadFormat::parsed`]. + pub fn upload( + &self, + document: bool, + publisher: Option, + expected_id: Option, + ) -> UploadFormat { + UploadFormat { + format: self.clone(), + document: document.then(|| Arc::new(OnceLock::new())), + publisher, + expected_id, + parsed: Arc::new(OnceLock::new()), + } + } +} + +/// See [`PluginFormat::upload`]. +pub struct UploadFormat { + format: PluginFormat, + /// The rest of the document the package came in, for an + /// [`api::Upload::Document`]. + pub document: Option>>>, + /// The username of the account publishing. + publisher: Option, + expected_id: Option, + /// `(canonical id, normalized version)` of the package, once parsed. + pub parsed: Arc>, +} + +impl Format for UploadFormat { + fn id(&self) -> &'static str { + self.format.id() + } + + fn package_asset_path(&self) -> &str { + self.format.package_asset_path() + } + + fn package_content_type(&self) -> &str { + self.format.package_content_type() + } + + fn manifest_asset_path(&self) -> &str { + self.format.manifest_asset_path() + } + + fn manifest_content_type(&self) -> &str { + self.format.manifest_content_type() + } + + fn parse_upload(&self, mut file: &mut dyn ReadSeek) -> Result { + file.rewind()?; + let parsed = match &self.document { + Some(document) => { + let document = document.get().ok_or_else(|| { + FormatError::InvalidPackage("the upload's document is incomplete".into()) + })?; + let upload = api::Uploaded { + document, + publisher: self.publisher.as_deref(), + }; + self.format.plugin.parse_upload(&upload, &mut file) + } + None => self.format.plugin.parse_package(&mut file), + } + .map_err(format_error) + .and_then(parsed_upload)?; + if let Some(expected) = &self.expected_id { + if &parsed.id != expected { + return Err(FormatError::InvalidPackage(format!( + "the package is {}, not {expected}", + parsed.id + ))); + } + } + let _ = self.parsed.set((parsed.id.clone(), parsed.version.clone())); + Ok(parsed) + } + + fn parse_upload_file(&self, path: &Path) -> Result { + let mut file = std::fs::File::open(path)?; + self.parse_upload(&mut file) + } + + fn normalize_version(&self, raw: &str) -> Result { + self.format.normalize_version(raw) + } + + fn compare_versions(&self, a: &str, b: &str) -> Ordering { + self.format.compare_versions(a, b) + } + + fn lower_id(&self, id: &str) -> String { + self.format.lower_id(id) + } + + fn validate_id(&self, id: &str) -> Result<(), FormatError> { + self.format.validate_id(id) + } + + fn manifest_summary(&self, manifest: &[u8]) -> Option { + self.format.manifest_summary(manifest) + } } fn format_error(e: api::FormatError) -> FormatError { @@ -202,6 +318,18 @@ impl Format for PluginFormat { .map_err(format_error) } + fn upstream_bearer_token(&self) -> bool { + self.upstream_token == UpstreamToken::Bearer + } + + fn readme(&self, manifest: &[u8]) -> Option { + self.plugin.readme(manifest) + } + + fn reserved_prefix(&self, prefix: &str) -> Result { + self.plugin.reserved_prefix(prefix).map_err(format_error) + } + fn manifest_summary(&self, manifest: &[u8]) -> Option { let text = self.plugin.manifest_summary(manifest)?; match serde_json::from_str::(&text) { diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index ba100537..16f174e9 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -19,7 +19,7 @@ use std::sync::Arc; pub use forge_plugin_api as api; pub use forge_plugin_api::FormatPlugin; -pub use format::PluginFormat; +pub use format::{PluginFormat, UploadFormat}; pub use manifest::Manifest; /// Errors installing a plugin. diff --git a/crates/plugin/src/manifest.rs b/crates/plugin/src/manifest.rs index 8df5cba4..281cac56 100644 --- a/crates/plugin/src/manifest.rs +++ b/crates/plugin/src/manifest.rs @@ -53,6 +53,7 @@ const RESERVED_MOUNTS: &[&str] = &[ "assets", "auth", "browse", + "cli-login", "healthz", "images", "invite", @@ -91,6 +92,21 @@ pub struct Manifest { pub storage: Storage, /// How a version's files are stored. pub assets: Assets, + /// How a proxy given a password and no username authenticates to its + /// upstream (`upstream_token`). + pub upstream_token: UpstreamToken, +} + +/// The `upstream_token` key: what a proxy does with a password stored +/// without a username. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum UpstreamToken { + /// Nothing: a proxy authenticates only with a username and a password + /// (Basic). + #[default] + None, + /// `bearer`: the password is a token, sent as `Authorization: Bearer`. + Bearer, } /// What a format's repositories hold (`storage`). @@ -155,6 +171,7 @@ struct RawPlugin { mount: Option, kinds: Option>, storage: Option, + upstream_token: Option, } impl Manifest { @@ -189,6 +206,15 @@ impl Manifest { } }; validate_assets(&raw.assets)?; + let upstream_token = match p.upstream_token.as_deref() { + None => UpstreamToken::None, + Some("bearer") => UpstreamToken::Bearer, + Some(other) => { + return Err(PluginError::Manifest(format!( + "upstream_token {other:?} is not supported (bearer)" + ))) + } + }; Ok(Manifest { id: p.id, name: p.name, @@ -199,6 +225,7 @@ impl Manifest { kinds, storage, assets: raw.assets, + upstream_token, }) } } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 9a857ed9..65ea6417 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,9 +22,10 @@ forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } forge-oci = { path = "../format-oci" } -# The built-in format plugins (crates/formats/nuget, crates/formats/raw), -# linked in. +# The built-in format plugins (crates/formats/nuget, crates/formats/npm, +# crates/formats/raw), linked in. forge-format-nuget = { path = "../formats/nuget" } +forge-format-npm = { path = "../formats/npm" } forge-format-raw = { path = "../formats/raw" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "signal"] } @@ -64,6 +65,8 @@ argon2 = { workspace = true, features = ["password-hash"] } tokio-util = { workspace = true } # Format-plugin uploads: the named file part of a multipart body, streamed. multer = "3" +# npm audits: a gzip-compressed request body. +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } # The file store: the MD5 of uploads (the `.md5` sidecar), file names in # headers, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content # types guessed from file names. diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index abe23813..ffca86d3 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use axum::extract::{Path, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; +use forge_core::format::Format; use forge_core::net::ssrf::validate_url; use forge_db::model::{ DeploymentPolicy, RepoKind, Repository, CONTENT_DISPOSITION_KEY, DEPLOYMENT_POLICY_KEY, @@ -70,7 +71,7 @@ pub struct RepoView { /// Hosted: the largest push it accepts (for docker, the largest blob), /// `None` for the default of its format (`GET /admin/formats`). pub max_upload_bytes: Option, - /// Hosted, of a files format (raw): `allow_redeploy` | + /// Hosted, of a files format (raw) or a package format: `allow_redeploy` | /// `disable_redeploy` | `read_only`. pub deployment_policy: Option<&'static str>, /// A files format (raw): `attachment` (files download) | `inline` @@ -112,8 +113,8 @@ impl RepoView { .writable_member_id() .and_then(|id| names.get(&id).cloned()), max_upload_bytes: r.max_upload_bytes(), - deployment_policy: (stores_files && r.kind() == RepoKind::Hosted) - .then(|| r.deployment_policy().as_str()), + deployment_policy: (!r.is_docker() && r.kind() == RepoKind::Hosted) + .then(|| deployment_policy_of(r).as_str()), content_disposition: stores_files.then(|| { if r.inline_content() { "inline" @@ -194,15 +195,17 @@ pub struct CreateRepo { /// Hosted docker only: with `immutable_tags`, `latest` may still move. #[serde(default)] pub latest_mutable: Option, - /// Docker group only: the hosted member a push to the group goes to. + /// Docker or package group only: the hosted member of its format a + /// push or publish to the group goes to. #[serde(default)] pub writable_member: Option, /// Hosted only: the largest push it accepts, in bytes (for docker, the /// largest blob). Absent: the default of its format. #[serde(default)] pub max_upload_bytes: Option, - /// Hosted raw only: `allow_redeploy`, `disable_redeploy` (default) or - /// `read_only`. + /// Hosted raw or package only: `allow_redeploy`, `disable_redeploy` + /// (default) or `read_only`. A package repository's `allow_redeploy` + /// says the same and may not contradict it. #[serde(default)] pub deployment_policy: Option, /// Raw only: `attachment` (default) or `inline`. @@ -238,14 +241,16 @@ pub struct CreateRepo { /// * Turning `anonymous_read` off also deletes the anonymous role's unscoped /// `read` grant on the repository (the grant migration 0007 synthesized from /// the flag), so anonymous access really ends. -/// * Docker options (`host`, `immutable_tags`, `latest_mutable`, -/// `writable_member`) follow the same rules (`host` and `writable_member`: -/// `null` clears) and are refused on NuGet repositories. +/// * Docker options (`host`, `immutable_tags`, `latest_mutable`) follow the +/// same rules (`host`: `null` clears) and are refused on other formats; a +/// docker or package group's `writable_member` too (`null` clears). /// * `max_upload_bytes` (hosted only): `null` clears it, back to the default /// of the repository's format. -/// * Raw options (`deployment_policy`, `content_disposition`, -/// `max_age_seconds`) are refused on other formats; `max_age_seconds: -/// null` goes back to the default. +/// * Raw options (`content_disposition`, `max_age_seconds`) are refused on +/// other formats; `max_age_seconds: null` goes back to the default. A +/// hosted raw or package repository's `deployment_policy` sets its +/// policy; a package repository's `allow_redeploy` follows it (and sets +/// it, when given alone). /// * Unknown fields are rejected (a misspelled field would otherwise keep the /// stored value). #[derive(Debug, Deserialize)] @@ -393,11 +398,17 @@ const MAX_PACKAGE_DELAY_SECONDS: i32 = 365 * 24 * 60 * 60; /// Most reserved prefixes a group may carry. const MAX_RESERVED_PREFIXES: usize = 100; -/// Validate and normalize a group's reserved id prefixes: each is a NuGet id -/// prefix (`A-Z a-z 0-9 . _ -`, at most 100 bytes, an optional trailing `*` -/// dropped), stored lowercase, deduplicated. A docker group's prefixes are -/// image-name prefixes and may contain `/` (a namespace such as `acme/`). -fn normalize_prefixes(raw: &[String], docker: bool) -> Result, String> { +/// Validate and normalize a group's reserved id prefixes: each is at most +/// 100 bytes, an optional trailing `*` dropped, deduplicated. A docker +/// group's are image-name prefixes (`A-Z a-z 0-9 . _ - /`, a namespace such +/// as `acme/`), stored lowercase; a package group's are what its format +/// makes of them ([`Format::reserved_prefix`]: NuGet's `A-Z a-z 0-9 . _ -`, +/// lowercase; npm's `@acme` is the scope `@acme/`). +fn normalize_prefixes( + raw: &[String], + docker: bool, + format: Option<&dyn Format>, +) -> Result, String> { if raw.len() > MAX_RESERVED_PREFIXES { return Err(format!( "at most {MAX_RESERVED_PREFIXES} reserved_prefixes are allowed" @@ -407,25 +418,38 @@ fn normalize_prefixes(raw: &[String], docker: bool) -> Result, Strin for prefix in raw { let prefix = prefix.trim(); let prefix = prefix.strip_suffix('*').unwrap_or(prefix); - let valid = !prefix.is_empty() - && prefix.len() <= 100 - && prefix.bytes().all(|b| { - b.is_ascii_alphanumeric() - || matches!(b, b'.' | b'_' | b'-') - || (docker && b == b'/') - }); - if !valid { - return Err(if docker { - format!( - "reserved_prefixes: {prefix:?} is not an image name prefix (letters, digits, '.', '_', '-', '/')" - ) - } else { - format!( - "reserved_prefixes: {prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" - ) - }); + if prefix.is_empty() || prefix.len() > 100 { + return Err(format!( + "reserved_prefixes: {prefix:?} must be 1-100 bytes long" + )); } - let prefix = prefix.to_ascii_lowercase(); + let prefix = match format.filter(|_| !docker) { + Some(format) => format.reserved_prefix(prefix).map_err(|e| match e { + forge_core::domain::FormatError::InvalidPackage(m) => { + format!("reserved_prefixes: {m}") + } + e => format!("reserved_prefixes: {e}"), + })?, + None => { + let valid = prefix.bytes().all(|b| { + b.is_ascii_alphanumeric() + || matches!(b, b'.' | b'_' | b'-') + || (docker && b == b'/') + }); + if !valid { + return Err(if docker { + format!( + "reserved_prefixes: {prefix:?} is not an image name prefix (letters, digits, '.', '_', '-', '/')" + ) + } else { + format!( + "reserved_prefixes: {prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" + ) + }); + } + prefix.to_ascii_lowercase() + } + }; if !out.contains(&prefix) { out.push(prefix); } @@ -433,6 +457,19 @@ fn normalize_prefixes(raw: &[String], docker: bool) -> Result, Strin Ok(out) } +/// What a hosted repository does with a publish of a version it has: the +/// stored `deployment_policy`, or, for a package repository that has none +/// yet, what `allow_redeploy` says. +fn deployment_policy_of(r: &Repository) -> DeploymentPolicy { + if r.config.get(DEPLOYMENT_POLICY_KEY).is_some() { + r.deployment_policy() + } else if r.allow_redeploy { + DeploymentPolicy::AllowRedeploy + } else { + DeploymentPolicy::DisableRedeploy + } +} + /// Validate and normalize a docker repository's `host`: a lowercase DNS name /// (no port, no scheme) other than the server's own public host. fn normalize_docker_host(raw: &str, public_base_url: &str) -> Result { @@ -461,13 +498,17 @@ fn normalize_docker_host(raw: &str, public_base_url: &str) -> Result Result { +/// A group's `writable_member`: an existing hosted repository of the +/// group's `format`, whose id is stored. +async fn validate_writable_member( + state: &AppState, + name: &str, + format: &str, +) -> Result { match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(r)) if r.is_docker() && r.kind() == RepoKind::Hosted => Ok(r.id), + Ok(Some(r)) if r.format == format && r.kind() == RepoKind::Hosted => Ok(r.id), Ok(_) => Err(bad_request(format!( - "writable_member {name:?} must be a hosted docker repository (and a member of the group)" + "writable_member {name:?} must be a hosted {format} repository (and a member of the group)" ))), Err(e) => Err(db_error(e)), } @@ -587,10 +628,14 @@ pub async fn create(State(state): State, Json(body): Json) )); } let docker = body.format == DOCKER_FORMAT; + let files = state.formats.stores_files(&body.format); + // A package format plugin's (NuGet's, npm's): package versions. + let package = !files && state.formats.plugin(&body.format).is_some(); let parsed_kind = RepoKind::from(kind.as_str()); let docker_config = match docker_options( &state, docker, + package.then_some(body.format.as_str()), parsed_kind, &body.name, DockerOptions { @@ -614,7 +659,6 @@ pub async fn create(State(state): State, Json(body): Json) "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); } - let files = state.formats.stores_files(&body.format); if files { if let Err(resp) = refuse_for_files( Some(body.package_delay_seconds), @@ -625,8 +669,25 @@ pub async fn create(State(state): State, Json(body): Json) return resp; } } + // A package repository's redeploys follow its deployment policy, when + // it is given; `allow_redeploy` says the same. + let policy = body + .deployment_policy + .as_deref() + .and_then(DeploymentPolicy::parse); + if package + && body.allow_redeploy + && policy.is_some_and(|p| p != DeploymentPolicy::AllowRedeploy) + { + return bad_request("allow_redeploy contradicts deployment_policy"); + } + let allow_redeploy = match policy { + Some(policy) if package => policy == DeploymentPolicy::AllowRedeploy, + _ => body.allow_redeploy, + }; let docker_config = match file_options( files, + package, parsed_kind, FileOptions { deployment_policy: body.deployment_policy.clone(), @@ -664,7 +725,7 @@ pub async fn create(State(state): State, Json(body): Json) let reserved_prefixes = match body .reserved_prefixes .as_deref() - .map(|p| normalize_prefixes(p, docker)) + .map(|p| normalize_prefixes(p, docker, state.formats.get(&body.format).as_deref())) { Some(Err(msg)) => return bad_request(&msg), Some(Ok(prefixes)) => Some(prefixes), @@ -712,7 +773,7 @@ pub async fn create(State(state): State, Json(body): Json) }, metadata_ttl_seconds: body.metadata_ttl_seconds, not_found_ttl_seconds: body.not_found_ttl_seconds, - allow_redeploy: body.allow_redeploy, + allow_redeploy, }; // The row and its sealed secret commit together: a failed seal must not @@ -786,10 +847,14 @@ impl DockerOptions { /// Validate the docker options against the repository's format and kind /// and merge them into `existing` (its stored config object). Returns the /// config object to store. +/// +/// A group of a package format (`writable`: its format) takes a writable +/// member too: where its clients' publishes and changes land. #[allow(clippy::result_large_err)] async fn docker_options( state: &AppState, docker: bool, + writable: Option<&str>, kind: RepoKind, name: &str, options: DockerOptions, @@ -797,12 +862,16 @@ async fn docker_options( ) -> Result, Response> { let mut config = existing.clone(); if !docker { - if options.any() { + let writable_only = options.host.is_none() + && options.immutable_tags.is_none() + && options.latest_mutable.is_none(); + if options.any() && !(writable_only && writable.is_some()) { return Err(bad_request( - "host, immutable_tags, latest_mutable and writable_member are only valid for docker repositories", + "host, immutable_tags and latest_mutable are only valid for docker repositories, writable_member for docker and package groups", )); } - return Ok(config); + return writable_member_option(state, writable, kind, options.writable_member, config) + .await; } if !forge_oci::reference::valid_name_component(name) { return Err(bad_request(format!( @@ -817,11 +886,6 @@ async fn docker_options( "immutable_tags and latest_mutable are only valid for hosted repositories", )); } - if options.writable_member.is_some() && kind != RepoKind::Group { - return Err(bad_request( - "writable_member is only valid for group repositories", - )); - } match options.host { Some(Some(host)) => { let host = normalize_docker_host(&host, &state.config.public_base_url()) @@ -839,15 +903,40 @@ async fn docker_options( if let Some(latest) = options.latest_mutable { config.insert(LATEST_MUTABLE_KEY.into(), json!(latest)); } - match options.writable_member { - Some(Some(member)) => { - let member = validate_writable_member(state, &member).await?; + writable_member_option( + state, + Some(DOCKER_FORMAT), + kind, + options.writable_member, + config, + ) + .await +} + +/// Merge a group's `writable_member` (a hosted repository of `format`; +/// `Some(None)`: cleared) into `config`. +#[allow(clippy::result_large_err)] +async fn writable_member_option( + state: &AppState, + format: Option<&str>, + kind: RepoKind, + member: Option>, + mut config: serde_json::Map, +) -> Result, Response> { + if member.is_some() && kind != RepoKind::Group { + return Err(bad_request( + "writable_member is only valid for group repositories", + )); + } + match (member, format) { + (Some(Some(member)), Some(format)) => { + let member = validate_writable_member(state, &member, format).await?; config.insert(WRITABLE_MEMBER_ID_KEY.into(), json!(member)); } - Some(None) => { + (Some(None), _) => { config.remove(WRITABLE_MEMBER_ID_KEY); } - None => {} + _ => {} } Ok(config) } @@ -872,21 +961,24 @@ impl FileOptions { const MAX_FILE_MAX_AGE_SECONDS: u64 = 10 * 366 * 24 * 60 * 60; /// Validate the files options against the repository's format and kind and -/// merge them into `config` (the config object to store). +/// merge them into `config` (the config object to store). A package +/// repository (`package`) takes the deployment policy too. #[allow(clippy::result_large_err)] fn file_options( files: bool, + package: bool, kind: RepoKind, options: FileOptions, mut config: serde_json::Map, ) -> Result, Response> { if !files { - if options.any() { + let policy_only = + options.content_disposition.is_none() && options.max_age_seconds.is_none(); + if options.any() && !(policy_only && package) { return Err(bad_request( - "deployment_policy, content_disposition and max_age_seconds are only valid for repositories of files (raw)", + "content_disposition and max_age_seconds are only valid for repositories of files (raw), deployment_policy for those and package repositories", )); } - return Ok(config); } if let Some(policy) = options.deployment_policy { if kind != RepoKind::Hosted { @@ -1064,10 +1156,13 @@ pub async fn update( "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); } + let files = state.formats.stores_files(&existing.format); + let package = !files && state.formats.plugin(&existing.format).is_some(); let stored_config = existing.config.as_object().cloned().unwrap_or_default(); let config = match docker_options( &state, docker, + package.then_some(existing.format.as_str()), existing.kind(), &final_name, DockerOptions { @@ -1083,11 +1178,40 @@ pub async fn update( Ok(config) => config, Err(resp) => return resp, }; + // A package repository's deployment policy and `allow_redeploy` say + // the same: setting one sets the other. + let policy = body + .deployment_policy + .as_deref() + .and_then(DeploymentPolicy::parse); + let mut allow_redeploy = body.allow_redeploy.unwrap_or(existing.allow_redeploy); + let mut deployment_policy = body.deployment_policy.clone(); + if package && existing.kind() == RepoKind::Hosted { + match (policy, body.allow_redeploy) { + (Some(policy), Some(allow)) if allow != (policy == DeploymentPolicy::AllowRedeploy) => { + return bad_request("allow_redeploy contradicts deployment_policy"); + } + (Some(policy), _) => allow_redeploy = policy == DeploymentPolicy::AllowRedeploy, + (None, Some(allow)) if deployment_policy.is_none() => { + deployment_policy = Some( + if allow { + DeploymentPolicy::AllowRedeploy + } else { + DeploymentPolicy::DisableRedeploy + } + .as_str() + .to_string(), + ); + } + _ => {} + } + } let mut config = match file_options( - state.formats.stores_files(&existing.format), + files, + package, existing.kind(), FileOptions { - deployment_policy: body.deployment_policy.clone(), + deployment_policy, content_disposition: body.content_disposition.clone(), max_age_seconds: body.max_age_seconds, }, @@ -1109,7 +1233,11 @@ pub async fn update( if existing.kind() != RepoKind::Group { return bad_request("reserved_prefixes is only valid for group repositories"); } - let prefixes = match normalize_prefixes(prefixes, docker) { + let prefixes = match normalize_prefixes( + prefixes, + docker, + state.formats.get(&existing.format).as_deref(), + ) { Ok(prefixes) => prefixes, Err(msg) => return bad_request(&msg), }; @@ -1187,7 +1315,7 @@ pub async fn update( not_found_ttl_seconds: body .not_found_ttl_seconds .unwrap_or(existing.not_found_ttl_seconds), - allow_redeploy: body.allow_redeploy.unwrap_or(existing.allow_redeploy), + allow_redeploy, }; if let Err(resp) = validate_timings( upd.package_delay_seconds, @@ -1227,6 +1355,10 @@ pub async fn update( if let Err(e) = forge_db::proxy_dao::reset_for_new_upstream(&mut tx, updated.id).await { return db_error(e); } + // Its tags were the previous upstream's. + if let Err(e) = forge_db::tag_dao::delete_repository_tags(&mut *tx, updated.id).await { + return db_error(e); + } // A docker proxy's tags name what the previous upstream served: // the repointed proxy starts with an empty cache. Locked first, as // the purge requires: a fetch from the previous upstream still diff --git a/crates/server/src/admin/retention.rs b/crates/server/src/admin/retention.rs index 96ccdaa2..8cb5f583 100644 --- a/crates/server/src/admin/retention.rs +++ b/crates/server/src/admin/retention.rs @@ -427,13 +427,19 @@ pub async fn preview( ) }) .await; - let (decisions, proxy_decisions) = match evaluation { + let (mut decisions, proxy_decisions) = match evaluation { Ok(d) => d, Err(e) => { tracing::error!(error = %e, repo = %repo.name, "retention preview evaluation failed"); return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(); } }; + // As the worker does: a tagged version (npm's dist-tags) is never touched. + if let Err(e) = + forge_core::retention::spare_protected(&state.pool, repo.id, &mut decisions).await + { + return db_error(e); + } use forge_core::retention::model::Action; let unlisted = decisions diff --git a/crates/server/src/auth/mod.rs b/crates/server/src/auth/mod.rs index 94901205..13518583 100644 --- a/crates/server/src/auth/mod.rs +++ b/crates/server/src/auth/mod.rs @@ -190,6 +190,10 @@ pub fn auth_router() -> Router { .route("/auth/me", get(me)) .route("/auth/tokens", get(list_tokens).post(create_token)) .route("/auth/tokens/{id}", delete(revoke_token)) + .route( + "/auth/cli-login/{secret}", + get(cli_login_info).post(cli_login_decide), + ) } /// Resolve the realm snapshot or produce the fail-closed error response. @@ -223,7 +227,7 @@ async fn oidc_login( headers: HeaderMap, Query(q): Query, ) -> Response { - let client_ip = login_client_ip(&state, connect, &headers); + let client_ip = login_client_ip(&state, connect.map(|Extension(ci)| ci.0), &headers); if !state.oidc_login_starts.allow(client_ip) { return ( StatusCode::TOO_MANY_REQUESTS, @@ -798,54 +802,181 @@ async fn revoke_token( } } -// ── password login + identifier-first UX ──────────────────────────────────── +// ── CLI logins in the browser (`npm login`, see `crate::cli_auth`) ────────── -/// `POST /auth/login` request body. +/// `GET /auth/cli-login/{secret}` answer: what the approval page shows of a +/// pending CLI login. +#[derive(Debug, Serialize)] +struct CliLoginView { + /// The client: the repository's format (`npm`). + client: String, + /// The repository the CLI logs in to. + repository: String, + /// The host name the CLI reported. + hostname: Option, + /// Where the request came from. + client_ip: Option, + created_at: DateTime, + expires_at: DateTime, +} + +/// `GET /auth/cli-login/{secret}` — the pending CLI login whose login URL +/// carries `secret`, for the signed-in user's approval page. 404 once it +/// expired, was decided or never existed. +async fn cli_login_info( + State(state): State, + headers: HeaderMap, + Path(secret): Path, +) -> Response { + let rt = SessionAuthCtx::of(&state); + if let Err(e) = require_session(&rt, &headers).await { + return e.into_response(); + } + let login = match forge_db::cli_login_dao::find(&state.pool, &hash_token(&secret)).await { + Ok(Some(login)) if login.user_id.is_none() => login, + Ok(_) => return (StatusCode::NOT_FOUND, "no such login request").into_response(), + Err(e) => { + tracing::error!(error = %e, "cli login lookup failed"); + return auth_unavailable(); + } + }; + let repository = + match forge_db::repository_dao::find_repository_by_id(&state.pool, login.repository_id) + .await + { + Ok(Some(repo)) => repo.name, + Ok(None) => return (StatusCode::NOT_FOUND, "no such login request").into_response(), + Err(e) => { + tracing::error!(error = %e, "cli login repository lookup failed"); + return auth_unavailable(); + } + }; + Json(CliLoginView { + client: login.client, + repository, + hostname: login.hostname, + client_ip: login.client_ip, + created_at: login.created_at, + expires_at: login.expires_at, + }) + .into_response() +} + +/// `POST /auth/cli-login/{secret}` body. #[derive(Debug, Deserialize)] -struct LoginBody { - username: String, - password: String, +#[serde(deny_unknown_fields)] +struct CliLoginDecision { + /// `true` signs the CLI in as the session's user; `false` refuses it. + approve: bool, } -/// `POST /auth/login` — authenticate a username/password through the realm -/// chain (local + credential-forwarding realms, in priority order) and mint a -/// browser session. -/// -/// This is the form-login counterpart of the wire Basic path: it runs the SAME -/// chain with the same fail-closed semantics, but is **not** gated by -/// `auth.allow_password_basic` (that flag governs API Basic auth only). The -/// response mirrors `/auth/me` so the SPA can bootstrap immediately; failures -/// are a uniform 401 (no username-exists oracle). A username over -/// [`MAX_USERNAME_BYTES`], or one containing `:`, is refused the same way -/// before any lookup or lockout bookkeeping (it is never counted, so it cannot -/// grow the lockout map). -async fn password_login( +/// `POST /auth/cli-login/{secret}` — approve (or refuse) a pending CLI +/// login as the session's user (CSRF-protected). The CLI collects its token +/// on its next poll. 204, or 404 when the request is gone. +async fn cli_login_decide( State(state): State, - connect: Option>>, headers: HeaderMap, - Json(body): Json, + Path(secret): Path, + Json(body): Json, ) -> Response { - let snap = match snapshot_or_resp(&state).await { + let rt = SessionAuthCtx::of(&state); + let ActiveSession { user, .. } = match require_session_csrf(&rt, &headers).await { Ok(s) => s, - Err(resp) => return resp, + Err(e) => return e.into_response(), }; + let hash = hash_token(&secret); + let decided = if body.approve { + forge_db::cli_login_dao::approve(&state.pool, &hash, user.id).await + } else { + forge_db::cli_login_dao::deny(&state.pool, &hash).await + }; + match decided { + Ok(true) => { + tracing::info!( + target: "forge::audit", + user_id = user.id, + approved = body.approve, + "cli login decided" + ); + StatusCode::NO_CONTENT.into_response() + } + Ok(false) => (StatusCode::NOT_FOUND, "no such login request").into_response(), + Err(e) => { + tracing::error!(error = %e, "cli login decision failed"); + auth_unavailable() + } + } +} + +// ── password login + identifier-first UX ──────────────────────────────────── + +/// `POST /auth/login` request body. +#[derive(Debug, Deserialize)] +struct LoginBody { + username: String, + password: String, +} + +/// Why a password login was refused ([`verify_password_login`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LoginRefusal { + /// Wrong credentials, or none the chain accepts: a uniform 401. + Invalid, + /// The lockout holds the `(client IP, username)` pair: 429, with how + /// many seconds to wait. + Throttled { retry_after: u64 }, + /// Forge could not decide (its database failed): 503. + Unavailable, +} + +impl IntoResponse for LoginRefusal { + fn into_response(self) -> Response { + match self { + LoginRefusal::Invalid => { + (StatusCode::UNAUTHORIZED, "invalid credentials").into_response() + } + LoginRefusal::Throttled { retry_after } => ( + StatusCode::TOO_MANY_REQUESTS, + [(header::RETRY_AFTER, retry_after.to_string())], + "too many failed sign-in attempts; try again later", + ) + .into_response(), + LoginRefusal::Unavailable => auth_unavailable(), + } + } +} + +/// Check a username and password as an interactive login does: through the +/// realm chain (local and credential-forwarding realms, in priority order), +/// under the login lockout, and never for a secret that is a token Forge +/// issued. Not gated by `allow_password_basic`, which governs passwords on +/// the feeds only. Returns the signed-in user and the canonical username. +/// `POST /auth/login` and `npm login` sign in through it. +pub(crate) async fn verify_password_login( + state: &AppState, + client_ip: IpAddr, + username: &str, + password: &str, +) -> Result<(forge_core::auth::principal::UserPrincipal, String), LoginRefusal> { + let snap = snapshot_or_resp(state) + .await + .map_err(|_| LoginRefusal::Unavailable)?; let Some(chain) = snap.chain.clone() else { // No enabled password-capable realm. Uniform denial (the UI learns // whether to even offer the form from /auth/login-options). - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); }; - if body.username.len() > MAX_USERNAME_BYTES || body.password.is_empty() { - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + if username.len() > MAX_USERNAME_BYTES || password.is_empty() { + return Err(LoginRefusal::Invalid); } // The lockout, the chain and the account lookup all key on the canonical // username (trimmed, NFC, lowercased). A name with a `:` cannot sign in // with a password (a forwarding realm's upstream would split it, and Basic // cannot carry one); like an oversized one it is never counted. - let Ok(username) = forge_core::auth::canonical_password_username(&body.username) else { - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + let Ok(username) = forge_core::auth::canonical_password_username(username) else { + return Err(LoginRefusal::Invalid); }; - let client_ip = login_client_ip(&state, connect, &headers); // Reserve the attempt against the (IP, username) lockout budget before // running the chain; a locked pair is refused without any provider work // (no argon2, nothing forwarded upstream). @@ -866,12 +997,7 @@ async fn password_login( 1 } }; - return ( - StatusCode::TOO_MANY_REQUESTS, - [(header::RETRY_AFTER, retry_after.to_string())], - "too many failed sign-in attempts; try again later", - ) - .into_response(); + return Err(LoginRefusal::Throttled { retry_after }); } }; // A token Forge issued is never a password, even one that cannot be used @@ -879,24 +1005,24 @@ async fn password_login( // credential-forwarding realm would send it upstream), and it does not // sign in either: a session carries its owner's rights, not the token's // scope. The same holds for a workload API key. - if forge_core::auth::is_workload_key(&body.password) { + if forge_core::auth::is_workload_key(password) { attempt.failed(); - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); } - match check_token(&state.pool, &body.password).await { + match check_token(&state.pool, password).await { Ok(TokenCheck::Unknown) => {} Ok(TokenCheck::Live(_) | TokenCheck::Unusable) => { attempt.failed(); - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); } Err(_) => { attempt.released(); - return auth_unavailable(); + return Err(LoginRefusal::Unavailable); } } let cred = PasswordCredential { username: &username, - password: &body.password, + password, client_ip, }; let identity = match chain.authenticate(&cred).await { @@ -907,16 +1033,16 @@ async fn password_login( // towards the lockout. ChainOutcome::Unavailable => { attempt.released(); - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); } // Forge's own database failed: not a verdict on the credentials. ChainOutcome::Failed => { attempt.released(); - return auth_unavailable(); + return Err(LoginRefusal::Unavailable); } ChainOutcome::Rejected | ChainOutcome::NoCredentials => { attempt.failed(); - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); } }; @@ -924,12 +1050,12 @@ async fn password_login( Ok(Some(p)) => p, Ok(None) => { attempt.failed(); - return (StatusCode::UNAUTHORIZED, "invalid credentials").into_response(); + return Err(LoginRefusal::Invalid); } Err(e) => { tracing::error!(error = %e, "login principal mapping failed"); attempt.released(); - return auth_unavailable(); + return Err(LoginRefusal::Unavailable); } }; attempt.succeeded(); @@ -938,6 +1064,33 @@ async fn password_login( // effective set so the new session sees it immediately. forge_core::authz::invalidate_user(&state.cache, principal.user_id).await; } + Ok((principal, username)) +} + +/// `POST /auth/login` — authenticate a username/password through the realm +/// chain (local + credential-forwarding realms, in priority order) and mint a +/// browser session. +/// +/// This is the form-login counterpart of the wire Basic path: it runs the SAME +/// chain with the same fail-closed semantics, but is **not** gated by +/// `auth.allow_password_basic` (that flag governs API Basic auth only). The +/// response mirrors `/auth/me` so the SPA can bootstrap immediately; failures +/// are a uniform 401 (no username-exists oracle). A username over +/// [`MAX_USERNAME_BYTES`], or one containing `:`, is refused the same way +/// before any lookup or lockout bookkeeping (it is never counted, so it cannot +/// grow the lockout map). +async fn password_login( + State(state): State, + connect: Option>>, + headers: HeaderMap, + Json(body): Json, +) -> Response { + let client_ip = login_client_ip(&state, connect.map(|Extension(ci)| ci.0), &headers); + let (principal, username) = + match verify_password_login(&state, client_ip, &body.username, &body.password).await { + Ok(verified) => verified, + Err(refusal) => return refusal.into_response(), + }; // Fresh session per login (fixation-proof), same as the OIDC callback. let issued = match state @@ -1020,7 +1173,7 @@ async fn lookup_invitation( headers: HeaderMap, Json(body): Json, ) -> Response { - let client_ip = login_client_ip(&state, connect, &headers); + let client_ip = login_client_ip(&state, connect.map(|Extension(ci)| ci.0), &headers); let Ok(attempt) = state .invitation_throttle .begin(client_ip, INVITATION_THROTTLE_KEY) @@ -1067,7 +1220,7 @@ async fn accept_invitation( ) .into_response(); } - let client_ip = login_client_ip(&state, connect, &headers); + let client_ip = login_client_ip(&state, connect.map(|Extension(ci)| ci.0), &headers); let Ok(attempt) = state .invitation_throttle .begin(client_ip, INVITATION_THROTTLE_KEY) @@ -1122,13 +1275,13 @@ async fn accept_invitation( /// every `X-Forwarded-For` line behind the configured trusted proxies and /// canonicalizes IPv4-mapped addresses, mirroring the principal middleware. /// Loopback when no connect info is available (in-process router tests). -fn login_client_ip( +pub(crate) fn login_client_ip( state: &AppState, - connect: Option>>, + connect: Option, headers: &HeaderMap, ) -> IpAddr { let peer = connect - .map(|Extension(ci)| ci.0.ip()) + .map(|addr| addr.ip()) .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); let xff = forwarded_for(headers); let trusted = state.config.trusted_proxy_cidrs().unwrap_or_default(); diff --git a/crates/server/src/browse.rs b/crates/server/src/browse.rs index 98142186..ab7e668d 100644 --- a/crates/server/src/browse.rs +++ b/crates/server/src/browse.rs @@ -56,10 +56,6 @@ use crate::dispatch::{offline_response, offline_serves_nothing}; use crate::middleware::principal_of; use crate::state::AppState; -/// Canonical asset path of the stored `.nupkg` (mirrors -/// `forge_core::repo::NUPKG_ASSET_PATH`). -const NUPKG_ASSET_PATH: &str = "package.nupkg"; - /// Hard cap on the cross-repo browse page size. const MAX_TAKE: usize = 100; /// Default browse page size. @@ -401,6 +397,18 @@ pub struct PackageDetail { /// Dependency groups keyed by normalized version string. pub dependency_groups_by_version: std::collections::BTreeMap>, + /// The package's tags (npm's dist-tags), tag → version. + #[serde(skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub tags: std::collections::BTreeMap, + /// The message of each deprecated version. + #[serde(skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub deprecated: std::collections::BTreeMap, + /// The latest version's description, when its manifest has one. + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The latest version's README, when its format reads one. + #[serde(skip_serializing_if = "Option::is_none")] + pub readme: Option, } /// `GET /browse/packages/{repo}/{id}` — package detail. @@ -498,7 +506,18 @@ pub async fn package_detail( .unwrap_or_else(|| id.clone()); let mut dependency_groups_by_version = std::collections::BTreeMap::new(); + let mut deprecated = std::collections::BTreeMap::new(); for entry in ®istration { + // Deprecated here, or (a proxy's) upstream. + let upstream_deprecation = entry + .manifest_summary + .as_ref() + .and_then(|s| s.get("deprecated")) + .and_then(serde_json::Value::as_str) + .filter(|m| !m.is_empty()); + if let Some(message) = entry.deprecation.as_deref().or(upstream_deprecation) { + deprecated.insert(entry.version.clone(), message.to_string()); + } let groups = forge_format_nuget::nuspec::summary_of( entry.manifest_summary.as_ref(), entry.nuspec_bytes.as_deref(), @@ -524,7 +543,7 @@ pub async fn package_detail( // Per-version size / published_at from the local component rows when // present (one query for every version, not two per version). let local: std::collections::HashMap, Option)> = - component_dao::version_sizes(&state.pool, repo.id, &lower, NUPKG_ASSET_PATH) + component_dao::version_sizes(&state.pool, repo.id, &lower, format.package_asset_path()) .await .unwrap_or_else(|e| { tracing::warn!(error = %e, "browse: version sizes lookup failed"); @@ -566,6 +585,50 @@ pub async fn package_detail( // the UI can render a delay badge alongside any downloadable versions. detail_versions.extend(delayed); + let tags = ops.tags(&repo.name, &lower).await.unwrap_or_else(|e| { + tracing::warn!(error = %e, "browse: tags lookup failed"); + Default::default() + }); + // The latest version: the one tagged so, else the highest. + let latest = tags + .get("latest") + .and_then(|l| registration.iter().find(|e| &e.version == l)) + .or_else(|| { + registration + .iter() + .max_by(|a, b| format.compare_versions(&a.version, &b.version)) + }); + let description = latest.and_then(|entry| { + let summary = entry.manifest_summary.as_ref(); + summary + .and_then(|s| s.get("description")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .or_else(|| { + forge_format_nuget::nuspec::summary_of(summary, entry.nuspec_bytes.as_deref()) + .metadata + .description + }) + .filter(|d| !d.trim().is_empty()) + }); + let readme = match latest { + Some(entry) => match ops.resolve_nuspec(&repo.name, &lower, &entry.version).await { + Ok(Some(manifest)) => { + let format = format.clone(); + tokio::task::spawn_blocking(move || format.readme(&manifest)) + .await + .ok() + .flatten() + } + Ok(None) => None, + Err(e) => { + tracing::warn!(error = %e, "browse: README lookup failed"); + None + } + }, + None => None, + }; + Json(PackageDetail { repo: repo.name.clone(), repo_kind: repo.kind.clone(), @@ -573,6 +636,10 @@ pub async fn package_detail( original_id, versions: detail_versions, dependency_groups_by_version, + tags, + deprecated, + description, + readme, }) .into_response() } @@ -640,6 +707,15 @@ fn read_err(e: forge_core::repo::RepoError) -> Response { ) .into_response() } + RepoError::BadUpstream(_) => { + tracing::warn!(error = %e, "browse: proxy upstream answered unusably"); + ( + StatusCode::BAD_GATEWAY, + "the upstream refused this proxy (are its credentials right?) or served a package \ + it does not take", + ) + .into_response() + } RepoError::FormatUnavailable(_) => ( StatusCode::SERVICE_UNAVAILABLE, "the repository's format is not installed on this server", diff --git a/crates/server/src/cli_auth.rs b/crates/server/src/cli_auth.rs new file mode 100644 index 00000000..fd6a79ee --- /dev/null +++ b/crates/server/src/cli_auth.rs @@ -0,0 +1,282 @@ +//! Logins of command-line clients, as format plugins ask for them +//! ([`AuthRequest`]): npm's `npm login`, `npm logout` and `npm whoami`. +//! +//! * A **password** login is checked through the realm chain like the web +//! UI's (local passwords, credential forwarding to a Nexus or a +//! directory, the login lockout); it mints a new API token. No account is +//! ever created here. +//! * A **web** login gives the client a login URL, Forge's page +//! `/cli-login/`, which it opens in the browser, and a secret it +//! polls with. The user signs in there as always (password or OIDC) and +//! approves the request ([`crate::auth`]'s `/auth/cli-login`); the next +//! poll mints the token. The two secrets differ: only the client that +//! started the login can collect the token, and only the approving +//! user's session can approve it. A request lives +//! `cli_login.web_login_ttl_seconds` (10 minutes). +//! * A **logout** revokes the token, when it is the caller's own. +//! +//! Tokens minted by a login carry the user's rights (no scope), expire +//! after `cli_login.token_lifetime_days`, and are listed and revocable on +//! the user's tokens page like every token. Logins, mints and revocations +//! are audit-logged (`forge::audit`). + +use std::net::IpAddr; + +use axum::http::{header, StatusCode}; +use axum::response::{IntoResponse, Response}; +use forge_core::auth::principal::Principal; +use forge_core::auth::token::{generate_token_secret, hash_token}; +use forge_db::cli_login_dao::{self, CliLoginState}; +use forge_db::model::Repository; +use forge_db::token_dao::{self, NewToken}; +use forge_db::user_dao; +use forge_plugin::api::{AuthOutcome, AuthRequest}; + +use crate::auth::{verify_password_login, LoginRefusal}; +use crate::state::AppState; + +/// The longest host name a web login keeps. +const MAX_HOSTNAME: usize = 255; + +fn unauthorized(message: &'static str) -> Response { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Basic realm=\"Forge\"")], + message, + ) + .into_response() +} + +fn unavailable() -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "30")], + "authentication temporarily unavailable", + ) + .into_response() +} + +fn db_error(e: sqlx::Error) -> Response { + tracing::error!(error = %e, "cli login failed"); + unavailable() +} + +/// Perform `request` for `principal` on `repo`, asked from `client_ip`: +/// what the format plugin renders, or the refusal to answer with. +pub async fn perform( + state: &AppState, + repo: &Repository, + principal: Principal, + client_ip: IpAddr, + request: AuthRequest, +) -> Result { + match request { + AuthRequest::Password { username, password } => { + password_login(state, repo, client_ip, &username, &password).await + } + AuthRequest::StartWebLogin { hostname } => { + start_web_login(state, repo, client_ip, hostname).await + } + AuthRequest::PollWebLogin { secret } => poll_web_login(state, repo, &secret).await, + AuthRequest::Logout { token } => logout(state, repo, principal, &token).await, + AuthRequest::Whoami => whoami(state, principal).await, + } +} + +/// Mint a login token for `user_id`, named `name`. +async fn mint( + state: &AppState, + repo: &Repository, + user_id: i64, + name: String, + how: &str, +) -> Result { + let user = user_dao::find_user_by_id(&state.pool, user_id) + .await + .map_err(db_error)? + .filter(|u| u.active) + .ok_or_else(|| unauthorized("the account is not active"))?; + let days = state.config.cli_login.token_lifetime_days; + let secret = generate_token_secret(); + let token = token_dao::insert_token( + &state.pool, + NewToken { + user_id, + name, + token_hash: hash_token(&secret), + expires_at: (days > 0) + .then(|| chrono::Utc::now() + chrono::Duration::days(i64::from(days))), + // A credential-forwarding account's login came through its + // upstream: an admin-forced reauth revokes the token. + minted_with_forwarded_credential: user.auth_source == "upstream-basic", + }, + ) + .await + .map_err(db_error)?; + tracing::info!( + target: "forge::audit", + user_id, + token_id = token.id, + repo = %repo.name, + client = %repo.format, + how, + "cli login: token minted" + ); + Ok(AuthOutcome::Token { + token: secret, + username: user.username, + }) +} + +async fn password_login( + state: &AppState, + repo: &Repository, + client_ip: IpAddr, + username: &str, + password: &str, +) -> Result { + let (principal, _) = match verify_password_login(state, client_ip, username, password).await { + Ok(verified) => verified, + Err(refusal) => { + tracing::info!( + target: "forge::audit", + repo = %repo.name, + client = %repo.format, + ip = %client_ip, + outcome = ?refusal, + "cli login refused" + ); + return Err(match refusal { + LoginRefusal::Invalid => unauthorized("invalid credentials"), + refusal => refusal.into_response(), + }); + } + }; + mint( + state, + repo, + principal.user_id, + format!("{} login ({})", repo.format, repo.name), + "password", + ) + .await +} + +/// Without the web UI there is no page to approve a web login on: 404, +/// and npm falls back to asking for a password. +async fn start_web_login( + state: &AppState, + repo: &Repository, + client_ip: IpAddr, + hostname: Option, +) -> Result { + if !state.config.web.enabled { + return Err(( + StatusCode::NOT_FOUND, + "logins approved in the browser need the web UI", + ) + .into_response()); + } + // Starting a login is free for anyone: bounded per client network, as + // OIDC logins are. + if !state.oidc_login_starts.allow(client_ip) { + return Err(( + StatusCode::TOO_MANY_REQUESTS, + [(header::RETRY_AFTER, "60")], + "too many logins started; try again later", + ) + .into_response()); + } + let hostname = hostname.map(|h| { + h.chars() + .filter(|c| !c.is_control()) + .take(MAX_HOSTNAME) + .collect::() + }); + let login_secret = generate_token_secret(); + let done_secret = generate_token_secret(); + let ttl = chrono::Duration::seconds(state.config.cli_login.web_login_ttl_seconds as i64); + cli_login_dao::insert( + &state.pool, + &hash_token(&login_secret), + &hash_token(&done_secret), + &repo.format, + repo.id, + hostname.as_deref(), + Some(&client_ip.to_string()), + chrono::Utc::now() + ttl, + ) + .await + .map_err(db_error)?; + let public = state.config.public_base_url(); + Ok(AuthOutcome::WebLoginStarted { + login_url: format!("{}/cli-login/{login_secret}", public.trim_end_matches('/')), + secret: done_secret, + }) +} + +async fn poll_web_login( + state: &AppState, + repo: &Repository, + secret: &str, +) -> Result { + match cli_login_dao::poll(&state.pool, &hash_token(secret)) + .await + .map_err(db_error)? + { + CliLoginState::Pending => Ok(AuthOutcome::WebLoginPending), + CliLoginState::Unknown => Ok(AuthOutcome::WebLoginUnknown), + CliLoginState::Approved { + user_id, hostname, .. + } => { + let name = match hostname.as_deref().filter(|h| !h.is_empty()) { + Some(host) => format!("{} login ({}) on {host}", repo.format, repo.name), + None => format!("{} login ({})", repo.format, repo.name), + }; + mint(state, repo, user_id, name, "web").await + } + } +} + +/// The caller must be authenticated as the user the token is of. +async fn logout( + state: &AppState, + repo: &Repository, + principal: Principal, + token: &str, +) -> Result { + let Principal::User { user_id, .. } = principal else { + return Err(unauthorized("authentication required")); + }; + let not_found = || (StatusCode::NOT_FOUND, "no such token").into_response(); + let row = token_dao::find_token_by_hash(&state.pool, &hash_token(token)) + .await + .map_err(db_error)? + .filter(|row| row.user_id == user_id) + .ok_or_else(not_found)?; + token_dao::revoke_token(&state.pool, row.id) + .await + .map_err(db_error)?; + tracing::info!( + target: "forge::audit", + user_id, + token_id = row.id, + repo = %repo.name, + client = %repo.format, + "cli logout: token revoked" + ); + Ok(AuthOutcome::LoggedOut) +} + +async fn whoami(state: &AppState, principal: Principal) -> Result { + let username = match principal { + Principal::User { user_id, .. } => user_dao::find_user_by_id(&state.pool, user_id) + .await + .map_err(db_error)? + .map(|u| u.username) + .ok_or_else(|| unauthorized("authentication required"))?, + Principal::Workload { .. } => forge_core::auth::WORKLOAD_USERNAME.to_string(), + _ => return Err(unauthorized("authentication required")), + }; + Ok(AuthOutcome::Identity { username }) +} diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 0dbe1d13..28b76ced 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -67,6 +67,13 @@ pub struct Config { /// and blob-link expiry. #[serde(default)] pub docker: DockerConfig, + /// npm registries (`/npm`): what the npm plugin reads of a request. + #[serde(default)] + pub npm: NpmConfig, + /// Logins of command-line clients (`npm login`): their tokens and the + /// logins approved in the browser. + #[serde(default)] + pub cli_login: CliLoginConfig, /// Static web UI (SPA) serving. #[serde(default)] pub web: WebConfig, @@ -1461,6 +1468,106 @@ impl DockerConfig { } } +/// `[npm]`: what the npm plugin reads of a request. +#[derive(Debug, Clone, Deserialize)] +pub struct NpmConfig { + /// The largest publish document, without its tarball (the tarball + /// follows the repository's upload limit), and the largest other + /// document a client sends (`npm deprecate`, `npm unpublish `, + /// an audit), in bytes. Default 32 MiB. + #[serde(default = "default_npm_max_document_bytes")] + pub max_document_bytes: u64, + /// The most a tarball may unpack to, in bytes. Default 1 GiB. + #[serde(default = "default_npm_max_unpacked_bytes")] + pub max_unpacked_bytes: u64, +} + +impl Default for NpmConfig { + fn default() -> Self { + NpmConfig { + max_document_bytes: default_npm_max_document_bytes(), + max_unpacked_bytes: default_npm_max_unpacked_bytes(), + } + } +} + +impl NpmConfig { + fn validate(&self) -> Result<(), ConfigError> { + if self.max_document_bytes == 0 || self.max_unpacked_bytes == 0 { + return Err(ConfigError::Invalid( + "npm.max_document_bytes and npm.max_unpacked_bytes must be greater than 0".into(), + )); + } + if usize::try_from(self.max_document_bytes).is_err() { + return Err(ConfigError::Invalid( + "npm.max_document_bytes is too large".into(), + )); + } + Ok(()) + } + + /// The limits handed to [`forge_format_npm::Npm::with_limits`]. + pub fn limits(&self) -> forge_format_npm::NpmLimits { + forge_format_npm::NpmLimits { + max_document_bytes: self.max_document_bytes, + max_unpacked_bytes: self.max_unpacked_bytes, + } + } +} + +fn default_npm_max_document_bytes() -> u64 { + forge_format_npm::NpmLimits::default().max_document_bytes +} +fn default_npm_max_unpacked_bytes() -> u64 { + forge_format_npm::NpmLimits::default().max_unpacked_bytes +} + +/// `[cli_login]`: logins of command-line clients (`npm login`), with a +/// password or approved in the browser. +#[derive(Debug, Clone, Deserialize)] +pub struct CliLoginConfig { + /// Lifetime, in days, of the token a login mints. `0`: the token does + /// not expire. Default 90. + #[serde(default = "default_cli_login_token_lifetime_days")] + pub token_lifetime_days: u32, + /// How long a login approved in the browser waits for the approval, in + /// seconds (60–3600). Default 600. + #[serde(default = "default_cli_login_web_login_ttl_seconds")] + pub web_login_ttl_seconds: u64, +} + +impl Default for CliLoginConfig { + fn default() -> Self { + CliLoginConfig { + token_lifetime_days: default_cli_login_token_lifetime_days(), + web_login_ttl_seconds: default_cli_login_web_login_ttl_seconds(), + } + } +} + +impl CliLoginConfig { + fn validate(&self) -> Result<(), ConfigError> { + if !(60..=3600).contains(&self.web_login_ttl_seconds) { + return Err(ConfigError::Invalid( + "cli_login.web_login_ttl_seconds must be between 60 and 3600".into(), + )); + } + if self.token_lifetime_days > 36_500 { + return Err(ConfigError::Invalid( + "cli_login.token_lifetime_days must be at most 36500 (0: no expiry)".into(), + )); + } + Ok(()) + } +} + +fn default_cli_login_token_lifetime_days() -> u32 { + 90 +} +fn default_cli_login_web_login_ttl_seconds() -> u64 { + 600 +} + /// The largest blob a docker proxy downloads unless `docker.max_blob_bytes` /// says otherwise: 1 GiB. pub const DEFAULT_DOCKER_MAX_BLOB_BYTES: u64 = 1024 * 1024 * 1024; @@ -2194,6 +2301,8 @@ impl Config { self.proxy.validate()?; self.nuget.validate()?; self.docker.validate()?; + self.npm.validate()?; + self.cli_login.validate()?; self.validate_workers()?; self.validate_insecure_defaults()?; if self.server.removed_max_upload_bytes.is_some() { diff --git a/crates/server/src/feed.rs b/crates/server/src/feed.rs index b1e4ebf4..6170e78c 100644 --- a/crates/server/src/feed.rs +++ b/crates/server/src/feed.rs @@ -1,13 +1,16 @@ //! What every feed response and package transfer shares, whatever the //! format: the browser hardening of feed responses, byte ranges of -//! downloads, and the bounds on a multipart upload's preamble. +//! downloads, streaming a stored package, and the bounds on a multipart +//! upload's preamble. use std::ops::Range; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use async_trait::async_trait; -use axum::http::{header, HeaderMap, HeaderValue}; +use axum::body::Body; +use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; use bytes::Bytes; use forge_core::repo::{NupkgLocation, RepoError}; use futures_util::{Stream, StreamExt}; @@ -76,6 +79,65 @@ pub enum ByteRange { Unsatisfiable, } +/// A header value of ASCII digits and punctuation. +pub(crate) fn numeric_header(value: String) -> HeaderValue { + HeaderValue::from_str(&value).expect("ASCII header value") +} + +/// Stream the stored package at `location` as the answer to a download +/// with `request`'s headers (`head`: a `HEAD`, answered from the metadata +/// alone): its content type and length, and a single `Range` when the blob +/// source serves ranges (with the local blob cache). +pub async fn serve_package( + state: &crate::state::AppState, + location: &NupkgLocation, + request: &HeaderMap, + head: bool, +) -> Response { + let size = u64::try_from(location.size).unwrap_or(0); + let blobs = state.blob_bytes(); + let ranges = blobs.serves_ranges(); + let mut headers = HeaderMap::new(); + if let Ok(value) = HeaderValue::from_str(&location.content_type) { + headers.insert(header::CONTENT_TYPE, value); + } + if ranges { + headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + } + let requested = if ranges && !head { + byte_range(request, size) + } else { + ByteRange::Full + }; + let (status, range) = match requested { + ByteRange::Full => (StatusCode::OK, None), + ByteRange::Partial(range) => { + headers.insert( + header::CONTENT_RANGE, + numeric_header(format!("bytes {}-{}/{size}", range.start, range.end - 1)), + ); + (StatusCode::PARTIAL_CONTENT, Some(range)) + } + ByteRange::Unsatisfiable => { + headers.insert( + header::CONTENT_RANGE, + numeric_header(format!("bytes */{size}")), + ); + return (StatusCode::RANGE_NOT_SATISFIABLE, headers).into_response(); + } + }; + let length = range.as_ref().map_or(size, |r| r.end - r.start); + headers.insert(header::CONTENT_LENGTH, numeric_header(length.to_string())); + // A HEAD needs the headers only; don't open (or bill) a GET for it. + if head { + return (status, headers).into_response(); + } + match blobs.open(location, range).await { + Ok(stream) => (status, headers, Body::from_stream(stream)).into_response(), + Err(e) => crate::plugins::repo_error(e), + } +} + /// The [`ByteRange`] a request asks of a `size`-byte body. Only one /// `bytes` range is honored; anything else is answered in full. pub fn byte_range(request: &HeaderMap, size: u64) -> ByteRange { diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index a67418c8..cd92336e 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -18,6 +18,7 @@ pub mod app; pub mod auth; pub mod browse; pub mod browse_images; +pub mod cli_auth; pub mod config; pub mod crypto; pub mod dispatch; diff --git a/crates/server/src/nexus_import/nexus.rs b/crates/server/src/nexus_import/nexus.rs index 54a13bbd..3eff1cd0 100644 --- a/crates/server/src/nexus_import/nexus.rs +++ b/crates/server/src/nexus_import/nexus.rs @@ -230,6 +230,7 @@ impl Nexus { origin: self.origin.clone(), username, password: password.expose_secret(), + bearer: false, }), _ => None, } diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index fca9d6e5..7363e3f8 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -2,12 +2,11 @@ //! formats under each format's mount, `{mount}/{repo}/...` (`/repository` //! unless its `plugin.toml` names another; see `docs/format-plugins.md`). //! -//! The dispatcher does what [`crate::dispatch`] does for the NuGet feed — -//! resolve the repository, authorize the request's action (GET/HEAD read, -//! PUT/POST write, DELETE delete), refuse an offline repository — and then -//! lets the format's plugin decide what the request is -//! ([`FormatPlugin::handle`]). -//! Everything that touches data stays here: +//! The dispatcher resolves the repository, authenticates the caller (an +//! `Authorization: Bearer` API token too, as npm sends it), refuses an +//! offline repository, and lets the format's plugin decide what the request +//! is ([`FormatPlugin::handle`]). It then checks the answer against what +//! the caller may do and does everything that touches data itself: //! //! * **Queries** the plugin asks for run through the same read ops as every //! other read ([`AppState::read_ops_for`]): package selectors, group @@ -15,43 +14,55 @@ //! apply, and an id the format does not accept is unknown. A files //! format's queries go to the file store ([`crate::files`]), with the same //! selectors on paths. A request may load at most [`MAX_QUERIES`] answers -//! in at most [`MAX_ROUNDS`] rounds. +//! in at most [`MAX_ROUNDS`] rounds. They, responses, downloads and +//! forwards need read. //! * **Downloads** are streamed from the blob store and counted, exactly as //! NuGet's, answering a single `Range` when the blob source serves ranges; //! a file is served by the file store ([`crate::files::serve`]). -//! * **Publishing, unlisting, relisting and deleting** go through the hosted -//! write path (CAS staging, size cap, selector and redeploy policy, -//! trusted-publishing audit), and receiving, storing and deleting files -//! through the file store's, only when the request's method was -//! authorized for that action and carries an API token or workload -//! identity. +//! * **Publishing and changes** (unlist, relist, delete, deprecate, tags) go +//! through the hosted write path (CAS staging, size cap, selector and +//! redeploy policy, trusted-publishing audit) of the repository, or of a +//! group's writable member, with the permission each needs, never for a +//! GET, and only with an API token or workload identity. A files format's +//! requests are authorized by their method up front (GET and HEAD read, +//! PUT and POST write, DELETE delete), and receiving, storing and deleting +//! files go through the file store's write path. +//! * **Bodies** are read when the plugin asks ([`Action::ReadBody`]), bounded, +//! for a caller with the permission their use needs. +//! * **Logins** ([`Action::Auth`]) are the host's ([`crate::cli_auth`]). +//! * **Forwards** ([`Action::Forward`]) go to the repository's upstream, or +//! a group's first proxy member, with its credentials. //! * **Responses** the plugin renders keep only the headers in //! [`PASSED_RESPONSE_HEADERS`] (no cookies, no CORS, no redirects or links //! off this repository) and get the feed's hardening headers. use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use crate::feed::{byte_range, ByteRange, PackageBytes}; use axum::body::Body; -use axum::extract::{Request, State}; -use axum::http::{header, HeaderValue, Method, StatusCode}; +use axum::extract::{ConnectInfo, Request, State}; +use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use bytes::Bytes; use forge_core::auth::principal::Principal; -use forge_core::authz::{Action as Permission, PackageFilter}; +use forge_core::auth::token::{check_token, TokenCheck}; +use forge_core::authz::Action as Permission; use forge_core::format::Format; +use forge_core::net::upstream::{RequestClass, SpooledBody}; use forge_core::repo::{ DeleteOutcome, RegistrationEntry, RepoError, RepoKind, RepoReadOps, RepoWriteOps, SearchParams, UploadReader, WriteError, }; -use forge_db::model::Repository; +use forge_db::model::{DeploymentPolicy, Repository}; use forge_db::raw_dao::RawFile; -use forge_db::repository_dao; +use forge_db::{component_dao, repository_dao, tag_dao, user_dao}; use forge_plugin::api::{ - self as protocol, Action, Answer, ChangeRequest, FormatPlugin, Loaded, PackageInfo, PackageRef, - PublishRequest, Query, Upload, + self as protocol, Action, Answer, AuthRequest, BodyRequest, BodyUse, Change, ChangeSet, + FormatPlugin, Forward, Loaded, PackageInfo, PackageRef, PublishRequest, Query, Unpacked, + Upload, }; use forge_plugin::Plugin; use futures_util::{StreamExt, TryStreamExt}; @@ -79,6 +90,12 @@ pub const MAX_QUERIES: usize = 2048; /// Queries answered at once. const CONCURRENT_QUERIES: usize = 16; +/// The largest body a plugin may ask for ([`Action::ReadBody`]). +pub const MAX_BODY_BYTES: u64 = 64 * 1024 * 1024; + +/// The largest body of a login ([`BodyUse::Login`]). +pub const MAX_LOGIN_BODY_BYTES: u64 = 64 * 1024; + /// Request headers a plugin sees. Never credentials or cookies. pub const PASSED_REQUEST_HEADERS: &[&str] = &[ "accept", @@ -102,19 +119,29 @@ pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ "last-modified", "link", "location", + "retry-after", "vary", ]; /// The format plugins built into this server: crates under -/// `crates/formats//`, linked in and always installed (NuGet, raw). NuGet -/// reads archives under `nuget_limits` (`[nuget]`). -pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec> { +/// `crates/formats//`, linked in and always installed (NuGet, npm, raw). +/// NuGet reads archives under `nuget_limits` (`[nuget]`), npm requests under +/// `npm_limits` (`[npm]`). +pub fn builtin( + nuget_limits: forge_format_nuget::NupkgLimits, + npm_limits: forge_format_npm::NpmLimits, +) -> Vec> { vec![ Plugin::builtin( forge_format_nuget::PLUGIN_TOML, forge_format_nuget::Nuget::with_limits(nuget_limits), ) .expect("crates/formats/nuget/plugin.toml is valid"), + Plugin::builtin( + forge_format_npm::PLUGIN_TOML, + forge_format_npm::Npm::with_limits(npm_limits), + ) + .expect("crates/formats/npm/plugin.toml is valid"), Plugin::builtin(forge_format_raw::PLUGIN_TOML, forge_format_raw::Raw) .expect("crates/formats/raw/plugin.toml is valid"), ] @@ -122,11 +149,82 @@ pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec /// Axum handler mounted at `/repository/{repo}` (and everything under it). pub async fn dispatch(State(state): State, req: Request) -> Response { + // The plugin the mount serves, for errors answered before the + // repository (and its plugin) is known. + let mounted = + split_path(req.uri().path(), &state.formats.mounts()).and_then(|(mount, _, _)| { + let mut serving = state + .formats + .plugins() + .filter(|plugin| plugin.manifest().mount == mount); + match (serving.next(), serving.next()) { + (Some(plugin), None) => Some(plugin.clone()), + _ => None, + } + }); + let head = req.method() == Method::HEAD; let mut response = dispatch_request(state, req).await; + let plugin = response + .extensions_mut() + .remove::() + .map(|s| s.0) + .or(mounted); + if let Some(plugin) = plugin { + if !head { + response = in_format(&plugin, response).await; + } + } crate::feed::harden_response_headers(response.headers_mut()); response } +/// The plugin that served a request, once its repository is known. +#[derive(Clone)] +struct ServedBy(Arc); + +/// Marks a response the plugin (or the upstream it forwarded to) wrote. +#[derive(Clone, Copy)] +struct PluginWritten; + +/// The longest host error [`in_format`] rewrites. +const MAX_ERROR_BYTES: usize = 64 * 1024; + +/// A host error in the shape the format's clients read errors in +/// ([`FormatPlugin::error_response`]), keeping its status and headers. +async fn in_format(plugin: &Plugin, response: Response) -> Response { + let status = response.status(); + if !(status.is_client_error() || status.is_server_error()) + || response.extensions().get::().is_some() + { + return response; + } + let (mut parts, body) = response.into_parts(); + let Ok(message) = axum::body::to_bytes(body, MAX_ERROR_BYTES).await else { + return (status, "internal error").into_response(); + }; + let message = String::from_utf8_lossy(&message); + let Some(rendered) = plugin + .implementation() + .error_response(status.as_u16(), message.trim()) + else { + return Response::from_parts(parts, Body::from(message.into_owned())); + }; + let content_type = rendered + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-type")) + .and_then(|(_, value)| HeaderValue::from_str(value).ok()); + match content_type { + Some(content_type) => parts.headers.insert(header::CONTENT_TYPE, content_type), + None => parts.headers.remove(header::CONTENT_TYPE), + }; + parts.headers.insert( + header::CONTENT_LENGTH, + HeaderValue::from(rendered.body.len()), + ); + Response::from_parts(parts, Body::from(rendered.body)) +} + /// `(mount, repository name, rest of the path)` of a `{mount}/{repo}/...` /// path, for one of `mounts`; the rest starts with `/`, or is empty when the /// path ends with the repository's name. @@ -141,10 +239,54 @@ fn split_path<'p>(path: &'p str, mounts: &[&str]) -> Option<(&'p str, &'p str, & }) } +/// The caller, with an `Authorization: Bearer` API token resolved when the +/// principal middleware left it anonymous (npm sends its `_authToken` so). +/// An unknown or unusable token is refused, never read as anonymous. +async fn with_bearer( + state: &AppState, + headers: &HeaderMap, + principal: Principal, +) -> Result { + if !matches!( + principal, + Principal::Anonymous { .. } | Principal::Unauthenticated + ) { + return Ok(principal); + } + let Some(token) = headers + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| { + let (scheme, token) = v.split_once(' ')?; + scheme.eq_ignore_ascii_case("bearer").then(|| token.trim()) + }) + .filter(|token| !token.is_empty()) + else { + return Ok(principal); + }; + match check_token(&state.pool, token).await { + Ok(TokenCheck::Live(user)) => Ok(Principal::from_user(user)), + Ok(TokenCheck::Unusable | TokenCheck::Unknown) => Err(( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Bearer realm=\"Forge\"")], + "invalid token", + ) + .into_response()), + Err(_) => Err(( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "30")], + "authentication temporarily unavailable", + ) + .into_response()), + } +} + async fn dispatch_request(state: AppState, req: Request) -> Response { - let principal = principal_of(&req); - let method = req.method().clone(); let uri = req.uri().clone(); + let principal = match with_bearer(&state, req.headers(), principal_of(&req)).await { + Ok(principal) => principal, + Err(response) => return response, + }; let mounts = state.formats.mounts(); let Some((mount, repo_name, rest)) = split_path(uri.path(), &mounts) else { return (StatusCode::NOT_FOUND, "repository not found").into_response(); @@ -164,23 +306,9 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { } }; - let permission = action_for_method(&method); - let allowed = match permission { - Permission::Read => state.authz.can_read(&principal, &repo).await, - Permission::Write => state.authz.can_write(&principal, &repo).await, - Permission::Delete => state.authz.can_delete(&principal, &repo).await, - Permission::Admin => false, - }; - if !allowed { - return authz_denied(&principal); - } - if offline_serves_nothing(&repo) { - return offline_response(); - } let Some(plugin) = state.formats.plugin(&repo.format) else { return if state.formats.is_installed(&repo.format) { - // A built-in format has its own mount (NuGet: `/v3`, Docker: - // `/v2`). + // A built-in format has its own mount (Docker: `/v2`). (StatusCode::NOT_FOUND, "repository is not served here").into_response() } else { ( @@ -194,20 +322,56 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { if plugin.manifest().mount != mount { return (StatusCode::NOT_FOUND, "repository is not served here").into_response(); } - let format = match state.formats.for_repository(&repo) { + let mut response = serve(&state, req, principal, &repo, plugin.clone(), mount, rest).await; + response.extensions_mut().insert(ServedBy(plugin)); + response +} + +/// Serve `req` from `repo`, whose format `plugin` is. +async fn serve( + state: &AppState, + req: Request, + principal: Principal, + repo: &Repository, + plugin: Arc, + mount: &str, + rest: &str, +) -> Response { + let method = req.method().clone(); + let uri = req.uri().clone(); + let format = match state.formats.for_repository(repo) { Ok(format) => format, Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(), }; let stores_files = format.stores_files(); - let package_filter = state - .authz - .package_filter_for(&principal, &repo, permission) - .await; + // A files format's request is authorized by its method; a package + // format's actions each by what they need (see `Ctx::act`). + let permission = action_for_method(&method); + if stores_files { + let allowed = match permission { + Permission::Read => state.authz.can_read(&principal, repo).await, + Permission::Write => state.authz.can_write(&principal, repo).await, + Permission::Delete => state.authz.can_delete(&principal, repo).await, + Permission::Admin => false, + }; + if !allowed { + return authz_denied(&principal); + } + } + if offline_serves_nothing(repo) { + // Only to a caller who may read it: the offline state reveals the + // repository. + return if state.authz.can_read(&principal, repo).await { + offline_response() + } else { + authz_denied(&principal) + }; + } // A files format reads the file store, never the package read ops. let ops = if stores_files { None } else { - match state.read_ops_for(&repo, principal).await { + match state.read_ops_for(repo, principal).await { Ok(ops) => Some(ops), Err(e) => return repo_error(e), } @@ -220,8 +384,13 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { RepoKind::Proxy => protocol::RepositoryKind::Proxy, RepoKind::Group => protocol::RepositoryKind::Group, }, - base_url: repository_url(&state, mount, &repo.name), + base_url: repository_url(state, mount, &repo.name), inline: stores_files && repo.inline_content(), + reserved_prefixes: if repo.kind() == RepoKind::Group { + repo.reserved_prefixes() + } else { + Vec::new() + }, }; let request = protocol::Request { method: method.as_str().to_string(), @@ -233,21 +402,26 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { .filter(|(name, _)| PASSED_REQUEST_HEADERS.contains(&name.as_str())) .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) .collect(), + body: None, }; + let peer = req + .extensions() + .get::>() + .map(|ci| ci.0); let ctx = Ctx { - state: &state, - repo: &repo, + state, + repo, mount, principal, - permission, - head: method == Method::HEAD, + method: method.clone(), + client_ip: crate::auth::login_client_ip(state, peer, req.headers()), headers: req.headers().clone(), - plugin: plugin.implementation().clone(), + plugin: plugin.clone(), format, + permission, stores_files, ops, - package_filter, target, body: Mutex::new(Some(req)), form: Mutex::new(None), @@ -262,7 +436,7 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { Ok(action) => ctx.perform(action, request).await, Err(response) => response, }; - if permission != Permission::Read { + if !matches!(method, Method::GET | Method::HEAD) { // The same audit trail as the NuGet feed's package mutations. tracing::info!( target: "forge::audit", @@ -284,22 +458,23 @@ struct Ctx<'a> { /// The format's mount (`/repository`). mount: &'a str, principal: Principal, - /// What the request's method was authorized for. - permission: Permission, - head: bool, + method: Method, + /// Where the request came from, as the login lockout sees it. + client_ip: std::net::IpAddr, /// The request's headers, for a download's `Range`. - headers: axum::http::HeaderMap, - plugin: Arc, + headers: HeaderMap, + plugin: Arc, format: Arc, + /// What the request's method is for; a files format's request was + /// authorized for it. + permission: Permission, /// Whether the format's repositories hold files (its plugin asks for /// files, not packages). stores_files: bool, /// The package read ops; `None` for a files format. ops: Option>, - /// The caller's package selector for [`Self::permission`]. - package_filter: PackageFilter, target: protocol::Repository, - /// The request, until a publish or a file query reads its body. + /// The request, until a publish, a body or a file query reads it. body: Mutex>>, /// The file parts of the form the plugin had received, by their index /// among its parts, until a store takes them. @@ -309,55 +484,78 @@ struct Ctx<'a> { } impl Ctx<'_> { + fn implementation(&self) -> Arc { + self.plugin.implementation().clone() + } + + fn head(&self) -> bool { + self.method == Method::HEAD + } + + /// Whether the request's method may change anything. + fn mutating(&self) -> bool { + !matches!(self.method, Method::GET | Method::HEAD) + } + /// Run a protocol call of the plugin off the async workers: rendering a /// large listing takes a while. A plugin that panics answers 500. async fn call( &self, f: impl FnOnce(&dyn FormatPlugin) -> R + Send + 'static, ) -> Result { - let plugin = Arc::clone(&self.plugin); + let plugin = self.implementation(); tokio::task::spawn_blocking(move || f(plugin.as_ref())) .await .map_err(|e| plugin_failed(self.repo, &e)) } - /// Do what the plugin decided, `handle` or the last round of `render`. - async fn perform(&self, action: Action, request: protocol::Request) -> Response { - if let Action::Load(queries) = action { - return self.render(queries, request).await; + /// Whether the caller may do `permission` on `repo`. + async fn may(&self, permission: Permission, repo: &Repository) -> bool { + match permission { + Permission::Read => self.state.authz.can_read(&self.principal, repo).await, + Permission::Write => self.state.authz.can_write(&self.principal, repo).await, + Permission::Delete => self.state.authz.can_delete(&self.principal, repo).await, + Permission::Admin => false, } - self.act(action).await } - /// Do an action other than loading data. - async fn act(&self, action: Action) -> Response { - let files_action = matches!(action, Action::ServeFile(_)); - let package_action = !matches!(action, Action::Respond(_) | Action::ServeFile(_)); - if (files_action && !self.stores_files) || (package_action && self.stores_files) { - return self.plugin_misused("an action of the other storage"); + /// The caller must read this repository. + async fn require_read(&self) -> Result<(), Response> { + if self.may(Permission::Read, self.repo).await { + Ok(()) + } else { + Err(authz_denied(&self.principal)) } + } + + /// Do what `handle` decided. + async fn perform(&self, action: Action, request: protocol::Request) -> Response { match action { - Action::Respond(response) => self.respond(response), - Action::Load(_) => self.plugin_misused("a load as an action"), - Action::Download(target) => { - if self.permission != Permission::Read { - return refused("download"); - } - self.download(&target).await - } + _ if self.stores_files => match action { + Action::Load(queries) => self.render(queries, &request).await, + action => self.act_files(action).await, + }, + Action::ReadBody(wanted) => self.read_body(wanted, request).await, Action::Publish(publish) => match self.take_body() { - Some(req) => self.publish(publish, req).await, + Some(req) => self.publish(publish, request, req).await, None => self.plugin_misused("a publish after the body was read"), }, - Action::Unlist(change) => self.change(change, Change::Unlist).await, - Action::Relist(change) => self.change(change, Change::Relist).await, - Action::Delete(change) => self.change(change, Change::Delete).await, + other => self.act(other, &request, false).await, + } + } + + /// An action of a files format's plugin: a response, or a file to serve. + async fn act_files(&self, action: Action) -> Response { + match action { + Action::Respond(response) => self.respond(response), Action::ServeFile(path) => { if self.permission != Permission::Read { return refused("serving a file"); } self.serve_file(&path).await } + Action::Load(_) => self.plugin_misused("a load as an action"), + _ => self.plugin_misused("an action of the other storage"), } } @@ -376,6 +574,58 @@ impl Ctx<'_> { (StatusCode::INTERNAL_SERVER_ERROR, "format plugin failed").into_response() } + /// Everything but a body or a publish, which need the request itself. + /// `login_body`: the request's body was read as a login's, so only a + /// login or a plain response may follow. + async fn act(&self, action: Action, request: &protocol::Request, login_body: bool) -> Response { + if login_body && !matches!(action, Action::Auth(_) | Action::Respond(_)) { + return plugin_failed(self.repo, &"a login body led to another action"); + } + match action { + Action::Auth(auth) => self.auth(auth, request).await, + // A login's own errors need no permission: they reveal nothing. + Action::Respond(response) if login_body => self.respond(response), + Action::Respond(response) => match self.require_read().await { + Ok(()) => self.respond(response), + Err(denied) => denied, + }, + Action::Load(queries) => match self.require_read().await { + Ok(()) => self.render(queries, request).await, + Err(denied) => denied, + }, + Action::Download(target) => { + if self.mutating() { + return refused("download"); + } + match self.require_read().await { + Ok(()) => self.download(&target).await, + Err(denied) => denied, + } + } + Action::Forward(forward) => match self.require_read().await { + Ok(()) => self.forward(forward).await, + Err(denied) => denied, + }, + Action::Unlist(change) => { + self.change_one(Change::Unlist(change.target), change.done) + .await + } + Action::Relist(change) => { + self.change_one(Change::Relist(change.target), change.done) + .await + } + Action::Delete(change) => { + self.change_one(Change::Delete(change.target), change.done) + .await + } + Action::Change(set) => self.changes(set).await, + Action::ReadBody(_) | Action::Publish(_) => { + plugin_failed(self.repo, &"a body was asked for twice") + } + Action::ServeFile(_) => self.plugin_misused("an action of the other storage"), + } + } + /// The canonical id the format makes of `raw`, if it is a valid id. fn id(&self, raw: &str) -> Option { let id = self.format.lower_id(raw); @@ -390,7 +640,7 @@ impl Ctx<'_> { } /// Load data and let the plugin render it, round by round. - async fn render(&self, mut queries: Vec, request: protocol::Request) -> Response { + async fn render(&self, mut queries: Vec, request: &protocol::Request) -> Response { let mut data: Vec = Vec::new(); for _ in 0..MAX_ROUNDS { if data.len() + queries.len() > MAX_QUERIES { @@ -426,7 +676,8 @@ impl Ctx<'_> { data = loaded; queries = more; } - Ok((action, _)) => return self.act(action).await, + Ok((action, _)) if self.stores_files => return self.act_files(action).await, + Ok((action, _)) => return Box::pin(self.act(action, request, false)).await, Err(response) => return response, } } @@ -649,7 +900,7 @@ impl Ctx<'_> { } } }; - crate::files::serve::file(self.state, self.repo, &file, &self.headers, self.head).await + crate::files::serve::file(self.state, self.repo, &file, &self.headers, self.head()).await } /// The answer to a package query, through the package read ops. @@ -683,6 +934,15 @@ impl Ctx<'_> { .collect(), None => Vec::new(), }), + Query::DatedPackages(id) => Answer::Packages(match self.id(id) { + Some(id) => ops + .dated_registration(repo, &id) + .await? + .into_iter() + .map(|entry| self.package_info(entry)) + .collect(), + None => Vec::new(), + }), Query::Package(target) => Answer::Package(match self.package(target) { Some((id, version)) => ops .registration_entry(repo, &id, &version) @@ -715,6 +975,15 @@ impl Ctx<'_> { Some((id, version)) => ops.resolve_nuspec(repo, &id, &version).await?, None => None, }), + Query::Tags(id) => Answer::Tags(match self.id(id) { + Some(id) => ops + .tags(repo, &id) + .await? + .into_iter() + .map(|(name, version)| protocol::Tag { name, version }) + .collect(), + None => Vec::new(), + }), Query::Search(search) => { // The read ops clamp `take` (0 is the default page). let params = SearchParams { @@ -776,7 +1045,7 @@ impl Ctx<'_> { return self.plugin_misused("a package download of a files format"); }; // A HEAD (a probe, a mirror tool) is not a download. - let resolved = if self.head { + let resolved = if self.head() { ops.resolve_nupkg_head(&self.repo.name, &id, &version).await } else { ops.resolve_nupkg(&self.repo.name, &id, &version).await @@ -786,48 +1055,7 @@ impl Ctx<'_> { Ok(None) => return (StatusCode::NOT_FOUND, "package not found").into_response(), Err(e) => return repo_error(e), }; - let size = u64::try_from(location.size).unwrap_or(0); - let blobs = self.state.blob_bytes(); - let ranges = blobs.serves_ranges(); - let mut headers = axum::http::HeaderMap::new(); - if let Ok(value) = HeaderValue::from_str(&location.content_type) { - headers.insert(header::CONTENT_TYPE, value); - } - if ranges { - headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); - } - let requested = if ranges && !self.head { - byte_range(&self.headers, size) - } else { - ByteRange::Full - }; - let (status, range) = match requested { - ByteRange::Full => (StatusCode::OK, None), - ByteRange::Partial(range) => { - headers.insert( - header::CONTENT_RANGE, - numeric_header(format!("bytes {}-{}/{size}", range.start, range.end - 1)), - ); - (StatusCode::PARTIAL_CONTENT, Some(range)) - } - ByteRange::Unsatisfiable => { - headers.insert( - header::CONTENT_RANGE, - numeric_header(format!("bytes */{size}")), - ); - return (StatusCode::RANGE_NOT_SATISFIABLE, headers).into_response(); - } - }; - let length = range.as_ref().map_or(size, |r| r.end - r.start); - headers.insert(header::CONTENT_LENGTH, numeric_header(length.to_string())); - // A HEAD needs the headers only; don't open (or bill) a GET for it. - if self.head { - return (status, headers).into_response(); - } - match blobs.open(&location, range).await { - Ok(stream) => (status, headers, Body::from_stream(stream)).into_response(), - Err(e) => repo_error(e), - } + crate::feed::serve_package(self.state, &location, &self.headers, self.head()).await } /// A version's metadata as the plugin sees it. @@ -844,6 +1072,7 @@ impl Ctx<'_> { sha256: entry.package_sha256, sha512: entry.package_hash_sha512_base64, size: entry.package_size.and_then(|s| u64::try_from(s).ok()), + deprecated: entry.deprecation, } } @@ -859,18 +1088,59 @@ impl Ctx<'_> { .map(|s| s.to_string()) } - /// Whether this request may change packages: a hosted repository, a - /// method authorized for `needed`, and a package-client credential. - fn may_change(&self, needed: Permission, what: &str) -> Result<(), Response> { - if self.permission != needed { + /// Where a change of `permission` to this repository's packages lands: + /// the repository itself when it is hosted, a group's writable member + /// (reading the group is required too), with a package-client + /// credential, and never for a GET or HEAD. + async fn write_target( + &self, + permission: Permission, + what: &str, + ) -> Result { + if !self.mutating() { return Err(refused(what)); } - if self.repo.kind() != RepoKind::Hosted { - return Err(( - StatusCode::METHOD_NOT_ALLOWED, - "packages can only be changed in hosted repositories", - ) - .into_response()); + let target = match self.repo.kind() { + RepoKind::Hosted => self.repo.clone(), + RepoKind::Proxy => { + return Err(( + StatusCode::METHOD_NOT_ALLOWED, + "packages can only be changed in hosted repositories", + ) + .into_response()) + } + RepoKind::Group => { + if !self.may(Permission::Read, self.repo).await { + return Err(authz_denied(&self.principal)); + } + let member = match self.repo.writable_member_id() { + Some(id) => repository_dao::find_repository_by_id(&self.state.pool, id) + .await + .map_err(|e| { + tracing::error!(error = %e, "writable member lookup failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() + })? + .filter(|m| m.kind() == RepoKind::Hosted && m.format == self.repo.format), + None => None, + }; + let Some(member) = member else { + return Err(( + StatusCode::METHOD_NOT_ALLOWED, + "this group takes no changes: its administrator can name a writable member", + ) + .into_response()); + }; + if !member.online { + return Err(offline_response()); + } + member + } + }; + if !self.may(permission, &target).await { + return Err(authz_denied(&self.principal)); + } + if target.deployment_policy() == DeploymentPolicy::ReadOnly { + return Err((StatusCode::FORBIDDEN, "the repository is read-only").into_response()); } // As for NuGet: a browser session is not a publishing credential. if !principal_allows_protocol_mutations(&self.principal) { @@ -881,46 +1151,83 @@ impl Ctx<'_> { ) .into_response()); } - Ok(()) + Ok(target) } - fn writer(&self) -> HostedWriteAdapter { - HostedWriteAdapter::new( - self.state, - self.format.clone(), - self.repo.id, - self.repo.name.clone(), - self.principal, - self.package_filter.clone(), - // Set by `publish` when a redeploy may replace a version. - PackageFilter::DenyAll, - ) - } - - async fn publish(&self, publish: PublishRequest, req: Request) -> Response { - if let Err(response) = self.may_change(Permission::Write, "publish") { - return response; - } + /// The hosted write path of `target`, with the caller's selectors. + async fn writer( + &self, + target: &Repository, + format: Arc, + permission: Permission, + ) -> HostedWriteAdapter { + let filter = self + .state + .authz + .package_filter_for(&self.principal, target, permission) + .await; // A redeploy replaces a version's bytes: it takes the delete // permission's selector as well. let replace_filter = self .state .authz - .package_filter_for(&self.principal, self.repo, Permission::Delete) + .package_filter_for(&self.principal, target, Permission::Delete) .await; - let writer = HostedWriteAdapter::new( + HostedWriteAdapter::new( self.state, - self.format.clone(), - self.repo.id, - self.repo.name.clone(), + format, + target.id, + target.name.clone(), self.principal, - self.package_filter.clone(), + filter, replace_filter, - ); - let limit = self.state.upload_limit(self.repo); + ) + } + + /// The username publishing, recorded by formats that keep it. + async fn publisher(&self) -> Option { + match self.principal { + Principal::User { user_id, .. } => user_dao::find_user_by_id(&self.state.pool, user_id) + .await + .ok() + .flatten() + .map(|u| u.username), + _ => None, + } + } + + async fn publish( + &self, + publish: PublishRequest, + request: protocol::Request, + req: Request, + ) -> Response { + let target = match self.write_target(Permission::Write, "publish").await { + Ok(target) => target, + Err(response) => return response, + }; + let format = forge_plugin::PluginFormat::new(&self.plugin); + let expected_id = publish.id.as_deref().and_then(|id| self.id(id)); + if publish.id.is_some() && expected_id.is_none() { + return (StatusCode::BAD_REQUEST, "invalid package id").into_response(); + } + let upload = Arc::new(format.upload( + publish.upload == Upload::Document, + self.publisher().await, + expected_id, + )); + let writer = self + .writer(&target, upload.clone(), Permission::Write) + .await; + let limit = self.state.upload_limit(&target); + if publish.upload == Upload::Document { + return self + .publish_document(publish, request, req, upload, writer, &target, limit) + .await; + } let flags = UploadFlags::default(); let reader = match &publish.upload { - Upload::Body => body_reader(req.into_body()), + Upload::Body | Upload::Document => body_reader(req.into_body()), Upload::MultipartField(name) => { match multipart_reader(req, Some(name), limit, &flags) { Ok(reader) => reader, @@ -932,7 +1239,7 @@ impl Ctx<'_> { Err(response) => return response, }, }; - match writer.store_upload(&self.repo.name, reader, limit).await { + match writer.store_upload(&target.name, reader, limit).await { Ok(_) => self.respond(publish.done), Err(_) if flags.too_large.load(Ordering::Relaxed) => { write_error(WriteError::TooLarge { limit }) @@ -951,36 +1258,510 @@ impl Ctx<'_> { } } - async fn change(&self, change: ChangeRequest, kind: Change) -> Response { - let (needed, what) = match kind { - Change::Unlist => (Permission::Delete, "unlist"), - Change::Relist => (Permission::Write, "relist"), - Change::Delete => (Permission::Delete, "delete"), + /// An [`Upload::Document`]: the body streams through the plugin's + /// [`FormatPlugin::unpack_upload`] (off the async workers) while the + /// package it writes out is staged and stored; the document it returns + /// reaches the parser before the package does. A body without a package + /// is handed back to the plugin as the request's body. + #[allow(clippy::too_many_arguments)] + async fn publish_document( + &self, + publish: PublishRequest, + request: protocol::Request, + req: Request, + upload: Arc, + writer: HostedWriteAdapter, + target: &Repository, + limit: u64, + ) -> Response { + let Some(document) = upload.document.clone() else { + return plugin_failed(self.repo, &"a document upload without its document"); + }; + let (body_tx, body_rx) = tokio::sync::mpsc::channel::>(4); + let (package_tx, package_rx) = tokio::sync::mpsc::channel::>(4); + let mut stream = req.into_body().into_data_stream(); + let pump = tokio::spawn(async move { + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(std::io::Error::other); + let failed = chunk.is_err(); + if body_tx.send(chunk).await.is_err() || failed { + break; + } + } + }); + let plugin = self.implementation(); + let unpack = tokio::task::spawn_blocking(move || { + let mut body = ChannelReader::new(body_rx); + let mut package = ChannelWriter { tx: package_tx }; + let unpacked = plugin.unpack_upload(&mut body, &mut package); + match &unpacked { + // Before the package stream ends, so that the parser has it. + Ok(Unpacked::Package { document: doc }) => { + let _ = document.set(doc.clone()); + } + Err(e) => { + let _ = package + .tx + .blocking_send(Err(std::io::Error::other(e.to_string()))); + } + Ok(Unpacked::Document(_)) => {} + } + drop(package); + unpacked + }); + let package = futures_util::stream::unfold(package_rx, |mut rx| async move { + rx.recv().await.map(|chunk| (chunk, rx)) + }); + let reader: UploadReader = Box::new(StreamReader::new(Box::pin(package))); + let (stored, unpacked) = + tokio::join!(writer.store_upload(&target.name, reader, limit), unpack); + pump.abort(); + let unpacked = match unpacked { + Ok(unpacked) => unpacked, + Err(e) => return plugin_failed(self.repo, &e), + }; + match unpacked { + Err(e) => match stored { + // The store stopped reading a package over the upload limit + // first, which failed the unpacking: the limit is why. + Err(too_large @ WriteError::TooLarge { .. }) => write_error(too_large), + _ => (StatusCode::BAD_REQUEST, e.to_string()).into_response(), + }, + Ok(Unpacked::Document(body)) => { + // No package: another kind of document, which the plugin + // reads as the request's body (write was required already). + let request = protocol::Request { + body: Some(body), + ..request + }; + let handled = { + let (target, request) = (self.target.clone(), request.clone()); + self.call(move |plugin| plugin.handle(&target, &request)) + .await + }; + match handled { + Ok(action) => self.act(action, &request, false).await, + Err(response) => response, + } + } + Ok(Unpacked::Package { document }) => { + if let Err(e) = stored { + return write_error(e); + } + let Some((id, version)) = upload.parsed.get().cloned() else { + return plugin_failed(self.repo, &"stored without its identity"); + }; + let stored = PackageRef { id, version }; + let changes = { + let stored = stored.clone(); + match self + .call(move |plugin| plugin.published(&document, &stored)) + .await + { + Ok(changes) => changes, + Err(response) => return response, + } + }; + if !changes.is_empty() { + let set = ChangeSet { + changes, + done: publish.done.clone(), + }; + let response = self.changes(set).await; + if !response.status().is_success() { + tracing::warn!(repo = %self.repo.name, package = %stored.id, version = %stored.version, status = %response.status(), "published, but its tags could not be set"); + } + return response; + } + self.respond(publish.done) + } + } + } + + /// Read the request body, as `wanted` says, and ask the plugin again + /// with it. + async fn read_body(&self, wanted: BodyRequest, request: protocol::Request) -> Response { + if request.body.is_some() { + return plugin_failed(self.repo, &"a body was asked for twice"); + } + let Some(req) = self.take_body() else { + return self.plugin_misused("a body after it was read"); + }; + let limit = match wanted.used_for { + BodyUse::Login => wanted.limit.min(MAX_LOGIN_BODY_BYTES), + BodyUse::Read => { + if let Err(denied) = self.require_read().await { + return denied; + } + wanted.limit.min(MAX_BODY_BYTES) + } + BodyUse::Change => { + let needed = action_for_method(&self.method); + let allowed = match self.repo.kind() { + // A group's changes go to its writable member, checked + // there; reading the group comes first. + RepoKind::Group => self.may(Permission::Read, self.repo).await, + _ => self.may(needed, self.repo).await, + }; + if !self.mutating() || !allowed { + return authz_denied(&self.principal); + } + wanted.limit.min(MAX_BODY_BYTES) + } + }; + let gzip = req + .headers() + .get(header::CONTENT_ENCODING) + .is_some_and(|v| v.as_bytes().eq_ignore_ascii_case(b"gzip")); + let too_large = || { + ( + StatusCode::PAYLOAD_TOO_LARGE, + format!("the request body is larger than {limit} bytes"), + ) + .into_response() + }; + let Ok(raw) = axum::body::to_bytes(req.into_body(), limit as usize).await else { + return too_large(); + }; + let body = if gzip { + let inflated = tokio::task::spawn_blocking(move || { + let mut out = Vec::new(); + flate2::read::GzDecoder::new(&raw[..]) + .take(limit + 1) + .read_to_end(&mut out) + .map(|_| out) + }) + .await; + match inflated { + Ok(Ok(out)) if out.len() as u64 <= limit => out, + Ok(Ok(_)) => return too_large(), + Ok(Err(_)) => { + return ( + StatusCode::BAD_REQUEST, + "the request body is not valid gzip", + ) + .into_response() + } + Err(e) => return plugin_failed(self.repo, &e), + } + } else { + raw.to_vec() }; - if let Err(response) = self.may_change(needed, what) { - return response; + let request = protocol::Request { + body: Some(body), + ..request + }; + let handled = { + let (target, request) = (self.target.clone(), request.clone()); + self.call(move |plugin| plugin.handle(&target, &request)) + .await + }; + match handled { + Ok(action) => { + self.act(action, &request, wanted.used_for == BodyUse::Login) + .await + } + Err(response) => response, + } + } + + /// A login, a logout or `whoami` ([`crate::cli_auth`]), answered by the + /// plugin. + async fn auth(&self, auth: AuthRequest, request: &protocol::Request) -> Response { + match crate::cli_auth::perform(self.state, self.repo, self.principal, self.client_ip, auth) + .await + { + Ok(outcome) => { + let (target, request) = (self.target.clone(), request.clone()); + match self + .call(move |plugin| plugin.auth_response(&target, &request, &outcome)) + .await + { + Ok(response) => self.respond(response), + Err(response) => response, + } + } + Err(refused) => refused, + } + } + + /// One change ([`Action::Unlist`], [`Action::Relist`], + /// [`Action::Delete`]). + async fn change_one(&self, change: Change, done: protocol::Response) -> Response { + self.changes(ChangeSet { + changes: vec![change], + done, + }) + .await + } + + /// Make `set`'s changes in order; the first that fails answers. + async fn changes(&self, set: ChangeSet) -> Response { + // Every permission the set needs first: a refusal changes nothing. + let mut checked: Vec = Vec::new(); + for change in &set.changes { + let (needed, what) = needs(change); + if !checked.contains(&needed) { + if let Err(response) = self.write_target(needed, what).await { + return response; + } + checked.push(needed); + } } + for change in set.changes { + if let Err(response) = self.change(change).await { + return response; + } + } + self.respond(set.done) + } + + async fn change(&self, change: Change) -> Result<(), Response> { + let not_found = || (StatusCode::NOT_FOUND, "package not found").into_response(); + let (needed, what) = needs(&change); + let target = self.write_target(needed, what).await?; + let filter = self + .state + .authz + .package_filter_for(&self.principal, &target, needed) + .await; // A package outside the caller's selector answers like an unknown // one, as for NuGet. - let Some((id, version)) = self - .package(&change.target) - .filter(|(id, _)| self.package_filter.allows(id)) - else { - return (StatusCode::NOT_FOUND, "package not found").into_response(); + let package = |r: &PackageRef| { + self.package(r) + .filter(|(id, _)| filter.allows(id)) + .ok_or_else(not_found) + }; + let db = |e: sqlx::Error| { + tracing::error!(error = %e, repo = %target.name, "package change failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() }; - let writer = self.writer(); - let outcome = match kind { - Change::Unlist => writer.delete_component(&self.repo.name, &id, &version, false), - Change::Delete => writer.delete_component(&self.repo.name, &id, &version, true), - Change::Relist => writer.relist(&self.repo.name, &id, &version), + match change { + Change::Unlist(r) => { + let (id, version) = package(&r)?; + let writer = self.writer(&target, self.format.clone(), needed).await; + outcome( + writer + .delete_component(&target.name, &id, &version, false) + .await, + ) + } + Change::Delete(r) => { + let (id, version) = package(&r)?; + let writer = self.writer(&target, self.format.clone(), needed).await; + outcome( + writer + .delete_component(&target.name, &id, &version, true) + .await, + )?; + tag_dao::delete_tags(&self.state.pool, target.id, &id, Some(&version)) + .await + .map_err(db)?; + Ok(()) + } + Change::Relist(r) => { + let (id, version) = package(&r)?; + let writer = self.writer(&target, self.format.clone(), needed).await; + outcome(writer.relist(&target.name, &id, &version).await) + } + Change::Deprecate { target: r, message } => { + let (id, version) = package(&r)?; + let message = message.filter(|m| !m.is_empty()); + if component_dao::set_deprecation( + &self.state.pool, + target.id, + &id, + &version, + message.as_deref(), + ) + .await + .map_err(db)? + { + Ok(()) + } else { + Err(not_found()) + } + } + Change::Tag { id, tag, version } => { + let (id, version) = package(&PackageRef { id, version })?; + if !valid_tag(&tag) { + return Err((StatusCode::BAD_REQUEST, "invalid tag").into_response()); + } + if component_dao::find_component(&self.state.pool, target.id, &id, &version) + .await + .map_err(db)? + .is_none() + { + return Err(( + StatusCode::NOT_FOUND, + format!("version {version} of {id} not found"), + ) + .into_response()); + } + tag_dao::set_tag(&self.state.pool, target.id, &id, &tag, &version) + .await + .map_err(db) + } + Change::Untag { id, tag } => { + let id = self + .id(&id) + .filter(|id| filter.allows(id)) + .ok_or_else(not_found)?; + if tag_dao::delete_tag(&self.state.pool, target.id, &id, &tag) + .await + .map_err(db)? + { + Ok(()) + } else { + Err((StatusCode::NOT_FOUND, format!("{id} has no tag {tag:?}")).into_response()) + } + } } - .await; - match outcome { - Ok(DeleteOutcome::NotFound) => { - (StatusCode::NOT_FOUND, "package not found").into_response() + } + + /// The proxy whose upstream a forward from `repo` goes to: the + /// repository itself, or a group's first proxy member (by priority) + /// the caller may read, through nested groups. + async fn forward_source( + &self, + repo: &Repository, + depth: u32, + ) -> Result, RepoError> { + match repo.kind() { + RepoKind::Proxy => Ok(Some(repo.clone())), + RepoKind::Hosted => Ok(None), + RepoKind::Group => { + let max = self.state.config.groups.max_nesting_depth.max(1) as u32; + if depth >= max { + return Ok(None); + } + for (member, _) in repository_dao::group_members(&self.state.pool, repo.id).await? { + if member.format != repo.format || !self.may(Permission::Read, &member).await { + continue; + } + if let Some(found) = Box::pin(self.forward_source(&member, depth + 1)).await? { + return Ok(Some(found)); + } + } + Ok(None) } - Ok(_) => self.respond(change.done), - Err(e) => write_error(e), + } + } + + /// Send `forward` on to the upstream, with the proxy's credentials (to + /// its origin only), and answer with what the upstream said. + async fn forward(&self, forward: Forward) -> Response { + let source = match self.forward_source(self.repo, 0).await { + Ok(Some(source)) => source, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + "this repository has no upstream to ask: ask a proxy, or a group with one", + ) + .into_response() + } + Err(e) => return repo_error(e), + }; + if !source.online { + return offline_response(); + } + if !forward.path.starts_with('/') { + return plugin_failed(self.repo, &"a forward's path does not start with '/'"); + } + let Some(remote) = source.remote_url.as_deref() else { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "the proxy has no remote_url", + ) + .into_response(); + }; + let url = format!("{}{}", remote.trim_end_matches('/'), forward.path); + let credentials = match forge_core::repo::proxy::upstream_auth( + &self.state.pool, + self.state.envelope.as_ref(), + &source, + self.format.upstream_bearer_token(), + ) + .await + { + Ok(credentials) => credentials, + Err(e) => return repo_error(e), + }; + let auth = credentials.as_ref().map(|c| c.auth()); + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(content_type) = reqwest::header::HeaderValue::from_str(&forward.content_type) { + headers.insert(reqwest::header::CONTENT_TYPE, content_type); + } + headers.insert( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ); + let resp = match self + .state + .upstream + .send_with_body( + reqwest::Method::POST, + &url, + RequestClass::Metadata, + headers, + auth.as_ref(), + Some(forward.body), + ) + .await + { + Ok(resp) => resp, + Err(e) => { + tracing::warn!(repo = %source.name, error = %e, "forward to the upstream failed"); + return ( + StatusCode::BAD_GATEWAY, + "the upstream did not answer; try again later", + ) + .into_response(); + } + }; + let status = resp.status().as_u16(); + let content_type = resp + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("application/json") + .to_string(); + let config = self.state.upstream.config(); + let answer = match SpooledBody::read( + resp, + &url, + config.max_metadata_bytes, + config.spool_memory_bytes, + ) + .await + { + Ok(answer) => answer, + Err(e) => { + tracing::warn!(repo = %source.name, error = %e, "forward: the upstream's answer could not be read"); + return ( + StatusCode::BAD_GATEWAY, + "the upstream's answer could not be read", + ) + .into_response(); + } + }; + let body = tokio::task::spawn_blocking(move || -> std::io::Result> { + let mut out = Vec::new(); + answer.into_reader()?.read_to_end(&mut out)?; + Ok(out) + }) + .await; + match body { + Ok(Ok(body)) => self.respond(protocol::Response { + status, + headers: vec![("content-type".into(), content_type)], + body, + }), + _ => ( + StatusCode::BAD_GATEWAY, + "the upstream's answer could not be read", + ) + .into_response(), } } @@ -1021,9 +1802,10 @@ impl Ctx<'_> { HeaderValue::from(response.body.len()), ); } - if !self.head { + if !self.head() { *out.body_mut() = Body::from(response.body); } + out.extensions_mut().insert(PluginWritten); out } @@ -1042,11 +1824,89 @@ impl Ctx<'_> { } } -#[derive(Clone, Copy)] -enum Change { - Unlist, - Relist, - Delete, +/// The permission `change` needs, and what it is called in a refusal. +fn needs(change: &Change) -> (Permission, &'static str) { + match change { + Change::Unlist(_) => (Permission::Delete, "unlist"), + Change::Delete(_) => (Permission::Delete, "delete"), + Change::Relist(_) => (Permission::Write, "relist"), + Change::Deprecate { .. } => (Permission::Write, "deprecate"), + Change::Tag { .. } | Change::Untag { .. } => (Permission::Write, "tag"), + } +} + +/// What a write path change came to. +fn outcome(result: Result) -> Result<(), Response> { + match result { + Ok(DeleteOutcome::NotFound) => { + Err((StatusCode::NOT_FOUND, "package not found").into_response()) + } + Ok(_) => Ok(()), + Err(e) => Err(write_error(e)), + } +} + +/// A tag: a short word of letters, digits, `.`, `_` and `-`. +fn valid_tag(tag: &str) -> bool { + !tag.is_empty() + && tag.len() <= 64 + && tag + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) +} + +/// The request body, as [`FormatPlugin::unpack_upload`] reads it: chunks +/// sent from the async side. +struct ChannelReader { + rx: tokio::sync::mpsc::Receiver>, + chunk: Bytes, +} + +impl ChannelReader { + fn new(rx: tokio::sync::mpsc::Receiver>) -> Self { + ChannelReader { + rx, + chunk: Bytes::new(), + } + } +} + +impl Read for ChannelReader { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + while self.chunk.is_empty() { + match self.rx.blocking_recv() { + Some(Ok(chunk)) => self.chunk = chunk, + Some(Err(e)) => return Err(e), + None => return Ok(0), + } + } + let n = out.len().min(self.chunk.len()); + out[..n].copy_from_slice(&self.chunk[..n]); + self.chunk = self.chunk.slice(n..); + Ok(n) + } +} + +/// The package [`FormatPlugin::unpack_upload`] writes, sent to the staging +/// on the async side. +struct ChannelWriter { + tx: tokio::sync::mpsc::Sender>, +} + +impl Write for ChannelWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.is_empty() { + return Ok(0); + } + self.tx + .blocking_send(Ok(Bytes::copy_from_slice(buf))) + .map_err(|_| std::io::Error::other("the package is no longer read"))?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } /// Why an upload stream failed, for the right status. @@ -1216,11 +2076,6 @@ fn links_within(value: &str, mount: &str, repo: &str, base_url: &str) -> bool { targets > 0 } -/// A header value of ASCII digits and punctuation. -fn numeric_header(value: String) -> HeaderValue { - HeaderValue::from_str(&value).expect("ASCII header value") -} - /// A plugin returned an action the request's method was not authorized for /// (a download for a POST, a publish for a GET). fn refused(what: &str) -> Response { @@ -1236,7 +2091,7 @@ fn plugin_failed(repo: &Repository, e: &dyn std::fmt::Display) -> Response { (StatusCode::INTERNAL_SERVER_ERROR, "format plugin failed").into_response() } -fn repo_error(e: RepoError) -> Response { +pub(crate) fn repo_error(e: RepoError) -> Response { match e { RepoError::RepoNotFound(_) => { (StatusCode::NOT_FOUND, "repository not found").into_response() @@ -1261,6 +2116,16 @@ fn repo_error(e: RepoError) -> Response { ) .into_response() } + // Not an outage: retrying will not help. + RepoError::BadUpstream(_) => { + tracing::warn!(error = %e, "proxy upstream answered unusably and nothing cached answers"); + ( + StatusCode::BAD_GATEWAY, + "the upstream refused this proxy (are its credentials right?) or served a package \ + it does not take", + ) + .into_response() + } RepoError::FormatUnavailable(_) => ( StatusCode::SERVICE_UNAVAILABLE, "the repository's format is not installed on this server", diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index aae89cd7..b3bc7885 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -140,7 +140,7 @@ impl Instance { /// Format registry: maps a format id (the `repository.format` column) to its /// [`Format`] and its plugin: the built-in format plugins -/// ([`crate::plugins::builtin`]: NuGet, raw) and Docker. +/// ([`crate::plugins::builtin`]: NuGet, npm, raw) and Docker. /// /// Docker ([`DOCKER_FORMAT`]) is installed but has no [`Format`]: its /// repositories hold images (manifests and blobs), not package versions, @@ -157,13 +157,17 @@ pub struct FormatRegistry { impl FormatRegistry { /// A registry holding the built-in format plugins, NuGet reading - /// archives and upstream manifests under `nuget_limits`. - pub fn new(nuget_limits: forge_format_nuget::NupkgLimits) -> Self { + /// archives and upstream manifests under `nuget_limits`, npm requests + /// under `npm_limits`. + pub fn new( + nuget_limits: forge_format_nuget::NupkgLimits, + npm_limits: forge_format_npm::NpmLimits, + ) -> Self { let mut registry = FormatRegistry { formats: BTreeMap::new(), plugins: BTreeMap::new(), }; - for plugin in crate::plugins::builtin(nuget_limits) { + for plugin in crate::plugins::builtin(nuget_limits, npm_limits) { registry = registry .with_plugin(plugin) .expect("built-in format ids are distinct"); @@ -357,7 +361,10 @@ impl AppState { cas, cache, authz, - formats: Arc::new(FormatRegistry::new(config.nuget.limits())), + formats: Arc::new(FormatRegistry::new( + config.nuget.limits(), + config.npm.limits(), + )), config, realms, sessions, @@ -462,7 +469,8 @@ impl AppState { /// tests' own). A plugin whose format id is taken (a built-in plugin, or /// another plugin) is an error. pub fn with_plugins(mut self, plugins: Vec>) -> Result { - let mut registry = FormatRegistry::new(self.config.nuget.limits()); + let mut registry = + FormatRegistry::new(self.config.nuget.limits(), self.config.npm.limits()); for plugin in plugins { registry = registry.with_plugin(plugin)?; } @@ -697,12 +705,12 @@ mod tests { #[test] fn docker_is_installed_without_a_package_format() { - let registry = FormatRegistry::new(Default::default()); + let registry = FormatRegistry::new(Default::default(), Default::default()); assert!(registry.is_installed(DOCKER_FORMAT)); assert!(registry.get(DOCKER_FORMAT).is_none()); assert_eq!( registry.ids().collect::>(), - ["docker", "nuget", "raw"] + ["docker", "npm", "nuget", "raw"] ); // Raw is a built-in plugin whose repositories hold files. assert!(registry.get("raw").is_some()); @@ -721,15 +729,23 @@ mod tests { registry.kinds("nuget"), Some(vec!["hosted", "proxy", "group"]) ); + // npm is a built-in plugin, served under its own mount. + assert_eq!( + registry.kinds("npm"), + Some(vec!["hosted", "proxy", "group"]) + ); + assert!(registry.plugin("npm").is_some()); + assert!(!registry.stores_files("npm")); + assert_eq!(registry.mounts(), ["/npm", "/raw", "/v3"]); assert_eq!(registry.kinds("helm"), None); - assert_eq!(registry.kinds("npm"), None); - assert!(!registry.is_installed("npm")); + assert_eq!(registry.kinds("maven"), None); + assert!(!registry.is_installed("maven")); } #[test] fn a_plugin_cannot_take_dockers_or_a_builtin_plugins_id() { - for id in ["docker", "nuget", "raw"] { - let registry = FormatRegistry::new(Default::default()); + for id in ["docker", "nuget", "npm", "raw"] { + let registry = FormatRegistry::new(Default::default(), Default::default()); let err = registry .with_format(Arc::new(Named(id))) .err() diff --git a/crates/server/src/web_ui.rs b/crates/server/src/web_ui.rs index 1f54784e..0b3b7df9 100644 --- a/crates/server/src/web_ui.rs +++ b/crates/server/src/web_ui.rs @@ -63,7 +63,7 @@ pub const UI_ADMIN_PREFIX: &str = "/manage"; /// The API and feed prefixes. No SPA route lives under them, so a path there /// that no API route matched answers 404 instead of `index.html`. -pub const API_PREFIXES: [&str; 6] = ["/admin", "/api", "/auth", "/browse", "/raw", "/v3"]; +pub const API_PREFIXES: [&str; 7] = ["/admin", "/api", "/auth", "/browse", "/npm", "/raw", "/v3"]; /// Whether `path` is `prefix` itself or lies under it (`/admin`, `/admin/x`; /// not `/administrator`). diff --git a/crates/server/src/workers.rs b/crates/server/src/workers.rs index 2e4e580e..f3b3db96 100644 --- a/crates/server/src/workers.rs +++ b/crates/server/src/workers.rs @@ -23,7 +23,8 @@ //! then read the freshly-populated row/cache. //! //! 4. **Auth reaper** — periodically deletes expired/used `oidc_flow` and -//! `email_oauth_flow` rows, revoked/expired `session` rows and `login_known_network` rows past the +//! `email_oauth_flow` rows, revoked/expired `session` rows, expired `cli_login` +//! rows and `login_known_network` rows past the //! login throttle's 30-day memory. All are pure `DELETE`s (no advisory //! locks, no leases): the auth-code-flow rows are single-use and //! short-lived, sessions carry their own validity predicate, so running the @@ -434,6 +435,13 @@ fn spawn_auth_reaper(pool: PgPool, interval: Duration) -> JoinHandle<()> { Ok(n) => tracing::debug!(reaped = n, "auth reaper deleted dead session rows"), Err(e) => tracing::error!(error = %e, "session reap failed"), } + // Command-line logins (`npm login`) nobody finished in the + // browser. + match forge_db::cli_login_dao::delete_expired(&pool).await { + Ok(0) => {} + Ok(n) => tracing::debug!(reaped = n, "auth reaper deleted expired cli logins"), + Err(e) => tracing::error!(error = %e, "cli login reap failed"), + } // Exchanged workload keys whose CI token has expired as well, // well past any clock skew a realm accepts (until then the row // records the token's redemption). diff --git a/crates/server/tests/npm.rs b/crates/server/tests/npm.rs new file mode 100644 index 00000000..29503ba3 --- /dev/null +++ b/crates/server/tests/npm.rs @@ -0,0 +1,1609 @@ +//! Integration tests of npm repositories (`/npm`): publishing and the +//! packument documents, the redeploy policy, dist-tags, deprecation and +//! unpublishing, Bearer tokens, `npm login` (legacy and web) and +//! `npm logout`, proxies of a mock registry (caching, stale serving, +//! integrity checks, the publication delay, Bearer credentials), groups +//! (merging, the reserved-scope and prefer-hosted guards, the writable +//! member), `npm audit` and retention. Requires Docker (Postgres + +//! versitygw via the shared harness), or `FORGE_TEST_DATABASE_URL` / +//! `FORGE_TEST_S3_ENDPOINT`. +//! +//! The same flows run with the real `npm` and `pnpm` in CI +//! (`.github/scripts/npm-e2e.sh`). + +mod common; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use base64::Engine as _; +use common::{unique, TestServer}; +use forge_format_npm::integrity::Digests; +use serde_json::{json, Value}; + +const ABBREVIATED: &str = "application/vnd.npm.install-v1+json"; + +/// `POST /admin/repositories`, expecting 201. +async fn create(srv: &TestServer, body: Value) -> Value { + let (u, p) = srv.admin_basic(); + let resp = srv + .client() + .post(format!("{}/admin/repositories", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&body) + .send() + .await + .unwrap(); + let status = resp.status(); + let text = resp.text().await.unwrap(); + assert_eq!(status, 201, "create {body}: {text}"); + serde_json::from_str(&text).unwrap() +} + +async fn admin( + srv: &TestServer, + method: reqwest::Method, + path: &str, + body: Value, +) -> reqwest::Response { + let (u, p) = srv.admin_basic(); + srv.client() + .request(method, format!("{}{path}", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&body) + .send() + .await + .unwrap() +} + +async fn add_member(srv: &TestServer, group: &str, member: &str, priority: i32) { + let resp = admin( + srv, + reqwest::Method::POST, + &format!("/admin/repositories/{group}/members"), + json!({"member": member, "priority": priority}), + ) + .await; + assert!(resp.status().is_success(), "{}", resp.text().await.unwrap()); +} + +/// A gzip-compressed npm tarball of `name@version`. +fn tgz(name: &str, version: &str) -> Vec { + let package_json = json!({"name": name, "version": version, "main": "index.js"}).to_string(); + let index = format!("module.exports = {:?};", format!("{name}@{version}")); + forge_format_npm::tarball::build(&[ + ("package/package.json", package_json.as_bytes()), + ("package/index.js", index.as_bytes()), + ("package/README.md", format!("# {name}").as_bytes()), + ]) +} + +/// The body `npm publish` sends for `name@version` with `tarball`, tagged +/// `tags`. +fn publish_body(name: &str, version: &str, tarball: &[u8], tags: &[&str]) -> Value { + let d = Digests::of(tarball).unwrap(); + let unscoped = name.rsplit('/').next().unwrap(); + let file = format!("{unscoped}-{version}.tgz"); + let dist_tags: serde_json::Map = tags + .iter() + .map(|t| (t.to_string(), json!(version))) + .collect(); + json!({ + "_id": name, + "name": name, + "description": format!("{name} for tests"), + "dist-tags": dist_tags, + "versions": {version: { + "name": name, "version": version, "main": "index.js", + "description": format!("{name} for tests"), + "readme": format!("# {name}\n\nversion {version}"), + "dependencies": {"left-pad": "^1.3.0"}, + "_id": format!("{name}@{version}"), + "dist": {"integrity": d.integrity(), "shasum": d.sha1_hex, "tarball": format!("http://localhost/{file}")} + }}, + "access": null, + "_attachments": {file: { + "content_type": "application/octet-stream", + "data": base64::engine::general_purpose::STANDARD.encode(tarball), + "length": tarball.len() + }} + }) +} + +/// An npm client of one repository, authenticated with a Bearer token. +struct Npm { + base: String, + root: String, + client: reqwest::Client, + token: Option, +} + +/// The escaped form of a package name in a URL (`@scope%2fname`). +fn escaped(name: &str) -> String { + name.replacen('/', "%2f", 1) +} + +impl Npm { + fn new(srv: &TestServer, repo: &str, token: Option) -> Self { + Npm { + base: format!("{}/npm/{repo}", srv.base_url), + root: srv.base_url.clone(), + client: srv.client(), + token, + } + } + + fn req(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { + let rb = self.client.request(method, format!("{}/{path}", self.base)); + match &self.token { + Some(t) => rb.bearer_auth(t), + None => rb, + } + } + + async fn publish(&self, name: &str, version: &str, tags: &[&str]) -> reqwest::Response { + self.publish_tarball(name, version, &tgz(name, version), tags) + .await + } + + async fn publish_tarball( + &self, + name: &str, + version: &str, + tarball: &[u8], + tags: &[&str], + ) -> reqwest::Response { + self.req(reqwest::Method::PUT, &escaped(name)) + .json(&publish_body(name, version, tarball, tags)) + .send() + .await + .unwrap() + } + + /// The packument of `name` (abbreviated or full), with its status. + async fn packument(&self, name: &str, abbreviated: bool) -> (StatusCode, Value) { + let accept = if abbreviated { + format!("{ABBREVIATED}; q=1.0, application/json; q=0.8, */*") + } else { + "application/json".into() + }; + let resp = self + .req(reqwest::Method::GET, &escaped(name)) + .header("accept", accept) + .send() + .await + .unwrap(); + let status = StatusCode::from_u16(resp.status().as_u16()).unwrap(); + (status, resp.json().await.unwrap_or(Value::Null)) + } + + async fn ok_packument(&self, name: &str, abbreviated: bool) -> Value { + let (status, doc) = self.packument(name, abbreviated).await; + assert_eq!(status, 200, "packument of {name}: {doc}"); + doc + } + + /// GET a URL the repository served (a tarball's), on the test server. + async fn fetch(&self, url: &str) -> reqwest::Response { + let path = url + .split_once("://") + .and_then(|(_, rest)| rest.find('/').map(|i| rest[i..].to_string())) + .unwrap(); + let rb = self.client.get(format!("{}{path}", self.root)); + match &self.token { + Some(t) => rb.bearer_auth(t), + None => rb, + } + .send() + .await + .unwrap() + } +} + +async fn token_of(srv: &TestServer, username: &str, password: &str) -> String { + let session = srv.login(username, password).await; + let resp = session + .apply( + srv.client() + .post(format!("{}/admin/me/tokens", srv.base_url)), + ) + .json(&json!({"name": "npm"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + resp.json::().await.unwrap()["token"] + .as_str() + .unwrap() + .to_string() +} + +#[tokio::test] +async fn publishes_and_serves_packuments_and_tarballs() { + let srv = common::start().await; + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + + // Unpublished: 404, as npm expects. + assert_eq!(npm.packument("@acme/ui", true).await.0, 404); + let resp = npm.publish("@acme/ui", "1.0.0", &["latest"]).await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + let resp = npm.publish("@acme/ui", "1.1.0-beta.1", &["beta"]).await; + assert_eq!(resp.status(), 201); + assert_eq!( + npm.publish("plain-pkg", "0.1.0", &["latest"]) + .await + .status(), + 201 + ); + + // Abbreviated metadata: what installs need, tarballs on this repository. + let resp = npm + .req(reqwest::Method::GET, "@acme%2Fui") + .header("accept", ABBREVIATED) + .send() + .await + .unwrap(); + assert_eq!(resp.headers()["content-type"], ABBREVIATED); + let corgi: Value = resp.json().await.unwrap(); + assert_eq!(corgi["name"], "@acme/ui"); + assert_eq!( + corgi["dist-tags"], + json!({"latest": "1.0.0", "beta": "1.1.0-beta.1"}) + ); + let v = &corgi["versions"]["1.0.0"]; + assert_eq!(v["dependencies"], json!({"left-pad": "^1.3.0"})); + assert!(v.get("readme").is_none() && v.get("description").is_none()); + let tarball_url = v["dist"]["tarball"].as_str().unwrap(); + assert!( + tarball_url.ends_with(&format!("/npm/{repo}/@acme/ui/-/ui-1.0.0.tgz")), + "{tarball_url}" + ); + let bytes = npm.fetch(tarball_url).await.bytes().await.unwrap(); + assert_eq!(bytes.as_ref(), tgz("@acme/ui", "1.0.0").as_slice()); + let digests = Digests::of(bytes.as_ref()).unwrap(); + assert_eq!(v["dist"]["integrity"], digests.integrity()); + assert_eq!(v["dist"]["shasum"], digests.sha1_hex); + assert_eq!(v["dist"]["fileCount"], 3); + + // The full document: README, publish times, the publisher. + let full = npm.ok_packument("@acme/ui", false).await; + assert_eq!(full["_id"], "@acme/ui"); + assert_eq!(full["readme"], "# @acme/ui\n\nversion 1.0.0"); + assert!(full["time"]["1.0.0"].is_string() && full["time"]["created"].is_string()); + assert_eq!( + full["versions"]["1.0.0"]["_npmUser"]["name"], + srv.admin_user.clone().unwrap() + ); + assert_eq!(full["description"], "@acme/ui for tests"); + // The unencoded scoped path, a version and a tag. + let resp = npm + .req(reqwest::Method::GET, "@acme/ui/beta") + .send() + .await + .unwrap(); + assert_eq!( + resp.json::().await.unwrap()["version"], + "1.1.0-beta.1" + ); + let resp = npm + .req(reqwest::Method::GET, "@acme/ui/1.0.0") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!( + npm.req(reqwest::Method::GET, "@acme/ui/9.9.9") + .send() + .await + .unwrap() + .status(), + 404 + ); + // A tarball of a version that does not exist, a HEAD. + assert_eq!( + npm.req(reqwest::Method::GET, "@acme/ui/-/ui-9.9.9.tgz") + .send() + .await + .unwrap() + .status(), + 404 + ); + let head = npm + .req(reqwest::Method::HEAD, "@acme/ui/-/ui-1.0.0.tgz") + .send() + .await + .unwrap(); + assert_eq!(head.status(), 200); + assert_eq!( + head.headers()["content-length"], + bytes.len().to_string().as_str() + ); + + // A version is published once. + let resp = npm.publish("@acme/ui", "1.0.0", &["latest"]).await; + assert_eq!(resp.status(), 409); + assert!(resp.json::().await.unwrap()["error"] + .as_str() + .unwrap() + .contains("1.0.0")); + + // Ping, search, whoami. + assert_eq!( + npm.req(reqwest::Method::GET, "-/ping") + .send() + .await + .unwrap() + .status(), + 200 + ); + let found: Value = npm + .req(reqwest::Method::GET, "-/v1/search?text=acme&size=5") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(found["total"], 1, "{found}"); + assert_eq!(found["objects"][0]["package"]["name"], "@acme/ui"); + assert_eq!(found["objects"][0]["package"]["scope"], "acme"); + assert!(found["objects"][0]["package"]["maintainers"].is_array()); + let me: Value = npm + .req(reqwest::Method::GET, "-/whoami") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(me["username"], srv.admin_user.clone().unwrap()); +} + +#[tokio::test] +async fn refuses_publishes_that_do_not_hold_up() { + let srv = common::start().await; + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + + // The integrity claimed is not the tarball's. + let mut body = publish_body("a-pkg", "1.0.0", &tgz("a-pkg", "1.0.0"), &["latest"]); + body["versions"]["1.0.0"]["dist"]["integrity"] = + json!(Digests::of(&b"x"[..]).unwrap().integrity()); + let resp = npm + .req(reqwest::Method::PUT, "a-pkg") + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "{}", resp.text().await.unwrap()); + // The tarball is another version than the document says. + let other = tgz("a-pkg", "2.0.0"); + let resp = npm + .publish_tarball("a-pkg", "1.0.0", &other, &["latest"]) + .await; + assert_eq!(resp.status(), 400); + // The URL names another package than the document. + let body = publish_body("b-pkg", "1.0.0", &tgz("b-pkg", "1.0.0"), &["latest"]); + let resp = npm + .req(reqwest::Method::PUT, "a-pkg") + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + // A name new packages may not take. + let resp = npm.publish("Upper-Case", "1.0.0", &["latest"]).await; + assert_eq!(resp.status(), 400); + // Not JSON. + let resp = npm + .req(reqwest::Method::PUT, "a-pkg") + .body("{\"name\": ") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + // Nothing was stored. + assert_eq!(npm.packument("a-pkg", true).await.0, 404); + + // Larger than the repository's upload limit. + let small = format!("small{}", unique()); + create( + &srv, + json!({"name": small, "kind": "hosted", "format": "npm", "max_upload_bytes": 64}), + ) + .await; + let resp = Npm::new(&srv, &small, npm.token.clone()) + .publish("c-pkg", "1.0.0", &["latest"]) + .await; + assert_eq!(resp.status(), 413); + // Also when the store gives up long before the document ends. + let mut noise = vec![0u8; 2 * 1024 * 1024]; + let mut x: u32 = 0x2545_f491; + for byte in &mut noise { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *byte = x as u8; + } + let package_json = json!({"name": "big-pkg", "version": "1.0.0"}).to_string(); + let big = forge_format_npm::tarball::build(&[ + ("package/package.json", package_json.as_bytes()), + ("package/noise.bin", &noise), + ]); + let resp = Npm::new(&srv, &small, npm.token.clone()) + .publish_tarball("big-pkg", "1.0.0", &big, &["latest"]) + .await; + assert_eq!(resp.status(), 413); + + // Publishing needs a token: no credentials, a session without CSRF, a + // reader's token. + let anon = Npm::new(&srv, &repo, None); + assert_eq!( + anon.publish("d-pkg", "1.0.0", &["latest"]).await.status(), + 401 + ); + let reader = format!("reader{}@example.com", unique()); + srv.create_user(&reader, "reader-password-1", "reader") + .await; + let reader_npm = Npm::new( + &srv, + &repo, + Some(token_of(&srv, &reader, "reader-password-1").await), + ); + assert_eq!( + reader_npm + .publish("d-pkg", "1.0.0", &["latest"]) + .await + .status(), + 403 + ); + assert_eq!(npm.packument("d-pkg", true).await.0, 404); + // An unknown token is refused, never read as anonymous. + let bogus = Npm::new(&srv, &repo, Some("not-a-token".into())); + assert_eq!(bogus.packument("a-pkg", true).await.0, 401); +} + +#[tokio::test] +async fn the_deployment_policy_decides_on_republishing() { + let srv = common::start().await; + let token = srv.admin_token().await; + let repo = format!("npm{}", unique()); + let view = create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm", "deployment_policy": "allow_redeploy"}), + ) + .await; + assert_eq!(view["deployment_policy"], "allow_redeploy"); + assert_eq!(view["allow_redeploy"], true); + let npm = Npm::new(&srv, &repo, Some(token)); + assert_eq!(npm.publish("pkg", "1.0.0", &["latest"]).await.status(), 201); + assert_eq!(npm.publish("pkg", "1.0.0", &["latest"]).await.status(), 201); + + let resp = admin( + &srv, + reqwest::Method::PUT, + &format!("/admin/repositories/{repo}"), + json!({"deployment_policy": "read_only"}), + ) + .await; + assert_eq!(resp.status(), 200); + assert_eq!(npm.publish("pkg", "1.0.1", &["latest"]).await.status(), 403); + assert_eq!(npm.packument("pkg", true).await.0, 200, "still readable"); + // allow_redeploy says the same as the deployment policy, and may not + // contradict it. + let resp = admin( + &srv, + reqwest::Method::PUT, + &format!("/admin/repositories/{repo}"), + json!({"allow_redeploy": true, "deployment_policy": "read_only"}), + ) + .await; + assert_eq!(resp.status(), 400); + let resp = admin( + &srv, + reqwest::Method::PUT, + &format!("/admin/repositories/{repo}"), + json!({"allow_redeploy": true}), + ) + .await; + assert_eq!(resp.status(), 200); + let view: Value = resp.json().await.unwrap(); + assert_eq!(view["deployment_policy"], "allow_redeploy"); +} + +#[tokio::test] +async fn dist_tags_deprecations_and_unpublishing() { + let srv = common::start().await; + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + for (v, tags) in [ + ("1.0.0", vec!["latest"]), + ("1.1.0", vec!["next"]), + ("2.0.0", vec!["latest"]), + ] { + assert_eq!(npm.publish("@acme/tags", v, &tags).await.status(), 201); + } + let tags = |npm: &Npm| { + let rb = npm.req(reqwest::Method::GET, "-/package/@acme%2ftags/dist-tags"); + async move { rb.send().await.unwrap().json::().await.unwrap() } + }; + assert_eq!( + tags(&npm).await, + json!({"latest": "2.0.0", "next": "1.1.0"}) + ); + + // npm dist-tag add / rm. + let resp = npm + .req( + reqwest::Method::PUT, + "-/package/@acme%2ftags/dist-tags/stable", + ) + .json(&json!("1.0.0")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + for (tag, version, status) in [("1.x", "1.0.0", 400), ("other", "9.9.9", 404)] { + let resp = npm + .req( + reqwest::Method::PUT, + &format!("-/package/@acme%2ftags/dist-tags/{tag}"), + ) + .json(&json!(version)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), status, "{tag}"); + } + let resp = npm + .req( + reqwest::Method::DELETE, + "-/package/@acme%2ftags/dist-tags/next", + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = npm + .req( + reqwest::Method::DELETE, + "-/package/@acme%2ftags/dist-tags/latest", + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert_eq!( + tags(&npm).await, + json!({"latest": "2.0.0", "stable": "1.0.0"}) + ); + + // npm deprecate: the full packument, PUT back with `deprecated` set. + let mut doc = npm.ok_packument("@acme/tags", false).await; + doc["versions"]["1.0.0"]["deprecated"] = json!("use 2.x"); + let resp = npm + .req(reqwest::Method::PUT, "@acme%2ftags") + .json(&doc) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let corgi = npm.ok_packument("@acme/tags", true).await; + assert_eq!(corgi["versions"]["1.0.0"]["deprecated"], "use 2.x"); + assert!(corgi["versions"]["2.0.0"].get("deprecated").is_none()); + // And undeprecated with an empty message. + doc["versions"]["1.0.0"]["deprecated"] = json!(""); + npm.req(reqwest::Method::PUT, "@acme%2ftags") + .json(&doc) + .send() + .await + .unwrap(); + let corgi = npm.ok_packument("@acme/tags", true).await; + assert!(corgi["versions"]["1.0.0"].get("deprecated").is_none()); + + // npm unpublish : the packument without it, PUT to `-rev`, + // then the tarball's DELETE, which may find it gone already. + let mut doc = npm.ok_packument("@acme/tags", false).await; + let rev = doc["_rev"].as_str().unwrap().to_string(); + doc["versions"].as_object_mut().unwrap().remove("1.0.0"); + doc["dist-tags"].as_object_mut().unwrap().remove("stable"); + let resp = npm + .req(reqwest::Method::PUT, &format!("@acme%2ftags/-rev/{rev}")) + .json(&doc) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let resp = npm + .req( + reqwest::Method::DELETE, + &format!("@acme/tags/-/tags-1.0.0.tgz/-rev/{rev}"), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); + let corgi = npm.ok_packument("@acme/tags", true).await; + let versions: Vec<&String> = corgi["versions"].as_object().unwrap().keys().collect(); + assert_eq!(versions, ["1.1.0", "2.0.0"]); + assert_eq!(corgi["dist-tags"], json!({"latest": "2.0.0"})); + // A version's tarball DELETE unpublishes it too. + let resp = npm + .req( + reqwest::Method::DELETE, + "@acme/tags/-/tags-1.1.0.tgz/-rev/1-x", + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // npm unpublish of the package. + let resp = npm + .req(reqwest::Method::DELETE, "@acme%2ftags/-rev/1-x") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(npm.packument("@acme/tags", true).await.0, 404); + // Republishing it starts over with a `latest`. + assert_eq!( + npm.publish("@acme/tags", "3.0.0", &["beta"]).await.status(), + 201 + ); + assert_eq!( + tags(&npm).await, + json!({"beta": "3.0.0", "latest": "3.0.0"}) + ); + // A version unpublished may be published again. + assert_eq!( + npm.publish("@acme/tags", "1.0.0", &["old"]).await.status(), + 201 + ); + assert_eq!( + tags(&npm).await, + json!({"beta": "3.0.0", "latest": "3.0.0", "old": "1.0.0"}) + ); +} + +#[tokio::test] +async fn unpublishing_takes_the_delete_permission() { + let srv = common::start().await; + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let admin_npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + for v in ["1.0.0", "2.0.0"] { + assert_eq!(admin_npm.publish("pkg", v, &["latest"]).await.status(), 201); + } + let writer = format!("writer{}@example.com", unique()); + srv.create_user(&writer, "writer-password-1", "writer") + .await; + let npm = Npm::new( + &srv, + &repo, + Some(token_of(&srv, &writer, "writer-password-1").await), + ); + // A writer deprecates and moves tags, but removes no version. + let mut doc = npm.ok_packument("pkg", false).await; + doc["versions"]["1.0.0"]["deprecated"] = json!("old"); + assert_eq!( + npm.req(reqwest::Method::PUT, "pkg") + .json(&doc) + .send() + .await + .unwrap() + .status(), + 200 + ); + doc["versions"].as_object_mut().unwrap().remove("1.0.0"); + assert_eq!( + npm.req(reqwest::Method::PUT, "pkg/-rev/1") + .json(&doc) + .send() + .await + .unwrap() + .status(), + 403 + ); + assert_eq!( + npm.req(reqwest::Method::DELETE, "pkg/-rev/1") + .send() + .await + .unwrap() + .status(), + 403 + ); + assert_eq!( + npm.ok_packument("pkg", true).await["versions"] + .as_object() + .unwrap() + .len(), + 2 + ); + // A document that leaves no version is an unpublish, which has its own + // request. + doc["versions"] = json!({}); + assert_eq!( + admin_npm + .req(reqwest::Method::PUT, "pkg/-rev/1") + .json(&doc) + .send() + .await + .unwrap() + .status(), + 400 + ); + assert_eq!( + admin_npm.ok_packument("pkg", true).await["versions"] + .as_object() + .unwrap() + .len(), + 2 + ); +} + +#[tokio::test] +async fn npm_login_legacy_and_web_and_logout() { + let assets = tempfile::tempdir().unwrap(); + std::fs::write(assets.path().join("index.html"), "").unwrap(); + let admin_user = format!("admin{}@example.com", unique()); + let srv = common::start_with( + common::pool().await, + common::ServerOpts { + bootstrap_admin: Some((admin_user.clone(), "adminpw".into())), + web_assets_dir: Some(assets.path().display().to_string()), + ..Default::default() + }, + ) + .await; + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let user = format!("dev{}@example.com", unique()); + srv.create_user(&user, "dev-password-123", "writer").await; + let anon = Npm::new(&srv, &repo, None); + let path = format!("-/user/org.couchdb.user:{}", user.replace('@', "%40")); + + // Legacy: the password through the realm chain, a token back. + let resp = anon + .req(reqwest::Method::PUT, &path) + .json(&json!({"_id": format!("org.couchdb.user:{user}"), "name": user, "password": "wrong", "type": "user", "roles": []})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401); + let resp = anon + .req(reqwest::Method::PUT, &path) + .json(&json!({"name": user, "password": "dev-password-123", "type": "user", "roles": []})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + let body: Value = resp.json().await.unwrap(); + assert_eq!(body["ok"], true); + let legacy = Npm::new( + &srv, + &repo, + Some(body["token"].as_str().unwrap().to_string()), + ); + assert_eq!( + legacy + .publish("logged-in", "1.0.0", &["latest"]) + .await + .status(), + 201 + ); + let me: Value = legacy + .req(reqwest::Method::GET, "-/whoami") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(me["username"], user.as_str()); + + // Web: login and done URLs; the done URL waits for the browser. + let start: Value = anon + .req(reqwest::Method::POST, "-/v1/login") + .json(&json!({"hostname": "laptop"})) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let login_url = start["loginUrl"].as_str().unwrap(); + let done_url = start["doneUrl"].as_str().unwrap(); + assert!( + login_url.starts_with("http://test.local/cli-login/"), + "{login_url}" + ); + let secret = login_url.rsplit('/').next().unwrap(); + let pending = anon.fetch(done_url).await; + assert_eq!(pending.status(), 202); + assert!(pending.headers().contains_key("retry-after")); + let session = srv.login(&user, "dev-password-123").await; + let info: Value = session + .apply( + srv.client() + .get(format!("{}/auth/cli-login/{secret}", srv.base_url)), + ) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(info["repository"], repo.as_str()); + assert_eq!(info["hostname"], "laptop"); + // Approving takes the session's CSRF token. + let resp = srv + .client() + .post(format!("{}/auth/cli-login/{secret}", srv.base_url)) + .header("cookie", &session.cookie) + .json(&json!({"approve": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); + let resp = session + .apply( + srv.client() + .post(format!("{}/auth/cli-login/{secret}", srv.base_url)), + ) + .json(&json!({"approve": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 204); + let done: Value = anon.fetch(done_url).await.json().await.unwrap(); + let web_token = done["token"].as_str().unwrap().to_string(); + assert_eq!(anon.fetch(done_url).await.status(), 404, "collected once"); + let web = Npm::new(&srv, &repo, Some(web_token.clone())); + let me: Value = web + .req(reqwest::Method::GET, "-/whoami") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(me["username"], user.as_str()); + + // Both tokens are the user's, with a lifetime, listed on their page. + let tokens: Value = session + .apply(srv.client().get(format!("{}/auth/tokens", srv.base_url))) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let names: Vec<&str> = tokens + .as_array() + .unwrap() + .iter() + .map(|t| t["name"].as_str().unwrap()) + .collect(); + assert!( + names.contains(&format!("npm login ({repo})").as_str()), + "{names:?}" + ); + assert!( + names.contains(&format!("npm login ({repo}) on laptop").as_str()), + "{names:?}" + ); + assert!(tokens + .as_array() + .unwrap() + .iter() + .all(|t| t["expires_at"].is_string())); + + // A refused web login collects nothing. + let start: Value = anon + .req(reqwest::Method::POST, "-/v1/login") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let secret = start["loginUrl"] + .as_str() + .unwrap() + .rsplit('/') + .next() + .unwrap() + .to_string(); + let resp = session + .apply( + srv.client() + .post(format!("{}/auth/cli-login/{secret}", srv.base_url)), + ) + .json(&json!({"approve": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 204); + assert_eq!( + anon.fetch(start["doneUrl"].as_str().unwrap()) + .await + .status(), + 404 + ); + + // npm logout revokes the token it presents, and only the caller's own. + let resp = legacy + .req( + reqwest::Method::DELETE, + &format!("-/user/token/{web_token}"), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "the same user's token"); + assert_eq!( + web.req(reqwest::Method::GET, "-/whoami") + .send() + .await + .unwrap() + .status(), + 401 + ); + let admin_npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + let resp = admin_npm + .req( + reqwest::Method::DELETE, + &format!("-/user/token/{}", legacy.token.clone().unwrap()), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404, "another user's token"); + assert_eq!( + legacy + .req(reqwest::Method::GET, "-/whoami") + .send() + .await + .unwrap() + .status(), + 200 + ); +} + +/// A mock npm registry. +#[derive(Default)] +struct Upstream { + packuments: Mutex>, + tarballs: Mutex>>, + packument_hits: AtomicUsize, + tarball_hits: AtomicUsize, + down: AtomicBool, + /// The Bearer token every request must carry, if any. + token: Mutex>, + /// Authorization headers seen, by path. + auth: Mutex)>>, + audits: Mutex>, +} + +impl Upstream { + /// Serve `name` with `versions` (version, published), its tarballs at + /// `{base}/tarballs/...`. + fn publish(&self, base: &str, name: &str, versions: &[(&str, &str)], latest: &str) { + let mut doc_versions = serde_json::Map::new(); + let mut time = serde_json::Map::new(); + for (version, published) in versions { + let tarball = tgz(name, version); + let d = Digests::of(&tarball[..]).unwrap(); + let file = format!("{}-{version}.tgz", name.rsplit('/').next().unwrap()); + let path = format!("/tarballs/{}/{file}", name.replace('/', "_")); + self.tarballs.lock().unwrap().insert(path.clone(), tarball); + doc_versions.insert( + version.to_string(), + json!({"name": name, "version": version, "readme": "per-version readme", + "dist": {"tarball": format!("{base}{path}"), "integrity": d.integrity(), "shasum": d.sha1_hex}}), + ); + time.insert(version.to_string(), json!(published)); + } + self.packuments.lock().unwrap().insert( + name.to_string(), + json!({"_id": name, "name": name, "dist-tags": {"latest": latest}, + "versions": doc_versions, "time": time, "readme": format!("# {name} upstream"), + "users": {"someone": true}}), + ); + } +} + +async fn serve_upstream(upstream: Arc) -> String { + async fn handle( + axum::extract::State(up): axum::extract::State>, + method: axum::http::Method, + uri: axum::http::Uri, + headers: HeaderMap, + body: axum::body::Bytes, + ) -> Response { + let path = uri.path().to_string(); + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + up.auth.lock().unwrap().push((path.clone(), auth.clone())); + if up.down.load(Ordering::SeqCst) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if let Some(token) = up.token.lock().unwrap().clone() { + if auth.as_deref() != Some(format!("Bearer {token}").as_str()) { + return StatusCode::UNAUTHORIZED.into_response(); + } + } + if method == axum::http::Method::POST && path.starts_with("/-/npm/v1/security/") { + up.audits + .lock() + .unwrap() + .push(serde_json::from_slice(&body).unwrap_or(Value::Null)); + return axum::Json(json!({"left-pad": [{"id": 1, "severity": "high"}]})) + .into_response(); + } + if let Some(tarball) = up.tarballs.lock().unwrap().get(&path).cloned() { + up.tarball_hits.fetch_add(1, Ordering::SeqCst); + return tarball.into_response(); + } + let name = percent_encoding::percent_decode_str(path.trim_start_matches('/')) + .decode_utf8_lossy() + .to_string(); + up.packument_hits.fetch_add(1, Ordering::SeqCst); + match up.packuments.lock().unwrap().get(&name) { + Some(doc) => axum::Json(doc.clone()).into_response(), + None => ( + StatusCode::NOT_FOUND, + axum::Json(json!({"error": "not found"})), + ) + .into_response(), + } + } + let router = axum::Router::new().fallback(handle).with_state(upstream); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + base +} + +#[tokio::test] +async fn a_proxy_caches_verifies_and_serves_stale() { + let srv = common::start().await; + let upstream = Arc::new(Upstream::default()); + let base = serve_upstream(upstream.clone()).await; + upstream.publish( + &base, + "left-pad", + &[ + ("1.2.0", "2016-01-01T00:00:00.000Z"), + ("1.3.0", "2018-04-09T01:57:31.231Z"), + ], + "1.3.0", + ); + upstream.publish( + &base, + "@types/x", + &[("1.0.0", "2020-01-01T00:00:00.000Z")], + "1.0.0", + ); + let repo = format!("proxy{}", unique()); + create(&srv, json!({"name": repo, "kind": "proxy", "format": "npm", "remote_url": base, "metadata_ttl_seconds": 3600})).await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + + let corgi = npm.ok_packument("left-pad", true).await; + assert_eq!(corgi["dist-tags"], json!({"latest": "1.3.0"})); + let url = corgi["versions"]["1.3.0"]["dist"]["tarball"] + .as_str() + .unwrap() + .to_string(); + assert!( + url.contains(&format!("/npm/{repo}/left-pad/-/left-pad-1.3.0.tgz")), + "{url}" + ); + let full = npm.ok_packument("left-pad", false).await; + assert_eq!(full["readme"], "# left-pad upstream"); + assert!(full["versions"]["1.3.0"].get("readme").is_none()); + assert_eq!(full["time"]["1.3.0"], "2018-04-09T01:57:31.231Z"); + assert!(corgi.get("time").is_none()); + // Within the TTL the upstream is asked once for each document: the + // abbreviated one installs read, the full one with the publish times. + npm.ok_packument("left-pad", true).await; + npm.ok_packument("left-pad", false).await; + assert_eq!(upstream.packument_hits.load(Ordering::SeqCst), 2); + let scoped = npm.ok_packument("@types/x", true).await; + assert_eq!(scoped["name"], "@types/x"); + + // A tarball is downloaded once, checked, and kept. + let bytes = npm.fetch(&url).await.bytes().await.unwrap(); + assert_eq!(bytes.as_ref(), tgz("left-pad", "1.3.0").as_slice()); + npm.fetch(&url).await.bytes().await.unwrap(); + assert_eq!(upstream.tarball_hits.load(Ordering::SeqCst), 1); + + // The upstream goes down: the packument and the cached tarball are + // still served (after the TTL: the proxy is invalidated). + upstream.down.store(true, Ordering::SeqCst); + let resp = admin( + &srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/proxy-cache/invalidate"), + json!({}), + ) + .await; + assert_eq!(resp.status(), 200); + let stale = npm.ok_packument("left-pad", true).await; + assert_eq!(stale["dist-tags"], json!({"latest": "1.3.0"})); + assert!( + upstream.packument_hits.load(Ordering::SeqCst) >= 2, + "it asked again" + ); + assert_eq!(npm.fetch(&url).await.status(), 200); + // A tarball never downloaded cannot be served now: try again later. + let resp = npm + .req(reqwest::Method::GET, "left-pad/-/left-pad-1.2.0.tgz") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 503); + assert!(resp.headers().contains_key("retry-after")); + let body: Value = resp.json().await.unwrap(); + assert!( + body["error"].as_str().unwrap().contains("upstream"), + "{body}" + ); + upstream.down.store(false, Ordering::SeqCst); + + // An unknown package is remembered as such. + assert_eq!(npm.packument("nope", true).await.0, 404); + let hits = upstream.packument_hits.load(Ordering::SeqCst); + assert_eq!(npm.packument("nope", true).await.0, 404); + assert_eq!(upstream.packument_hits.load(Ordering::SeqCst), hits); + + // A tarball that is not what its packument says is refused. + upstream.publish( + &base, + "tampered", + &[("1.0.0", "2020-01-01T00:00:00.000Z")], + "1.0.0", + ); + upstream.tarballs.lock().unwrap().insert( + "/tarballs/tampered/tampered-1.0.0.tgz".into(), + tgz("tampered", "6.6.6"), + ); + let resp = npm + .req(reqwest::Method::GET, "tampered/-/tampered-1.0.0.tgz") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 502); + let stored: i64 = sqlx::query_scalar("SELECT count(*) FROM component c JOIN repository r ON r.id = c.repository_id WHERE r.name = $1 AND c.name = 'tampered'") + .bind(&repo) + .fetch_one(&srv.pool) + .await + .unwrap(); + assert_eq!(stored, 0); + + // A proxy is read-only. + assert_eq!( + npm.publish("left-pad", "9.0.0", &["latest"]).await.status(), + 405 + ); +} + +#[tokio::test] +async fn a_proxy_holds_back_versions_inside_the_publication_delay() { + let srv = common::start().await; + let upstream = Arc::new(Upstream::default()); + let base = serve_upstream(upstream.clone()).await; + let recent = (chrono::Utc::now() - chrono::Duration::hours(1)).to_rfc3339(); + upstream.publish( + &base, + "fresh", + &[ + ("1.0.0", "2020-01-01T00:00:00.000Z"), + ("1.1.0", "2021-01-01T00:00:00.000Z"), + ("2.0.0", &recent), + ], + "2.0.0", + ); + let repo = format!("proxy{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "proxy", "format": "npm", "remote_url": base, "package_delay_seconds": 86400}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + let corgi = npm.ok_packument("fresh", true).await; + let versions: Vec<&String> = corgi["versions"].as_object().unwrap().keys().collect(); + assert_eq!(versions, ["1.0.0", "1.1.0"]); + assert_eq!(corgi["dist-tags"], json!({"latest": "1.1.0"})); + let resp = npm + .req(reqwest::Method::GET, "fresh/-/fresh-2.0.0.tgz") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); + assert_eq!(upstream.tarball_hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn a_proxy_sends_its_token_to_its_upstream_only() { + let srv = common::start().await; + let upstream = Arc::new(Upstream::default()); + let base = serve_upstream(upstream.clone()).await; + // The tarballs live on another origin (localhost, not 127.0.0.1). + let other = base.replace("127.0.0.1", "localhost"); + upstream.publish( + &other, + "private-pkg", + &[("1.0.0", "2020-01-01T00:00:00.000Z")], + "1.0.0", + ); + let repo = format!("proxy{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "proxy", "format": "npm", "remote_url": base, "upstream_password": "upstream-token"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + let corgi = npm.ok_packument("private-pkg", true).await; + let url = corgi["versions"]["1.0.0"]["dist"]["tarball"] + .as_str() + .unwrap() + .to_string(); + assert_eq!(npm.fetch(&url).await.status(), 200); + let seen = upstream.auth.lock().unwrap().clone(); + assert!( + seen.iter() + .any(|(p, a)| p == "/private-pkg" && a.as_deref() == Some("Bearer upstream-token")), + "{seen:?}" + ); + assert!( + seen.iter() + .any(|(p, a)| p.starts_with("/tarballs/") && a.is_none()), + "no token off-origin: {seen:?}" + ); + + // An upstream that wants it: a proxy without it fails, one with it + // serves. + *upstream.token.lock().unwrap() = Some("upstream-token".into()); + upstream.publish( + &base, + "guarded", + &[("1.0.0", "2020-01-01T00:00:00.000Z")], + "1.0.0", + ); + assert_eq!(npm.packument("guarded", true).await.0, 200); + let open = format!("open{}", unique()); + create( + &srv, + json!({"name": open, "kind": "proxy", "format": "npm", "remote_url": base}), + ) + .await; + assert_eq!( + Npm::new(&srv, &open, npm.token.clone()) + .packument("guarded", true) + .await + .0, + 502 + ); +} + +#[tokio::test] +async fn a_group_merges_and_keeps_internal_scopes_internal() { + let srv = common::start().await; + let token = srv.admin_token().await; + let upstream = Arc::new(Upstream::default()); + let base = serve_upstream(upstream.clone()).await; + // A public registry that claims an internal scope and an internal name. + upstream.publish( + &base, + "@acme/ui", + &[ + ("1.0.0", "2020-01-01T00:00:00.000Z"), + ("99.0.0", "2020-02-01T00:00:00.000Z"), + ], + "99.0.0", + ); + upstream.publish( + &base, + "@acme/not-yet-internal", + &[("1.0.0", "2020-01-01T00:00:00.000Z")], + "1.0.0", + ); + upstream.publish( + &base, + "internal-lib", + &[("5.0.0", "2020-01-01T00:00:00.000Z")], + "5.0.0", + ); + upstream.publish( + &base, + "left-pad", + &[("1.3.0", "2018-01-01T00:00:00.000Z")], + "1.3.0", + ); + let n = unique(); + let (hosted, proxy, group) = (format!("hosted{n}"), format!("proxy{n}"), format!("all{n}")); + create( + &srv, + json!({"name": hosted, "kind": "hosted", "format": "npm"}), + ) + .await; + create( + &srv, + json!({"name": proxy, "kind": "proxy", "format": "npm", "remote_url": base}), + ) + .await; + let view = create( + &srv, + json!({"name": group, "kind": "group", "format": "npm", "reserved_prefixes": ["@acme"]}), + ) + .await; + assert_eq!(view["reserved_prefixes"], json!(["@acme/"])); + add_member(&srv, &group, &hosted, 10).await; + add_member(&srv, &group, &proxy, 5).await; + let resp = admin( + &srv, + reqwest::Method::PUT, + &format!("/admin/repositories/{group}"), + json!({"writable_member": hosted}), + ) + .await; + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + + let internal = Npm::new(&srv, &hosted, Some(token.clone())); + assert_eq!( + internal + .publish("@acme/ui", "1.0.0", &["latest"]) + .await + .status(), + 201 + ); + assert_eq!( + internal + .publish("internal-lib", "1.0.0", &["latest"]) + .await + .status(), + 201 + ); + let npm = Npm::new(&srv, &group, Some(token.clone())); + + // The scope is reserved: the public 99.0.0 never appears, nor a scoped + // package no hosted member has yet. + let ui = npm.ok_packument("@acme/ui", true).await; + let versions: Vec<&String> = ui["versions"].as_object().unwrap().keys().collect(); + assert_eq!(versions, ["1.0.0"]); + assert_eq!(ui["dist-tags"], json!({"latest": "1.0.0"})); + let url = ui["versions"]["1.0.0"]["dist"]["tarball"].as_str().unwrap(); + assert!( + url.contains(&format!("/npm/{group}/@acme/ui/-/ui-1.0.0.tgz")), + "{url}" + ); + assert_eq!( + npm.fetch(url).await.bytes().await.unwrap().as_ref(), + tgz("@acme/ui", "1.0.0").as_slice() + ); + assert_eq!(npm.packument("@acme/not-yet-internal", true).await.0, 404); + assert_eq!( + npm.req(reqwest::Method::GET, "@acme/ui/-/ui-99.0.0.tgz") + .send() + .await + .unwrap() + .status(), + 404 + ); + // Prefer hosted: an internal name a hosted member has is not mixed with + // the public one. + let lib = npm.ok_packument("internal-lib", true).await; + assert_eq!( + lib["versions"] + .as_object() + .unwrap() + .keys() + .collect::>(), + ["1.0.0"] + ); + // A public package comes from the proxy, with its tags. + let pad = npm.ok_packument("left-pad", true).await; + assert_eq!(pad["dist-tags"], json!({"latest": "1.3.0"})); + let url = pad["versions"]["1.3.0"]["dist"]["tarball"] + .as_str() + .unwrap(); + assert!(url.contains(&format!("/npm/{group}/left-pad/")), "{url}"); + assert_eq!(npm.fetch(url).await.status(), 200); + + // Publishing to the group lands in its writable member. + assert_eq!( + npm.publish("@acme/ui", "1.1.0", &["latest"]).await.status(), + 201 + ); + let ui = internal.ok_packument("@acme/ui", true).await; + assert_eq!(ui["dist-tags"], json!({"latest": "1.1.0"})); + + // npm audit through the group goes to the proxy's upstream, without the + // internal scope's packages. + let resp = npm + .req(reqwest::Method::POST, "-/npm/v1/security/advisories/bulk") + .json(&json!({"@acme/ui": ["1.1.0"], "left-pad": ["1.3.0"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await.unwrap()["left-pad"][0]["severity"], + "high" + ); + assert_eq!( + upstream.audits.lock().unwrap().clone(), + vec![json!({"left-pad": ["1.3.0"]})] + ); + // A hosted repository has no audit source. + let resp = internal + .req(reqwest::Method::POST, "-/npm/v1/security/advisories/bulk") + .json(&json!({})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); +} + +#[tokio::test] +async fn retention_never_deletes_a_tagged_version() { + let srv = common::start().await; + let repo = format!("npm{}", unique()); + let view = create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(srv.admin_token().await)); + for v in ["1.0.0", "1.1.0", "1.2.0", "1.3.0", "1.4.0"] { + assert_eq!(npm.publish("kept", v, &["latest"]).await.status(), 201); + } + // `legacy` holds an old version; `latest` is the newest. + let resp = npm + .req(reqwest::Method::PUT, "-/package/kept/dist-tags/legacy") + .json(&json!("1.0.0")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + // Two days old, so that a one-day trigger takes them all. + sqlx::query( + "UPDATE component SET created_at = now() - interval '2 days' WHERE repository_id = $1", + ) + .bind(view["id"].as_i64().unwrap()) + .execute(&srv.pool) + .await + .unwrap(); + let policy = json!({"policy": {"version": 1, "rules": [{"id": "keep-two", "action": "hard_delete", "criteria": {"keep_last_n": 2, "older_than_days": 1}}]}}); + let resp = admin( + &srv, + reqwest::Method::PUT, + &format!("/admin/repositories/{repo}/retention"), + policy, + ) + .await; + assert!(resp.status().is_success(), "{}", resp.text().await.unwrap()); + let preview: Value = admin( + &srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/retention/preview"), + json!({}), + ) + .await + .json() + .await + .unwrap(); + let mut previewed: Vec<&str> = preview["decisions"] + .as_array() + .unwrap() + .iter() + .map(|d| d["version"].as_str().unwrap()) + .collect(); + previewed.sort(); + assert_eq!(previewed, ["1.1.0", "1.2.0"], "{preview}"); + assert_eq!( + admin( + &srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/retention/run"), + json!({}) + ) + .await + .status(), + 202 + ); + let worker = forge_core::retention::RetentionWorker::new( + srv.pool.clone(), + forge_server::state::RegistryResolver(srv.state.formats.clone()), + Arc::new(forge_core::repo::NoopCacheInvalidator), + forge_core::retention::RetentionConfig::default(), + ); + let summary = worker + .tick_repo(view["id"].as_i64().unwrap()) + .await + .unwrap() + .expect("a run"); + assert_eq!(summary.deleted, 2, "{summary:?}"); + let corgi = npm.ok_packument("kept", true).await; + let versions: Vec<&String> = corgi["versions"].as_object().unwrap().keys().collect(); + assert_eq!(versions, ["1.0.0", "1.3.0", "1.4.0"]); + assert_eq!( + corgi["dist-tags"], + json!({"latest": "1.4.0", "legacy": "1.0.0"}) + ); +} + +#[tokio::test] +async fn routes_only_npm_repositories() { + let srv = common::start().await; + let nuget = format!("nuget{}", unique()); + create(&srv, json!({"name": nuget, "kind": "hosted"})).await; + let token = srv.admin_token().await; + let npm = Npm::new(&srv, &nuget, Some(token.clone())); + assert_eq!(npm.packument("left-pad", true).await.0, 404); + let missing = Npm::new(&srv, "no-such-repo", None); + assert_eq!(missing.packument("left-pad", true).await.0, 401); + let repo = format!("npm{}", unique()); + create( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let npm = Npm::new(&srv, &repo, Some(token)); + for bad in ["..%2f..%2fadmin", "%2e%2e", "a%2fb", "-/nope"] { + let status = npm + .req(reqwest::Method::GET, bad) + .send() + .await + .unwrap() + .status(); + assert!(status == 404 || status == 400, "{bad}: {status}"); + } + // The format is listed with its kinds and mount. + let formats: Value = admin(&srv, reqwest::Method::GET, "/admin/formats", json!({})) + .await + .json() + .await + .unwrap(); + let npm_format = formats + .as_array() + .unwrap() + .iter() + .find(|f| f["id"] == "npm") + .expect("npm listed"); + assert_eq!(npm_format["mount"], "/npm"); + assert_eq!(npm_format["kinds"], json!(["hosted", "proxy", "group"])); +} diff --git a/crates/server/tests/plugin_contract.rs b/crates/server/tests/plugin_contract.rs index 9fff98fa..89bf4b53 100644 --- a/crates/server/tests/plugin_contract.rs +++ b/crates/server/tests/plugin_contract.rs @@ -124,10 +124,12 @@ impl FormatPlugin for Probe { match (req.method.as_str(), segments.as_slice()) { ("PUT", ["upload"]) => Action::Publish(PublishRequest { upload: Upload::FirstPart, + id: None, done: Response::text(201, "saved"), }), ("POST", ["upload"]) => Action::Publish(PublishRequest { upload: Upload::Body, + id: None, done: Response::text(201, "saved"), }), (_, ["files", id, version]) if read => Action::Download(target(id, version)), diff --git a/crates/server/tests/raw.rs b/crates/server/tests/raw.rs index 95eb4bc5..bf6a967b 100644 --- a/crates/server/tests/raw.rs +++ b/crates/server/tests/raw.rs @@ -1024,7 +1024,7 @@ async fn forms_the_web_ui_and_the_admin_api() { assert_eq!(raw["kinds"], json!(["hosted", "proxy", "group"])); for (body, why) in [ ( - json!({"name": format!("x{}", unique()), "kind": "hosted", "deployment_policy": "read_only"}), + json!({"name": format!("x{}", unique()), "kind": "hosted", "content_disposition": "inline"}), "a raw option on nuget", ), ( diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 221eb419..4e754ac4 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -2495,7 +2495,7 @@ async fn admin_validates_docker_repositories() { 400 ); assert_eq!( - bad(json!({"name": format!("x{n}"), "kind": "hosted", "format": "npm"})).await, + bad(json!({"name": format!("x{n}"), "kind": "hosted", "format": "pypi"})).await, 400 ); assert_eq!( diff --git a/crates/server/tests/release_hardening.rs b/crates/server/tests/release_hardening.rs index a38d0653..05907dd0 100644 --- a/crates/server/tests/release_hardening.rs +++ b/crates/server/tests/release_hardening.rs @@ -690,7 +690,7 @@ async fn repository_formats_that_are_not_installed_are_rejected() { .client() .post(format!("{}/admin/repositories", srv.base_url)) .basic_auth(&admin, Some(&password)) - .json(&json!({ "name": format!("npm{}", common::unique()), "kind": "hosted", "format": "npm" })) + .json(&json!({ "name": format!("pypi{}", common::unique()), "kind": "hosted", "format": "pypi" })) .send() .await .unwrap(); diff --git a/docs/changes/2.0.0/283.fixed.md b/docs/changes/2.0.0/283.fixed.md index 16b6fe1b..30c2e1e9 100644 --- a/docs/changes/2.0.0/283.fixed.md +++ b/docs/changes/2.0.0/283.fixed.md @@ -4,5 +4,4 @@ answered a 5xx or 429, with nothing cached for the request, answered 500 and logged an error, as if Forge were broken. It answers 503 with `Retry-After`, like the docker registry's proxy, and logs a warning naming the upstream. A group whose member failed this way answers the same. -Other upstream failures, such as credentials the upstream refuses, still -answer 500. +Credentials the upstream refuses answer 502. diff --git a/docs/changes/2.0.0/41-package-options.feature.md b/docs/changes/2.0.0/41-package-options.feature.md new file mode 100644 index 00000000..b1d95021 --- /dev/null +++ b/docs/changes/2.0.0/41-package-options.feature.md @@ -0,0 +1,6 @@ +Hosted NuGet repositories have a deployment policy, like raw and npm ones: +allow redeploys, refuse them (the default), or read-only, which refuses +every push, unlist and delete. `allow_redeploy` says the same and follows +it. A NuGet group may have a writable member, a hosted member that pushes, +unlists and deletes through the group go to. The web UI's repository form +offers both. diff --git a/docs/changes/2.0.0/41.feature.md b/docs/changes/2.0.0/41.feature.md new file mode 100644 index 00000000..636b48cf --- /dev/null +++ b/docs/changes/2.0.0/41.feature.md @@ -0,0 +1,19 @@ +npm registries: repositories of `"format": "npm"`, served at +`/npm//` for npm, pnpm, Yarn and Bun. Hosted ones take +`npm publish` (the tarball checked against the integrity the client claims +and the version it names, under the repository's deployment policy), +dist-tags, `npm deprecate` and `npm unpublish`. Proxies cache +`registry.npmjs.org` or any registry, with Basic credentials or a token +sent as `Authorization: Bearer` to its own origin only, and serve the +upstream's dist-tags, deprecations and publish times, verified tarballs and +the publication delay. Groups merge their members, take publishes into a +writable member, keep reserved scopes (`@acme`) away from proxy members and +pass `npm audit` on to their proxy without the reserved names. `npm login` +is approved in the browser on a page of the web UI, or checks a password +(`yarn npm login`), and mints an API token that `npm logout` revokes; npm +clients send API tokens as `Authorization: Bearer`, which every format +plugin's repositories now accept. Retention never deletes a version a +dist-tag names. The web UI shows a package's dist-tags, deprecated versions +and README, and a repository's `.npmrc` and `.yarnrc.yml` lines and +install commands. npm is a built-in format plugin (`crates/formats/npm`). See +[docs/npm.md](docs/npm.md). diff --git a/docs/changes/2.0.0/41.fixed.md b/docs/changes/2.0.0/41.fixed.md new file mode 100644 index 00000000..7fb0e2b2 --- /dev/null +++ b/docs/changes/2.0.0/41.fixed.md @@ -0,0 +1,6 @@ +Proxies of a format plugin (NuGet, npm): an upstream that refuses the +proxy's credentials (401, 403), or serves a package the proxy does not take +(too large, unreadable, not the version asked for, not matching the digest +the upstream publishes), with nothing cached for the request, answered 500. +It answers 502: the upstream or the proxy's settings need fixing, retrying +does not help. diff --git a/docs/changes/2.0.0/41.limitation.md b/docs/changes/2.0.0/41.limitation.md new file mode 100644 index 00000000..805a267a --- /dev/null +++ b/docs/changes/2.0.0/41.limitation.md @@ -0,0 +1,6 @@ +npm's password login (`npm login --auth-type=legacy`) refuses a username +with an `@`, which every Forge account has: log in in the browser +(`npm login`), with `yarn npm login`, or with an API token from the web UI. +A publish with a provenance statement (`npm publish --provenance`) is +refused, and `npm access`, `npm owner`, `npm team`, `npm org`, `npm star` +and `npm token` are not served. diff --git a/docs/format-plugins.md b/docs/format-plugins.md index 4b00ef53..7bf43d5b 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -11,9 +11,10 @@ streaming of package bytes. Every plugin is built in: a crate under `crates/formats//` linked into the server and called directly, like any other Rust code. Plugins need no -configuration. Forge ships two: **NuGet** -([`crates/formats/nuget`](../crates/formats/nuget)) and **raw files** -([`crates/formats/raw`](../crates/formats/raw), +configuration. Forge ships three: **NuGet** +([`crates/formats/nuget`](../crates/formats/nuget)), **npm** +([`crates/formats/npm`](../crates/formats/npm), [npm.md](npm.md)) and **raw +files** ([`crates/formats/raw`](../crates/formats/raw), [Files formats](#files-formats)). Helm charts are served by the registry ([docker.md](docker.md#helm-charts)). @@ -21,7 +22,9 @@ configuration. Forge ships two: **NuGet** > change before 1.x of the contract; a plugin declares the contract version it > was built for, and the server refuses one it does not implement. 0.4.0 added > files formats ([Files formats](#files-formats)), and `render` returns an -> action, like `handle`. +> action, like `handle`; and for npm, request bodies, document uploads, +> changes (tags, deprecations), logins and forwards +> ([npm as a plugin](#npm-as-a-plugin)). ## Contents @@ -71,10 +74,12 @@ The pieces: | | Forge (the host) | The plugin | | --- | --- | --- | | Identity | Calls the format for every id and version it sees | `canonical_id`, `validate_id`, `normalize_version`, `compare_versions` | -| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file, returns id, version, manifest, dependencies | +| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file, returns id, version, manifest, dependencies. For a package that comes inside a document (npm's JSON with a base64 tarball), `unpack_upload` takes the package out while the body streams by, and `parse_upload` reads it with the rest of the document | | Reads | Runs the queries the plugin asks for through the repository read path (hosted, proxy or group). Package selectors, group membership, prefer-hosted and reserved prefixes all apply | `handle` decides what a request is; `render` turns the loaded data into the response body | | Downloads | Streams the blob from S3, counts the download, answers `HEAD` without touching S3, and a single `Range` when the local blob cache is on | Names the version (`download`) | | Unlist, relist, delete | Hosted write path, with the repository's permissions and selectors; a package outside the caller's selector answers 404, like an unknown one | Names the version and the success response | +| Tags and deprecations | Keeps a package's tags (npm's dist-tags: a name for one version) and each version's deprecation, drops a version's tags with the version, and never lets retention delete a tagged version | Names the changes (`tag`, `untag`, `deprecate`); `published` names those a publish document carries | +| Logins | Checks a password through the realm chain, or runs a login approved in the browser, and mints, lists and revokes the API tokens; accepts `Authorization: Bearer` API tokens | Says what a request asks (`auth`) and renders the outcome (`auth_response`) | | Proxies | Sends every upstream request (credentials only to the upstream's origin, bounded, cached), downloads and verifies packages, keeps the package delay, the not-found markers and the offline mode | `upstream`: which documents an operation needs, and what they say ([Proxy repositories](#proxy-repositories)) | | Groups | Merges members by priority; the dependency-confusion guards apply | Nothing: a group of plugin repositories just works | | Retention | Evaluates policies, ordering versions with the plugin's `compare_versions` | Nothing more | @@ -88,26 +93,46 @@ The dispatcher ([`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs)) handles it in this order: -1. Resolve the repository and authorize the method: - GET and HEAD need read, PUT and POST need write, DELETE needs delete. - Unknown repositories answer like unreadable ones, and offline ones get 503. -2. Call the plugin's `handle` with the repository (name, kind, base URL) and - the request: method, path, query, and a short allowlist of headers. The - plugin never sees credentials or cookies. Protocol calls run on a - blocking thread, never on the async workers. -3. Act on the plugin's answer: - - `respond`: send the response as given, keeping only the allowed headers. +1. Resolve the repository and the caller (a session, Basic credentials, or + an `Authorization: Bearer` API token). Unknown repositories answer like + unreadable ones, and offline ones get 503 (to a reader). A files + format's request is authorized by its method first: GET and HEAD need + read, PUT and POST write, DELETE delete; a package format's actions are + checked one by one (below). +2. Call the plugin's `handle` with the repository (name, kind, base URL, a + group's reserved prefixes) and the request: method, path, query, and a + short allowlist of headers. The plugin never sees credentials or + cookies. Protocol calls run on a blocking thread, never on the async + workers. +3. Act on the plugin's answer, checked against what the caller may do: + - `respond`: send the response as given, keeping only the allowed + headers. Needs read. - `load(queries)`: run the queries, then call `render` with every query and its answer. `render` answers like `handle` does: a response, more data to load, or one of the actions below. One request gets at most 4 - rounds and 2,048 answers. - - `download`: stream the stored package. + rounds and 2,048 answers. Needs read. + - `download`: stream the stored package. Needs read. - `serve-file`: stream a stored file (a files format). - - `publish`, `unlist`, `relist`, `delete`: go through the hosted write - path, but only when the request's method was authorized for that action - (a plugin cannot turn a GET into a delete). The repository must be - hosted, and the caller must present an API token or a workload + - `read body`: read the request's body, bounded (at most 64 MiB, 64 KiB + for a login), and ask `handle` again with it. Its use says what it + needs: a login nothing, a read read, a change the permission of the + method. + - `publish`, `change` (`unlist`, `relist`, `delete`, `deprecate`, `tag`, + `untag`): go through the hosted write path, but only for a mutating + method, with the permission each needs (delete for a delete, write for + the rest), never for a GET. The repository must be hosted, or a group + with a writable member (where they go), its deployment policy must not + be read-only, and the caller must present an API token or a workload identity. + - `auth`: a login, a logout or a `whoami`, performed by the host; the + plugin renders the outcome (`auth_response`). + - `forward`: a POST to the repository's upstream (a proxy's, or a + group's first proxy member's), with the proxy's credentials; its answer + is passed back. npm's audits go so. Needs read. + + A host error (a refusal, a conflict, a missing package) is rendered by + the plugin's `error_response` when it has one: npm clients print + `{"error": "..."}`, NuGet's plain text. The contract is [`crates/plugin-api/src/lib.rs`](../crates/plugin-api/src/lib.rs). Its queries are targeted: @@ -115,11 +140,13 @@ Its queries are targeted: | Query | Answer | | --- | --- | | `versions(id)` | Every version of a package | -| `packages(id)` | Every version with its metadata: publish time, SHA-256, SHA-512, size, the stored manifest summary | +| `packages(id)` | Every version with its metadata: publish time, SHA-256, SHA-512, size, the stored manifest summary, the deprecation | +| `dated-packages(id)` | `packages`, with every version's publish time even where that costs more: a proxy asks its upstream with `dating` instead of `metadata` (npm's full packument, whose `time` installs with a minimum release age read) | | `packages-of(id, versions)` | Some versions with their metadata (a page of a long listing) | | `package(id, version)` | One version with its metadata | | `manifest(id, version)` | The manifest stored at publish | | `search(text, skip, take, prerelease, semver2, upstream)` | Matching packages with their versions, the latest version's summary and the download total | +| `tags(id)` | The package's tags: a hosted repository's own, a proxy's upstream's, a group's from its first member that has the package and tags for it | A files format asks for files instead ([Files formats](#files-formats)); a query of the other kind fails the request. @@ -134,9 +161,11 @@ with milliseconds. A `take` of 0 is the default page (20). At publish, Forge stores the JSON object the plugin's `manifest_summary` returns with the version. `render` gets it back as `summary`, so a listing never re-reads manifests (for a version stored before summaries were, the -host asks `manifest_summary` of its stored manifest). The web UI's package detail reads two keys from it, -if present: `description`, and `dependency_groups` (the same shape as -NuGet's). +host asks `manifest_summary` of its stored manifest). The web UI's package detail reads three keys from it, +if present: `description`, `deprecated` (a proxy's upstream deprecation), +and `dependency_groups` (the same shape as NuGet's). It shows the latest +version's README, which the plugin's `readme` reads from the stored +manifest, and the package's tags. Every listing hands each version's summary to the plugin, so a summary should stay small. @@ -209,27 +238,32 @@ what they say: | --- | --- | | `index` | A JSON object of its own making, from `remote_url` (NuGet: the service index's resources). The host persists it, refreshes it once per metadata TTL, falls back to the last good one when a refresh fails, and hands it to every other operation | | `versions(id)` | The versions the upstream lists, or none when it does not know the package | -| `package(id, version)` | The package's URL. The host downloads it, checks it with `parse_package` against the version asked for, and stores it | +| `package(id, version)` | The package's URL, and the SHA-512 or SHA-1 the upstream publishes for it when it does (`located`). The host downloads it, checks the digest and, with `parse_package`, the version asked for, and stores it | | `manifest(id, version)` | The manifest, verified to be that version's, without downloading the package | | `metadata(versions)`, `dating(versions)` | Per version: listing state, publish time, a manifest summary for a version not downloaded yet, and (`dating`) when it was created upstream, which the package delay counts from | | `search(query)` | A page of the upstream's search results | | `file(path)` | A files format: the file's URL, or none when the upstream cannot have it. The host downloads it (following redirects; credentials go to `remote_url`'s origin only), caches it, and re-validates it with a conditional GET once the repository's `max_age_seconds` has passed (raw: `/`) | +| `tags(id)` | The tags the upstream gives the package. The host keeps the last ones read, for when the upstream is down | Each operation is a conversation: `upstream` either answers or asks for GETs (`fetch`), and is called again with every fetch so far and how it went (a body, a 404, or a failure), so a plugin keeps no state between calls. A fetch says how long its document may be reused: for the repository's -metadata TTL, or for a day when it never changes once published. One -conversation gets at most 4 rounds and 256 fetches, 4 at a time. +metadata TTL, or for a day when it never changes once published, and may +name the `Accept` it sends (npm's abbreviated and full packuments share a +URL). One conversation gets at most 4 rounds and 256 fetches, 4 at a time. An unavailable upstream is an outage, not a fault of Forge. Unavailable means a fetch or a download got no answer (connection refused, a name that does not resolve, a timeout) or a 5xx or 429. When nothing cached answers in its place, the request answers **503** with `Retry-After`, and the host logs a warning that names the upstream URL. So does a group whose answer waits -on such a member. Any other failure answers 500: an answer the plugin -cannot use, a 401 or 403 for the proxy's credentials, a document over its -size bound. +on such a member. An upstream that refuses the proxy (a 401 or 403 for its +credentials), or serves a package the proxy does not take (too large, +unreadable, not the version asked for, not matching its published digest), +answers **502**: retrying does not help until the upstream or the proxy +changes. Any other failure answers 500: an answer the plugin cannot use, a +document over its size bound. The host sends a fetch only where the upstream points: to the origin (scheme, host and port) of `remote_url`, or to an origin that an absolute `http` or @@ -262,7 +296,7 @@ kinds it supports, what its repositories hold (`storage`: `packages`, `files`, or Docker's `images`) and where its repositories are served. The web UI's *New repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream -side (NuGet and raw do). Docker has proxies of its own. +side (NuGet, npm and raw do). Docker has proxies of its own. A repository of a plugin's format is created like any other, with the plugin's format id, and served under the plugin's mount (`/repository` @@ -285,8 +319,8 @@ read as another format, and retention skips it (a failed run is recorded). The web UI's *Browse* screen filters repositories by format (`GET /browse/repos?format=`), and a repository's screen shows its -client commands: `dotnet nuget` for NuGet, the repository's URL for other -plugin formats. +client commands: `dotnet nuget` for NuGet, the `.npmrc` / `.yarnrc.yml` +lines and logins for npm, the repository's URL for other plugin formats. **Upload limits.** Each hosted repository has its own upload limit, the largest push it accepts (413 above it), whatever its format: set it under @@ -306,9 +340,13 @@ The host enforces these for every plugin: visibility. An id the format does not validate is unknown, not an error. - **No mutations it was not asked for.** The host checks every mutating - action against the method the caller was authorized for, the repository - kind and the credential type, before the hosted write path applies its - own selector and redeploy checks. + action against the request's method, the permission it needs, the + repository kind and the credential type, before the hosted write path + applies its own selector and redeploy checks. +- **No credentials.** The plugin never sees a password or a token: a + login's body goes to the host's login (`auth`), which checks it and mints + the token the plugin renders. A body is read only when the plugin asks, + bounded, and for a caller with the permission its use needs. - **Bounded ids and versions.** Whatever the plugin's own rules, an id longer than 512 bytes or a version longer than 256 bytes, as given or as the plugin normalizes it, is not one: a push of a package carrying one @@ -327,8 +365,8 @@ The host enforces these for every plugin: through the proxy's metadata. - **Responses cannot attack the origin.** Only `allow`, `cache-control`, `content-disposition`, `content-language`, `content-type`, `etag`, - `last-modified`, `link`, `location` and `vary` survive; `location` and - `link` survive only when they stay inside the repository. Every response gets the feed's hardening + `last-modified`, `link`, `location`, `retry-after` and `vary` survive; + `location` and `link` survive only when they stay inside the repository. Every response gets the feed's hardening headers (`Content-Security-Policy: sandbox`, `nosniff`, `Content-Disposition: attachment`). A plugin cannot set cookies, CORS headers, or redirect a client elsewhere. @@ -362,9 +400,10 @@ pub struct MyFormat; impl FormatPlugin for MyFormat { // identity: canonical_id, validate_id, normalize_version, compare_versions - // packages: parse_package, manifest_summary + // packages: parse_package, manifest_summary (optional: readme, reserved_prefix) // (a files format keeps the defaults of these, which refuse everything) - // protocol: handle, render + // protocol: handle, render (optional: error_response, auth_response) + // uploads in a document (optional): unpack_upload, parse_upload, published // proxies (optional): upstream } ``` @@ -380,6 +419,8 @@ mount = "/repository" # optional: served at {mount}/{repo}/… kinds = ["hosted", "group"] # optional: the repository kinds it serves # ("proxy" too when it implements `upstream`) storage = "packages" # optional: or "files" (Files formats) +upstream_token = "bearer" # optional: a proxy's password without a + # username is a token, sent as Bearer [assets] # optional: how a version's files are stored package = "package" @@ -426,24 +467,14 @@ These are the current limits of the preview, roughly in the order we expect to lift them: - **Proxies fetch whole documents with GET.** An upstream request carries - no headers but the repository's Basic credentials, and its body is read - whole up to `proxy.max_metadata_bytes` before the plugin sees it. An - upstream that needs a token exchange (Docker Hub's) or streams a listing - too large to hold needs more from the contract. -- **Requests carry no body into the plugin.** A publish streams its body - straight into the store, and other requests' bodies are not passed on. - npm's login and audit endpoints, for example, need a small body. -- **The uploaded body must be the package.** It can be the whole body or one - multipart part. npm publishes the tarball base64-encoded inside a JSON - document, and Cargo frames it after a JSON length prefix; both need a - host-side extraction step (a byte range, base64 inside a range) that the - contract does not have yet. + no headers but the repository's credentials (Basic, or a Bearer token) and + the `Accept` the fetch names, and its body is read whole up to + `proxy.max_metadata_bytes` before the plugin sees it; only a `forward` + POSTs. An upstream that needs a token exchange (Docker Hub's) or streams a + listing too large to hold needs more from the contract. - **One package file per version.** Formats that publish several files per version (PyPI wheels per platform, Maven's jar, pom and sources) need multi-asset versions. -- **Basic authentication only.** Clients that send API tokens as - `Authorization: Bearer` (npm's `_authToken`, Cargo) need the principal - middleware to accept Bearer API tokens. - **Prerelease is "contains a `-`".** This is right for SemVer formats; a format with another rule (PEP 440's `1.0a1`) needs a contract function for it. @@ -470,6 +501,33 @@ index's resources, the flat container, the registration (bounded to the newest pages a request needs), catalog entries for the package delay of unlisted versions, and the search service. +### npm as a plugin + +npm is a built-in plugin too: [`crates/formats/npm`](../crates/formats/npm) serves +registries at `/npm/{repo}/...` for npm, pnpm, Yarn and Bun ([npm.md](npm.md)). +Contract 0.4.0 grew for it too, and every piece is general: + +- **Bodies on demand** (`read body`): logins, audits, dist-tag changes and + packument updates are small JSON bodies the plugin reads. +- **Packages inside a document** (`Upload::Document`): `npm publish` sends + the tarball base64-encoded inside a JSON document. The plugin's + `unpack_upload` decodes it while the body streams into the store, and the + rest of the document (its `dist-tags`, its integrity claims) comes back + to `parse_upload` and `published`. +- **Tags and deprecations** (`change`, `tags`): stored by the host for any + format, kept by retention, shown by the web UI. +- **Logins** (`auth`): a password login through the realm chain, and a + login approved in the browser (`/cli-login/`), each minting an API + token; a logout revokes it, and `whoami` names the caller. +- **Forwards**: an audit goes to the upstream with the proxy's credentials, + without a group's reserved names. +- **Proxies**: tags from the upstream, packages located with their digests, + documents fetched with an `Accept` of their own, and a Bearer token for a + registry that wants one. +- **Groups**: a writable member, where publishes and changes through the + group go, and reserved prefixes in the format's own words (`@acme` is the + scope `@acme/`, `reserved_prefix`). + ### Why Docker is not a plugin The contract models a repository as package versions, each one file with an diff --git a/docs/npm.md b/docs/npm.md new file mode 100644 index 00000000..632b40fd --- /dev/null +++ b/docs/npm.md @@ -0,0 +1,303 @@ +# npm registries + +Forge serves npm packages to npm, pnpm, Yarn (1 and 2+) and Bun: publish, +install, dist-tags, deprecations, unpublishing, search, `npm audit` and the +logins. Repositories of `format: "npm"` are served at +`https:///npm//`, in the usual three kinds: + +- **hosted**: publish and install your own packages; +- **proxy**: a cache of another registry, `https://registry.npmjs.org` or a + private one; +- **group**: one registry URL over several members, which keeps your scopes + internal and passes `npm audit` on to the public registry. + +npm is a format plugin built into Forge ([`crates/formats/npm`](../crates/formats/npm), +[format-plugins.md](format-plugins.md)): tarballs are blobs in the +content-addressed store, and repositories, permissions, package selectors, +retention and the web UI are the same as for every other format. + +## Contents + +- [Repositories](#repositories) +- [Connecting a client](#connecting-a-client) +- [Logging in](#logging-in) +- [Publishing](#publishing) +- [Dist-tags, deprecations and unpublishing](#dist-tags-deprecations-and-unpublishing) +- [Installing and searching](#installing-and-searching) +- [Proxy repositories](#proxy-repositories) +- [Group repositories](#group-repositories) +- [Audits](#audits) +- [Permissions](#permissions) +- [Retention](#retention) +- [Limits](#limits) +- [Limitations](#limitations) + +## Repositories + +Create them in the web UI (**Admin → Repositories → New repository**, format +*npm*) or with the admin API: + +```bash +api() { curl -fsS -u admin@example.com: -H 'content-type: application/json' "$@"; } +api https://forge.example.com/admin/repositories \ + -d '{"name": "npm-hosted", "kind": "hosted", "format": "npm"}' +api https://forge.example.com/admin/repositories \ + -d '{"name": "npm-proxy", "kind": "proxy", "format": "npm", "remote_url": "https://registry.npmjs.org"}' +api https://forge.example.com/admin/repositories \ + -d '{"name": "npm-all", "kind": "group", "format": "npm", + "reserved_prefixes": ["@acme"], "writable_member": "npm-hosted"}' +api https://forge.example.com/admin/repositories/npm-all/members -d '{"member": "npm-hosted", "priority": 10}' +api https://forge.example.com/admin/repositories/npm-all/members -d '{"member": "npm-proxy", "priority": 5}' +``` + +Most teams point their clients at the group: it serves the internal packages +and the public ones, takes publishes (into its writable member) and audits. + +## Connecting a client + +A repository's page in the web UI shows its registry URL and these lines, +ready to copy. + +**npm, pnpm, Bun and Yarn 1** read `.npmrc` (the project's, or `~/.npmrc`): + +```ini +registry=https://forge.example.com/npm/npm-all/ +//forge.example.com/npm/npm-all/:_authToken= +``` + +Only one scope from Forge, the rest from elsewhere: + +```ini +@acme:registry=https://forge.example.com/npm/npm-all/ +//forge.example.com/npm/npm-all/:_authToken= +``` + +**Yarn 2 and later** read `.yarnrc.yml`: + +```yaml +npmRegistryServer: "https://forge.example.com/npm/npm-all/" +npmAlwaysAuth: true +npmAuthToken: "" +``` + +The token is an API token: one `npm login` mints (below), or one you create +under **Your tokens** in the web UI. Any API token of yours works for every +repository you may read; Forge sees whose it is, not which repository it +was minted for. A repository that allows anonymous reads needs no token to +install from. + +## Logging in + +**`npm login`** (npm 9 and later, and `pnpm login`) opens a page of Forge in +the browser. Sign in there as always (a password or your organization's +OIDC provider), check the client, repository and host name it shows, and +approve; the client then stores a new API token in `.npmrc`: + +```bash +npm login --registry https://forge.example.com/npm/npm-all/ +npm whoami --registry https://forge.example.com/npm/npm-all/ +``` + +Without a browser on the machine (`--browser=false`), npm prints the page's +URL to open elsewhere. A login waits for its approval for +`cli_login.web_login_ttl_seconds` (10 minutes), and the page answers only +the signed-in user who opens it. Logins in the browser need the web UI +(`[web] enabled`); without it npm falls back to asking for a password. + +**A username and a password**: `yarn npm login` (Yarn 2+) and `npm login +--auth-type=legacy` send them to Forge, which checks them like the web +login does (local passwords, credential forwarding, the lockout after +failed attempts) and mints a token. No account is ever created by a login. +npm's legacy login refuses a username containing an `@`, which every Forge +account has (an e-mail address): use the browser login, Yarn's, or a token +from the web UI. + +A token a login mints carries your rights, expires after +`cli_login.token_lifetime_days` (90 days; `0`: never), and is listed and +revocable under **Your tokens** like every other. **`npm logout`** revokes +the token it logged in with. Logins, the tokens they mint and logouts are +audit-logged (`forge::audit`). + +```toml +[cli_login] +token_lifetime_days = 90 +web_login_ttl_seconds = 600 +``` + +## Publishing + +```bash +npm publish --registry https://forge.example.com/npm/npm-hosted/ +pnpm publish --registry https://forge.example.com/npm/npm-hosted/ +yarn npm publish # with npmRegistryServer / npmPublishRegistry set +bun publish --registry https://forge.example.com/npm/npm-hosted/ +``` + +Publishing needs the **write** permission and an API token (a password typed +at the web login is not a publishing credential); CI jobs can use a +[trusted publishing](trusted-publishing.md) token instead. Publishing to a +group goes to its writable member. + +Forge checks what it stores: + +- the tarball's `package/package.json` names the package and version the + document publishes, and the integrity (`sha512`) and `shasum` the client + claims match the tarball; +- the name is a valid npm name (scoped or not; at most 214 characters, + lowercase for new names) and the version valid SemVer; +- the tarball is a gzip'd tar that unpacks to at most `npm.max_unpacked_bytes`; +- the tarball fits the repository's upload limit (*Maximum upload size*, + 512 MiB by default), and the rest of the document `npm.max_document_bytes`. + +A version that exists is refused (409), as on the public registry, unless +the repository's **deployment policy** allows redeploys (which needs the +delete permission too). The policy *read-only* refuses every publish, +unpublish, deprecation and dist-tag change: + +```bash +api -X PUT https://forge.example.com/admin/repositories/npm-hosted \ + -d '{"deployment_policy": "read_only"}' +``` + +The dist-tags of the publish (`npm publish --tag next`; `latest` by default) +are set with it. Package selectors apply as for every format: a caller +whose selector covers the `@acme` scope only cannot publish `left-pad`. + +## Dist-tags, deprecations and unpublishing + +```bash +npm dist-tag add @acme/ui@2.0.0-rc.1 next +npm dist-tag rm @acme/ui next +npm dist-tag ls @acme/ui +npm deprecate @acme/ui@1.0.0 "use 2.x" +npm deprecate @acme/ui@1.0.0 "" # undeprecate +npm unpublish @acme/ui@1.0.0 +npm unpublish @acme/ui --force # every version +``` + +- A **dist-tag** names one version. `latest` is always there: it cannot be + removed, and when the version it named is gone it moves to the highest + version left. Deleting a version drops its tags. +- A **deprecated** version still installs; clients warn with its message. +- **Unpublishing** deletes the version (or every version) and needs the + **delete** permission. Unlike on the public registry there is no time + limit, and a version unpublished may be published again. + +Dist-tags and deprecations need the write permission. The web UI shows a +package's dist-tags, deprecated versions with their messages, and the +README of its `latest` version. + +## Installing and searching + +Every client installs from a hosted repository, a proxy or a group: +abbreviated packuments (`application/vnd.npm.install-v1+json`) for clients +that ask for them, full ones (with each version's publish time and the +README of `latest`) for the others, tarballs, `npm view`, `npm search` +(`GET /-/v1/search`, over the repository's packages; a proxy searches its +upstream too) and `npm ping`. Tarballs are served from the store with +counted as downloads. Tarball URLs in a packument point at the +repository it was read from, so a lockfile written through the group keeps +resolving through it. + +## Proxy repositories + +A proxy reads another registry on demand, `https://registry.npmjs.org` or +any registry that speaks npm's protocol: + +- **Credentials.** With a username and a password, it sends Basic + authentication; with a password alone (a registry token, such as an npm + automation token or a GitHub Packages token), `Authorization: Bearer`. + They go to the origin of the remote URL only, never to a tarball host + elsewhere. +- **Caching.** A packument is read again after the repository's metadata + TTL; installs read the abbreviated one, `npm view` and clients that want + publish times (pnpm's `minimumReleaseAge`, `npm install --before`) the + full one. A package the upstream does not know is remembered for the + not-found TTL. A tarball is downloaded once, checked against the + `integrity` (or `shasum`) the upstream publishes and against its version, + and kept; one that does not hold up is refused (502) and not stored. +- **The upstream's dist-tags and deprecations** are served as it gives + them, and the last dist-tags read are kept for when it is down. +- **Publication delay.** `package_delay_seconds` holds back versions + published upstream less than that long ago (counted from the upstream's + publish time, not from when Forge first saw them): a new release, possibly + a compromised one, reaches your builds only after the delay. +- **Outages.** While the upstream is down, what is cached is served: the + packument as last read, and every tarball downloaded before. A tarball + never downloaded answers 503 with `Retry-After`. An upstream that refuses + the proxy's credentials answers 502. +- **Invalidate cache** on the repository's page (or + `POST /admin/repositories//proxy-cache/invalidate`) makes it read + the upstream again now. + +Publishing, dist-tag changes and unpublishing are refused on a proxy. + +## Group repositories + +A group serves its members, by priority: + +- a packument merges the versions of every member that has the package (the + highest-priority member's copy of a version wins), with the **dist-tags** + of the first member, by priority, that has the package and tags for it; +- **reserved prefixes** (`@acme`, the scope `@acme/`, or a name prefix such + as `acme-`) are never served from proxy members, also before the first + internal version is published, so a public package that claims your + scope or name cannot slip into a build (dependency confusion); +- **prefer hosted** (on by default) hides proxy versions of a package a + hosted member has; +- the **writable member**, a hosted npm repository that is also a member, + takes publishes, dist-tag changes, deprecations and unpublishing through + the group; without one they are refused. + +## Audits + +`npm audit` (and `pnpm audit`) asks the registry for advisories. A proxy +passes the request on to its upstream with its credentials, and a group to +its first proxy member; a group leaves the packages under its reserved +prefixes out, so your internal package names stay inside. A hosted +repository has no upstream to ask (404). Auditing needs the read +permission. + +## Permissions + +| What | Permission | +| --- | --- | +| Install, `npm view`, search, audit | read | +| Publish, `npm dist-tag add` / `rm`, `npm deprecate` | write | +| Unpublish (a version or the whole package); a redeploy | delete | +| `npm login`, `npm logout`, `npm whoami` | none (a login checks the password) | + +An unknown repository answers like one the caller may not read. A +read-only deployment policy refuses every change, whatever the caller's +permissions. + +## Retention + +Retention policies apply to npm repositories like to every other format, +with one rule more: **a version a dist-tag names is never deleted** (a +proxy's tags are its upstream's, as last read), so `latest`, `next` and +every other tag keep installing. The preview of a policy lists such +versions as kept. + +## Limits + +```toml +[npm] +max_document_bytes = 33554432 # 32 MiB: a publish document without its tarball, an update, an audit +max_unpacked_bytes = 1073741824 # 1 GiB: the most a tarball may unpack to +``` + +A tarball's size is bounded by its repository's upload limit. A login's +body is at most 64 KiB. + +## Limitations + +- **No provenance.** A publish with a provenance statement (`npm publish + --provenance`) attaches a second file, which Forge refuses: publish + without it. +- **No package access management.** `npm access`, `npm owner`, `npm team`, + `npm org`, `npm star` and `npm token` are not served; access is Forge's + roles and package selectors, tokens are managed in the web UI. +- **npm's legacy login and `@` usernames.** See [Logging in](#logging-in). +- **One registry per URL.** Clients address a repository by its URL; a + scope maps to one registry, as in npm. diff --git a/forge.toml.example b/forge.toml.example index b00e0ce9..e936b9d5 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -159,6 +159,18 @@ max_total_uncompressed_bytes = 17179869184 # (default: 16 GiB) total declared s max_nuspec_bytes = 1048576 # (default: 1 MiB) also caps an upstream .nuspec a proxy verifies max_nuspec_compression_ratio = 200 # (default: 200) +[npm] +# What the npm registries (/npm, see docs/npm.md) read of a request. A +# tarball itself is bounded by its repository's upload limit. +max_document_bytes = 33554432 # (default: 32 MiB) a publish document without its tarball, an unpublish, an audit +max_unpacked_bytes = 1073741824 # (default: 1 GiB) the most a published or proxied tarball may unpack to + +[cli_login] +# Logins of command-line clients (`npm login`, `yarn npm login`), with a +# password or approved in the browser (which needs the web UI). +token_lifetime_days = 90 # (default: 90; 0: never expires) the API token a login mints +web_login_ttl_seconds = 600 # (default: 600; 60-3600) how long a login approved in the browser waits + [docker] # The Docker / OCI registry under /v2 (see docs/docker.md). # max_blob_bytes = 1073741824 # (default: 1 GiB, or what an explicit server.max_staging_bytes holds) largest layer a proxy downloads; staged on disk (pushes follow the repository's upload limit) diff --git a/migrations/0046_package_tags_cli_logins.sql b/migrations/0046_package_tags_cli_logins.sql new file mode 100644 index 00000000..ec60dbed --- /dev/null +++ b/migrations/0046_package_tags_cli_logins.sql @@ -0,0 +1,50 @@ +-- --------------------------------------------------------------------------- +-- 0046: package tags, deprecations and command-line logins (#41, npm) +-- --------------------------------------------------------------------------- +-- +-- What a format plugin may keep about a package beyond its versions: the +-- tags that name versions (npm's dist-tags, `latest`, `next`), and whether +-- a version is deprecated. And the logins a command-line client asks the +-- user to approve in the browser (`npm login`). + +-- A repository's tags of its packages. A hosted repository's are set by its +-- clients; a proxy's are its upstream's, as last read (served while the +-- upstream is unreachable). Retention never deletes a version a tag of a +-- hosted repository names. +CREATE TABLE package_tag ( + repository_id bigint NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + package text NOT NULL, + tag text NOT NULL CHECK (tag <> ''), + version text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (repository_id, package, tag) +); + +CREATE INDEX package_tag_version_idx ON package_tag (repository_id, package, version); + +-- Why a version should not be used any more (`npm deprecate`); NULL when it +-- is not deprecated. +ALTER TABLE component ADD COLUMN deprecation text; + +-- A command-line client's login, approved in the browser: the client is +-- given a login URL to open and a done URL to poll. Both carry a secret; +-- only their SHA-256 is stored. Signing in approves the login (`user_id`), +-- and the next poll of the done URL mints the client's token and deletes +-- the row. +CREATE TABLE cli_login ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + login_hash text NOT NULL UNIQUE, + done_hash text NOT NULL UNIQUE, + -- The client (the repository's format: `npm`) and where it signs in. + client text NOT NULL, + repository_id bigint NOT NULL REFERENCES repository(id) ON DELETE CASCADE, + -- What the client said about itself, shown on the approval page. + hostname text, + client_ip text, + user_id bigint REFERENCES app_user(id) ON DELETE CASCADE, + approved_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL +); + +CREATE INDEX cli_login_expires_idx ON cli_login (expires_at); diff --git a/web/src/api/cliLogin.ts b/web/src/api/cliLogin.ts new file mode 100644 index 00000000..f747d2e8 --- /dev/null +++ b/web/src/api/cliLogin.ts @@ -0,0 +1,53 @@ +import { queryOptions } from '@tanstack/react-query' + +import { ApiError, httpClient } from '~/lib/httpClient' + +/** + * A command-line client's pending web login (`npm login`, which opens + * `/cli-login/` in the browser): what the signed-in user approves or + * refuses. The secret in the URL is the request's only handle. + */ +export interface CliLoginRequest { + /** The client that asked: `npm`. */ + client: string + /** The repository it signs in to. */ + repository: string + /** The host name the client reported, if any. */ + hostname: string | null + /** Where the request came from. */ + client_ip: string | null + created_at: string + expires_at: string +} + +export const cliLoginKeys = { + request: (secret: string) => ['cli-login', secret] as const, +} + +function path(secret: string): string { + return `/auth/cli-login/${encodeURIComponent(secret)}` +} + +/** `GET /auth/cli-login/{secret}`; 404 once it is decided, expired or unknown. */ +export function fetchCliLogin(secret: string, signal?: AbortSignal): Promise { + return httpClient.get(path(secret), { signal }) +} + +export function cliLoginQueryOptions(secret: string) { + return queryOptions({ + queryKey: cliLoginKeys.request(secret), + queryFn: ({ signal }) => fetchCliLogin(secret, signal), + // A decided request is gone: never refetch it behind the user's back. + staleTime: Infinity, + refetchOnWindowFocus: false, + retry: false, + }) +} + +/** + * `POST /auth/cli-login/{secret}` — sign the client in as the current user + * (`approve`), or refuse it. The client collects its token on its next poll. + */ +export function decideCliLogin(secret: string, approve: boolean): Promise { + return httpClient.post(path(secret), { approve }) +} diff --git a/web/src/api/packages.ts b/web/src/api/packages.ts index bf816b09..67949fd4 100644 --- a/web/src/api/packages.ts +++ b/web/src/api/packages.ts @@ -116,7 +116,11 @@ export interface DetailDependency { range: string | null } -/** A dependency group (per target framework) for one version. */ +/** + * A dependency group for one version: per target framework (NuGet), or per + * dependency kind (npm: `dependencies`, `optionalDependencies`, + * `peerDependencies`). + */ export interface DetailDependencyGroup { target_framework: string | null dependencies: DetailDependency[] @@ -131,6 +135,14 @@ export interface PackageDetail { versions: DetailVersion[] /** Dependency groups keyed by normalized version string. */ dependency_groups_by_version: Record + /** The package's tags (npm's dist-tags), tag → version; absent when none. */ + tags?: Record + /** Version → its deprecation message; absent when none is deprecated. */ + deprecated?: Record + /** The latest version's description, when its manifest has one. */ + description?: string + /** The latest version's README (Markdown), when its format reads one. */ + readme?: string } /** Search parameters for the browse list (mirrors `BrowseQuery`). */ diff --git a/web/src/api/repositories.ts b/web/src/api/repositories.ts index 4d4c1cd7..55d89caf 100644 --- a/web/src/api/repositories.ts +++ b/web/src/api/repositories.ts @@ -20,8 +20,10 @@ import { httpClient } from '~/lib/httpClient' * `writable_member` — which the server refuses on a NuGet repository (and * `package_delay_seconds` / `allow_redeploy` on a docker one). Raw * repositories (`format: "raw"`, served at `/raw/`) likewise — - * `deployment_policy`, `content_disposition`, `max_age_seconds`. Both - * bodies reject unknown fields. + * `deployment_policy`, `content_disposition`, `max_age_seconds`. npm + * repositories (`format: "npm"`, served at `/npm/`) take a hosted + * repository's `deployment_policy` (not `allow_redeploy`) and a group's + * `writable_member`. Both bodies reject unknown fields. * - `MemberView` / `AddMember` / `UpdateMember` * (`crates/server/src/admin/members.rs`) drive the group editor; `priority` * is unique within a group (a duplicate → 409). A member may itself be a diff --git a/web/src/components/DependencyGroups.tsx b/web/src/components/DependencyGroups.tsx index d13aec02..af02f729 100644 --- a/web/src/components/DependencyGroups.tsx +++ b/web/src/components/DependencyGroups.tsx @@ -22,17 +22,23 @@ export function DependencyGroups({ repo, byVersion, caption, + npm = false, }: { repo: string /** Dependency groups keyed by version, as the detail API returns them. */ byVersion: Record caption?: string + /** + * An npm package's: the groups are dependency kinds, and package names are + * exact (NuGet ids are linked in lower case). + */ + npm?: boolean }) { return ( Version - Target framework + {npm ? 'Kind' : 'Target framework'} Package Version range @@ -71,7 +77,7 @@ export function DependencyGroups({ {dep.id} diff --git a/web/src/components/PackageConnect.test.ts b/web/src/components/PackageConnect.test.ts index 312f236e..5f41159e 100644 --- a/web/src/components/PackageConnect.test.ts +++ b/web/src/components/PackageConnect.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { connectSnippets } from './PackageConnect' +import { connectSnippets, npmrc, yarnrc } from './PackageConnect' describe('connectSnippets', () => { it('gives a format without commands of its own its URL', () => { @@ -21,4 +21,28 @@ describe('connectSnippets', () => { expect(labels(true)).toEqual(['Add the source', 'Push a package']) expect(labels(false)).toEqual(['Add the source']) }) + + it('points npm at the registry URL with its trailing slash', () => { + const snippets = connectSnippets('npm', 'npm-all', 'https://f.example/npm/npm-all', false) + expect(snippets.map((s) => s.command)).toEqual([ + 'npm config set registry https://f.example/npm/npm-all/', + 'npm config set @:registry https://f.example/npm/npm-all/', + 'npm login --registry https://f.example/npm/npm-all/', + ]) + expect( + connectSnippets('npm', 'npm-hosted', 'https://f.example/npm/npm-hosted', true).at(-1) + ?.command, + ).toBe('npm publish --registry https://f.example/npm/npm-hosted/') + }) +}) + +describe('npmrc', () => { + it('scopes the token to the registry, without the scheme', () => { + expect(npmrc('https://f.example/npm/npm-all')).toBe( + 'registry=https://f.example/npm/npm-all/\n//f.example/npm/npm-all/:_authToken=', + ) + expect(yarnrc('http://localhost:8080/npm/x')).toContain( + 'npmRegistryServer: "http://localhost:8080/npm/x/"', + ) + }) }) diff --git a/web/src/components/PackageConnect.tsx b/web/src/components/PackageConnect.tsx index 9dd15787..0755bc9e 100644 --- a/web/src/components/PackageConnect.tsx +++ b/web/src/components/PackageConnect.tsx @@ -1,6 +1,6 @@ import { Link } from '@tanstack/react-router' -import { CommandSnippet, CopyField, Panel } from '~/components/ui' +import { CodeBlock, CommandSnippet, CopyField, Panel } from '~/components/ui' /** One copyable command of a format's connect panel. */ interface Snippet { @@ -39,6 +39,33 @@ export function connectSnippets( ] : []), ] + case 'npm': + return [ + { + label: 'Use the registry', + command: `npm config set registry ${url}/`, + copyLabel: 'Copy set-registry command', + }, + { + label: 'Or only for a scope', + command: `npm config set @:registry ${url}/`, + copyLabel: 'Copy scope-registry command', + }, + { + label: 'Sign in', + command: `npm login --registry ${url}/`, + copyLabel: 'Copy login command', + }, + ...(canPush + ? [ + { + label: 'Publish a package', + command: `npm publish --registry ${url}/`, + copyLabel: 'Copy publish command', + }, + ] + : []), + ] default: return [ { @@ -51,11 +78,26 @@ export function connectSnippets( } /** - * "Connect" panel of a package repository (NuGet, or another format - * plugin's): the address to copy, the client commands for it, and where the - * password comes from — clients authenticate with an API token (from the - * Your tokens screen), not the UI sign-in. Docker repositories have - * `DockerConnect`. + * The `.npmrc` lines that point npm, pnpm, Bun and Yarn 1 at the npm + * registry at `url`, with an API token: `npm login` writes the same + * `_authToken` line. + */ +export function npmrc(url: string): string { + const path = url.replace(/^[a-z]+:/i, '') + return `registry=${url}/\n${path}/:_authToken=` +} + +/** The `.yarnrc.yml` settings of Yarn 2 and later for the registry at `url`. */ +export function yarnrc(url: string): string { + return `npmRegistryServer: "${url}/"\nnpmAlwaysAuth: true\nnpmAuthToken: ""` +} + +/** + * "Connect" panel of a package repository (NuGet, npm, or another format + * plugin's): the address to copy, the client commands for it, and + * where the password comes from — clients authenticate with an API token + * (from the Your tokens screen, or for npm one `npm login` mints), not the + * UI sign-in. Docker repositories have `DockerConnect`. */ export function PackageConnect({ format, @@ -69,18 +111,23 @@ export function PackageConnect({ canPush?: boolean }) { const snippets = connectSnippets(format, repo, url, canPush) + const tokens = ( + + Your tokens + + ) return ( - Use your username and an API token as the password; create one - under{' '} - - Your tokens - - . - + format === 'npm' ? ( + <> + Sign in with npm login, or put an API token in{' '} + .npmrc; create one under {tokens}. + + ) : ( + <>Use your username and an API token as the password; create one under {tokens}. + ) } > {format === 'nuget' && ( @@ -90,9 +137,28 @@ export function PackageConnect({ copyLabel="Copy service index" /> )} + {format === 'npm' && ( + + )} {snippets.map((snippet) => ( ))} + {format === 'npm' && ( + <> + ']} + /> + ']} + /> + + )} ) } diff --git a/web/src/components/RepositoryForm.test.tsx b/web/src/components/RepositoryForm.test.tsx index f0afded7..e6acee49 100644 --- a/web/src/components/RepositoryForm.test.tsx +++ b/web/src/components/RepositoryForm.test.tsx @@ -5,6 +5,7 @@ import userEvent from '@testing-library/user-event' import type { FormatView, RepoView } from '~/api/repositories' import { DOCKER_FORMAT, + NPM_FORMAT, NUGET_FORMAT, RAW_FORMAT, RepositoryForm, @@ -20,7 +21,7 @@ import { /** * RepositoryForm: * - a create-hosted submission produces the right `CreateRepo` body (kind + - * hosted-only `allow_redeploy`, no proxy fields); + * hosted-only `deployment_policy`, no proxy fields); * - the proxy upstream password is **write-only**: never seeded from an * existing repo's GET, and only sent when typed; * - validation blocks an empty name and a proxy without a remote URL; @@ -44,7 +45,7 @@ function lastValues(onSubmit: ReturnType): RepoFormValues { } describe('RepositoryForm', () => { - it('builds a hosted CreateRepo body with allow_redeploy and no proxy fields', async () => { + it('builds a hosted CreateRepo body with a deployment policy and no proxy fields', async () => { const user = userEvent.setup() const onSubmit = vi.fn() render( @@ -57,10 +58,8 @@ describe('RepositoryForm', () => { ) await user.type(screen.getByLabelText('Name'), 'my-hosted') - // kind defaults to hosted; toggle allow_redeploy on. - await user.click( - screen.getByLabelText(/Allow redeploy/), - ) + // kind defaults to hosted; let versions be pushed again. + await user.selectOptions(screen.getByLabelText('Deployment policy'), 'allow_redeploy') await user.click(screen.getByRole('button', { name: 'Create repository' })) expect(onSubmit).toHaveBeenCalledTimes(1) @@ -68,8 +67,9 @@ describe('RepositoryForm', () => { expect(body).toMatchObject({ name: 'my-hosted', kind: 'hosted', - allow_redeploy: true, + deployment_policy: 'allow_redeploy', }) + expect(body).not.toHaveProperty('allow_redeploy') expect(body).not.toHaveProperty('remote_url') expect(body).not.toHaveProperty('upstream_password') // NuGet is the server's default format: not sent. @@ -784,7 +784,7 @@ describe('docker repositories: update body', () => { const values = { ...lastValuesOf(dockerProxy), package_delay_seconds: '3600', - allow_redeploy: true, + deployment_policy: 'allow_redeploy' as const, } expect(toUpdateBody(values, dockerProxy)).toEqual({}) @@ -1053,7 +1053,115 @@ describe('raw repositories', () => { } const values = lastValuesOf(hosted) expect( - toUpdateBody({ ...values, deployment_policy: 'read_only', allow_redeploy: true }, hosted), + toUpdateBody({ ...values, deployment_policy: 'read_only' }, hosted), ).toEqual({ deployment_policy: 'read_only' }) }) }) + +describe('npm repositories', () => { + function renderNpmCreate() { + const onSubmit = vi.fn() + render( + {}} + />, + ) + return onSubmit + } + + it('hosted: a deployment policy instead of allow_redeploy', async () => { + const user = userEvent.setup() + const onSubmit = renderNpmCreate() + + await user.selectOptions(screen.getByLabelText('Format'), 'npm') + expect(screen.queryByLabelText(/Allow redeploy \(re-push/)).toBeNull() + await user.type(screen.getByLabelText('Name'), 'npm-hosted') + await user.selectOptions(screen.getByLabelText('Deployment policy'), 'read_only') + await create(user) + + expect(toCreateBody(lastValues(onSubmit))).toEqual({ + name: 'npm-hosted', + kind: 'hosted', + format: 'npm', + anonymous_read: false, + online: true, + deployment_policy: 'read_only', + }) + }) + + it('proxy: the public registry as placeholder, with a package delay', async () => { + const user = userEvent.setup() + const onSubmit = renderNpmCreate() + + await user.selectOptions(screen.getByLabelText('Format'), 'npm') + await user.selectOptions(screen.getByLabelText('Kind'), 'proxy') + expect(screen.getByLabelText('Remote URL')).toHaveAttribute( + 'placeholder', + 'https://registry.npmjs.org', + ) + await user.type(screen.getByLabelText('Name'), 'npmjs') + await user.type(screen.getByLabelText('Remote URL'), 'https://registry.npmjs.org') + await user.clear(screen.getByLabelText('Package delay (seconds)')) + await user.type(screen.getByLabelText('Package delay (seconds)'), '86400') + await create(user) + + expect(toCreateBody(lastValues(onSubmit))).toMatchObject({ + kind: 'proxy', + format: 'npm', + remote_url: 'https://registry.npmjs.org', + package_delay_seconds: 86400, + }) + }) + + it('group: reserved scopes and the writable member publishes go to', async () => { + const user = userEvent.setup() + const onSubmit = renderNpmCreate() + + await user.selectOptions(screen.getByLabelText('Format'), 'npm') + await user.selectOptions(screen.getByLabelText('Kind'), 'group') + await user.type(screen.getByLabelText('Name'), 'npm-all') + await user.type(screen.getByLabelText('Reserved prefixes'), '@acme, @acme-internal') + await user.type(screen.getByLabelText('Writable member'), 'npm-hosted') + await create(user) + + expect(toCreateBody(lastValues(onSubmit))).toMatchObject({ + kind: 'group', + format: 'npm', + prefer_hosted: true, + reserved_prefixes: ['@acme', '@acme-internal'], + writable_member: 'npm-hosted', + }) + }) + + it('on edit, diffs the deployment policy and the writable member', () => { + const hosted: RepoView = { + ...storedProxy, + name: 'npm-hosted', + format: 'npm', + kind: 'hosted', + remote_url: null, + deployment_policy: 'disable_redeploy', + } + const values = lastValuesOf(hosted) + expect(toUpdateBody(values, hosted)).toEqual({}) + expect( + toUpdateBody({ ...values, deployment_policy: 'allow_redeploy' }, hosted), + ).toEqual({ deployment_policy: 'allow_redeploy' }) + + const group: RepoView = { + ...storedProxy, + name: 'npm-all', + format: 'npm', + kind: 'group', + remote_url: null, + writable_member: 'npm-hosted', + } + expect(toUpdateBody({ ...lastValuesOf(group), writable_member: '' }, group)).toEqual({ + writable_member: null, + }) + }) +}) diff --git a/web/src/components/RepositoryForm.tsx b/web/src/components/RepositoryForm.tsx index caa45be7..179bc922 100644 --- a/web/src/components/RepositoryForm.tsx +++ b/web/src/components/RepositoryForm.tsx @@ -23,8 +23,8 @@ import { /** * Shared repository create/edit form. Kind-conditional fields: * - * - **hosted** → `allow_redeploy` (NuGet) / `immutable_tags` + - * `latest_mutable` (docker), and the upload limit + * - **hosted** → the deployment policy (raw, NuGet, npm) / `immutable_tags` + * + `latest_mutable` (docker), and the upload limit * (`max_upload_bytes`, entered in MiB; empty = the format's * default) * - **proxy** → `remote_url`, `package_delay_seconds` (not docker), @@ -32,15 +32,15 @@ import { * (the password is never populated from a GET; the form * offers a "set new password" affordance and only sends it * when the admin types one) - * - **group** → `prefer_hosted`, `reserved_prefixes`, and for docker the - * optional `writable_member`; members are managed on a + * - **group** → `prefer_hosted`, `reserved_prefixes`, and the optional + * `writable_member` (not raw); members are managed on a * separate screen (`GroupMembers`) * - **docker** (any kind) → the optional `host` the repository also answers * on at the root - * - **raw** → instead of the NuGet options: the deployment policy (hosted), - * whether files display in the browser (any kind), and the - * maximum age of a proxy's cached files; a raw group has no - * dependency-confusion options + * - **raw** → instead of the package options: whether files display in + * the browser (any kind), and the maximum age of a proxy's + * cached files; a raw group has no dependency-confusion + * options and no writable member * * On **create** `format` and `kind` are selectable (the kinds the format * supports: format plugins have no proxies); on **edit** both are fixed @@ -160,6 +160,18 @@ export const RAW_FORMAT: FormatView = { mount: '/raw', } +/** + * npm registries, built in: packages for npm, pnpm, Yarn and Bun, served + * under `/npm`. Hosted ones have a deployment policy, groups a writable + * member. + */ +export const NPM_FORMAT: FormatView = { + id: 'npm', + name: 'npm', + kinds: ['hosted', 'proxy', 'group'], + mount: '/npm', +} + /** Default maximum age of a raw proxy's cached files: a day. */ export const DEFAULT_RAW_MAX_AGE_SECONDS = 86400 @@ -178,7 +190,6 @@ export interface RepoFormValues { kind: RepoKind anonymous_read: boolean online: boolean - allow_redeploy: boolean prefer_hosted: boolean /** Comma- or newline-separated id prefixes (group). */ reserved_prefixes: string @@ -197,11 +208,11 @@ export interface RepoFormValues { immutable_tags: boolean /** Hosted docker with immutable tags: `latest` may still be moved. */ latest_mutable: boolean - /** Docker group: hosted member pushes go to; blank = pushes refused. */ + /** Docker or package group: hosted member pushes go to; blank = pushes refused. */ writable_member: string /** Hosted: the upload limit in MiB; blank = the format's default. */ max_upload_mib: string - /** Hosted raw: what an upload to an existing path does. */ + /** Hosted raw or package: what an upload to an existing path or version does. */ deployment_policy: DeploymentPolicy /** Raw: files display in the browser instead of downloading. */ inline_content: boolean @@ -236,7 +247,6 @@ function seed(existing?: RepoView): RepoFormValues { kind: (existing?.kind as RepoKind) ?? 'hosted', anonymous_read: existing?.anonymous_read ?? false, online: existing?.online ?? true, - allow_redeploy: existing?.allow_redeploy ?? false, prefer_hosted: existing?.prefer_hosted ?? true, reserved_prefixes: (existing?.reserved_prefixes ?? []).join(', '), remote_url: existing?.remote_url ?? '', @@ -259,7 +269,9 @@ function seed(existing?: RepoView): RepoFormValues { writable_member: existing?.writable_member ?? '', max_upload_mib: existing?.max_upload_bytes != null ? bytesToMib(existing.max_upload_bytes) : '', - deployment_policy: existing?.deployment_policy ?? 'disable_redeploy', + deployment_policy: + existing?.deployment_policy ?? + (existing?.allow_redeploy ? 'allow_redeploy' : 'disable_redeploy'), inline_content: existing?.content_disposition === 'inline', max_age_seconds: existing?.max_age_seconds != null && @@ -301,6 +313,7 @@ function formatHelp(format: FormatView): string { if (format.id === RAW_FORMAT.id) { return 'Files under arbitrary paths (binaries, installers, fixtures), served under /raw.' } + if (format.id === NPM_FORMAT.id) return 'npm packages, for npm, pnpm, Yarn and Bun.' return 'A format plugin.' } @@ -406,7 +419,7 @@ export function RepositoryForm({ // load, or a plugin is not installed any more): the built-in formats are // known; any other one must have been a plugin format. const format: FormatView = available.find((f) => f.id === values.format) ?? - [NUGET_FORMAT, DOCKER_FORMAT, RAW_FORMAT].find((f) => f.id === values.format) ?? { + [NUGET_FORMAT, DOCKER_FORMAT, RAW_FORMAT, NPM_FORMAT].find((f) => f.id === values.format) ?? { id: values.format, name: values.format, kinds: ['hosted', 'group'], @@ -427,6 +440,9 @@ export function RepositoryForm({ const isGroup = values.kind === 'group' const isDocker = values.format === DOCKER_FORMAT.id const isRaw = values.format === RAW_FORMAT.id + const isNpm = values.format === NPM_FORMAT.id + // NuGet, npm: packages with versions (not docker images, not raw files). + const isPackage = !isDocker && !isRaw // What the repository holds, for the help texts. const things = isDocker ? 'images' : isRaw ? 'files' : 'packages' @@ -605,25 +621,25 @@ export function RepositoryForm({ } /> - ) : isRaw ? ( + ) : ( set('deployment_policy', e.target.value as DeploymentPolicy) } - description="What an upload to a path that holds a file does: refused (uploading the same content again is accepted), replaces it (which needs the delete permission too), or every upload and delete is refused." + description={ + isRaw + ? 'What an upload to a path that holds a file does: refused (uploading the same content again is accepted), replaces it (which needs the delete permission too), or every upload and delete is refused.' + : isNpm + ? 'What publishing a version that exists does: refused (as on the public registry, where a published version never changes), replaces it (which needs the delete permission too), or every publish, unpublish and dist-tag change is refused.' + : 'What pushing a version that exists does: refused, replaces it (which needs the delete permission too), or every push, unlist and delete is refused.' + } > - ) : ( - set('allow_redeploy', c)} - /> )} <remote URL>/<path>, following redirects. Credentials below are sent to this URL's host only. + ) : isNpm ? ( + <> + The upstream registry, e.g. https://registry.npmjs.org. + For a token, leave the username empty and put the token in the + password: it is sent as a Bearer token, to this host only. + ) : undefined } /> @@ -863,9 +887,18 @@ export function RepositoryForm({ mono value={values.reserved_prefixes} onChange={(e) => set('reserved_prefixes', e.target.value)} - placeholder={isDocker ? 'acme/, team/' : 'Acme., Contoso.Internal.'} + placeholder={ + isDocker ? 'acme/, team/' : isNpm ? '@acme, @acme-internal' : 'Acme., Contoso.Internal.' + } description={ - isDocker ? ( + isNpm ? ( + <> + Scopes such as @acme, or name prefixes, whose packages + are never served from proxy members — also before the first internal + version is published — and stay out of the audits sent upstream. + Comma separated. + + ) : isDocker ? ( <> Image names starting with one of these — namespaces such as{' '} acme/ — are never served from proxy members, also @@ -880,7 +913,7 @@ export function RepositoryForm({ ) } /> - {isDocker && ( + {(isDocker || isPackage) && ( set('writable_member', e.target.value)} - placeholder="docker-hosted" + placeholder={`${format.id}-hosted`} description={ - <> - Optional. The hosted docker repository that docker push{' '} - to this group stores images in (make it a member too). Leave empty - to refuse pushes to the group. - + isNpm ? ( + <> + Optional. The hosted npm repository that npm publish, + dist-tag changes and unpublishes through this group go to (make it + a member too). Leave empty to refuse them. + + ) : isPackage ? ( + <> + Optional. The hosted repository of this format that pushes, unlists + and deletes through this group go to (make it a member too). Leave + empty to refuse them. + + ) : ( + <> + Optional. The hosted docker repository that docker push{' '} + to this group stores images in (make it a member too). Leave empty + to refuse pushes to the group. + + ) } /> )} @@ -973,6 +1020,7 @@ const DEPLOYMENT_POLICY_FACTS: Record = { export function repositoryFacts(repo: RepoView): string[] { const docker = repo.format === DOCKER_FORMAT.id const raw = repo.format === RAW_FORMAT.id + const npm = repo.format === NPM_FORMAT.id const facts: string[] = [] if (repo.kind === 'proxy' && repo.remote_url) { facts.push(`caches ${hostOf(repo.remote_url)}`) @@ -984,11 +1032,11 @@ export function repositoryFacts(repo: RepoView): string[] { if (docker && repo.immutable_tags) { facts.push('immutable tags') if (repo.latest_mutable) facts.push('latest may move') - } else if (raw) { - const policy = DEPLOYMENT_POLICY_FACTS[repo.deployment_policy ?? 'disable_redeploy'] + } else if (!docker) { + const policy = DEPLOYMENT_POLICY_FACTS[ + repo.deployment_policy ?? (repo.allow_redeploy ? 'allow_redeploy' : 'disable_redeploy') + ] if (policy) facts.push(policy) - } else if (!docker && repo.allow_redeploy) { - facts.push('redeploy allowed') } if (repo.max_upload_bytes != null) { facts.push(`${bytesToMib(repo.max_upload_bytes)} MiB max. upload`) @@ -998,7 +1046,9 @@ export function repositoryFacts(repo: RepoView): string[] { if (repo.prefer_hosted) facts.push(`prefers hosted ${docker ? 'images' : 'packages'}`) const n = repo.reserved_prefixes.length if (n > 0) facts.push(`${n} reserved ${n === 1 ? 'prefix' : 'prefixes'}`) - if (docker && repo.writable_member) facts.push(`pushes go to ${repo.writable_member}`) + if (repo.writable_member) { + facts.push(`${npm ? 'publishes' : 'pushes'} go to ${repo.writable_member}`) + } } if (docker && repo.host) facts.push(`also at ${repo.host}`) return facts @@ -1012,7 +1062,7 @@ export function capitalize(text: string): string { /** * Build a `CreateRepo` body from validated form values. Sends the format and * only the fields relevant to it and the kind — docker options never for a - * NuGet repository, `allow_redeploy` / `package_delay_seconds` never for a + * NuGet repository, `deployment_policy` / `package_delay_seconds` never for a * docker one (the server refuses both combinations); an empty TTL becomes * `null` (server default), and an empty proxy password, docker host or * writable member is omitted (none set). Exported so create + edit map @@ -1037,10 +1087,8 @@ export function toCreateBody(values: RepoFormValues): CreateRepo { if (docker) { body.immutable_tags = values.immutable_tags body.latest_mutable = values.immutable_tags && values.latest_mutable - } else if (raw) { - body.deployment_policy = values.deployment_policy } else { - body.allow_redeploy = values.allow_redeploy + body.deployment_policy = values.deployment_policy } const maxUpload = mibToBytes(values.max_upload_mib) if (maxUpload != null) body.max_upload_bytes = maxUpload @@ -1049,7 +1097,7 @@ export function toCreateBody(values: RepoFormValues): CreateRepo { body.prefer_hosted = values.prefer_hosted const prefixes = parsePrefixes(values.reserved_prefixes) if (prefixes.length > 0) body.reserved_prefixes = prefixes - if (docker && values.writable_member.trim() !== '') { + if (values.writable_member.trim() !== '') { body.writable_member = values.writable_member.trim() } } @@ -1094,8 +1142,8 @@ export function parsePrefixes(raw: string): string[] { * - the username is sent when changed (`null` when emptied); the password * only when a new one was typed (it is write-only, so blank = keep); * - "Remove stored credentials" sends both as `null`; - * - a docker `host` / `writable_member` is sent when changed, `null` when - * emptied (removes it); + * - a docker `host`, a docker or package group's `writable_member` is sent + * when changed, `null` when emptied (removes it); * - the upload limit is sent when changed, `null` when emptied (back to the * format's default). * @@ -1118,22 +1166,15 @@ export function toUpdateBody( if (values.online !== before.online) body.online = values.online if ( !docker && - !raw && values.kind === 'hosted' && - values.allow_redeploy !== before.allow_redeploy + values.deployment_policy !== before.deployment_policy ) { - body.allow_redeploy = values.allow_redeploy + body.deployment_policy = values.deployment_policy } if (raw) { if (values.inline_content !== before.inline_content) { body.content_disposition = values.inline_content ? 'inline' : 'attachment' } - if ( - values.kind === 'hosted' && - values.deployment_policy !== before.deployment_policy - ) { - body.deployment_policy = values.deployment_policy - } if ( values.kind === 'proxy' && values.max_age_seconds.trim() !== before.max_age_seconds.trim() @@ -1160,7 +1201,7 @@ export function toUpdateBody( ) { body.max_upload_bytes = mibToBytes(values.max_upload_mib) } - if (docker && values.kind === 'group') { + if (!raw && values.kind === 'group') { const member = values.writable_member.trim() if (member !== before.writable_member.trim()) { body.writable_member = member === '' ? null : member diff --git a/web/src/components/VersionTable.tsx b/web/src/components/VersionTable.tsx index 38e1c9d2..c3153f78 100644 --- a/web/src/components/VersionTable.tsx +++ b/web/src/components/VersionTable.tsx @@ -22,17 +22,30 @@ import { RetentionBadge } from './RetentionBadge' * Forge first downloaded it from the upstream, not when it was published * there (and unknown for versions not downloaded yet), so the column is * labelled "Cached" for a proxy repository. + * + * An npm package's versions also show the dist-tags that point at them and + * whether they are deprecated (the message in the badge's title). */ export function VersionTable({ versions, repoKind, caption, + distTags, + deprecated, }: { versions: DetailVersion[] /** The package's repository kind: `hosted` | `proxy` | `group`. */ repoKind: string caption?: string + /** npm: tag → version. */ + distTags?: Record + /** npm: version → deprecation message. */ + deprecated?: Record }) { + const tagsOf = (version: string) => + Object.entries(distTags ?? {}) + .filter(([, v]) => v === version) + .map(([tag]) => tag) return (
@@ -61,6 +74,16 @@ export function VersionTable({ + {tagsOf(v.version).map((tag) => ( + + {tag} + + ))} + {deprecated?.[v.version] != null && ( + + deprecated + + )} {!v.listed && unlisted} diff --git a/web/src/components/ui/badges.tsx b/web/src/components/ui/badges.tsx index 7a1a14ea..560319d8 100644 --- a/web/src/components/ui/badges.tsx +++ b/web/src/components/ui/badges.tsx @@ -35,6 +35,7 @@ const FORMATS: Record = { nuget: { label: 'NuGet', icon: 'Package' }, docker: { label: 'Docker', icon: 'Layers' }, raw: { label: 'Raw', icon: 'Folder' }, + npm: { label: 'npm', icon: 'Package' }, } /** Display name of a repository format (`NuGet`, `Docker`, a plugin's id). */ diff --git a/web/src/routeTree.gen.ts b/web/src/routeTree.gen.ts index debc4752..c313dec5 100644 --- a/web/src/routeTree.gen.ts +++ b/web/src/routeTree.gen.ts @@ -15,6 +15,7 @@ import { Route as LoginRouteImport } from './routes/Login' import { Route as BrowseRouteImport } from './routes/Browse' import { Route as AdminLayoutRouteImport } from './routes/AdminLayout' import { Route as TokensRouteImport } from './routes/Tokens' +import { Route as CliLoginRouteImport } from './routes/CliLogin' import { Route as adminAdminHomeRouteImport } from './routes/admin/AdminHome' import { Route as adminSelectorsListRouteImport } from './routes/admin/selectors/List' import { Route as adminRealmsListRouteImport } from './routes/admin/realms/List' @@ -70,6 +71,11 @@ const TokensRoute = TokensRouteImport.update({ path: '/tokens', getParentRoute: () => AppLayoutRoute, } as any) +const CliLoginRoute = CliLoginRouteImport.update({ + id: '/cli-login/$secret', + path: '/cli-login/$secret', + getParentRoute: () => AppLayoutRoute, +} as any) const adminAdminHomeRoute = adminAdminHomeRouteImport.update({ id: '/', path: '/', @@ -206,6 +212,7 @@ export interface FileRoutesByFullPath { '/manage': typeof AdminLayoutRouteWithChildren '/tokens': typeof TokensRoute '/manage/': typeof adminAdminHomeRoute + '/cli-login/$secret': typeof CliLoginRoute '/manage/content-selectors': typeof adminSelectorsListRoute '/manage/realms': typeof adminRealmsListRoute '/manage/repositories': typeof adminRepositoriesListRoute @@ -237,6 +244,7 @@ export interface FileRoutesByTo { '/': typeof BrowseRoute '/tokens': typeof TokensRoute '/manage': typeof adminAdminHomeRoute + '/cli-login/$secret': typeof CliLoginRoute '/manage/content-selectors': typeof adminSelectorsListRoute '/manage/realms': typeof adminRealmsListRoute '/manage/repositories': typeof adminRepositoriesListRoute @@ -271,6 +279,7 @@ export interface FileRoutesById { '/_app/manage': typeof AdminLayoutRouteWithChildren '/_app/tokens': typeof TokensRoute '/_app/manage/': typeof adminAdminHomeRoute + '/_app/cli-login/$secret': typeof CliLoginRoute '/_app/manage/content-selectors': typeof adminSelectorsListRoute '/_app/manage/realms': typeof adminRealmsListRoute '/_app/manage/repositories': typeof adminRepositoriesListRoute @@ -305,6 +314,7 @@ export interface FileRouteTypes { | '/manage' | '/tokens' | '/manage/' + | '/cli-login/$secret' | '/manage/content-selectors' | '/manage/realms' | '/manage/repositories' @@ -336,6 +346,7 @@ export interface FileRouteTypes { | '/' | '/tokens' | '/manage' + | '/cli-login/$secret' | '/manage/content-selectors' | '/manage/realms' | '/manage/repositories' @@ -369,6 +380,7 @@ export interface FileRouteTypes { | '/_app/manage' | '/_app/tokens' | '/_app/manage/' + | '/_app/cli-login/$secret' | '/_app/manage/content-selectors' | '/_app/manage/realms' | '/_app/manage/repositories' @@ -445,6 +457,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof TokensRouteImport parentRoute: typeof AppLayoutRoute } + '/_app/cli-login/$secret': { + id: '/_app/cli-login/$secret' + path: '/cli-login/$secret' + fullPath: '/cli-login/$secret' + preLoaderRoute: typeof CliLoginRouteImport + parentRoute: typeof AppLayoutRoute + } '/_app/manage/': { id: '/_app/manage/' path: '/' @@ -681,6 +700,7 @@ interface AppLayoutRouteChildren { BrowseRoute: typeof BrowseRoute AdminLayoutRoute: typeof AdminLayoutRouteWithChildren TokensRoute: typeof TokensRoute + CliLoginRoute: typeof CliLoginRoute RepoBrowseRoute: typeof RepoBrowseRoute ImageDetailRoute: typeof ImageDetailRoute PackageDetailRoute: typeof PackageDetailRoute @@ -690,6 +710,7 @@ const AppLayoutRouteChildren: AppLayoutRouteChildren = { BrowseRoute: BrowseRoute, AdminLayoutRoute: AdminLayoutRouteWithChildren, TokensRoute: TokensRoute, + CliLoginRoute: CliLoginRoute, RepoBrowseRoute: RepoBrowseRoute, ImageDetailRoute: ImageDetailRoute, PackageDetailRoute: PackageDetailRoute, diff --git a/web/src/routes.ts b/web/src/routes.ts index 961e2d05..5deb3225 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -21,6 +21,8 @@ import { index, layout, rootRoute, route } from '@tanstack/virtual-file-routes' * the splat is the image name, * which may contain `/`) * /tokens -> Tokens.tsx (self-service registry tokens) + * /cli-login/$secret -> CliLogin.tsx (approve a command-line client's + * web login: `npm login`) * /manage -> AdminLayout.tsx (beforeLoad: require is_admin) * * The admin screens live under `/manage/*`, NOT `/admin/*`: `/admin/*` is the @@ -38,6 +40,7 @@ export const routes = rootRoute('root.tsx', [ route('/packages/$repo/$id', 'PackageDetail.tsx'), route('/images/$repo/$', 'ImageDetail.tsx'), route('/tokens', 'Tokens.tsx'), + route('/cli-login/$secret', 'CliLogin.tsx'), route('/manage', 'AdminLayout.tsx', [ index('admin/AdminHome.tsx'), route('/repositories', 'admin/repositories/List.tsx'), diff --git a/web/src/routes/CliLogin.test.tsx b/web/src/routes/CliLogin.test.tsx new file mode 100644 index 00000000..b44df3ac --- /dev/null +++ b/web/src/routes/CliLogin.test.tsx @@ -0,0 +1,122 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { RouterProvider, createMemoryHistory, createRouter } from '@tanstack/react-router' + +import { clearSession } from '~/lib/auth' +import type { Me } from '~/api/me' +import type { CliLoginRequest } from '~/api/cliLogin' +import { routeTree } from '~/routeTree.gen' + +/** + * The page `npm login` opens: it shows who asks, signs the client in as the + * current user or refuses it (`POST /auth/cli-login/{secret}`), and says so + * when the request is no longer pending (404). + */ + +const me: Me = { + user: { id: 4, username: 'dev@example.com', is_admin: false, active: true }, + is_admin: false, + capabilities: ['repo.read'], +} + +const pending: CliLoginRequest = { + client: 'npm', + repository: 'npm-all', + hostname: 'laptop-42', + client_ip: '192.0.2.7', + created_at: '2026-10-23T10:00:00Z', + expires_at: '2026-10-23T10:10:00Z', +} + +const fetchMock = vi.fn() + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +function mockApi(login: () => Response, decide: () => Response = () => new Response(null, { status: 204 })) { + fetchMock.mockImplementation((input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input.toString(), 'http://localhost') + if (url.pathname.endsWith('/admin/me')) return Promise.resolve(json(me)) + if (url.pathname === '/auth/me') { + return Promise.resolve(json({ username: me.user.username, csrf_token: 'csrf' })) + } + if (url.pathname === '/auth/cli-login/s3cret') { + return Promise.resolve((init?.method ?? 'GET') === 'POST' ? decide() : login()) + } + if (url.pathname.endsWith('/browse/repos')) return Promise.resolve(json([])) + return Promise.resolve(new Response('not found', { status: 404 })) + }) +} + +async function renderPage() { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const router = createRouter({ + routeTree, + context: { queryClient }, + history: createMemoryHistory({ initialEntries: ['/cli-login/s3cret'] }), + }) + render( + + + , + ) + await waitFor(() => expect(router.state.status).toBe('idle')) +} + +function decisions() { + return fetchMock.mock.calls + .filter(([input, init]) => String(input).includes('/auth/cli-login/') && init?.method === 'POST') + .map(([, init]) => JSON.parse(String(init?.body))) +} + +describe('CliLogin', () => { + beforeEach(() => { + vi.stubGlobal('fetch', fetchMock) + fetchMock.mockReset() + clearSession() + }) + + afterEach(() => { + vi.unstubAllGlobals() + clearSession() + }) + + it('shows the request and signs the client in on approval', async () => { + const user = userEvent.setup() + mockApi(() => json(pending)) + await renderPage() + + expect(await screen.findByText('npm asks to sign in')).toBeInTheDocument() + expect(screen.getByText('npm-all')).toBeInTheDocument() + expect(screen.getByText('laptop-42')).toBeInTheDocument() + expect(screen.getByText('192.0.2.7')).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Sign in' })) + expect(await screen.findByText('Signed in')).toBeInTheDocument() + expect(decisions()).toEqual([{ approve: true }]) + }) + + it('refuses the client', async () => { + const user = userEvent.setup() + mockApi(() => json(pending)) + await renderPage() + + await user.click(await screen.findByRole('button', { name: 'Refuse' })) + expect(await screen.findByText('Refused')).toBeInTheDocument() + expect(decisions()).toEqual([{ approve: false }]) + }) + + it('says when the request is no longer pending', async () => { + mockApi(() => new Response('no such login request', { status: 404 })) + await renderPage() + + expect(await screen.findByText('This login request is no longer pending')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Sign in' })).toBeNull() + }) +}) diff --git a/web/src/routes/CliLogin.tsx b/web/src/routes/CliLogin.tsx new file mode 100644 index 00000000..7d1deae8 --- /dev/null +++ b/web/src/routes/CliLogin.tsx @@ -0,0 +1,150 @@ +import { useState } from 'react' +import { useMutation, useQuery } from '@tanstack/react-query' +import { createFileRoute } from '@tanstack/react-router' + +import { cliLoginQueryOptions, decideCliLogin } from '~/api/cliLogin' +import { ApiError } from '~/lib/httpClient' +import { useMe } from '~/queries/useMe' +import { EmptyState } from '~/components/EmptyState' +import { ErrorState } from '~/components/ErrorState' +import { + Button, + Code, + FactList, + FormActions, + Notice, + Page, + PageHeader, + Panel, + Skeleton, +} from '~/components/ui' + +export const Route = createFileRoute('/_app/cli-login/$secret')({ + component: CliLogin, +}) + +/** + * The page `npm login` opens in the browser (its web login): the signed-in + * user sees which client asks, for which repository and from where, and + * signs it in as themselves or refuses. The client polls the registry and + * receives a token once approved. A request that was decided, expired or + * never existed answers 404, shown as "no longer pending". + */ +function CliLogin() { + const { secret } = Route.useParams() + const me = useMe() + const request = useQuery(cliLoginQueryOptions(secret)) + const [decided, setDecided] = useState(null) + const decide = useMutation({ + mutationFn: (approve: boolean) => decideCliLogin(secret, approve), + onSuccess: (_, approve) => setDecided(approve), + }) + const header = + + if (decided !== null) { + return ( + + {header} + + {decided + ? 'The client is signed in as you. Return to your terminal; you can close this page.' + : 'The client was not signed in. Return to your terminal; you can close this page.'} + + + ) + } + + if (request.isLoading) { + return ( + + {header} + + + ) + } + + if (request.isError) { + const gone = request.error instanceof ApiError && request.error.status === 404 + return ( + + {header} + {gone ? ( + + It was answered already, or it expired. Run npm login again + for a new one. + + ) : ( + void request.refetch()} + /> + )} + + ) + } + + const login = request.data + if (!login) return null + const gone = decide.error instanceof ApiError && decide.error.status === 404 + const username = me.data?.user.username + + return ( + + {header} + + Approve only a login you started yourself, with npm login. The + client gets a token that acts as{' '} + {username ? {username} : 'you'} on this server. + + } + > + + {gone ? ( + + It was answered already, or it expired. Run npm login again for a + new one. + + ) : ( + decide.isError && ( + + {decide.error.message} + + ) + )} + + + + + + + ) +} diff --git a/web/src/routes/PackageDetail.test.tsx b/web/src/routes/PackageDetail.test.tsx index 26f7b8da..cb9acfe5 100644 --- a/web/src/routes/PackageDetail.test.tsx +++ b/web/src/routes/PackageDetail.test.tsx @@ -66,9 +66,40 @@ const proxyDetail: PackageDetail = { }, } +/** A scoped npm package of a hosted repository. */ +const npmDetail: PackageDetail = { + repo: 'npm-hosted', + repo_kind: 'hosted', + id: '@acme/ui', + original_id: '@acme/ui', + versions: ['1.0.0', '1.1.0', '2.0.0-rc.1'].map((version) => ({ + version, + version_original: version, + listed: true, + size: 2048, + published_at: '2026-06-01T00:00:00Z', + delay_status: null, + delayed_until: null, + retention_status: null, + })), + dependency_groups_by_version: { + '1.1.0': [ + { target_framework: 'dependencies', dependencies: [{ id: '@acme/core', range: '^2.0.0' }] }, + { target_framework: 'peerDependencies', dependencies: [{ id: 'react', range: '>=18' }] }, + ], + }, + tags: { latest: '1.1.0', next: '2.0.0-rc.1' }, + deprecated: { '1.0.0': 'use 1.1.0' }, + readme: '# @acme/ui\n\nButtons.', + description: 'Acme UI components', +} + const fetchMock = vi.fn() -function mockApi(detail: (repo: string, id: string) => Response) { +function mockApi( + detail: (repo: string, id: string) => Response, + repos: { name: string; format: string }[] = [], +) { fetchMock.mockImplementation((input: RequestInfo | URL) => { const raw = typeof input === 'string' ? input : input.toString() const url = new URL(raw, 'http://localhost') @@ -80,6 +111,13 @@ function mockApi(detail: (repo: string, id: string) => Response) { }), ) } + if (url.pathname.endsWith('/browse/repos')) { + return Promise.resolve( + jsonResponse( + repos.map((r, i) => ({ id: i + 1, kind: 'hosted', online: true, ...r })), + ), + ) + } const m = url.pathname.match(/\/browse\/packages\/([^/]+)\/([^/]+)$/) if (m) return Promise.resolve(detail(m[1], m[2])) return Promise.resolve(new Response('not found', { status: 404 })) @@ -246,4 +284,37 @@ describe('PackageDetail', () => { const alert = await screen.findByRole('alert') expect(within(alert).getByText('Could not load package')).toBeInTheDocument() }) + + it('shows an npm package\'s dist-tags, deprecations, README and install commands', async () => { + mockApi(() => jsonResponse(npmDetail), [{ name: 'npm-hosted', format: 'npm' }]) + + await renderDetail('/packages/npm-hosted/%40acme%2Fui') + + await screen.findByRole('heading', { name: '@acme/ui' }) + const detailCalls = fetchMock.mock.calls.map(([input]) => String(input)) + expect(detailCalls.some((u) => u.endsWith('/browse/packages/npm-hosted/%40acme%2Fui'))).toBe( + true, + ) + const table = screen.getByRole('table', { name: 'Versions of @acme/ui' }) + const row = (v: string) => within(table).getByText(textContent(v)).closest('tr') as HTMLElement + expect(within(row('1.1.0')).getByText('latest')).toBeInTheDocument() + expect(within(row('2.0.0-rc.1')).getByText('next')).toBeInTheDocument() + expect(within(row('1.0.0')).getByText('deprecated')).toHaveAttribute('title', 'use 1.1.0') + + // The README, as text. + expect(screen.getByLabelText('README of @acme/ui')).toHaveTextContent('# @acme/ui Buttons.') + // Dependency kinds stand where target frameworks do; names stay exact. + expect(screen.getByRole('columnheader', { name: 'Kind' })).toBeInTheDocument() + expect(screen.getByText('peerDependencies')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '@acme/core' })).toHaveAttribute( + 'href', + '/packages/npm-hosted/%40acme%2Fcore', + ) + // Install commands start at `latest`, not the newest prerelease. + expect(await screen.findByText(textContent('npm install @acme/ui@1.1.0'))).toBeInTheDocument() + expect(screen.getByText(textContent('pnpm add @acme/ui@1.1.0'))).toBeInTheDocument() + // The dist-tags, and the latest version's description. + expect(screen.getByRole('heading', { name: 'Dist-tags' })).toBeInTheDocument() + expect(screen.getByText('Acme UI components')).toBeInTheDocument() + }) }) diff --git a/web/src/routes/PackageDetail.tsx b/web/src/routes/PackageDetail.tsx index 110268b6..d9afed05 100644 --- a/web/src/routes/PackageDetail.tsx +++ b/web/src/routes/PackageDetail.tsx @@ -29,8 +29,10 @@ export const Route = createFileRoute('/_app/packages/$repo/$id')({ /** * Package-detail view: versions with per-version metadata + delay/retention - * badges, and dependency groups per version. Loading / empty / not-found / - * repository-offline / error states are all handled. Reachable by any reader. + * badges, and dependency groups per version; the package's tags (npm's + * dist-tags), deprecations and README when its format has them. Loading / + * empty / not-found / repository-offline / error states are all handled. + * Reachable by any reader. */ function PackageDetailPage() { const { repo, id } = Route.useParams() @@ -82,6 +84,9 @@ function PackageDetailPage() { const pkg = detail.data if (!pkg) return null const hasDependencies = Object.keys(pkg.dependency_groups_by_version).length > 0 + const npm = format === 'npm' + const tags = pkg.tags ?? {} + const latest = tags.latest return ( @@ -97,24 +102,53 @@ function PackageDetailPage() { format={format} meta={[ pkg.versions.length === 1 ? '1 version' : `${pkg.versions.length} versions`, + ...(latest ? [`latest ${latest}`] : []), ]} /> {pkg.versions.length > 0 && format && ( - + )} + {Object.keys(tags).length > 0 && ( + + What npm install {pkg.original_id}@<tag> installs;{' '} + latest is the default. + + ) : undefined + } + > + ({ + term: tag, + value: version, + mono: true, + }))} + /> + + )} } asideLabel="Install and details" @@ -123,7 +157,11 @@ function PackageDetailPage() { title="Versions" count={pkg.versions.length} flush - description="Unlisted versions stay restorable but are hidden from search and from floating version ranges." + description={ + npm + ? 'A deprecated version still installs; npm warns about it with the message shown on its badge.' + : 'Unlisted versions stay restorable but are hidden from search and from floating version ranges.' + } > {pkg.versions.length === 0 ? ( @@ -132,20 +170,41 @@ function PackageDetailPage() { versions={pkg.versions} repoKind={pkg.repo_kind} caption={`Versions of ${pkg.original_id}`} + distTags={pkg.tags} + deprecated={pkg.deprecated} /> )} + {(npm || pkg.readme) && ( + + {pkg.readme ? ( + 
+                {pkg.readme}
+
+ ) : ( + + The latest version was published without one. + + )} + + )} + {hasDependencies ? ( ) : ( @@ -159,39 +218,48 @@ function PackageDetailPage() { } /** - * The version the install commands start with: the last listed one that can - * be downloaded, a stable one before a prerelease (as `dotnet add package` - * without `--version` picks), else the last one. The detail API lists the - * versions oldest first. + * The version the install commands start with: the one npm's `latest` + * dist-tag names, else the last listed one that can be downloaded, a stable + * one before a prerelease (as `dotnet add package` without `--version` + * picks), else the last one. The detail API lists the versions oldest first. */ -function defaultVersion(versions: DetailVersion[]): string { +function defaultVersion(versions: DetailVersion[], latest?: string): string { + const tagged = latest ? versions.find((v) => v.version === latest) : undefined + if (tagged) return tagged.version_original const ready = versions.filter((v) => v.listed && v.delay_status !== 'delayed_until') const stable = ready.filter((v) => !v.version_original.includes('-')) return (stable.at(-1) ?? ready.at(-1) ?? versions.at(-1))?.version_original ?? '' } /** - * The install commands of one version, for the format Forge knows a client - * command of (NuGet). The source is added once per machine with the - * repository's Connect commands. + * The install commands of one version, for the formats Forge knows a client + * command of (NuGet, npm). The source or registry is added once per machine + * with the repository's Connect commands. */ function InstallPanel({ format, repo, id, versions, + latest, }: { format: string repo: string id: string versions: DetailVersion[] + /** npm: the version the `latest` dist-tag names. */ + latest?: string }) { - const [version, setVersion] = useState(() => defaultVersion(versions)) - if (format !== 'nuget') return null + const [version, setVersion] = useState(() => defaultVersion(versions, latest)) + if (format !== 'nuget' && format !== 'npm') return null return ( {versions.length > 1 && ( setVersion(e.target.value)}> @@ -202,17 +270,36 @@ function InstallPanel({ ))} )} - - `} - placeholders={[]} - /> + {format === 'npm' ? ( + npmInstallCommands(id, version).map((snippet) => ( + + )) + ) : ( + <> + + `} + placeholders={[]} + /> + + )} ) } + +/** The commands that add `id` at `version` to a project, per npm client. */ +function npmInstallCommands(id: string, version: string) { + const spec = `${id}@${version}` + return [ + { label: 'npm', command: `npm install ${spec}`, copyLabel: 'Copy npm command' }, + { label: 'pnpm', command: `pnpm add ${spec}`, copyLabel: 'Copy pnpm command' }, + { label: 'Yarn', command: `yarn add ${spec}`, copyLabel: 'Copy Yarn command' }, + { label: 'Bun', command: `bun add ${spec}`, copyLabel: 'Copy Bun command' }, + ] +} diff --git a/web/src/styles/forge-ui.css b/web/src/styles/forge-ui.css index e64b8348..871433a6 100644 --- a/web/src/styles/forge-ui.css +++ b/web/src/styles/forge-ui.css @@ -423,6 +423,7 @@ .fg-subtitle { margin: 0; color: var(--ink); font-size: 14px; line-height: 20px; font-weight: 600; } .fg-text { margin: 0; color: var(--ink-muted); font-size: 14px; line-height: 20px; } .fg-text--small { font-size: 12px; line-height: 16px; } + .fg-readme { margin: 0; max-height: 640px; overflow: auto; color: var(--ink); font-family: var(--font-mono); font-size: 13px; line-height: 20px; white-space: pre-wrap; overflow-wrap: anywhere; } .fg-strong { color: var(--ink); font-weight: 600; } .fg-divider { height: 1px; margin: 0; border: 0; background: var(--line); } .fg-code-inline { padding: 1px 5px; border-radius: var(--radius-xs); background: var(--surface-sunken); color: var(--ink); font-family: var(--font-mono); font-size: 12.5px; line-height: 18px; overflow-wrap: anywhere; } diff --git a/web/vite.config.ts b/web/vite.config.ts index 1a6251db..fbec1cba 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -48,6 +48,7 @@ export default defineConfig({ '/v3': { target: apiTarget, changeOrigin: false }, '/v2': { target: apiTarget, changeOrigin: false }, '/raw': { target: apiTarget, changeOrigin: false }, + '/npm': { target: apiTarget, changeOrigin: false }, '/auth': { target: apiTarget, changeOrigin: false }, }, }, From fa91037c37faf28503e3a81487ab8f1dd82dc505 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 08:30:00 +0000 Subject: [PATCH 2/2] npm e2e: republish with --force so that the server refuses it npm 11 (CI's Node 26) checks the registry's versions before publishing and refuses a known one itself, so the republish never reached the server and its message did not match. With --force the client skips its check and the server's 409 is what refuses the republish, with npm 10 and 11 alike. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BaGpNGgkQV54XvgL5D9ay2 --- .github/scripts/npm-e2e.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/npm-e2e.sh b/.github/scripts/npm-e2e.sh index 1eba917a..e10b2ffe 100755 --- a/.github/scripts/npm-e2e.sh +++ b/.github/scripts/npm-e2e.sh @@ -327,7 +327,10 @@ package "$work/src/pnpm-made" pnpm-made 1.0.0 (cd "$work/src/pnpm-made" && pnpm publish --no-git-checks --registry "$(reg npm-hosted)") >/dev/null npm view @acme/core version --registry "$(reg npm-hosted)" | grep -qx 1.0.0 \ || fail "a publish through the group did not land in npm-hosted" -if (cd "$work/src/hosted-1.0.0" && npm publish --registry "$(reg npm-hosted)") > "$work/republish.out" 2>&1; then +# --force: npm 11 refuses a version the registry lists by itself; the +# server must refuse it too. +if (cd "$work/src/hosted-1.0.0" && npm publish --force --registry "$(reg npm-hosted)") \ + > "$work/republish.out" 2>&1; then fail "a republish of e2e-hosted@1.0.0 was accepted" fi grep -q "409 Conflict.*e2e-hosted 1.0.0 already exists" "$work/republish.out" \