diff --git a/.env.example b/.env.example index c327153d..fbcdf919 100644 --- a/.env.example +++ b/.env.example @@ -151,15 +151,6 @@ FORGE_AUTH__BOOTSTRAP_ADMIN__PASSWORD=changeme # FORGE_DOCKER__UNREFERENCED_BLOB_TTL_SECONDS=86400 # FORGE_DOCKER__SWEEP_INTERVAL_SECONDS=300 -# --------------------------------------------------------------------------- -# WebAssembly format plugins (optional; docs/format-plugins.md) -# Load further package formats from //, each pinned to the SHA-256 of -# its component (a mismatch fails startup). The built-in format (NuGet) needs -# nothing here. `cargo xtask plugins` builds components and prints digests. -# --------------------------------------------------------------------------- -# FORGE_PLUGINS__DIR=/etc/forge/plugins -# FORGE_PLUGINS__FORMATS__EXAMPLE=sha256:<64 hex digits> - # --------------------------------------------------------------------------- # Cache # --------------------------------------------------------------------------- diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3c39fb19..44e6b013 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,12 +12,6 @@ updates: interval: weekly open-pull-requests-limit: 10 groups: - # wasmtime-wasi of a major builds only against the wasmtime of the same - # major, so a major update of one alone fails to compile. Every update - # of the Wasmtime crates comes as one pull request; listed first, since - # a dependency joins the first group it matches. - wasmtime: - patterns: ["wasmtime", "wasmtime-*"] cargo-minor-and-patch: update-types: [minor, patch] diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index 31b58963..cd6d52de 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -40,7 +40,7 @@ COMMON=(.github/scripts/ci-changes.sh) # jobs would hide a change from the jobs using it; ci.yml has none.) CI=.github/workflows/ci.yml declare -A CI_JOB_GROUP=( - [fmt]=changelog [clippy]=rust [audit]=audit [plugins]=rust [test]=rust + [fmt]=changelog [clippy]=rust [audit]=audit [test]=rust [docker]=image [kubernetes]=kubernetes [web]=web [oci-e2e]=rust [raw-e2e]=rust [e2e]=rust [web-e2e]=web ) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 629974f4..93105470 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,29 +231,6 @@ jobs: - name: cargo audit run: cargo audit - # ── Format plugins as WebAssembly components (plugins//) ────────────── - # The plugins are workspace crates (fmt, clippy and their tests run with the - # workspace); this checks that each one also lints clean and builds as a - # component for wasm32-wasip2. The tests that load a component run in `test`. - plugins: - name: plugins - runs-on: ubuntu-latest - needs: changes - if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install Rust toolchain (rust-toolchain.toml) (with clippy, wasm32-wasip2) - uses: dtolnay/rust-toolchain@d124f86f28ff42855784ff3319f48ac182c248f8 # 1.99.1 - with: - components: clippy - targets: wasm32-wasip2 - - name: Cache cargo registry + dependency builds - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - save-if: ${{ github.event_name == 'push' }} - - name: cargo xtask plugins-check - run: cargo xtask plugins-check - # ── Unit + integration tests ─────────────────────────────────────────────── # The integration tests start their own Postgres, versitygw and Redis with # testcontainers (the runner has a Docker daemon). There is no separate @@ -281,9 +258,6 @@ jobs: df -h / - name: Install Rust toolchain (rust-toolchain.toml) uses: dtolnay/rust-toolchain@d124f86f28ff42855784ff3319f48ac182c248f8 # 1.99.1 - with: - # The format-plugin tests build plugins/ as WebAssembly components. - targets: wasm32-wasip2 - name: Cache cargo registry + dependency builds uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00940c39..03fd5236 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The `name:` of every ci.yml job that runs on the default branch. The # required status checks in DEVELOPMENT.md (merge queue) list the same. - REQUIRED_JOBS: fmt clippy audit plugins test docker kubernetes web oci-e2e raw-e2e e2e web-e2e + REQUIRED_JOBS: fmt clippy audit test docker kubernetes web oci-e2e raw-e2e e2e web-e2e steps: - name: Wait for the commit's CI run and check its jobs shell: bash # -eo pipefail: a failing gh call fails the step diff --git a/.gitignore b/.gitignore index b8101b80..186985ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ /target -# Format plugins build in their own workspaces (`cargo xtask plugins`). -/plugins/*/target **/*.rs.bk Cargo.lock.orig .env diff --git a/AGENTS.md b/AGENTS.md index 2b034646..28a00766 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,14 +53,12 @@ its pull request's description says why. ## Formats: every format plugin lives under plugins/ A package format is a crate in `plugins//`, the directory named after its -format id, whether it runs built in only, as a WebAssembly component, or -both. Built in only is no reason to put it elsewhere: NuGet runs built in -only and lives in `plugins/nuget`. Never add a format as a crate -under `crates/` (no `crates/format-`) or as a module of the server: it -implements `FormatPlugin` with the layout -[plugins/README.md](plugins/README.md) describes, and has a `component` -feature unless it is built in only. Docker / OCI (`crates/format-oci`) and -raw (`crates/server/src/raw/`) are not there yet and are to move, to +format id, linked into the server like `plugins/nuget` (Forge loads no +WebAssembly plugins). Never add a format as a crate under `crates/` (no +`crates/format-`) or as a module of the server: it implements +`FormatPlugin` with the layout [plugins/README.md](plugins/README.md) +describes. Docker / OCI (`crates/format-oci`) and raw +(`crates/server/src/raw/`) are not there yet and are to move, to `plugins/docker` and `plugins/raw`, each in a pull request of its own; until then they are not the model for a new format. diff --git a/Cargo.lock b/Cargo.lock index c02ebab0..47c0d88a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aead" version = "0.6.1" @@ -69,12 +54,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "ambient-authority" -version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" - [[package]] name = "android_system_properties" version = "0.1.6" @@ -90,12 +69,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" - [[package]] name = "arc-swap" version = "1.9.2" @@ -123,12 +96,6 @@ dependencies = [ "password-hash", ] -[[package]] -name = "arraydeque" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - [[package]] name = "asn1-rs" version = "0.7.2" @@ -980,9 +947,6 @@ name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -dependencies = [ - "allocator-api2", -] [[package]] name = "bytemuck" @@ -1012,24 +976,6 @@ dependencies = [ "either", ] -[[package]] -name = "cap-primitives" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" -dependencies = [ - "ambient-authority", - "fs-set-times", - "io-extras", - "io-lifetimes 3.0.1", - "ipnet", - "maybe-owned", - "rustix", - "rustix-linux-procfs", - "windows-sys 0.61.2", - "winx", -] - [[package]] name = "cc" version = "1.4.7" @@ -1118,15 +1064,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "cobs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" -dependencies = [ - "thiserror", -] - [[package]] name = "combine" version = "4.6.8" @@ -1185,15 +1122,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" -[[package]] -name = "cpp_demangle" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" -dependencies = [ - "cfg-if", -] - [[package]] name = "cpubits" version = "0.1.1" @@ -1218,152 +1146,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cranelift-assembler-x64" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c587da2696fbaa58e7a2fa2e5bfdd365d0928219f36992304ac1f15e5bc254" -dependencies = [ - "cranelift-assembler-x64-meta", -] - -[[package]] -name = "cranelift-assembler-x64-meta" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a60027b7556b680b051651291053ec381edf4ffbc01784997ae51a64269bb" -dependencies = [ - "cranelift-srcgen", -] - -[[package]] -name = "cranelift-bforest" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bfdb846acebfd591d601994ea283c98d525ac0bc9a84ca3f2c36bcf5b69ff1" -dependencies = [ - "cranelift-entity", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-bitset" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9f8cc1080739d5f72b4ee7e00aff7d589f1284beaf0a19472c07878be7f299" -dependencies = [ - "serde", - "serde_derive", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-codegen" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85417d493ecec833b5134546b4a252c78cc0bd5be11729a35d25dcff9e4e70b" -dependencies = [ - "bumpalo", - "cranelift-assembler-x64", - "cranelift-bforest", - "cranelift-bitset", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-control", - "cranelift-entity", - "cranelift-isle", - "gimli", - "hashbrown 0.17.1", - "libm", - "log", - "postcard", - "pulley-interpreter", - "regalloc2", - "rustc-hash", - "serde", - "serde_derive", - "sha2 0.10.9", - "smallvec", - "target-lexicon", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-codegen-meta" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01c8ed0098360bae69ad6b82952d4247516799ce8050d45ad168d6689f4eae6" -dependencies = [ - "cranelift-assembler-x64-meta", - "cranelift-codegen-shared", - "cranelift-srcgen", - "heck", - "pulley-interpreter", -] - -[[package]] -name = "cranelift-codegen-shared" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f9b86d809cec9c4e88b201f1b3ed3b0347674c9ccfbb8c355228669e34be81" - -[[package]] -name = "cranelift-control" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce182038fb10b719841e65fdf5deeee258c684f63439f5be680c2be059274b58" -dependencies = [ - "arbitrary", -] - -[[package]] -name = "cranelift-entity" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bca812b03a7b942796391f25bc02aa70a1a007fee3ce2756490c788429e9c39" -dependencies = [ - "cranelift-bitset", - "serde", - "serde_derive", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-frontend" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c30563d2aef95e8d1a00a920199c99117b350339599f0e9487a7a1cf5d59de" -dependencies = [ - "cranelift-codegen", - "hashbrown 0.17.1", - "log", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-isle" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c54184333a275698edb1d3bdff226131a25d47133d792aa716e47e32b8b7f05" - -[[package]] -name = "cranelift-native" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb67a4c8a3180b5f1d77dffcd6317c05386ced42e5a31c7004523a9d99dc85c1" -dependencies = [ - "cranelift-codegen", - "libc", - "target-lexicon", -] - -[[package]] -name = "cranelift-srcgen" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b533db1679b4b728568bd490da6533cf50974c48a4d7534e78f1e0e02bc39769" - [[package]] name = "crc" version = "3.4.0" @@ -1407,16 +1189,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.21" @@ -1738,18 +1510,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - [[package]] name = "encoding_rs" version = "0.8.41" @@ -1866,8 +1626,6 @@ version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ - "crc32fast", - "miniz_oxide", "zlib-rs", ] @@ -2035,16 +1793,6 @@ dependencies = [ "zip", ] -[[package]] -name = "forge-format-helm" -version = "0.1.0" -dependencies = [ - "flate2", - "forge-plugin-api", - "serde_json", - "yaml-rust2", -] - [[package]] name = "forge-format-nuget" version = "1.0.0" @@ -2093,30 +1841,18 @@ dependencies = [ name = "forge-plugin" version = "1.1.0" dependencies = [ - "flate2", "forge-core", - "forge-format-helm", - "forge-plugin", "forge-plugin-api", - "hex", "serde", "serde_json", - "sha2 0.11.1", - "tempfile", "thiserror", "toml 1.1.8+spec-1.1.0", "tracing", - "wasm-encoder 0.258.3", - "wasmtime", - "wasmtime-wasi", ] [[package]] name = "forge-plugin-api" version = "1.1.0" -dependencies = [ - "wit-bindgen 0.63.0", -] [[package]] name = "forge-server" @@ -2136,7 +1872,6 @@ dependencies = [ "forge-core", "forge-crypto", "forge-db", - "forge-format-helm", "forge-format-nuget", "forge-oci", "forge-plugin", @@ -2217,17 +1952,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-set-times" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" -dependencies = [ - "io-lifetimes 2.0.4", - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -2393,18 +2117,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "gimli" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" -dependencies = [ - "fnv", - "hashbrown 0.16.1", - "indexmap 2.14.2", - "stable_deref_trait", -] - [[package]] name = "globset" version = "0.4.20" @@ -2474,8 +2186,6 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash", - "serde", - "serde_core", ] [[package]] @@ -2487,15 +2197,6 @@ dependencies = [ "hashbrown 0.16.1", ] -[[package]] -name = "hashlink" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "heck" version = "0.5.0" @@ -2834,12 +2535,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -2905,28 +2600,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "io-extras" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" -dependencies = [ - "io-lifetimes 3.0.1", - "windows-sys 0.52.0", -] - -[[package]] -name = "io-lifetimes" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" - -[[package]] -name = "io-lifetimes" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" - [[package]] name = "ipnet" version = "2.12.2" @@ -3099,12 +2772,6 @@ dependencies = [ "spin 0.9.9", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "lettre" version = "0.11.23" @@ -3197,12 +2864,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" -[[package]] -name = "mach2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" - [[package]] name = "matchers" version = "0.2.0" @@ -3218,12 +2879,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "maybe-owned" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" - [[package]] name = "md-5" version = "0.11.0" @@ -3240,15 +2895,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "memfd" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" -dependencies = [ - "rustix", -] - [[package]] name = "mime" version = "0.3.17" @@ -3271,16 +2917,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.3" @@ -3511,18 +3147,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" -dependencies = [ - "crc32fast", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "memchr", -] - [[package]] name = "oid-registry" version = "0.8.1" @@ -3791,18 +3415,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "postcard" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" -dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "serde", -] - [[package]] name = "potential_utf" version = "0.1.6" @@ -3827,16 +3439,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" -dependencies = [ - "proc-macro2", - "syn 3.0.6", -] - [[package]] name = "primeorder" version = "0.13.6" @@ -3901,42 +3503,19 @@ dependencies = [ ] [[package]] -name = "pulley-interpreter" -version = "49.0.4" +name = "quick-xml" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd3de43f4f4cff536de8902097edbdf749321b97dda548f36a37918eb85bba3" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ - "cranelift-bitset", - "log", - "pulley-macros", - "wasmtime-internal-core", + "memchr", ] [[package]] -name = "pulley-macros" -version = "49.0.4" +name = "quinn" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8319a8fa5269b4895957f8795861c42822bd26218e495f70060a963323d94847" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "quick-xml" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" -dependencies = [ - "memchr", -] - -[[package]] -name = "quinn" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -4100,26 +3679,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "rcgen" version = "0.14.10" @@ -4194,21 +3753,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "regalloc2" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" -dependencies = [ - "allocator-api2", - "bumpalo", - "hashbrown 0.17.1", - "log", - "rustc-hash", - "serde", - "smallvec", -] - [[package]] name = "regex" version = "1.13.1" @@ -4331,12 +3875,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustc-demangle" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" - [[package]] name = "rustc-hash" version = "2.1.3" @@ -4374,16 +3912,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustix-linux-procfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" -dependencies = [ - "once_cell", - "rustix", -] - [[package]] name = "rustls" version = "0.23.46" @@ -4572,10 +4100,6 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "serde" @@ -4906,7 +4430,7 @@ dependencies = [ "futures-io", "futures-util", "hashbrown 0.16.1", - "hashlink 0.11.1", + "hashlink", "indexmap 2.14.2", "log", "memchr", @@ -5185,12 +4709,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "target-lexicon" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" - [[package]] name = "tempfile" version = "3.27.0" @@ -5204,15 +4722,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "testcontainers" version = "0.27.3" @@ -5854,7 +5363,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", + "wit-bindgen", ] [[package]] @@ -5912,50 +5421,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64765e43c000fae008a48db8b288e175e237fc9ce20a26586836878c1bee89f" -dependencies = [ - "leb128fmt", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasm-encoder" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a" -dependencies = [ - "leb128fmt", - "wasmparser 0.262.0", -] - -[[package]] -name = "wasm-metadata" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e10808eb27efde692f1b1706089fe19296728bf872291f5b4e0d50fba88f2f" -dependencies = [ - "anyhow", - "indexmap 2.14.2", - "wasm-encoder 0.258.3", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasm-metadata" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c86ff6668190d9fb70372b306b4f9fe8930f0e2f57ce72e4922bf2f117900949" -dependencies = [ - "anyhow", - "indexmap 2.14.2", - "wasm-encoder 0.262.0", - "wasmparser 0.262.0", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -5969,282 +5434,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9269dd817e0e7021bf534ccd9761bc2d64778b6708e41228b8439d75eaa0b1" -dependencies = [ - "bitflags 2.13.2", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "semver", - "serde", -] - -[[package]] -name = "wasmparser" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b" -dependencies = [ - "bitflags 2.13.2", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "semver", -] - -[[package]] -name = "wasmprinter" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a457ba8670a3db3d357c1aeac3ed4fe378a76054a221789f0df0ea507e292b" -dependencies = [ - "anyhow", - "termcolor", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasmtime" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36abb25de5518707e5ba96b586366b09db168d2d4976cb9dac57265a317497ea" -dependencies = [ - "addr2line", - "async-trait", - "bitflags 2.13.2", - "bumpalo", - "cc", - "encoding_rs", - "futures", - "libc", - "log", - "mach2", - "memfd", - "object", - "once_cell", - "postcard", - "pulley-interpreter", - "rayon", - "rustix", - "semver", - "serde", - "serde_derive", - "smallvec", - "target-lexicon", - "wasmparser 0.258.3", - "wasmtime-environ", - "wasmtime-internal-component-macro", - "wasmtime-internal-component-util", - "wasmtime-internal-core", - "wasmtime-internal-cranelift", - "wasmtime-internal-fiber", - "wasmtime-internal-jit-debug", - "wasmtime-internal-jit-icache-coherence", - "wasmtime-internal-unwinder", - "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-environ" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92f16a4d09fb1fbf2dcfeddb9818e1517a258e13b505b3bfdadcf47a19bf31ca" -dependencies = [ - "anyhow", - "cpp_demangle", - "cranelift-bforest", - "cranelift-bitset", - "cranelift-entity", - "gimli", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "log", - "object", - "postcard", - "rustc-demangle", - "semver", - "serde", - "serde_derive", - "sha2 0.10.9", - "smallvec", - "target-lexicon", - "wasm-encoder 0.258.3", - "wasmparser 0.258.3", - "wasmprinter", - "wasmtime-internal-component-util", - "wasmtime-internal-core", -] - -[[package]] -name = "wasmtime-internal-component-macro" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fca03bd4425856574c0807d1dee16fad43f947feaf4e9ae1d2129596ee4de4" -dependencies = [ - "anyhow", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasmtime-internal-component-util", - "wasmtime-internal-wit-bindgen", - "wit-parser 0.258.3", -] - -[[package]] -name = "wasmtime-internal-component-util" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4006009fc3598af4e895ab18c6c33d2c6b7c657b1bd0a755ca7ee1abef6f0c16" - -[[package]] -name = "wasmtime-internal-core" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947a732ad659f42c10ec639706fc8cb1f068dd9a55eebde8a1f24dfd06dfa525" -dependencies = [ - "hashbrown 0.17.1", - "libm", - "serde", -] - -[[package]] -name = "wasmtime-internal-cranelift" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfb8b455d524939e03df42e3e749f8e26e26ba2acd8b08fa00b3bda55686750" -dependencies = [ - "cranelift-codegen", - "cranelift-control", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "gimli", - "itertools", - "log", - "object", - "pulley-interpreter", - "smallvec", - "target-lexicon", - "thiserror", - "wasmparser 0.258.3", - "wasmtime-environ", - "wasmtime-internal-core", - "wasmtime-internal-unwinder", - "wasmtime-internal-versioned-export-macros", -] - -[[package]] -name = "wasmtime-internal-fiber" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8aa8711f68b2cdd94c2b85ba5c8317edaa1ac4ee0a0534b957123680239307" -dependencies = [ - "cc", - "libc", - "rustix", - "wasmtime-environ", - "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-internal-jit-debug" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e873e4504faf50aa8ce255d6ea7210d87b39d5f379c2486782e2c5f179c70ca8" -dependencies = [ - "cc", - "wasmtime-internal-versioned-export-macros", -] - -[[package]] -name = "wasmtime-internal-jit-icache-coherence" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197ab1fda5da958a5c0b8a97eb241beafc48ac568b82ef4f12f38d6866c74173" -dependencies = [ - "libc", - "wasmtime-internal-core", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-internal-unwinder" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5577f3a5a979a7c3b1b23c1e9f13e70a8583acc8828471ec56f282117fc4952b" -dependencies = [ - "cranelift-codegen", - "log", - "object", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-internal-versioned-export-macros" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6399c9c4050ed431cf66e21c05fadd6f9731db44476f452f8c7142b4597b883" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "wasmtime-internal-wit-bindgen" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eaf45d4da7681d234bb6efd846b4a528b08a75384a7fc23f125d0145364b06" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "heck", - "indexmap 2.14.2", - "wit-component 0.258.3", - "wit-parser 0.258.3", -] - -[[package]] -name = "wasmtime-wasi" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "821e45765c35555d3b0bdf9277cb99478f7d1f29a17b3315ba76e01220f02a38" -dependencies = [ - "async-trait", - "bitflags 2.13.2", - "bytes", - "cap-primitives", - "futures", - "rand 0.10.3", - "rustix", - "rustix-linux-procfs", - "thiserror", - "tokio", - "tracing", - "url", - "wasmtime", - "wasmtime-wasi-io", - "windows-sys 0.61.2", - "winx", -] - -[[package]] -name = "wasmtime-wasi-io" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdff38d1c730bdaf4fcfbe0eedf3265fb9c7a4deb119deecbfc1c1778950716" -dependencies = [ - "async-trait", - "bytes", - "futures", - "tracing", - "wasmtime", -] - [[package]] name = "web-sys" version = "0.3.105" @@ -6478,16 +5667,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -[[package]] -name = "winx" -version = "0.36.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" -dependencies = [ - "bitflags 2.13.2", - "windows-sys 0.52.0", -] - [[package]] name = "wiremock" version = "0.6.5" @@ -6517,133 +5696,6 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85a987ab61d365382d25c7d47e26edd48941b3bf6e3fbf76eff918bc71dab06" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e122e390603abfacd548027016cd0113a6e2bf2f8ce6bdd3658b013456ea8" -dependencies = [ - "anyhow", - "heck", - "wit-parser 0.262.0", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e26e13b9d972026778d2ffb7985736de116026fdc8d9464f966e12df2eb240" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.2", - "prettyplease", - "syn 3.0.6", - "wasm-metadata 0.262.0", - "wit-bindgen-core", - "wit-component 0.262.0", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a008a0d6b4840677326beb4edbeb210a2f1b7d0183fb7cc33f466b90fd7fb6" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 3.0.6", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fca88950af0c1f42bc77e17315e48798b8d68340171b6d204f62bc19a878a6" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "indexmap 2.14.2", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder 0.258.3", - "wasm-metadata 0.258.3", - "wasmparser 0.258.3", - "wit-parser 0.258.3", -] - -[[package]] -name = "wit-component" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078def49fdb49cab405c3006e05793b3fdaf7069126ba6b9c97d01ec8c98fe4c" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "indexmap 2.14.2", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder 0.262.0", - "wasm-metadata 0.262.0", - "wasmparser 0.262.0", - "wit-parser 0.262.0", -] - -[[package]] -name = "wit-parser" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2127663dcf3ee16614fb0e3c33d5c6c66fec9dad317a8b0fdd3869487a533" -dependencies = [ - "anyhow", - "hashbrown 0.17.1", - "id-arena", - "indexmap 2.14.2", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-ident", - "wasmparser 0.258.3", -] - -[[package]] -name = "wit-parser" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f0dc051466ffe46787721fca3d088536f158ae400be044f1b1c386ba9397ad" -dependencies = [ - "anyhow", - "hashbrown 0.17.1", - "id-arena", - "indexmap 2.14.2", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-ident", - "wasmparser 0.262.0", -] - [[package]] name = "writeable" version = "0.6.4" @@ -6694,16 +5746,6 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" -[[package]] -name = "yaml-rust2" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e5b818a27a4cd30884ea380857a5e56f7ec3ba24a3990a3cc0b95af3238e18" -dependencies = [ - "arraydeque", - "hashlink 0.12.2", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index a2fdd9ed..585ee8ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ members = [ "crates/format-oci", "crates/server", "crates/e2e", - "plugins/helm", "plugins/nuget", "xtask", ] @@ -184,33 +183,6 @@ tempfile = { version = "3" } # Date/time (shared with forge-db's model types) chrono = { version = "0.4", default-features = false, features = ["clock"] } -# WebAssembly runtime for format plugins (forge-plugin). 49 is a regular -# Wasmtime release: it gets security fixes for two months, an LTS release -# (every 12th, like 48) for two years, so the next major is due within two -# months. wasmtime-wasi of a major builds only against the wasmtime of the same -# major: bump both together (Dependabot groups them). Only what the host uses: -# the component model compiled by Cranelift, no WAT text parser, no GC -# proposals, no profiling or debugging. -wasmtime = { version = "49.0.4", default-features = false, features = [ - "runtime", - "cranelift", - "component-model", - "std", - "parallel-compilation", -] } -# WASI 0.2 for plugin components (a Rust guest's std imports it); the host -# grants no filesystem, network or environment. -wasmtime-wasi = { version = "49.0.4", default-features = false, features = ["p2"] } - -# Format plugins built as WebAssembly components (`cargo xtask plugins`): -# small, self-contained `.wasm` files. -[profile.plugin] -inherits = "release" -opt-level = "s" -lto = true -codegen-units = 1 -strip = true - # Tests build unoptimized, yet the mock IdPs generate RSA keys and every test # server hashes its bootstrap admin password: ~30x (RSA-2048 keygen, ~3 s) and # ~15x (Argon2 default, ~0.3 s) slower than optimized. Optimize just those diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 798176bc..fa289034 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -9,9 +9,7 @@ Locally it runs as a single `forge-server` process backed by **PostgreSQL** ## Prerequisites - **Docker** (for Postgres + versitygw, and for the test suites). -- **Rust** stable toolchain (`rustup`, pinned in `rust-toolchain.toml`), with - the `wasm32-wasip2` target for the format plugins' component builds and the - tests that load them (`rustup target add wasm32-wasip2`). +- **Rust** stable toolchain (`rustup`, pinned in `rust-toolchain.toml`). - **Node 26 + pnpm 11** (for the web UI in `web/`; `corepack enable` activates the pinned `pnpm@11.5.0` from `web/package.json`. Node 25+ no longer bundles corepack: `npm install -g corepack` first). @@ -251,27 +249,16 @@ groups. See [docs/raw.md](docs/raw.md). ### Repositories of other formats (format plugins) -Every package format is a format plugin, a crate in -[`plugins/`](./plugins) (see [docs/format-plugins.md](./docs/format-plugins.md)), -whether it runs built in only, as a WebAssembly component, or both: NuGet runs -built in only. `plugins/helm`, classic Helm chart repositories, is the example -plugin the plugin tests run; it is not built in (Helm charts are pushed to -docker repositories, see [docs/docker.md](docs/docker.md#helm-charts)). A new -format goes in `plugins/` too, never in a `crates/format-` crate or the -server. Docker / OCI (`crates/format-oci`) and raw (`crates/server/src/raw/`) -are not there yet and are to move to `plugins/docker` and `plugins/raw`. - -To run a plugin as a WebAssembly component: - -```bash -cargo xtask plugins # builds plugins/* as components, prints the [plugins] section for forge.toml -``` - -A repository of a plugin format is created with `"format": ""` (hosted, -proxy or group; a plugin format has proxies when it implements the upstream -side) and served under `/repository//` unless the plugin names another -mount. A built-in plugin's component loads only under another id (rename its -installed directory and the `id` in its `plugin.toml`). +Every package format is a format plugin, a crate in [`plugins/`](./plugins) +linked into the server (see [docs/format-plugins.md](./docs/format-plugins.md)); +NuGet is the one there is. A new format goes in `plugins/` too, never in a +`crates/format-` crate or the server. Docker / OCI (`crates/format-oci`) +and raw (`crates/server/src/raw/`) are not there yet and are to move to +`plugins/docker` and `plugins/raw`. A repository of a plugin format is created +with `"format": ""` (hosted, proxy or group; a plugin format has proxies +when it implements the upstream side) and served under `/repository//` +unless the plugin names another mount (NuGet's is `/v3`). Helm charts are +pushed to docker repositories, see [docs/docker.md](docs/docker.md#helm-charts). ## Configuration @@ -613,14 +600,8 @@ way (`mirror.gcr.io/library/` for an official image). cargo xtask test # unit + integration (cargo test --workspace) cargo xtask e2e # end-to-end suite (cargo test -p forge-e2e --features forge-e2e) cargo xtask check # cargo fmt --check + clippy -D warnings -cargo xtask plugins-check # the format plugins' component builds: clippy and build ``` -The format-plugin tests (`crates/plugin/tests`, `crates/server/tests/plugins.rs`) -run the example plugin, `plugins/helm`: built in for the test, and built as a -component themselves into `target/plugins/`; they need the `wasm32-wasip2` -target. - The e2e suite includes an optional `dotnet` smoke test that is skipped when the .NET SDK isn't installed. @@ -750,9 +731,6 @@ server hashes its bootstrap admin password. `clippy` · `build` · `prepare` (regenerate sqlx offline data) · `psql` · `logs` · `ps` · `clean`. -Format plugins as components: `plugins` (build + install, prints the config) · -`plugins-check` (clippy and build). - Web UI: `web` (build) · `web-install` · `web-dev` (Vite HMR) · `web-test` (Vitest) · `web-e2e` (Playwright). @@ -840,7 +818,7 @@ backups, upgrades) follow [docs/deployment.md](docs/deployment.md). ## CI and the merge queue CI (`.github/workflows/ci.yml`) has the jobs `fmt`, `clippy`, `audit`, -`plugins`, `test`, `docker`, `kubernetes`, `web` and `oci-e2e`, and the slow +`test`, `docker`, `kubernetes`, `web`, `oci-e2e` and `raw-e2e`, and the slow `e2e` and `web-e2e`, which pull requests skip: those two run in the merge queue and on every push to `main`. It runs on pull requests that target `main` only; a stacked one waits (see [Stacked pull @@ -874,7 +852,7 @@ It sorts the change into groups of paths: A change to `ci-changes.sh` counts for every group. A change to `ci.yml` counts for the groups whose jobs it changes (#365): `fmt` is in Changelog; -`clippy`, `plugins`, `test`, `oci-e2e`, `raw-e2e` and `e2e` in Rust; `audit` +`clippy`, `test`, `oci-e2e`, `raw-e2e` and `e2e` in Rust; `audit` in Audit; `web` and `web-e2e` in Web; `docker` in Image and `kubernetes` in Kubernetes. A change outside the jobs (the triggers, the settings, the `changes` job) or to a job missing from `CI_JOB_GROUP` in `ci-changes.sh` @@ -920,8 +898,8 @@ To turn it on, edit the ruleset (or branch protection rule) of `main`: 1. Enable **Require merge queue**. Keep its **status check timeout** above the run time of `e2e` and `web-e2e` (60 minutes by default). 2. Under **Require status checks to pass**, require `fmt`, `clippy`, `audit`, - `plugins`, `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, - `e2e` and `web-e2e` from GitHub Actions: every job of `ci.yml` but + `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, `e2e` and + `web-e2e` from GitHub Actions: every job of `ci.yml` but `changes`, the jobs the release workflow requires of the tagged commit (`REQUIRED_JOBS` in `.github/workflows/release.yml`). A job left out here can fail on a pull request that merges anyway, and the release then diff --git a/README.md b/README.md index 76ef9aec..12cbdb70 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ without stored API keys through Forge's **direct-token OIDC trusted-publishing** flow for GitHub Actions, GitLab CI, and compatible workload issuers; its bearer token, cache, and audit limitations are documented below. A **retention** engine reaps cold versions and garbage-collects dereferenced -blobs. **Format plugins** (preview) add further package formats that reuse all -of the above, either built into the server or loaded as sandboxed WebAssembly -components ([docs/format-plugins.md](docs/format-plugins.md)). The admin + browse UI is +blobs. **Format plugins** (preview), crates built into the server, add +further package formats that reuse all of the above +([docs/format-plugins.md](docs/format-plugins.md)). The admin + browse UI is a **TanStack Router SPA** served by the same Rust process on a single origin — there is no Node server in production. @@ -51,19 +51,18 @@ A Cargo workspace of small crates plus the web SPA: | `cache` | `forge-cache` | In-process L1 + optional Redis L2 with cross-instance invalidation (no Postgres advisory locks). | | `core` | `forge-core` | Domain logic: repo dispatch, auth chain, authz/RBAC, sessions, OIDC, retention. | | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | -| `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait, its WIT twin and the component glue. | -| `plugin` | `forge-plugin` | The host side of format plugins: built-in and WebAssembly plugins (Wasmtime) behind one trait. | +| `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | +| `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | | `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`), the Docker registry, raw file repositories (`/raw`), `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | -| `plugins/` | `forge-format-*` | Format plugins, built in or as WebAssembly components: `nuget` (built in only), and `helm`, the example plugin the plugin tests run (not built in). | +| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | **Every format is a plugin in `plugins/`.** A package format is a crate in -`plugins//` implementing `FormatPlugin`, whether it runs built in only, as -a WebAssembly component, or both. Docker / OCI (`crates/format-oci`) and raw -(in `forge-server`) are not there yet and are to move to `plugins/docker` and -`plugins/raw`. +`plugins//` implementing `FormatPlugin`, linked into the server. Docker / +OCI (`crates/format-oci`) and raw (in `forge-server`) are not there yet and +are to move to `plugins/docker` and `plugins/raw`. **Single-origin serving.** `forge-server` mounts each format plugin's feed (NuGet at `/v3`, others at `/repository`), the Docker / OCI registry (`/v2`), @@ -212,8 +211,7 @@ curl -u admin@example.com: http://localhost:8421/raw/binaries/tools/1.0/t workload identity for GitHub Actions, GitLab CI, and compatible OIDC issuers, and exchange a job's OIDC token for a short-lived API key (`POST /api/v2/token`). - **[Format plugins](./docs/format-plugins.md)** — further package formats, - built in or as sandboxed WebAssembly components: design, operation, and - writing one. + built into the server: design, operation, and writing one. - **[Importing from Nexus](./docs/nexus-import.md)** — copy a Nexus hosted repository (NuGet, Docker) into Forge, with its publish dates: dry run, conflicts, resuming, and the cutover. diff --git a/crates/core/src/net/origins.rs b/crates/core/src/net/origins.rs index 9b8f0496..5f016f1b 100644 --- a/crates/core/src/net/origins.rs +++ b/crates/core/src/net/origins.rs @@ -2,12 +2,11 @@ //! //! A proxy repository's format says which documents to fetch from its //! upstream, and where a version's package is -//! ([`crate::repo::proxy::ProxyRepo`]). The format may be a WebAssembly -//! plugin, code from elsewhere with no network access of its own: these -//! requests are its only reach beyond its sandbox, the host sends them from -//! Forge's network position, and the plugin reads the answers back into what -//! the proxy serves. So they go only where the upstream itself points -//! ([`UpstreamOrigins`]): +//! ([`crate::repo::proxy::ProxyRepo`]). The host sends these requests from +//! Forge's network position, and the format reads the answers back into what +//! the proxy serves, so a document an upstream serves must not be able to +//! steer them at an internal service. They go only where the upstream itself +//! points ([`UpstreamOrigins`]): //! //! * the origin (scheme, host, port) of the repository's `remote_url`; //! * an origin that an absolute `http`/`https` URL names in a document the diff --git a/crates/plugin-api/Cargo.toml b/crates/plugin-api/Cargo.toml index 53ccd221..17d55565 100644 --- a/crates/plugin-api/Cargo.toml +++ b/crates/plugin-api/Cargo.toml @@ -5,16 +5,9 @@ edition.workspace = true rust-version.workspace = true license.workspace = true publish.workspace = true -description = "The contract of a Forge format plugin: built in, or a WebAssembly component" +description = "The contract of a Forge format plugin, built into the server" -# Nothing but std: a plugin crate builds against this for the server and for -# `wasm32-wasip2` alike. +# Nothing but std. [features] -# `export_component!`: build the plugin crate as a WebAssembly component of -# the `format-plugin` world (`wit/format.wit`). -component = ["dep:wit-bindgen"] # `testing`: run a plugin's protocol against canned answers, without a server. testing = [] - -[dependencies] -wit-bindgen = { version = "0.63", optional = true, default-features = false, features = ["macros", "realloc", "std"] } diff --git a/crates/plugin-api/src/component.rs b/crates/plugin-api/src/component.rs deleted file mode 100644 index 680ee751..00000000 --- a/crates/plugin-api/src/component.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Building a plugin crate as a WebAssembly component (feature `component`). -//! -//! ```ignore -//! #[cfg(feature = "component")] -//! forge_plugin_api::export_component!(Helm); -//! ``` -//! -//! and build the crate as a `cdylib` for `wasm32-wasip2` with the feature on -//! (`cargo xtask plugins` does): the component exports the `format-plugin` -//! world of `wit/format.wit`, each function calling the plugin's -//! [`FormatPlugin`] method of the same name. - -use std::io::{self, Read, Seek, SeekFrom}; - -use crate::FormatPlugin; - -/// The guest bindings of `wit/format.wit`. -#[doc(hidden)] -pub mod bindings { - wit_bindgen::generate!({ - world: "format-plugin", - path: "wit", - pub_export_macro: true, - default_bindings_module: "forge_plugin_api::component::bindings", - }); -} - -crate::__wit_conversions!(crate::component::bindings); - -use bindings::exports::forge::format::{format, protocol, upstream}; -use bindings::forge::format::package_source::PackageReader; -use bindings::forge::format::types; - -/// The plugin a component exports (implemented by [`export_component!`]). -#[doc(hidden)] -pub trait Exported { - fn plugin() -> &'static dyn FormatPlugin; -} - -/// Export `$plugin` (an expression of a type implementing [`FormatPlugin`], -/// built once on the first call) as this component's `format-plugin` world. -#[macro_export] -macro_rules! export_component { - ($plugin:expr) => { - #[doc(hidden)] - pub struct __ForgeFormatComponent; - - impl $crate::component::Exported for __ForgeFormatComponent { - fn plugin() -> &'static dyn $crate::FormatPlugin { - static PLUGIN: ::std::sync::OnceLock<::std::boxed::Box> = - ::std::sync::OnceLock::new(); - PLUGIN - .get_or_init(|| ::std::boxed::Box::new($plugin)) - .as_ref() - } - } - - $crate::component::bindings::export!( - __ForgeFormatComponent with_types_in $crate::component::bindings - ); - }; -} - -/// The package reader the host lends, as [`Read`] and [`Seek`]. -struct Lent<'a> { - reader: &'a PackageReader, - size: u64, - offset: u64, -} - -impl Read for Lent<'_> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let len = u32::try_from(buf.len()).unwrap_or(u32::MAX); - let chunk = self - .reader - .read_at(self.offset, len) - .map_err(io::Error::other)?; - let n = chunk.len().min(buf.len()); - buf[..n].copy_from_slice(&chunk[..n]); - self.offset += n as u64; - Ok(n) - } -} - -impl Seek for Lent<'_> { - fn seek(&mut self, pos: SeekFrom) -> io::Result { - let target = match pos { - SeekFrom::Start(n) => Some(n), - SeekFrom::End(d) => self.size.checked_add_signed(d), - SeekFrom::Current(d) => self.offset.checked_add_signed(d), - }; - self.offset = target - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "seek before the start"))?; - Ok(self.offset) - } -} - -impl format::Guest for T { - fn canonical_id(id: String) -> String { - T::plugin().canonical_id(&id) - } - - fn validate_id(id: String) -> Result<(), types::FormatError> { - T::plugin().validate_id(&id).map_err(Into::into) - } - - fn normalize_version(version: String) -> Result { - T::plugin().normalize_version(&version).map_err(Into::into) - } - - fn compare_versions(a: String, b: String) -> i8 { - T::plugin().compare_versions(&a, &b) as i8 - } - - fn parse_package(source: &PackageReader) -> Result { - let mut lent = Lent { - reader: source, - size: source.size(), - offset: 0, - }; - T::plugin() - .parse_package(&mut lent) - .map(Into::into) - .map_err(Into::into) - } - - fn manifest_summary(manifest: Vec) -> Option { - T::plugin().manifest_summary(&manifest) - } -} - -impl protocol::Guest for T { - fn handle(repo: protocol::Repository, req: protocol::Request) -> protocol::Action { - T::plugin().handle(&repo.into(), &req.into()).into() - } - - fn render( - repo: protocol::Repository, - req: protocol::Request, - data: Vec, - ) -> protocol::Rendered { - let data: Vec = data.into_iter().map(Into::into).collect(); - T::plugin().render(&repo.into(), &req.into(), &data).into() - } -} - -impl upstream::Guest for T { - fn upstream( - remote: upstream::Remote, - op: upstream::UpstreamOp, - fetched: Vec, - ) -> Result { - let fetched: Vec = fetched.into_iter().map(Into::into).collect(); - T::plugin() - .upstream(&remote.into(), &op.into(), &fetched) - .map(Into::into) - .map_err(Into::into) - } -} diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 5e57f4f1..68c77a72 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -8,22 +8,11 @@ //! storage key, or a whole package in memory. //! //! A plugin is a crate under `plugins//` implementing [`FormatPlugin`], -//! with a `plugin.toml` naming it. The same crate runs either way: -//! -//! * **built in** — linked into the server and called directly (NuGet); -//! * **as a WebAssembly component** — built for `wasm32-wasip2` with this -//! crate's `component` feature and `export_component!`, then loaded at -//! startup into a sandbox, from a digest the operator pinned. -//! -//! The trait and its types mirror the `forge:format` WIT package -//! (`wit/format.wit`) one to one: a component exports exactly these -//! functions, and the host adapts a loaded component back into a -//! [`FormatPlugin`], so the server serves both kinds through one code path. +//! with a `plugin.toml` naming it, linked into the server and called +//! directly (NuGet). mod types; -#[cfg(feature = "component")] -pub mod component; #[cfg(feature = "testing")] pub mod testing; @@ -33,8 +22,8 @@ use std::path::Path; pub use types::*; -/// The `forge:format` WIT package version this crate implements: what a -/// plugin built against it declares as `api` in its `plugin.toml`. +/// The `forge:format` contract version this crate implements: what a plugin +/// built against it declares as `api` in its `plugin.toml`. pub const API: &str = "0.3.0"; /// The `plugin.toml` of the crate this is expanded in: how a built-in plugin @@ -91,8 +80,7 @@ pub trait FormatPlugin: Send + Sync + 'static { /// A JSON object summarizing `manifest`, stored with the version so that /// rendering never re-reads the manifest ([`PackageInfo::summary`]). /// `None` when there is nothing worth keeping. Every listing hands each - /// version's summary back, so keep it small: the host stores none larger - /// than 64 KiB from a WebAssembly plugin. + /// version's summary back, so keep it small. fn manifest_summary(&self, manifest: &[u8]) -> Option; /// What a request is: a response right away, data to load first (then @@ -130,214 +118,3 @@ pub trait FormatPlugin: Send + Sync + 'static { )) } } - -/// Conversions between the types of this crate and the bindings of -/// `wit/format.wit`, for both sides of the component boundary: the -/// `wit-bindgen` guest bindings (module `component`) and the host's Wasmtime -/// bindings. Both generators name every type, field and case the same, so -/// one definition serves both. Expanded once per bindings module: -/// `forge_plugin_api::__wit_conversions!(crate::bindings);`. -#[doc(hidden)] -#[macro_export] -macro_rules! __wit_conversions { - ($($root:tt)*) => { - const _: () = { - use $crate as api; - use $crate::__private::{into, opt, vec}; - use $($root)*::forge::format::types as t; - use $($root)*::exports::forge::format::format as f; - use $($root)*::exports::forge::format::protocol as p; - use $($root)*::exports::forge::format::upstream as u; - - $crate::__record!(t, Dependency { id: into, range: into }); - $crate::__record!(t, DependencyGroup { target: into, dependencies: vec }); - $crate::__variant!(t, FormatError {} { - InvalidPackage: into, - MissingManifest: into, - MalformedManifest: into, - InvalidVersion: into, - InvalidId: into, - Internal: into, - }); - $crate::__record!(f, ParsedPackage { - id: into, - original_id: into, - version: into, - version_original: into, - listed: into, - manifest: into, - dependencies: vec, - }); - $crate::__variant!(p, RepositoryKind { Hosted, Proxy, Group } {}); - $crate::__record!(p, Repository { name: into, kind: into, base_url: into }); - $crate::__record!(p, Request { method: into, path: into, query: into, headers: into }); - $crate::__record!(p, Response { status: into, headers: into, body: into }); - $crate::__record!(p, PackageRef { id: into, version: into }); - $crate::__record!(p, PackageVersions { id: into, versions: into }); - $crate::__record!(p, SearchQuery { - text: into, - skip: into, - take: into, - prerelease: into, - semver2: into, - upstream: into, - }); - $crate::__variant!(p, Query {} { - Versions: into, - Packages: into, - Package: into, - PackagesOf: into, - Manifest: into, - Search: into, - }); - $crate::__record!(p, VersionInfo { - original_id: into, - version: into, - version_original: into, - listed: into, - }); - $crate::__record!(p, PackageInfo { - original_id: into, - version: into, - version_original: into, - listed: into, - published: into, - sha256: into, - sha512: into, - size: into, - summary: into, - }); - $crate::__record!(p, SearchHit { - original_id: into, - latest_version: into, - versions: vec, - summary: into, - total_downloads: into, - }); - $crate::__record!(p, SearchResult { total: into, hits: vec }); - $crate::__variant!(p, Answer {} { - Versions: vec, - Packages: vec, - Package: opt, - Manifest: into, - Search: into, - }); - $crate::__record!(p, Loaded { query: into, answer: into }); - $crate::__variant!(p, Upload { Body, FirstPart } { MultipartField: into }); - $crate::__record!(p, PublishRequest { upload: into, done: into }); - $crate::__record!(p, ChangeRequest { target: into, done: into }); - $crate::__variant!(p, Action {} { - Respond: into, - Load: vec, - Download: into, - Publish: into, - Unlist: into, - Relist: into, - Delete: into, - }); - $crate::__variant!(p, Rendered {} { Respond: into, Load: vec }); - $crate::__record!(u, Remote { url: into, index: into }); - $crate::__variant!(u, UpstreamOp { Index } { - Versions: into, - Package: into, - Manifest: into, - Metadata: into, - Dating: into, - Search: into, - }); - $crate::__variant!(u, Freshness { Metadata, Immutable } {}); - $crate::__record!(u, Fetch { url: into, freshness: into }); - $crate::__variant!(u, Fetched { NotFound } { Found: into, Failed: into }); - $crate::__record!(u, FetchResult { fetch: into, outcome: into }); - $crate::__record!(u, VersionMetadata { - version: into, - original_id: into, - listed: into, - published: into, - created: into, - summary: into, - }); - $crate::__record!(u, PackageMetadata { original_id: into, versions: vec }); - $crate::__record!(u, UpstreamHit { - id: into, - versions: into, - summary: into, - total_downloads: into, - }); - $crate::__record!(u, UpstreamSearch { total: into, hits: vec }); - $crate::__variant!(u, UpstreamAnswer {} { - Index: into, - Versions: into, - Package: into, - Manifest: into, - Metadata: into, - Search: opt, - Failed: into, - }); - $crate::__variant!(u, UpstreamStep {} { Fetch: vec, Done: into }); - }; - }; -} - -/// `From` both ways between `api::$ty` and `$m::$ty`, a record. -#[doc(hidden)] -#[macro_export] -macro_rules! __record { - ($m:ident, $ty:ident { $($field:ident: $how:ident),* $(,)? }) => { - impl From for $m::$ty { - fn from(x: api::$ty) -> Self { - $m::$ty { $($field: $how(x.$field)),* } - } - } - impl From<$m::$ty> for api::$ty { - fn from(x: $m::$ty) -> Self { - api::$ty { $($field: $how(x.$field)),* } - } - } - }; -} - -/// `From` both ways between `api::$ty` and `$m::$ty`, a variant: its cases -/// without a payload, then its cases with one. -#[doc(hidden)] -#[macro_export] -macro_rules! __variant { - ($m:ident, $ty:ident { $($unit:ident),* $(,)? } { $($case:ident: $how:ident),* $(,)? }) => { - impl From for $m::$ty { - fn from(x: api::$ty) -> Self { - match x { - $(api::$ty::$unit => $m::$ty::$unit,)* - $(api::$ty::$case(v) => $m::$ty::$case($how(v)),)* - } - } - } - impl From<$m::$ty> for api::$ty { - fn from(x: $m::$ty) -> Self { - match x { - $($m::$ty::$unit => api::$ty::$unit,)* - $($m::$ty::$case(v) => api::$ty::$case($how(v)),)* - } - } - } - }; -} - -#[doc(hidden)] -pub mod __private { - //! What [`__wit_conversions!`](crate::__wit_conversions) expands to. - - /// A field or payload converted as a whole. - pub fn into>(a: A) -> B { - B::from(a) - } - - /// A list converted element by element. - pub fn vec>(v: Vec) -> Vec { - v.into_iter().map(B::from).collect() - } - - /// An option converted inside. - pub fn opt>(o: Option) -> Option { - o.map(B::from) - } -} diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 437fb43c..30c8d5af 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -1,5 +1,4 @@ -//! The data a plugin and the host exchange: the records and variants of -//! `wit/format.wit`, in Rust. +//! The data a plugin and the host exchange. use std::fmt; diff --git a/crates/plugin-api/wit/format.wit b/crates/plugin-api/wit/format.wit deleted file mode 100644 index 95f9b46c..00000000 --- a/crates/plugin-api/wit/format.wit +++ /dev/null @@ -1,482 +0,0 @@ -// The contract between Forge and a format plugin: what teaches Forge one more -// package format. -// -// A plugin is a Rust crate implementing `forge_plugin_api::FormatPlugin`, the -// same contract in Rust (crates/plugin-api/src/lib.rs). Forge links its -// built-in plugins natively; any plugin can also be built as a WebAssembly -// component of the `format-plugin` world below and loaded at startup. -// -// The division of labor is fixed: the host owns everything a repository is — -// storage (the content-addressed store), metadata (Postgres), authentication, -// authorization and package selectors, hosted/group semantics, retention, -// streaming of package bytes in both directions. The plugin owns what is -// specific to its format: the identity rules (ids, versions), reading a -// package's manifest, and the client protocol (which URL answers what, and how -// a response looks). -// -// A plugin never sees credentials, never touches storage, and never holds a -// whole package in memory: the host stages uploads on its disk and lends the -// plugin a reader, and the host streams downloads itself. -// -// Versioning: a plugin is built against one version of this package and -// declares it in its plugin.toml (`api`). A host supports a fixed set of -// versions; changing anything in this file means a new package version. -package forge:format@0.3.0; - -/// Types shared by the interfaces below. -interface types { - /// A dependency on another package of the same format. - record dependency { - /// The package id, as written. - id: string, - /// The version range in the format's syntax; none for any version. - range: option, - } - - /// The dependencies that apply to one target (a framework, a platform); - /// a format without targets uses a single group without one. - record dependency-group { - target: option, - dependencies: list, - } - - /// Why a package, an id or a version was refused. The text is shown to - /// the client that sent it. - variant format-error { - invalid-package(string), - missing-manifest(string), - malformed-manifest(string), - invalid-version(string), - invalid-id(string), - /// The plugin itself failed (a bug, an exhausted resource): the - /// client gets a server error, not this text. - internal(string), - } -} - -/// Read access to a package the host staged on its local disk. -interface package-source { - resource package-reader { - /// The package's size in bytes. - size: func() -> u64; - /// Up to `len` bytes starting at `offset`: fewer only at the end of - /// the package, none at or past it. `len` is clamped to 1 MiB. - read-at: func(offset: u64, len: u32) -> result, string>; - } -} - -/// The format's identity rules and package parsing. Every function is pure: -/// the host may call them in any order, on any instance, any number of times. -interface format { - use types.{dependency-group, format-error}; - use package-source.{package-reader}; - - /// What a package declares about itself. - record parsed-package { - /// The canonical id: what `canonical-id` returns for `original-id`. - id: string, - /// The id exactly as the package writes it. - original-id: string, - /// The normalized version (what `normalize-version` returns). - version: string, - /// The version exactly as the package writes it. - version-original: string, - /// Whether the version shows up in searches and as "latest". - listed: bool, - /// The manifest (e.g. Chart.yaml, package.json), stored next to the - /// package and handed back through the `manifest` query. Keep it - /// small: it is held in memory. - manifest: list, - dependencies: list, - } - - /// The canonical form of a package id, used for lookups and uniqueness - /// (e.g. lowercased). - canonical-id: func(id: string) -> string; - - /// Whether `id` (canonical) is a well-formed id. The host treats a query - /// or an action naming any other id like one naming an unknown package - /// (an empty answer, a 404). - validate-id: func(id: string) -> result<_, format-error>; - - /// The normalized form of a version, used for lookups and uniqueness. - /// Prerelease versions must contain a `-` (the host filters on it). - normalize-version: func(version: string) -> result; - - /// The order of two normalized versions: negative when `a` sorts before - /// `b`, zero when equal, positive when after. - compare-versions: func(a: string, b: string) -> s8; - - /// Read a package's identity, manifest and dependencies. - parse-package: func(source: borrow) -> result; - - /// A JSON object summarizing `manifest`, stored with the version so that - /// rendering never re-reads the manifest (the `summary` of package-info). - /// None when there is nothing worth keeping. - manifest-summary: func(manifest: list) -> option; -} - -/// The client protocol of the format. -/// -/// The host has authenticated the caller and checked that it may perform the -/// request's action on the repository (GET/HEAD read, PUT/POST write, DELETE -/// delete) before `handle` is called. `handle` decides what the request is: -/// a response it can give right away, data it needs first (then `render` is -/// called with that data), or an action only the host can perform (serve a -/// package, publish one, unlist, relist or delete a version). -interface protocol { - /// How the repository stores its packages. - enum repository-kind { - hosted, - proxy, - group, - } - - /// The repository a request is for. - record repository { - name: string, - kind: repository-kind, - /// The absolute URL the repository is served at, without a trailing - /// slash (e.g. https://forge.example.com/repository/charts). - base-url: string, - } - - /// An HTTP request from a package client. - record request { - /// The method, upper case. - method: string, - /// The path below `base-url`, starting with "/", still - /// percent-encoded as the client sent it. - path: string, - /// The query string without its "?", empty when there is none. - query: string, - /// The request headers the host passes on (lowercase names): accept, - /// accept-language, content-type, if-match, if-modified-since, - /// if-none-match, user-agent. Never credentials or cookies. - headers: list>, - } - - /// An HTTP response. The host keeps only the headers it allows (content - /// type, caching and validators, content-disposition, location) and adds - /// its own security headers. - record response { - status: u16, - headers: list>, - body: list, - } - - /// One version of one package (canonical id, normalized version). - record package-ref { - id: string, - version: string, - } - - /// Some versions of one package. - record package-versions { - id: string, - versions: list, - } - - record search-query { - /// Free text; empty matches every package. `packageid:x` and `id:x` - /// match an id exactly; several terms must all match. - text: string, - skip: u32, - /// The page size: 0 means the default (20), more than 1000 is 1000. - take: u32, - /// Include prerelease versions. - prerelease: bool, - /// Include SemVer 2.0.0 versions (dotted prerelease labels, build - /// metadata). - semver2: bool, - /// Let a proxy repository forward the search to its upstream, so - /// packages it has not cached yet are found too. - upstream: bool, - } - - /// Data the plugin can ask for. Package ids are canonical, versions - /// normalized. Everything the caller may not read (package selectors, - /// group members it cannot see) is left out, exactly as for an unknown - /// package. - variant query { - /// Every version of a package, ascending. - versions(string), - /// Every version of a package with its metadata, ascending. - packages(string), - /// One version with its metadata. - %package(package-ref), - /// Some versions of a package with their metadata, ascending; - /// versions it does not have are left out. - packages-of(package-versions), - /// The manifest stored with one version. - manifest(package-ref), - /// The packages matching a search. - search(search-query), - } - - record version-info { - /// The id as the package writes it. - original-id: string, - version: string, - version-original: string, - listed: bool, - } - - record package-info { - original-id: string, - version: string, - version-original: string, - listed: bool, - /// When the version was published into the repository: RFC 3339 in - /// UTC with milliseconds (2026-10-21T12:00:00.000+00:00). - published: option, - /// The package's SHA-256, lowercase hex. - sha256: option, - /// The package's SHA-512, base64. - sha512: option, - /// The package's size in bytes. - size: option, - /// What `manifest-summary` returned at publish. - summary: option, - } - - record search-hit { - original-id: string, - latest-version: string, - /// Ascending. - versions: list, - /// The summary of the latest version. - summary: option, - /// Downloads of the package, all versions, when the repository counts - /// them. - total-downloads: option, - } - - record search-result { - /// Matching packages before paging. - total: u32, - hits: list, - } - - variant answer { - versions(list), - /// The answer to `packages` and to `packages-of`. - packages(list), - %package(option), - manifest(option>), - search(search-result), - } - - /// A query with its answer. - record loaded { - query: query, - answer: answer, - } - - /// Where a publish request carries the package. - variant upload { - /// The request body is the package. - body, - /// The package is the multipart/form-data part of this name. - multipart-field(string), - /// The package is the first part of a multipart/form-data body, - /// whatever its name. - first-part, - } - - /// A publish, unlist, relist or delete, and the response when it succeeds. - /// When it fails the host answers with its own status (400 invalid - /// package, 403 not allowed, 404 unknown version, 409 version exists, - /// 413 too large). - record publish-request { - upload: upload, - done: response, - } - - record change-request { - target: package-ref, - done: response, - } - - variant action { - /// Answer with this response. - respond(response), - /// Load this data, then call `render` with it. - load(list), - /// Serve the stored package of this version, or 404 when the caller - /// cannot read it. The host streams it and counts the download. - download(package-ref), - /// Publish the package the request carries (hosted repositories, - /// write permission). The host parses it with `parse-package`. - publish(publish-request), - /// Hide a version from searches and "latest" (delete permission). - unlist(change-request), - /// Show an unlisted version again (write permission). - relist(change-request), - /// Remove a version and its bytes (delete permission). - delete(change-request), - } - - variant rendered { - /// Answer with this response. - respond(response), - /// Load more data, then call `render` again with everything loaded - /// so far (the host bounds the rounds and the number of queries). - load(list), - } - - handle: func(repo: repository, req: request) -> action; - - render: func(repo: repository, req: request, data: list) -> rendered; -} - -/// The upstream side of the format's proxy repositories. -/// -/// The host performs every request: it sends the repository's credentials -/// only to the upstream's own origin, bounds each response's size and time, -/// inflates gzip-encoded bodies, refuses link-local and cloud-metadata -/// targets, and caches what it fetched as the plugin says. It also keeps -/// everything else a proxy is: downloading and verifying packages (with -/// `parse-package`), the package delay, the not-found markers, the offline -/// mode. The plugin says what to fetch and reads what came back, so the -/// conversation is stateless: every call gets everything fetched so far. -interface upstream { - use types.{format-error}; - use protocol.{package-ref, package-versions, search-query}; - - /// The upstream a proxy repository mirrors. - record remote { - /// The upstream's URL as the repository is configured with it. - url: string, - /// What `index` answered for it (a JSON object of the plugin's own - /// making); none while the index is being resolved. - index: option, - } - - /// What the host wants to know about the upstream. - variant upstream-op { - /// Resolve the upstream into the index every other operation gets. - /// The host keeps it, refreshes it once per metadata TTL and falls - /// back to the last good one when a refresh fails. - index, - /// The versions the upstream lists of a package (canonical id). - versions(string), - /// The URL of a version's package. The host downloads it, checks it - /// with `parse-package` against the version asked for, and stores it. - %package(package-ref), - /// A version's manifest without its package, verified to be that - /// version's; none sends the host to the package instead. - manifest(package-ref), - /// The upstream's metadata of some versions (listing state, publish - /// time, manifest summary), best effort. - metadata(package-versions), - /// `metadata`, plus when each version was created upstream: the - /// package delay counts from it. It may cost more requests. - dating(package-versions), - /// A page of the upstream's search results. - search(search-query), - } - - /// How long the host may reuse a fetched document. - enum freshness { - /// For the repository's metadata TTL: listings, indexes, search pages. - metadata, - /// For a day: a document that never changes once published. - immutable, - } - - /// A GET the plugin asks the host to send to the upstream. - record fetch { - url: string, - freshness: freshness, - } - - variant fetched { - /// A 2xx, with its body. - found(list), - /// A 404. - not-found, - /// Anything else: a transport error, another status, a body over the - /// size limit. - failed(string), - } - - record fetch-result { - fetch: fetch, - outcome: fetched, - } - - record version-metadata { - /// Normalized. - version: string, - /// The id as this version spells it, when the upstream says. - original-id: option, - listed: bool, - /// When the upstream published the version: RFC 3339, shown to - /// clients. - published: option, - /// When the version was created upstream, RFC 3339 (`dating` only). - created: option, - /// A manifest summary for a version the proxy has not downloaded. - summary: option, - } - - record package-metadata { - /// The id as the upstream spells the package (its newest version's). - original-id: option, - versions: list, - } - - record upstream-hit { - /// The id as the upstream spells it. - id: string, - /// Normalized, ascending. - versions: list, - /// A manifest summary of the latest version. - summary: option, - total-downloads: option, - } - - record upstream-search { - /// Matching packages before paging, as the upstream counts them. - total: u64, - hits: list, - } - - variant upstream-answer { - /// The answer to `index`: a JSON object. - index(string), - /// None: the upstream does not know the package. - versions(option>), - /// The package's URL. - %package(string), - manifest(option>), - /// The answer to `metadata` and `dating`. - metadata(package-metadata), - /// None: the upstream offers no search. - search(option), - /// The upstream answered something unusable: the host serves what - /// it has cached. - failed(string), - } - - variant upstream-step { - /// Fetch these, then call `upstream` again with everything fetched - /// so far (the host bounds the rounds and the number of fetches). - fetch(list), - done(upstream-answer), - } - - upstream: func(remote: remote, op: upstream-op, fetched: list) -> result; -} - -/// What a format plugin is: it exports the format, its protocol and its -/// upstream side, and imports nothing but the package reader (and WASI, -/// which the host provides without any file system, network, environment or -/// arguments). -world format-plugin { - import package-source; - export format; - export protocol; - export upstream; -} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index 1cb7be01..087ad2da 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -6,11 +6,6 @@ rust-version.workspace = true license.workspace = true publish.workspace = true -[features] -# `testing::example_plugin`: builds a plugin of `plugins/` as a component for -# tests. -testing = [] - [dependencies] forge-core = { path = "../core" } forge-plugin-api = { path = "../plugin-api" } @@ -19,20 +14,4 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } -sha2 = { workspace = true } -hex = { workspace = true } toml = "1.1" -# Spools a package handed over as a reader into a file the plugin can be lent. -tempfile = { workspace = true } - -wasmtime = { workspace = true } -wasmtime-wasi = { workspace = true } - -[dev-dependencies] -forge-plugin = { path = ".", features = ["testing"] } -# The Helm example plugin built in, to compare with its component. -forge-format-helm = { path = "../../plugins/helm" } -flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } -# Hand-built components for the sandbox tests: the version Wasmtime itself -# depends on, so bump it with `wasmtime`. -wasm-encoder = "0.258.3" diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index 350cc930..f4483cc2 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -19,38 +19,22 @@ use crate::Plugin; /// larger than about 2,700 bytes: a plugin without a limit of its own let a /// multi-kilobyte version through to that insert, so a push answered 500 /// instead of 400, and so did a proxy asked for such a version. 256 is above -/// the plugins' own limits (NuGet's 64 characters, Helm's 128) and, +/// NuGet's own limit (64 characters) and, /// with [`MAX_ID_LENGTH`], keeps an entry far below PostgreSQL's. const MAX_VERSION_LENGTH: usize = 256; /// The longest package id the host takes from a format plugin, in bytes: /// an id to validate, and a parsed package's id as written and canonical. -/// Ids share the indexes of [`MAX_VERSION_LENGTH`]. 512 is above the -/// plugins' own limits (Helm's 250 ASCII characters, NuGet's 100 -/// characters, at most 400 bytes in UTF-8). +/// Ids share the indexes of [`MAX_VERSION_LENGTH`]. 512 is above NuGet's +/// own limit (100 characters, at most 400 bytes in UTF-8). const MAX_ID_LENGTH: usize = 512; -/// The largest manifest summary, as JSON text, stored from a WebAssembly -/// plugin. Every listing hands each version's summary to the plugin, so a -/// plugin that summarizes without a bound lets a publisher make every -/// listing take gigabytes. Several times the 16 KiB a Helm chart's summary -/// is cut down to, and many times a typical summary. -/// -/// A built-in plugin is Forge's own code and bounds its own summaries, or -/// needs them whole: NuGet's carries a package's dependency groups, and a -/// version served without them would install without its dependencies. -const MAX_SUMMARY: usize = 64 * 1024; - -/// A format plugin, built in or WebAssembly, as a [`Format`]: what the -/// hosted write path, the read ops, the proxy read path and retention work -/// with. +/// A format plugin as a [`Format`]: what the hosted write path, the read +/// ops, the proxy read path and retention work with. pub struct PluginFormat { id: &'static str, plugin: Arc, assets: Assets, - /// The largest manifest summary stored: [`MAX_SUMMARY`] for a - /// WebAssembly plugin, none for a built-in one. - max_summary: Option, } impl PluginFormat { @@ -60,7 +44,6 @@ impl PluginFormat { id: plugin.id(), plugin: plugin.implementation().clone(), assets: plugin.manifest().assets.clone(), - max_summary: (!plugin.is_builtin()).then_some(MAX_SUMMARY), } } } @@ -215,15 +198,6 @@ impl Format for PluginFormat { fn manifest_summary(&self, manifest: &[u8]) -> Option { let text = self.plugin.manifest_summary(manifest)?; - if let Some(max) = self.max_summary.filter(|max| text.len() > *max) { - tracing::warn!( - plugin = self.id, - bytes = text.len(), - max, - "manifest-summary returned more than the host keeps; not stored" - ); - return None; - } match serde_json::from_str::(&text) { Ok(summary @ serde_json::Value::Object(_)) => Some(summary), _ => { @@ -240,7 +214,6 @@ impl Format for PluginFormat { #[cfg(test)] mod tests { use super::*; - use crate::{Manifest, Origin}; /// A format without limits of its own: every id is valid, every version /// normalizes to `normalized` (or to itself), and every package parses @@ -386,83 +359,4 @@ mod tests { assert!(matches!(parse(parsed), Err(FormatError::InvalidPackage(_)))); } } - - /// A plugin whose summary of a manifest is `{"description": }`. - struct Describes; - - impl FormatPlugin for Describes { - fn canonical_id(&self, id: &str) -> String { - id.to_string() - } - - fn validate_id(&self, _id: &str) -> Result<(), api::FormatError> { - Ok(()) - } - - fn normalize_version(&self, version: &str) -> Result { - Ok(version.to_string()) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - a.cmp(b) - } - - fn parse_package( - &self, - _package: &mut dyn api::PackageSource, - ) -> Result { - Err(api::FormatError::InvalidPackage("not a package".into())) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - Some( - serde_json::json!({ "description": String::from_utf8_lossy(manifest) }).to_string(), - ) - } - - fn handle(&self, _repo: &api::Repository, _req: &api::Request) -> api::Action { - api::Action::Respond(api::Response::text(404, "not found")) - } - - fn render( - &self, - _repo: &api::Repository, - _req: &api::Request, - _data: &[api::Loaded], - ) -> api::Rendered { - api::Rendered::Respond(api::Response::text(404, "not found")) - } - } - - fn plugin_toml() -> String { - format!( - "[plugin]\nid = \"describes\"\nname = \"Describes\"\nversion = \"1.0.0\"\n\ - api = \"{}\"\ncomponent = \"describes.wasm\"\n", - api::API - ) - } - - #[test] - fn a_webassembly_plugins_oversized_summary_is_not_stored() { - let manifest = Manifest::parse(&plugin_toml()).unwrap(); - let wasm = Plugin::new( - manifest, - Origin::Wasm { - sha256: "0".repeat(64), - }, - Arc::new(Describes), - ); - let format = PluginFormat::new(&wasm); - // `{"description":"…"}` takes 18 bytes more than the description. - let fits = "x".repeat(MAX_SUMMARY - 18); - let summary = format.manifest_summary(fits.as_bytes()).unwrap(); - assert_eq!(summary["description"], serde_json::json!(fits)); - let over = "x".repeat(MAX_SUMMARY - 17); - assert_eq!(format.manifest_summary(over.as_bytes()), None); - - // A built-in plugin bounds its own. - let builtin = Plugin::builtin(&plugin_toml(), Describes).unwrap(); - let format = PluginFormat::new(&builtin); - assert!(format.manifest_summary(over.as_bytes()).is_some()); - } } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index b5a31cf2..5e67b9b4 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -6,70 +6,28 @@ //! storage, metadata, authentication and authorization, hosted/group //! semantics, retention, the streaming of package bytes. //! -//! A [`Plugin`] is one installed plugin, of either kind: -//! -//! * **built in** ([`Plugin::builtin`]): a crate under `plugins//` -//! linked into the server, called directly; -//! * **WebAssembly** ([`Plugin::load`]): the same kind of crate built as a -//! component, loaded from a plugin directory against the SHA-256 the -//! operator pinned, and run in a sandbox ([`WasmPlugin`], which implements -//! [`FormatPlugin`] by calling into the component). -//! -//! Either way the server holds an `Arc` and cannot tell the -//! two apart. [`PluginFormat`] adapts a plugin to [`forge_core::Format`], so -//! hosted publishing, reads, groups and retention work for it unchanged. +//! A [`Plugin`] is one installed plugin: a crate under `plugins//` linked +//! into the server ([`Plugin::builtin`]) and called directly. The server holds +//! it as an `Arc`; [`PluginFormat`] adapts it to +//! [`forge_core::Format`], so hosted publishing, reads, groups and retention +//! work for it unchanged. mod format; pub mod manifest; -#[cfg(feature = "testing")] -pub mod testing; -pub mod wasm; -use std::path::Path; use std::sync::Arc; pub use forge_plugin_api as api; pub use forge_plugin_api::FormatPlugin; pub use format::PluginFormat; pub use manifest::Manifest; -pub use wasm::{HostLimits, Limits, WasmPlugin}; -/// Errors loading a plugin. +/// Errors installing a plugin. #[derive(Debug, thiserror::Error)] pub enum PluginError { - /// The plugin directory or component could not be read. - #[error("{0}")] - Io(String), /// `plugin.toml` is missing a key, malformed, or asks for too much. #[error("invalid plugin.toml: {0}")] Manifest(String), - /// The component's SHA-256 is not the pinned one. - #[error("component digest {actual} does not match the pinned {expected}")] - Digest { - /// The pinned digest. - expected: String, - /// The component's digest. - actual: String, - }, - /// The component does not compile or does not implement the world. - #[error("invalid component: {0}")] - Component(String), - /// A call into a component trapped, ran out of fuel or time, or its - /// instance could not be created. - #[error("plugin call failed: {0}")] - Trap(String), -} - -/// Where a plugin's code comes from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Origin { - /// Linked into the server. - Builtin, - /// A WebAssembly component. - Wasm { - /// The component's SHA-256 (lowercase hex). - sha256: String, - }, } /// An installed format plugin: its `plugin.toml` and its implementation. @@ -78,7 +36,6 @@ pub struct Plugin { /// The format id, leaked once per plugin so [`forge_core::Format::id`] /// can hand out a `&'static str`. id: &'static str, - origin: Origin, implementation: Arc, } @@ -87,7 +44,6 @@ impl std::fmt::Debug for Plugin { f.debug_struct("Plugin") .field("id", &self.id) .field("version", &self.manifest.version) - .field("origin", &self.origin) .finish_non_exhaustive() } } @@ -101,55 +57,11 @@ impl Plugin { implementation: impl FormatPlugin, ) -> Result, PluginError> { let manifest = Manifest::parse(plugin_toml)?; - Ok(Self::new( - manifest, - Origin::Builtin, - Arc::new(implementation), - )) - } - - /// Load the WebAssembly plugin in `dir`: read `plugin.toml`, check the - /// component against `expected_sha256` (hex, optionally `sha256:`-prefixed; - /// an operator pins every plugin), resolve its limits against `host`, and - /// compile it. - pub fn load( - dir: &Path, - expected_sha256: &str, - host: &HostLimits, - ) -> Result, PluginError> { - let manifest = Manifest::load(dir)?; - let Some(component) = &manifest.component else { - return Err(PluginError::Manifest( - "a WebAssembly plugin names its `component`".into(), - )); - }; - let path = dir.join(component); - let bytes = std::fs::read(&path) - .map_err(|e| PluginError::Io(format!("read {}: {e}", path.display())))?; - Self::from_parts(manifest, &bytes, expected_sha256, host) - } - - /// [`Self::load`] from a manifest and the component's bytes. - pub fn from_parts( - manifest: Manifest, - component: &[u8], - expected_sha256: &str, - host: &HostLimits, - ) -> Result, PluginError> { - let wasm = WasmPlugin::new(&manifest, component, expected_sha256, host)?; - let origin = Origin::Wasm { - sha256: wasm.sha256().to_string(), - }; - Ok(Self::new(manifest, origin, Arc::new(wasm))) - } - - fn new(manifest: Manifest, origin: Origin, implementation: Arc) -> Arc { - Arc::new(Plugin { + Ok(Arc::new(Plugin { id: Box::leak(manifest.id.clone().into_boxed_str()), manifest, - origin, - implementation, - }) + implementation: Arc::new(implementation), + })) } /// The format id. @@ -162,24 +74,6 @@ impl Plugin { &self.manifest } - /// Where the plugin's code comes from. - pub fn origin(&self) -> &Origin { - &self.origin - } - - /// Whether the plugin is linked into the server. - pub fn is_builtin(&self) -> bool { - self.origin == Origin::Builtin - } - - /// The component's SHA-256 (WebAssembly plugins only). - pub fn sha256(&self) -> Option<&str> { - match &self.origin { - Origin::Builtin => None, - Origin::Wasm { sha256 } => Some(sha256), - } - } - /// The format, as the plugin implements it. pub fn implementation(&self) -> &Arc { &self.implementation diff --git a/crates/plugin/src/manifest.rs b/crates/plugin/src/manifest.rs index dc161c5d..b77d8217 100644 --- a/crates/plugin/src/manifest.rs +++ b/crates/plugin/src/manifest.rs @@ -1,16 +1,13 @@ //! `plugin.toml`: what a plugin declares about itself, next to its crate's -//! `Cargo.toml` (`plugins//plugin.toml`) and in an installed plugin -//! directory. +//! `Cargo.toml` (`plugins//plugin.toml`). //! //! ```toml //! [plugin] -//! id = "helm" # the format id repositories are created with -//! name = "Helm charts" -//! version = "0.1.0" # the plugin's own version (informational) +//! id = "nuget" # the format id repositories are created with +//! name = "NuGet" +//! version = "1.0.0" # the plugin's own version (informational) //! api = "0.3.0" # the forge:format contract it is built against -//! component = "helm.wasm" # built as a component: its file, relative to -//! # the directory (unused when built in) -//! mount = "/repository" # optional: served at {mount}/{repository}/… +//! mount = "/v3" # optional: served at {mount}/{repository}/… //! kinds = ["hosted", "group"] # optional: the repository kinds it supports //! # ("proxy" too, when it implements `upstream`) //! @@ -19,15 +16,8 @@ //! package_type = "application/octet-stream" //! manifest = "manifest" //! manifest_type = "application/octet-stream" -//! -//! [limits] # as a component; each capped by the host's maximum -//! memory_mib = 64 -//! fuel = 2000000000 -//! timeout_ms = 2000 //! ``` -use std::path::{Component, Path, PathBuf}; - use forge_plugin_api::RepositoryKind; use serde::Deserialize; @@ -92,9 +82,6 @@ pub struct Manifest { pub api: String, /// Optional one-line description. pub description: Option, - /// Path of the component, relative to the plugin directory (WebAssembly - /// plugins). - pub component: Option, /// Where repositories of the format are served: `{mount}/{repository}/…`. pub mount: String, /// The repository kinds the format supports. `proxy` means the plugin @@ -102,8 +89,6 @@ pub struct Manifest { pub kinds: Vec, /// How a version's files are stored. pub assets: Assets, - /// Requested limits as a component (`None` → the host default). - pub limits: RequestedLimits, } /// The `[assets]` table: the names (within a version) and content types of @@ -128,27 +113,12 @@ impl Default for Assets { } } -/// The `[limits]` table. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RequestedLimits { - /// Memory ceiling per instance, in MiB: its linear memories and tables - /// together, and the most one call returns to the host. - pub memory_mib: Option, - /// Fuel (roughly, WebAssembly instructions) per call. - pub fuel: Option, - /// Wall-clock budget per call, in milliseconds. - pub timeout_ms: Option, -} - #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawManifest { plugin: RawPlugin, #[serde(default)] assets: Assets, - #[serde(default)] - limits: RequestedLimits, } #[derive(Deserialize)] @@ -158,7 +128,6 @@ struct RawPlugin { name: String, version: String, api: String, - component: Option, description: Option, mount: Option, kinds: Option>, @@ -177,18 +146,6 @@ impl Manifest { p.api ))); } - // The component is a file inside the plugin directory: a plain - // relative path, never `..`, absolute, or a drive/root prefix. - let component = p.component.map(|raw| { - let path = PathBuf::from(&raw); - if raw.is_empty() || !path.components().all(|c| matches!(c, Component::Normal(_))) { - return Err(PluginError::Manifest(format!( - "component {raw:?} must be a relative path inside the plugin directory" - ))); - } - Ok(path) - }); - let component = component.transpose()?; if p.name.trim().is_empty() { return Err(PluginError::Manifest("name must not be empty".into())); } @@ -205,21 +162,11 @@ impl Manifest { version: p.version, api: p.api, description: p.description, - component, mount, kinds, assets: raw.assets, - limits: raw.limits, }) } - - /// Read and parse `/plugin.toml`. - pub fn load(dir: &Path) -> Result { - let path = dir.join("plugin.toml"); - let text = std::fs::read_to_string(&path) - .map_err(|e| PluginError::Io(format!("read {}: {e}", path.display())))?; - Self::parse(&text) - } } /// A mount: `/` and one path segment of 2–32 lowercase letters, digits and @@ -341,19 +288,14 @@ mod tests { name = "Helm charts" version = "0.1.0" api = "0.3.0" - component = "helm.wasm" - - [limits] - memory_mib = 32 "#; #[test] fn parses_a_manifest() { let m = Manifest::parse(GOOD).unwrap(); assert_eq!(m.id, "helm"); - assert_eq!(m.component, Some(PathBuf::from("helm.wasm"))); - assert_eq!(m.limits.memory_mib, Some(32)); - assert_eq!(m.limits.fuel, None); + assert_eq!(m.name, "Helm charts"); + assert_eq!(m.description, None); } #[test] @@ -376,19 +318,6 @@ mod tests { assert!(validate_id("go-modules").is_ok()); } - #[test] - fn refuses_components_outside_the_directory() { - for path in ["../helm.wasm", "/etc/passwd", "a/../../b.wasm", ""] { - let text = GOOD.replace("\"helm.wasm\"", &format!("{path:?}")); - assert!(Manifest::parse(&text).is_err(), "{path:?} accepted"); - } - let text = GOOD.replace("\"helm.wasm\"", "\"lib/helm.wasm\""); - assert!(Manifest::parse(&text).is_ok()); - // A built-in plugin needs none. - let text = GOOD.replace("component = \"helm.wasm\"", ""); - assert_eq!(Manifest::parse(&text).unwrap().component, None); - } - #[test] fn mounts_kinds_and_assets_default_and_validate() { let m = Manifest::parse(GOOD).unwrap(); @@ -398,8 +327,8 @@ mod tests { let with = |plugin_keys: &str, tables: &str| { let text = GOOD.replace( - "component = \"helm.wasm\"", - &format!("component = \"helm.wasm\"\n{plugin_keys}"), + "api = \"0.3.0\"", + &format!("api = \"0.3.0\"\n{plugin_keys}"), ); Manifest::parse(&format!("{text}\n{tables}")) }; @@ -476,6 +405,8 @@ mod tests { // 0.1.0 was a preview that no release shipped. assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"0.2.0\"")).is_err()); assert!(Manifest::parse(&format!("{GOOD}\nunknown = 1\n")).is_err()); - assert!(Manifest::parse(&GOOD.replace("memory_mib", "memory_gib")).is_err()); + // What a WebAssembly component named is gone with it. + assert!(Manifest::parse(&format!("{GOOD}component = \"helm.wasm\"\n")).is_err()); + assert!(Manifest::parse(&format!("{GOOD}\n[limits]\nmemory_mib = 32\n")).is_err()); } } diff --git a/crates/plugin/src/testing.rs b/crates/plugin/src/testing.rs deleted file mode 100644 index d4a6d5ef..00000000 --- a/crates/plugin/src/testing.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Test support: build the plugins of `plugins/` as WebAssembly components -//! (feature `testing`). - -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::Mutex; - -/// Build `plugins/` as a component (once per process) and install it as -/// an operator would: a plugin directory holding its `plugin.toml` and the -/// component. -/// -/// Needs the `wasm32-wasip2` target (`rustup target add wasm32-wasip2`). -pub fn example_plugin(name: &str) -> PathBuf { - let source = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../plugins") - .join(name); - let id = crate::Manifest::load(&source) - .expect("example plugin.toml") - .id; - example_plugin_as(name, &id) -} - -/// [`example_plugin`], installed under format id `id` (its `plugin.toml` -/// rewritten): a component of a plugin that is also built in, under an id of -/// its own. -pub fn example_plugin_as(name: &str, id: &str) -> PathBuf { - example_plugin_with(name, id, "") -} - -/// [`example_plugin_as`], with `plugin_keys` (TOML lines such as -/// `mount = "/charts"`) added to its `[plugin]` table. -pub fn example_plugin_with(name: &str, id: &str, plugin_keys: &str) -> PathBuf { - static BUILT: Mutex> = Mutex::new(Vec::new()); - let mut built = BUILT.lock().unwrap_or_else(|p| p.into_inner()); - if let Some((_, _, dir)) = built - .iter() - .find(|(i, keys, _)| i == id && keys == plugin_keys) - { - return dir.clone(); - } - let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let source = workspace.join("plugins").join(name); - let target = workspace.join("target/plugins"); - let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); - let status = Command::new(cargo) - .args([ - "rustc", - "--locked", - "--lib", - "--crate-type", - "cdylib", - "--profile", - "plugin", - "--target", - "wasm32-wasip2", - "--features", - "component", - ]) - .arg("--manifest-path") - .arg(source.join("Cargo.toml")) - .arg("--target-dir") - .arg(&target) - // The outer build's flags are for the host, not for wasm. - .env_remove("RUSTFLAGS") - .env_remove("CARGO_ENCODED_RUSTFLAGS") - .env_remove("CARGO_BUILD_TARGET") - .status() - .expect("run cargo to build the example plugin"); - assert!( - status.success(), - "building plugins/{name} failed (is the wasm32-wasip2 target installed? \ - `rustup target add wasm32-wasip2`)" - ); - let text = std::fs::read_to_string(source.join("plugin.toml")).expect("read plugin.toml"); - let text: String = text - .lines() - .map(|line| { - let key = line.split('=').next().unwrap_or("").trim(); - if key == "id" { - format!("id = {id:?}\n") - } else if line.trim() == "[plugin]" { - format!("{line}\n{plugin_keys}\n") - } else { - format!("{line}\n") - } - }) - .collect(); - let manifest = crate::Manifest::parse(&text).expect("example plugin.toml"); - let component = manifest - .component - .expect("the example plugin.toml names its component"); - let dir = target.join("installed").join(id); - std::fs::create_dir_all(&dir).expect("create the plugin directory"); - std::fs::write(dir.join("plugin.toml"), text).expect("write plugin.toml"); - let wasm = target.join("wasm32-wasip2/plugin").join(&component); - std::fs::copy(&wasm, dir.join(&component)).expect("copy the component"); - built.push((id.to_string(), plugin_keys.to_string(), dir.clone())); - dir -} - -/// The SHA-256 (lowercase hex) of the component in plugin directory `dir`, -/// what an operator pins. -pub fn component_sha256(dir: &Path) -> String { - use sha2::Digest; - let manifest = crate::Manifest::load(dir).expect("plugin.toml"); - let component = manifest.component.expect("plugin.toml names a component"); - let bytes = std::fs::read(dir.join(component)).expect("read the component"); - hex::encode(sha2::Sha256::digest(bytes)) -} diff --git a/crates/plugin/src/wasm/bindings.rs b/crates/plugin/src/wasm/bindings.rs deleted file mode 100644 index abbae03e..00000000 --- a/crates/plugin/src/wasm/bindings.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Host bindings generated from `crates/plugin-api/wit/format.wit`. - -wasmtime::component::bindgen!({ - world: "format-plugin", - path: "../plugin-api/wit", - // The package reader the host lends to `parse-package` is a staged file. - with: { - "forge:format/package-source.package-reader": crate::wasm::host::StagedPackage, - }, - // A host function may trap the guest (an unknown resource handle). - imports: { default: trappable }, -}); diff --git a/crates/plugin/src/wasm/executor.rs b/crates/plugin/src/wasm/executor.rs deleted file mode 100644 index c2636624..00000000 --- a/crates/plugin/src/wasm/executor.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! The threads guest code runs on. -//! -//! Guest code never runs on a tokio worker: the WASI bindings answer a -//! guest's stdout/stderr writes (a panic message, say) by blocking on a tokio -//! runtime, which panics inside an asynchronous context. Every call into a -//! component is therefore handed to a dedicated thread, and the caller -//! blocks until the result is there. -//! -//! There are two pools, so that the calls a request makes synchronously — the -//! [`forge_core::Format`] rules, microseconds each, waited for on a tokio -//! worker — never queue behind long ones (rendering a large index, parsing a -//! package) and stall the server's workers while they wait. - -use std::sync::mpsc::{self, Receiver, Sender}; -use std::sync::{Arc, Mutex, OnceLock}; - -use crate::PluginError; - -type Job = Box; - -/// Which pool a call runs on. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Pool { - /// The format's rules (ids, versions, summaries): short calls a request - /// waits for synchronously. - Quick, - /// Parsing packages and the client protocol: calls that may take as long - /// as the plugin's time limit. - Slow, -} - -/// At most this many threads per pool (calls queue beyond it). -const MAX_THREADS: usize = 8; - -struct Executor { - jobs: Sender, -} - -fn executor(pool: Pool) -> &'static Executor { - static QUICK: OnceLock = OnceLock::new(); - static SLOW: OnceLock = OnceLock::new(); - let (cell, name) = match pool { - Pool::Quick => (&QUICK, "quick"), - Pool::Slow => (&SLOW, "slow"), - }; - cell.get_or_init(|| { - let (jobs, queue) = mpsc::channel::(); - let queue = Arc::new(Mutex::new(queue)); - let threads = std::thread::available_parallelism() - .map_or(2, |n| n.get()) - .clamp(2, MAX_THREADS); - for i in 0..threads { - let queue = Arc::clone(&queue); - std::thread::Builder::new() - .name(format!("forge-plugin-{name}-{i}")) - .spawn(move || worker(&queue)) - .expect("spawn a plugin thread"); - } - Executor { jobs } - }) -} - -fn worker(queue: &Mutex>) { - loop { - // Hold the lock only to take a job, not while running it. - let job = match queue.lock() { - Ok(queue) => queue.recv(), - Err(poisoned) => poisoned.into_inner().recv(), - }; - let Ok(job) = job else { - return; - }; - // A panicking job (a host bug) must not take the thread with it; the - // caller sees its result channel closed. - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)); - } -} - -/// Run `f` on a thread of `pool` and block until it returns. -pub(crate) fn run_blocking(pool: Pool, f: F) -> Result -where - R: Send + 'static, - F: FnOnce() -> R + Send + 'static, -{ - let (tx, rx) = mpsc::sync_channel(1); - executor(pool) - .jobs - .send(Box::new(move || { - let _ = tx.send(f()); - })) - .map_err(|_| PluginError::Trap("the plugin threads are gone".into()))?; - rx.recv() - .map_err(|_| PluginError::Trap("the plugin call panicked".into())) -} diff --git a/crates/plugin/src/wasm/host.rs b/crates/plugin/src/wasm/host.rs deleted file mode 100644 index 78784626..00000000 --- a/crates/plugin/src/wasm/host.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! The per-instance host state and the functions a plugin imports. - -use std::fs::File; -use std::io::{Read, Seek, SeekFrom}; - -use wasmtime::component::{Resource, ResourceTable}; -use wasmtime::ResourceLimiter; -use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; - -use super::bindings::forge::format::{package_source, types}; - -/// The most `read-at` returns in one call. -const MAX_READ: u32 = 1024 * 1024; - -/// What a plugin instance's store holds: an empty WASI context (no files, -/// network, environment or arguments; stdout and stderr discard), the -/// resource table, and the memory budget. -pub(crate) struct HostState { - wasi: WasiCtx, - table: ResourceTable, - pub(crate) budget: Budget, -} - -impl HostState { - /// The state of an instance whose memories and tables may hold - /// `memory_bytes` together. WASI's random bytes are capped the same way: - /// they are the one thing a guest can ask the host to allocate for it. - pub(crate) fn new(memory_bytes: usize) -> Self { - let wasi = WasiCtx::builder() - .allow_tcp(false) - .allow_udp(false) - .allow_ip_name_lookup(false) - .max_random_size(u64::try_from(memory_bytes).unwrap_or(u64::MAX)) - .build(); - HostState { - wasi, - table: ResourceTable::new(), - budget: Budget::new(memory_bytes), - } - } - - /// Lend `package` to the guest for one call. - pub(crate) fn lend( - &mut self, - package: StagedPackage, - ) -> wasmtime::Result> { - Ok(self.table.push(package)?) - } - - /// Take a lent package back after the call. - pub(crate) fn reclaim(&mut self, package: Resource) { - let _ = self.table.delete(package); - } -} - -impl WasiView for HostState { - fn ctx(&mut self) -> WasiCtxView<'_> { - WasiCtxView { - ctx: &mut self.wasi, - table: &mut self.table, - } - } -} - -/// Core instances a store may create: a Rust component instantiates a few -/// (its module, WASI adapters and shims). -const MAX_INSTANCES: usize = 64; - -/// Tables a store may create. -const MAX_TABLES: usize = 64; - -/// Linear memories a store may create. They share one [`Budget`], so this -/// bounds bookkeeping, not bytes. -const MAX_MEMORIES: usize = 16; - -/// What one table element holds on the host: a pointer, in Wasmtime. -const TABLE_ELEMENT_BYTES: usize = std::mem::size_of::(); - -/// One memory ceiling for everything an instance allocates on its behalf: -/// all its linear memories and all its tables together, counted in bytes. -/// -/// Wasmtime's own limits apply per memory and leave tables unbounded, so an -/// instance with several memories, or a `table.grow` to hundreds of millions -/// of elements, would take far more host memory than its ceiling. A growth -/// past the budget traps the guest, which fails the call and discards the -/// instance. -pub(crate) struct Budget { - /// What the memories and tables may hold together, in bytes. - limit: usize, - /// What they hold now. - used: usize, - /// What the last growth allowed added, given back when that growth then - /// fails (past a memory's declared maximum, say). - pending: usize, -} - -impl Budget { - fn new(limit: usize) -> Self { - Budget { - limit, - used: 0, - pending: 0, - } - } - - /// Take `bytes` more out of the budget, or refuse with a trap. - fn grow(&mut self, bytes: usize, what: &str) -> wasmtime::Result { - match self.used.checked_add(bytes) { - Some(used) if used <= self.limit => { - self.used = used; - self.pending = bytes; - Ok(true) - } - _ => Err(wasmtime::format_err!( - "growing a {what} by {bytes} bytes exceeds the plugin's memory limit of {} \ - bytes ({} in use)", - self.limit, - self.used - )), - } - } - - /// Give back what the growth that just failed took. - fn grow_failed(&mut self) { - self.used -= std::mem::take(&mut self.pending); - } -} - -// Wasmtime asks before it creates a memory or table (`current` 0) as well as -// before every growth, so the budget sees every byte. -impl ResourceLimiter for Budget { - fn memory_growing( - &mut self, - current: usize, - desired: usize, - _maximum: Option, - ) -> wasmtime::Result { - self.grow(desired.saturating_sub(current), "linear memory") - } - - fn memory_grow_failed(&mut self, _error: wasmtime::Error) -> wasmtime::Result<()> { - self.grow_failed(); - Ok(()) - } - - fn table_growing( - &mut self, - current: usize, - desired: usize, - _maximum: Option, - ) -> wasmtime::Result { - let elements = desired.saturating_sub(current); - self.grow(elements.saturating_mul(TABLE_ELEMENT_BYTES), "table") - } - - fn table_grow_failed(&mut self, _error: wasmtime::Error) -> wasmtime::Result<()> { - self.grow_failed(); - Ok(()) - } - - fn instances(&self) -> usize { - MAX_INSTANCES - } - - fn tables(&self) -> usize { - MAX_TABLES - } - - fn memories(&self) -> usize { - MAX_MEMORIES - } -} - -/// A package the host staged on disk, lent to `parse-package`. -pub struct StagedPackage { - file: File, - size: u64, -} - -impl StagedPackage { - /// Wrap an open package file. - pub(crate) fn new(file: File) -> std::io::Result { - let size = file.metadata()?.len(); - Ok(StagedPackage { file, size }) - } - - fn read_at(&mut self, offset: u64, len: u32) -> std::io::Result> { - if offset >= self.size { - return Ok(Vec::new()); - } - let len = u64::from(len.min(MAX_READ)).min(self.size - offset); - self.file.seek(SeekFrom::Start(offset))?; - let mut buf = Vec::with_capacity(len as usize); - (&mut self.file).take(len).read_to_end(&mut buf)?; - Ok(buf) - } -} - -// Both interfaces are imported for their types; only the reader has functions. -impl types::Host for HostState {} - -impl package_source::Host for HostState {} - -impl package_source::HostPackageReader for HostState { - fn size(&mut self, package: Resource) -> wasmtime::Result { - Ok(self.table.get(&package)?.size) - } - - fn read_at( - &mut self, - package: Resource, - offset: u64, - len: u32, - ) -> wasmtime::Result, String>> { - let package = self.table.get_mut(&package)?; - Ok(package.read_at(offset, len).map_err(|e| e.to_string())) - } - - fn drop(&mut self, package: Resource) -> wasmtime::Result<()> { - self.table.delete(package)?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[test] - fn reads_are_bounded_by_the_file_and_the_chunk_limit() { - let mut file = tempfile::tempfile().unwrap(); - file.write_all(b"0123456789").unwrap(); - let mut staged = StagedPackage::new(file).unwrap(); - assert_eq!(staged.size, 10); - assert_eq!(staged.read_at(0, 4).unwrap(), b"0123"); - assert_eq!(staged.read_at(8, 100).unwrap(), b"89"); - assert!(staged.read_at(10, 1).unwrap().is_empty()); - assert!(staged.read_at(u64::MAX, u32::MAX).unwrap().is_empty()); - } -} diff --git a/crates/plugin/src/wasm/mod.rs b/crates/plugin/src/wasm/mod.rs deleted file mode 100644 index baef11b3..00000000 --- a/crates/plugin/src/wasm/mod.rs +++ /dev/null @@ -1,826 +0,0 @@ -//! WebAssembly format plugins: a [component] implementing the `forge:format` -//! world (`crates/plugin-api/wit/format.wit`), run by Wasmtime. -//! -//! [`WasmPlugin`] implements [`FormatPlugin`] by calling into the component, -//! so a loaded component serves exactly like a built-in plugin. -//! -//! ## Sandbox -//! Every call runs in an instance with a memory ceiling, a fuel budget and a -//! wall-clock deadline (epoch interruption), on a dedicated thread (see -//! `executor`). The memory ceiling covers the instance's linear memories and -//! tables together, and what one call may hand back to the host (see -//! [`store`]). A trap — out of fuel, past the deadline, out of memory, a -//! guest panic — fails that call and discards the instance. The WASI context -//! is empty: no files, no sockets, no environment, no arguments; stdout and -//! stderr go nowhere. A component reaches the network only through the GETs -//! the host sends for a proxy's upstream conversation, and the host sends -//! them only to the origins that upstream points at -//! (`forge_core::net::origins`). -//! -//! A failed call cannot fail the infallible trait methods: they log it and -//! answer as for an unknown input — [`FormatPlugin::canonical_id`] the id -//! unchanged, [`FormatPlugin::compare_versions`] byte order, -//! [`FormatPlugin::manifest_summary`] none — and the protocol methods answer -//! with a 500 response. The fallible ones return -//! [`FormatError::Internal`]. -//! -//! [component]: https://component-model.bytecodealliance.org/ - -mod bindings; -mod executor; -mod host; - -use std::cmp::Ordering; -use std::collections::HashMap; -use std::io::SeekFrom; -use std::path::Path; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::Duration; - -use forge_plugin_api::{ - Action, FetchResult, FormatError, FormatPlugin, Loaded, PackageSource, ParsedPackage, Remote, - Rendered, Repository, Request, Response, UpstreamOp, UpstreamStep, -}; -use sha2::{Digest, Sha256}; -use wasmtime::component::{Component, Linker}; -use wasmtime::{Config, Engine, Store, Trap}; - -use crate::{manifest, Manifest, PluginError}; -use bindings::{FormatPlugin as Bindings, FormatPluginPre}; -use executor::Pool; -use host::{HostState, StagedPackage}; - -forge_plugin_api::__wit_conversions!(crate::wasm::bindings); - -/// The most a plugin may ask for, and what it gets when it asks for nothing. -#[derive(Debug, Clone)] -pub struct HostLimits { - /// Memory ceiling per instance when the plugin names none (MiB). - pub default_memory_mib: u64, - /// The largest memory ceiling a plugin may request (MiB). - pub max_memory_mib: u64, - /// Fuel per call when the plugin names none. - pub default_fuel: u64, - /// The most fuel per call a plugin may request. - pub max_fuel: u64, - /// Wall-clock budget per call when the plugin names none. - pub default_timeout: Duration, - /// The longest wall-clock budget a plugin may request. - pub max_timeout: Duration, -} - -impl Default for HostLimits { - fn default() -> Self { - HostLimits { - default_memory_mib: 64, - max_memory_mib: 512, - default_fuel: 5_000_000_000, - max_fuel: 50_000_000_000, - default_timeout: Duration::from_secs(5), - max_timeout: Duration::from_secs(30), - } - } -} - -/// The limits one plugin runs under. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Limits { - /// Memory ceiling per instance, in bytes: its linear memories and tables - /// together, and the most one call hands back to the host. - pub memory_bytes: usize, - /// Fuel per call. - pub fuel: u64, - /// Wall-clock budget per call. - pub timeout: Duration, -} - -impl HostLimits { - /// What `requested` resolves to, or why it is refused. - pub fn resolve(&self, requested: &manifest::RequestedLimits) -> Result { - let memory_mib = requested.memory_mib.unwrap_or(self.default_memory_mib); - if memory_mib == 0 || memory_mib > self.max_memory_mib { - return Err(PluginError::Manifest(format!( - "limits.memory_mib {memory_mib} is outside 1..={}", - self.max_memory_mib - ))); - } - let fuel = requested.fuel.unwrap_or(self.default_fuel); - if fuel == 0 || fuel > self.max_fuel { - return Err(PluginError::Manifest(format!( - "limits.fuel {fuel} is outside 1..={}", - self.max_fuel - ))); - } - let timeout = requested - .timeout_ms - .map_or(self.default_timeout, Duration::from_millis); - if timeout.is_zero() || timeout > self.max_timeout { - return Err(PluginError::Manifest(format!( - "limits.timeout_ms {} is outside 1..={}", - timeout.as_millis(), - self.max_timeout.as_millis() - ))); - } - let memory_bytes = usize::try_from(memory_mib * 1024 * 1024) - .map_err(|_| PluginError::Manifest("limits.memory_mib is too large".into()))?; - Ok(Limits { - memory_bytes, - fuel, - timeout, - }) - } -} - -/// How often the engine's epoch advances: the granularity of call deadlines. -const EPOCH_TICK: Duration = Duration::from_millis(10); - -/// An instance is replaced after this many calls, so whatever a guest leaks -/// (allocator fragmentation, a growing cache) is bounded. -const MAX_CALLS_PER_INSTANCE: u32 = 10_000; - -/// Idle instances kept per plugin. -const MAX_IDLE_INSTANCES: usize = 16; - -/// The engine every plugin is compiled for, and the thread advancing its -/// epoch. -fn engine() -> Result<&'static Engine, PluginError> { - static ENGINE: OnceLock> = OnceLock::new(); - ENGINE - .get_or_init(|| { - let mut config = Config::new(); - config - .wasm_component_model(true) - .consume_fuel(true) - .epoch_interruption(true) - .max_wasm_stack(1024 * 1024); - let engine = Engine::new(&config).map_err(|e| e.to_string())?; - let ticker = engine.clone(); - std::thread::Builder::new() - .name("forge-plugin-epoch".into()) - .spawn(move || loop { - std::thread::sleep(EPOCH_TICK); - ticker.increment_epoch(); - }) - .map_err(|e| e.to_string())?; - Ok(engine) - }) - .as_ref() - .map_err(|e| PluginError::Component(format!("engine: {e}"))) -} - -/// Compile `bytes` (whose SHA-256 is `sha256`), or reuse the compilation of -/// the same component: loading a plugin again (other limits, a reload) does -/// not pay for Cranelift twice. -fn compile(engine: &Engine, sha256: &str, bytes: &[u8]) -> Result { - static COMPILED: Mutex> = Mutex::new(Vec::new()); - let mut compiled = COMPILED.lock().unwrap_or_else(|p| p.into_inner()); - if let Some((_, component)) = compiled.iter().find(|(sha, _)| sha == sha256) { - return Ok(component.clone()); - } - let component = - Component::new(engine, bytes).map_err(|e| PluginError::Component(format!("{e:#}")))?; - compiled.push((sha256.to_string(), component.clone())); - Ok(component) -} - -/// A store for one instance of a plugin running under `limits`. -/// -/// `memory_bytes` bounds what the instance takes from the host: -/// -/// * its linear memories and tables, all together ([`host::Budget`]); -/// * what one call hands back: Wasmtime copies every string and list of a -/// result into the host, once per reference, so a result whose list -/// elements all point at the same large string would otherwise cost the -/// host its length times their number. Wasmtime charges each copy against -/// the store's hostcall fuel and fails the call once it is spent; -/// * the random bytes it may ask WASI for at once. -fn store(engine: &Engine, limits: &Limits) -> Store { - let mut store = Store::new(engine, HostState::new(limits.memory_bytes)); - store.limiter(|state| &mut state.budget); - store.set_hostcall_fuel(limits.memory_bytes); - store -} - -/// Comparisons remembered per plugin: sorting a package's versions asks the -/// plugin once per pair, and the same packages are listed over and over. -const MAX_REMEMBERED_COMPARISONS: usize = 65_536; - -/// A format plugin running as a WebAssembly component. -pub struct WasmPlugin { - inner: Arc, - comparisons: Mutex>, -} - -/// What the plugin threads share of a [`WasmPlugin`]. -struct Inner { - /// The format id, for logs. - id: String, - sha256: String, - limits: Limits, - pre: FormatPluginPre, - idle: Mutex>, -} - -/// An instantiated plugin, reused across calls. -struct Warm { - store: Store, - plugin: Bindings, - calls: u32, -} - -impl std::fmt::Debug for WasmPlugin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WasmPlugin") - .field("id", &self.inner.id) - .field("sha256", &self.inner.sha256) - .field("limits", &self.inner.limits) - .finish_non_exhaustive() - } -} - -impl WasmPlugin { - /// Check `component` against `expected_sha256` (hex, optionally - /// `sha256:`-prefixed), resolve the limits `manifest` asks for against - /// `host`, compile it, and instantiate it once. - pub fn new( - manifest: &Manifest, - component: &[u8], - expected_sha256: &str, - host: &HostLimits, - ) -> Result { - let actual = hex::encode(Sha256::digest(component)); - let expected = expected_sha256.trim().to_ascii_lowercase(); - let expected = expected.strip_prefix("sha256:").unwrap_or(&expected); - if actual != expected { - return Err(PluginError::Digest { - expected: expected.to_string(), - actual, - }); - } - let limits = host.resolve(&manifest.limits)?; - - let engine = engine()?; - let component = compile(engine, &actual, component)?; - let mut linker = Linker::::new(engine); - wasmtime_wasi::p2::add_to_linker_sync(&mut linker) - .map_err(|e| PluginError::Component(format!("link WASI: {e:#}")))?; - Bindings::add_to_linker::<_, wasmtime::component::HasSelf>( - &mut linker, - |state| state, - ) - .map_err(|e| PluginError::Component(format!("link host functions: {e:#}")))?; - let pre = linker - .instantiate_pre(&component) - .map_err(|e| PluginError::Component(format!("{e:#}")))?; - let pre = - FormatPluginPre::new(pre).map_err(|e| PluginError::Component(format!("{e:#}")))?; - - let inner = Arc::new(Inner { - id: manifest.id.clone(), - sha256: actual, - limits, - pre, - idle: Mutex::new(Vec::new()), - }); - // Instantiate once now, so a component that cannot start fails the - // load instead of the first request (on a plugin thread, like every - // run of guest code). - let starting = Arc::clone(&inner); - let warm = executor::run_blocking(Pool::Slow, move || starting.instantiate())??; - inner.park(warm); - Ok(WasmPlugin { - inner, - comparisons: Mutex::new(HashMap::new()), - }) - } - - /// The component's SHA-256 (lowercase hex). - pub fn sha256(&self) -> &str { - &self.inner.sha256 - } - - /// The limits every call runs under. - pub fn limits(&self) -> Limits { - self.inner.limits - } - - /// Run `f` against an instance on a thread of `pool`, blocking the - /// caller. - fn call(&self, pool: Pool, what: &'static str, f: F) -> Result - where - R: Send + 'static, - F: FnOnce(&mut Store, &Bindings) -> wasmtime::Result + Send + 'static, - { - let inner = Arc::clone(&self.inner); - executor::run_blocking(pool, move || inner.call(what, f))? - } -} - -impl Inner { - fn instantiate(&self) -> Result { - let mut store = store(self.pre.engine(), &self.limits); - self.arm(&mut store)?; - let plugin = self - .pre - .instantiate(&mut store) - .map_err(|e| PluginError::Trap(format!("instantiate: {}", describe(&e))))?; - Ok(Warm { - store, - plugin, - calls: 0, - }) - } - - /// Give `store` a fresh fuel budget and deadline for one call. - fn arm(&self, store: &mut Store) -> Result<(), PluginError> { - store - .set_fuel(self.limits.fuel) - .map_err(|e| PluginError::Trap(e.to_string()))?; - let ticks = self - .limits - .timeout - .as_millis() - .div_ceil(EPOCH_TICK.as_millis()) - + 1; - store.set_epoch_deadline(u64::try_from(ticks).unwrap_or(u64::MAX)); - store.epoch_deadline_trap(); - Ok(()) - } - - fn park(&self, warm: Warm) { - if warm.calls >= MAX_CALLS_PER_INSTANCE { - return; - } - let mut idle = self.idle.lock().unwrap_or_else(|p| p.into_inner()); - if idle.len() < MAX_IDLE_INSTANCES { - idle.push(warm); - } - } - - /// Run `f` against an instance, on the calling thread (which must be a - /// plugin thread, see `executor`). A failed call discards the instance. - fn call( - &self, - what: &str, - f: impl FnOnce(&mut Store, &Bindings) -> wasmtime::Result, - ) -> Result { - let idle = self.idle.lock().unwrap_or_else(|p| p.into_inner()).pop(); - let mut warm = match idle { - Some(warm) => warm, - None => self.instantiate()?, - }; - self.arm(&mut warm.store)?; - match f(&mut warm.store, &warm.plugin) { - Ok(value) => { - warm.calls += 1; - self.park(warm); - Ok(value) - } - Err(e) => { - let reason = describe(&e); - tracing::warn!(plugin = %self.id, call = what, %reason, "format plugin call failed"); - Err(PluginError::Trap(format!("{what}: {reason}"))) - } - } - } -} - -fn internal(e: PluginError) -> FormatError { - FormatError::Internal(e.to_string()) -} - -/// What a protocol call that failed answers. -fn failed() -> Response { - Response::new(500, "text/plain; charset=utf-8", "format plugin failed") -} - -impl FormatPlugin for WasmPlugin { - fn canonical_id(&self, id: &str) -> String { - let raw = id.to_string(); - self.call(Pool::Quick, "canonical-id", move |store, plugin| { - plugin.forge_format_format().call_canonical_id(store, &raw) - }) - .unwrap_or_else(|_| id.to_string()) - } - - fn validate_id(&self, id: &str) -> Result<(), FormatError> { - let id = id.to_string(); - self.call(Pool::Quick, "validate-id", move |store, plugin| { - plugin.forge_format_format().call_validate_id(store, &id) - }) - .map_err(internal)? - .map_err(Into::into) - } - - fn normalize_version(&self, version: &str) -> Result { - let version = version.to_string(); - self.call(Pool::Quick, "normalize-version", move |store, plugin| { - plugin - .forge_format_format() - .call_normalize_version(store, &version) - }) - .map_err(internal)? - .map_err(Into::into) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - let key = (a.to_string(), b.to_string()); - if let Some(order) = self - .comparisons - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&key) - { - return *order; - } - let (x, y) = key.clone(); - let order = match self.call(Pool::Quick, "compare-versions", move |store, plugin| { - plugin - .forge_format_format() - .call_compare_versions(store, &x, &y) - }) { - Ok(n) => n.cmp(&0), - Err(_) => return a.cmp(b), - }; - let mut remembered = self.comparisons.lock().unwrap_or_else(|p| p.into_inner()); - if remembered.len() >= MAX_REMEMBERED_COMPARISONS { - remembered.clear(); - } - remembered.insert(key, order); - order - } - - fn parse_package(&self, package: &mut dyn PackageSource) -> Result { - // The component reads a file it is lent; spool a reader into one. - let spool = |e: std::io::Error| FormatError::Internal(format!("spool the package: {e}")); - let mut spooled = tempfile::NamedTempFile::new().map_err(spool)?; - package.seek(SeekFrom::Start(0)).map_err(spool)?; - std::io::copy(package, &mut spooled).map_err(spool)?; - self.parse_package_file(spooled.path()) - } - - fn parse_package_file(&self, path: &Path) -> Result { - let file = std::fs::File::open(path) - .map_err(|e| FormatError::Internal(format!("open the staged package: {e}")))?; - let package = StagedPackage::new(file) - .map_err(|e| FormatError::Internal(format!("open the staged package: {e}")))?; - self.call(Pool::Slow, "parse-package", move |store, plugin| { - let lent = store.data_mut().lend(package)?; - let borrowed = wasmtime::component::Resource::new_borrow(lent.rep()); - let result = plugin - .forge_format_format() - .call_parse_package(&mut *store, borrowed); - store.data_mut().reclaim(lent); - result - }) - .map_err(internal)? - .map(Into::into) - .map_err(Into::into) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - let manifest = manifest.to_vec(); - self.call(Pool::Quick, "manifest-summary", move |store, plugin| { - plugin - .forge_format_format() - .call_manifest_summary(store, &manifest) - }) - .ok()? - } - - fn handle(&self, repo: &Repository, req: &Request) -> Action { - let repo = repo.clone().into(); - let req = req.clone().into(); - match self.call(Pool::Slow, "handle", move |store, plugin| { - plugin - .forge_format_protocol() - .call_handle(store, &repo, &req) - }) { - Ok(action) => action.into(), - Err(_) => Action::Respond(failed()), - } - } - - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let repo = repo.clone().into(); - let req = req.clone().into(); - let data: Vec = - data.iter().cloned().map(Into::into).collect(); - match self.call(Pool::Slow, "render", move |store, plugin| { - plugin - .forge_format_protocol() - .call_render(store, &repo, &req, &data) - }) { - Ok(rendered) => rendered.into(), - Err(_) => Rendered::Respond(failed()), - } - } - - fn upstream( - &self, - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], - ) -> Result { - let remote = remote.clone().into(); - let op = op.clone().into(); - let fetched: Vec = - fetched.iter().cloned().map(Into::into).collect(); - self.call(Pool::Slow, "upstream", move |store, plugin| { - plugin - .forge_format_upstream() - .call_upstream(store, &remote, &op, &fetched) - }) - .map_err(internal)? - .map(Into::into) - .map_err(Into::into) - } -} - -/// A readable reason for a failed call. -fn describe(e: &wasmtime::Error) -> String { - match e.downcast_ref::() { - Some(Trap::OutOfFuel) => "ran out of fuel".into(), - Some(Trap::Interrupt) => "exceeded its time limit".into(), - Some(trap) => format!("trapped: {trap}"), - None => format!("{e:#}"), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use manifest::RequestedLimits; - - #[test] - fn limits_default_and_are_capped() { - let host = HostLimits::default(); - let l = host.resolve(&RequestedLimits::default()).unwrap(); - assert_eq!(l.memory_bytes, 64 * 1024 * 1024); - assert_eq!(l.fuel, host.default_fuel); - assert_eq!(l.timeout, host.default_timeout); - - let asks = |memory_mib, fuel, timeout_ms| RequestedLimits { - memory_mib, - fuel, - timeout_ms, - }; - assert!(host.resolve(&asks(Some(513), None, None)).is_err()); - assert!(host.resolve(&asks(Some(0), None, None)).is_err()); - assert!(host.resolve(&asks(None, Some(0), None)).is_err()); - assert!(host.resolve(&asks(None, Some(u64::MAX), None)).is_err()); - assert!(host.resolve(&asks(None, None, Some(30_001))).is_err()); - assert!(host.resolve(&asks(None, None, Some(0))).is_err()); - let l = host - .resolve(&asks(Some(128), Some(1_000), Some(250))) - .unwrap(); - assert_eq!(l.memory_bytes, 128 * 1024 * 1024); - assert_eq!(l.fuel, 1_000); - assert_eq!(l.timeout, Duration::from_millis(250)); - } - - #[test] - fn a_component_with_another_digest_is_refused() { - let manifest = Manifest::parse( - r#" - [plugin] - id = "demo" - name = "Demo" - version = "0.1.0" - api = "0.3.0" - component = "demo.wasm" - "#, - ) - .unwrap(); - let err = WasmPlugin::new( - &manifest, - b"not a component", - &"0".repeat(64), - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Digest { .. }), "{err}"); - } - - /// Hand-built components that test the store's limits ([`store`]), each - /// against a memory ceiling of 1 MiB. - mod sandbox { - use super::*; - use wasm_encoder::{ - CanonicalOption, CodeSection, ComponentBuilder, ComponentExportKind, ComponentValType, - ConstExpr, DataSection, ExportKind, ExportSection, Function, FunctionSection, HeapType, - MemArg, MemorySection, MemoryType, Module, ModuleArg, PrimitiveValType, RefType, - TableSection, TableType, TypeSection, ValType, - }; - use wasmtime::component::Instance; - - /// The limits every test component runs under. - const CEILING: Limits = Limits { - memory_bytes: 1024 * 1024, - fuel: 1_000_000_000, - timeout: Duration::from_secs(10), - }; - - /// A store for [`CEILING`], armed for calls. - fn armed_store() -> Store { - let mut store = store(engine().unwrap(), &CEILING); - store.set_fuel(CEILING.fuel).unwrap(); - store.set_epoch_deadline(1_000); - store - } - - /// Instantiate `component` (its bytes) in `store`. - fn instantiate( - store: &mut Store, - component: &[u8], - ) -> wasmtime::Result { - let engine = engine().unwrap(); - let component = Component::new(engine, component).unwrap(); - Linker::new(engine).instantiate(store, &component) - } - - /// A memory section of one memory of `pages` 64 KiB pages. - fn memories(pages: u64) -> MemorySection { - let mut memories = MemorySection::new(); - memories.memory(MemoryType { - minimum: pages, - maximum: None, - memory64: false, - shared: false, - page_size_log2: None, - }); - memories - } - - /// A core module with one memory of `pages` 64 KiB pages. - fn memory_module(pages: u64) -> Module { - let mut module = Module::new(); - module.section(&memories(pages)); - module - } - - /// A component instantiating each of `modules` once, exporting - /// nothing. - fn component_of(modules: &[Module]) -> Vec { - let mut component = ComponentBuilder::default(); - for module in modules { - let index = component.core_module(None, module); - component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - } - component.finish() - } - - #[test] - fn memories_share_the_ceiling() { - // 768 KiB fits, twice does not: each memory alone is below the - // ceiling. - let one = component_of(&[memory_module(12)]); - assert!(instantiate(&mut armed_store(), &one).is_ok()); - let two = component_of(&[memory_module(12), memory_module(12)]); - let err = instantiate(&mut armed_store(), &two).unwrap_err(); - assert!(format!("{err:#}").contains("memory limit"), "{err:#}"); - } - - #[test] - fn tables_count_against_the_ceiling() { - // `grow: func(n: u32) -> s32`: `table.grow` by `n` null elements. - let mut types = TypeSection::new(); - types.ty().function([ValType::I32], [ValType::I32]); - let mut functions = FunctionSection::new(); - functions.function(0); - let mut tables = TableSection::new(); - tables.table(TableType { - element_type: RefType::FUNCREF, - table64: false, - minimum: 0, - maximum: None, - shared: false, - }); - let mut exports = ExportSection::new(); - exports.export("grow", ExportKind::Func, 0); - let mut grow = Function::new([]); - grow.instructions() - .ref_null(HeapType::FUNC) - .local_get(0) - .table_grow(0) - .end(); - let mut code = CodeSection::new(); - code.function(&grow); - let mut module = Module::new(); - module - .section(&types) - .section(&functions) - .section(&tables) - .section(&exports) - .section(&code); - - let mut component = ComponentBuilder::default(); - let index = component.core_module(None, &module); - let instance = component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - let core = component.core_alias_export(None, instance, "grow", ExportKind::Func); - let (ty, mut encoder) = component.type_function(None); - encoder - .async_(false) - .params([("n", PrimitiveValType::U32)]) - .result(Some(PrimitiveValType::S32.into())); - let lifted = component.lift_func(None, core, ty, []); - component.export("grow", ComponentExportKind::Func, lifted, None); - let component = component.finish(); - - let mut store = armed_store(); - let instance = instantiate(&mut store, &component).unwrap(); - let grow = instance - .get_typed_func::<(u32,), (i32,)>(&mut store, "grow") - .unwrap(); - // 1,000 elements fit; 10 million would take 80 MB of the host's. - assert_eq!(grow.call(&mut store, (1_000,)).unwrap(), (0,)); - let err = grow.call(&mut store, (10_000_000,)).unwrap_err(); - assert!(format!("{err:#}").contains("memory limit"), "{err:#}"); - } - - #[test] - fn results_are_bounded_by_the_ceiling() { - // `strings: func(n: u32) -> list`: the first `n` of 1,000 - // list elements that all point at the same 16 KiB string. - const ELEMENTS: u32 = 1_000; - const STRING: u32 = 16 * 1024; - let list = 8; - let string = list + 8 * ELEMENTS; - let mut data = Vec::new(); - data.extend_from_slice(&list.to_le_bytes()); - data.extend_from_slice(&0u32.to_le_bytes()); - for _ in 0..ELEMENTS { - data.extend_from_slice(&string.to_le_bytes()); - data.extend_from_slice(&STRING.to_le_bytes()); - } - data.resize(data.len() + STRING as usize, b'x'); - - let mut types = TypeSection::new(); - types.ty().function([ValType::I32], [ValType::I32]); - let mut functions = FunctionSection::new(); - functions.function(0); - let mut exports = ExportSection::new(); - exports.export("strings", ExportKind::Func, 0); - exports.export("memory", ExportKind::Memory, 0); - // Store `n` as the list's length, return the return area. - let mut strings = Function::new([]); - strings - .instructions() - .i32_const(4) - .local_get(0) - .i32_store(MemArg { - offset: 0, - align: 2, - memory_index: 0, - }) - .i32_const(0) - .end(); - let mut code = CodeSection::new(); - code.function(&strings); - let mut segments = DataSection::new(); - segments.active(0, &ConstExpr::i32_const(0), data); - let mut module = Module::new(); - module - .section(&types) - .section(&functions) - .section(&memories(1)) - .section(&exports) - .section(&code) - .section(&segments); - - let mut component = ComponentBuilder::default(); - let index = component.core_module(None, &module); - let instance = component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - let core = component.core_alias_export(None, instance, "strings", ExportKind::Func); - let memory = component.core_alias_export(None, instance, "memory", ExportKind::Memory); - let (list_of_strings, encoder) = component.type_defined(None); - encoder.list(PrimitiveValType::String); - let (ty, mut encoder) = component.type_function(None); - encoder - .async_(false) - .params([("n", PrimitiveValType::U32)]) - .result(Some(ComponentValType::Type(list_of_strings))); - let lifted = component.lift_func( - None, - core, - ty, - [CanonicalOption::Memory(memory), CanonicalOption::UTF8], - ); - component.export("strings", ComponentExportKind::Func, lifted, None); - let component = component.finish(); - - let mut store = armed_store(); - let instance = instantiate(&mut store, &component).unwrap(); - let strings = instance - .get_typed_func::<(u32,), (Vec,)>(&mut store, "strings") - .unwrap(); - // 10 copies (160 KiB) fit the ceiling. - let (ten,) = strings.call(&mut store, (10,)).unwrap(); - assert_eq!(ten.len(), 10); - assert!(ten.iter().all(|s| s.len() == STRING as usize)); - // 1,000 copies (16 MiB) do not, though the guest holds 16 KiB. - let err = strings.call(&mut store, (ELEMENTS,)).unwrap_err(); - assert!(format!("{err:#}").contains("too much data"), "{err:#}"); - } - } -} diff --git a/crates/plugin/tests/component.rs b/crates/plugin/tests/component.rs deleted file mode 100644 index 133fc7e8..00000000 --- a/crates/plugin/tests/component.rs +++ /dev/null @@ -1,522 +0,0 @@ -//! The Helm plugin (`plugins/helm`) built as a WebAssembly component and -//! loaded into the host: its format rules, its package parser, its protocol, -//! the sandbox around it — and that it answers exactly like the same crate -//! built in. - -use std::cmp::Ordering; -use std::io::Write; -use std::path::PathBuf; -use std::sync::Arc; - -use forge_core::Format; -use forge_plugin::api::{ - Action, Answer, FormatPlugin, Loaded, PackageInfo, Query, Rendered, Repository, RepositoryKind, - Request, SearchHit, SearchResult, Upload, VersionInfo, -}; -use forge_plugin::testing::{component_sha256, example_plugin}; -use forge_plugin::{HostLimits, Manifest, Origin, Plugin, PluginError, PluginFormat}; - -fn helm() -> (PathBuf, Arc) { - let dir = example_plugin("helm"); - let plugin = Plugin::load(&dir, &component_sha256(&dir), &HostLimits::default()) - .expect("load the helm plugin"); - (dir, plugin) -} - -fn builtin() -> Arc { - Plugin::builtin(forge_format_helm::PLUGIN_TOML, forge_format_helm::Helm).unwrap() -} - -/// A ustar header of a regular file, with its checksum. -fn tar_header(path: &str, size: u64) -> [u8; 512] { - let mut header = [0u8; 512]; - header[..path.len()].copy_from_slice(path.as_bytes()); - header[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); - header[156] = b'0'; - header[257..263].copy_from_slice(b"ustar\0"); - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - header -} - -/// A gzipped ustar archive of `(path, contents)` entries. -fn tgz(entries: &[(&str, &[u8])]) -> Vec { - let mut tar = Vec::new(); - for (path, data) in entries { - tar.extend_from_slice(&tar_header(path, data.len() as u64)); - tar.extend_from_slice(data); - tar.resize(tar.len().div_ceil(512) * 512, 0); - } - tar.extend_from_slice(&[0u8; 1024]); - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(&tar).unwrap(); - gz.finish().unwrap() -} - -fn chart(name: &str, version: &str) -> Vec { - let yaml = format!( - "apiVersion: v2\nname: {name}\nversion: {version}\nappVersion: \"1.27\"\n\ - description: The {name} chart\ndependencies:\n - name: redis\n version: \">=17\"\n" - ); - tgz(&[ - (&format!("{name}/values.yaml"), b"replicas: 1\n"), - (&format!("{name}/Chart.yaml"), yaml.as_bytes()), - ]) -} - -fn repo() -> Repository { - Repository { - name: "charts".into(), - kind: RepositoryKind::Hosted, - base_url: "https://forge.example.com/repository/charts".into(), - } -} - -fn request(method: &str, path: &str) -> Request { - Request { - method: method.into(), - path: path.into(), - query: String::new(), - headers: Vec::new(), - } -} - -#[test] -fn loads_only_the_pinned_component() { - let (dir, plugin) = helm(); - assert_eq!(plugin.id(), "helm"); - assert!(!plugin.is_builtin()); - assert_eq!( - plugin.origin(), - &Origin::Wasm { - sha256: component_sha256(&dir) - } - ); - let err = Plugin::load(&dir, &"0".repeat(64), &HostLimits::default()).unwrap_err(); - assert!(matches!(err, PluginError::Digest { .. }), "{err}"); - // The pin may carry its algorithm. - let pinned = format!("sha256:{}", component_sha256(&dir).to_uppercase()); - assert!(Plugin::load(&dir, &pinned, &HostLimits::default()).is_ok()); -} - -#[test] -fn the_format_rules_come_from_the_plugin() { - let (_, plugin) = helm(); - let format = PluginFormat::new(&plugin); - assert_eq!(format.id(), "helm"); - assert_eq!(format.lower_id("Ingress-NGINX"), "ingress-nginx"); - assert!(format.validate_id("ingress-nginx").is_ok()); - assert!(format.validate_id("../etc").is_err()); - assert_eq!(format.normalize_version("v1.2").unwrap(), "1.2.0"); - assert!(format.normalize_version("one").is_err()); - assert_eq!( - format.compare_versions("1.10.0", "1.9.0"), - Ordering::Greater - ); - assert_eq!( - format.compare_versions("1.0.0-rc.1", "1.0.0"), - Ordering::Less - ); - // Remembered comparisons answer the same. - assert_eq!( - format.compare_versions("1.10.0", "1.9.0"), - Ordering::Greater - ); - assert_eq!(format.package_asset_path(), "package"); -} - -#[test] -fn parses_a_chart_through_the_lent_reader() { - let (_, plugin) = helm(); - let format = PluginFormat::new(&plugin); - let mut staged = tempfile::NamedTempFile::new().unwrap(); - staged.write_all(&chart("NGINX", "v2.1")).unwrap(); - - let parsed = format.parse_upload_file(staged.path()).unwrap(); - assert_eq!(parsed.id, "nginx"); - assert_eq!(parsed.original_id, "NGINX"); - assert_eq!(parsed.version, "2.1.0"); - assert_eq!(parsed.version_original, "v2.1"); - assert!(parsed.listed); - assert!(String::from_utf8_lossy(&parsed.nuspec_bytes).contains("name: NGINX")); - assert_eq!(parsed.deps.len(), 1); - assert_eq!(parsed.deps[0].dependencies[0].id, "redis"); - - // The reader-based entry point spools into a file first. - let mut cursor = std::io::Cursor::new(chart("demo", "1.0.0")); - assert_eq!(format.parse_upload(&mut cursor).unwrap().id, "demo"); - - let summary = format.manifest_summary(&parsed.nuspec_bytes).unwrap(); - assert_eq!(summary["chart"]["appVersion"], "1.27"); - assert_eq!(summary["description"], "The NGINX chart"); - assert_eq!( - summary["dependency_groups"][0]["dependencies"][0]["id"], - "redis" - ); - - let mut not_a_chart = tempfile::NamedTempFile::new().unwrap(); - not_a_chart.write_all(b"definitely not gzip").unwrap(); - let err = format.parse_upload_file(not_a_chart.path()).unwrap_err(); - assert!(err.to_string().contains("not a chart archive"), "{err}"); -} - -#[test] -fn routes_requests_to_host_actions() { - let (_, plugin) = helm(); - let plugin = plugin.implementation(); - - let action = plugin.handle( - &repo(), - &request("GET", "/charts/nginx/v2.1/nginx-2.1.0.tgz"), - ); - let Action::Download(target) = action else { - panic!("expected a download, got {action:?}"); - }; - assert_eq!( - (target.id.as_str(), target.version.as_str()), - ("nginx", "2.1.0") - ); - - let action = plugin.handle(&repo(), &request("POST", "/api/charts")); - assert!( - matches!(&action, Action::Publish(p) if matches!(p.upload, Upload::Body)), - "{action:?}" - ); - let mut multipart = request("POST", "/api/charts"); - multipart.headers = vec![( - "content-type".into(), - "multipart/form-data; boundary=x".into(), - )]; - let action = plugin.handle(&repo(), &multipart); - assert!( - matches!(&action, Action::Publish(p) if matches!(&p.upload, Upload::MultipartField(f) if f == "chart")), - "{action:?}" - ); - - let action = plugin.handle(&repo(), &request("DELETE", "/api/charts/NGINX/2.1.0")); - assert!( - matches!(&action, Action::Delete(c) if c.target.id == "nginx"), - "{action:?}" - ); - - for (method, path) in [ - ("GET", "/nope"), - ("PUT", "/index.yaml"), - ("GET", "/charts/..%2Fx/1.0.0/x.tgz"), - ] { - let action = plugin.handle(&repo(), &request(method, path)); - assert!( - matches!(&action, Action::Respond(r) if r.status == 404), - "{method} {path}: {action:?}" - ); - } -} - -#[test] -fn renders_the_index_from_what_it_loads() { - let (_, plugin) = helm(); - let plugin = plugin.implementation(); - let req = request("GET", "/index.yaml"); - let Action::Load(first) = plugin.handle(&repo(), &req) else { - panic!("the index loads data"); - }; - assert!(matches!(&first[..], [Query::Search(s)] if s.skip == 0 && s.prerelease)); - - let version = |v: &str| VersionInfo { - original_id: "NGINX".into(), - version: v.into(), - version_original: v.into(), - listed: true, - }; - let mut data = vec![Loaded { - query: first[0].clone(), - answer: Answer::Search(SearchResult { - total: 1, - hits: vec![SearchHit { - original_id: "NGINX".into(), - latest_version: "2.1.0".into(), - versions: vec![version("1.0.0"), version("2.1.0")], - summary: None, - total_downloads: None, - }], - }), - }]; - let Rendered::Load(second) = plugin.render(&repo(), &req, &data) else { - panic!("the index loads each chart's versions"); - }; - assert!( - matches!(&second[..], [Query::Packages(id)] if id == "nginx"), - "{second:?}" - ); - - let package = |v: &str, listed: bool| PackageInfo { - original_id: "NGINX".into(), - version: v.into(), - version_original: v.into(), - listed, - published: Some("2026-10-01T12:00:00+00:00".into()), - sha256: Some("ab".repeat(32)), - sha512: None, - size: Some(1234), - summary: Some(r#"{"chart":{"apiVersion":"v2","appVersion":"1.27"}}"#.into()), - }; - data.push(Loaded { - query: second[0].clone(), - answer: Answer::Packages(vec![ - package("1.0.0", true), - package("1.5.0", false), - package("2.1.0", true), - ]), - }); - let Rendered::Respond(response) = plugin.render(&repo(), &req, &data) else { - panic!("the index renders once everything is loaded"); - }; - assert_eq!(response.status, 200); - let index: serde_json::Value = serde_json::from_slice(&response.body).unwrap(); - let entries = index["entries"]["NGINX"].as_array().unwrap(); - // Newest first; unlisted versions are not offered. - let versions: Vec<&str> = entries - .iter() - .map(|e| e["version"].as_str().unwrap()) - .collect(); - assert_eq!(versions, ["2.1.0", "1.0.0"]); - assert_eq!( - entries[0]["urls"][0], - "https://forge.example.com/repository/charts/charts/nginx/2.1.0/NGINX-2.1.0.tgz" - ); - assert_eq!(entries[0]["digest"], "ab".repeat(32)); - assert_eq!(entries[0]["appVersion"], "1.27"); - assert_eq!(index["apiVersion"], "v1"); -} - -#[test] -fn runs_inside_its_limits() { - let dir = example_plugin("helm"); - let sha = component_sha256(&dir); - let manifest = |limits: &str| { - let text = std::fs::read_to_string(dir.join("plugin.toml")).unwrap(); - let text = text.split("[limits]").next().unwrap().to_string() + "[limits]\n" + limits; - Manifest::parse(&text).unwrap() - }; - let component = std::fs::read(dir.join("forge_format_helm.wasm")).unwrap(); - - // Not enough fuel to even start. - let err = Plugin::from_parts( - manifest("fuel = 1\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(err.to_string().contains("ran out of fuel"), "{err}"); - - // Not enough memory for the guest's initial heap. - let err = Plugin::from_parts( - manifest("memory_mib = 1\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Trap(_)), "{err}"); - - // More than the host allows. - let err = Plugin::from_parts( - manifest("memory_mib = 4096\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Manifest(_)), "{err}"); - - // A decompression bomb runs out of time or fuel instead of hanging the - // host, and the plugin keeps serving afterwards. - let plugin = Plugin::from_parts( - manifest("fuel = 20000000\ntimeout_ms = 2000\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap(); - let format = PluginFormat::new(&plugin); - let bomb = { - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - // A tar entry claiming 32 MiB of zeros before any Chart.yaml. - gz.write_all(&tar_header("bomb/zeros", 32 << 20)).unwrap(); - let zeros = vec![0u8; 1 << 20]; - for _ in 0..32 { - gz.write_all(&zeros).unwrap(); - } - gz.finish().unwrap() - }; - let mut staged = tempfile::NamedTempFile::new().unwrap(); - staged.write_all(&bomb).unwrap(); - let err = format.parse_upload_file(staged.path()).unwrap_err(); - assert!( - err.to_string().contains("fuel") || err.to_string().contains("time limit"), - "{err}" - ); - assert_eq!(format.normalize_version("1.0").unwrap(), "1.0.0"); -} - -#[test] -fn a_builtin_plugin_has_no_component() { - let plugin = builtin(); - assert_eq!(plugin.id(), "helm"); - assert!(plugin.is_builtin()); - assert_eq!(plugin.sha256(), None); - assert_eq!(plugin.manifest().name, "Helm charts"); - let format = PluginFormat::new(&plugin); - assert_eq!(format.lower_id("NGINX"), "nginx"); -} - -/// One contract, two ways to run it: the component and the built-in crate -/// answer every call the same. -#[test] -fn the_component_answers_like_the_builtin_plugin() { - let (_, component) = helm(); - let builtin = builtin(); - let (wasm, native): (&dyn FormatPlugin, &dyn FormatPlugin) = ( - component.implementation().as_ref(), - builtin.implementation().as_ref(), - ); - - for id in ["Ingress-NGINX", "nginx", "../etc", ""] { - assert_eq!(wasm.canonical_id(id), native.canonical_id(id), "{id:?}"); - assert_eq!(wasm.validate_id(id), native.validate_id(id), "{id:?}"); - } - for version in ["v1.2", "1.0.0-rc.1+build.5", "one", ""] { - assert_eq!( - wasm.normalize_version(version), - native.normalize_version(version), - "{version:?}" - ); - } - for (a, b) in [ - ("1.10.0", "1.9.0"), - ("1.0.0-rc.1", "1.0.0"), - ("2.0.0", "2.0.0"), - ] { - assert_eq!(wasm.compare_versions(a, b), native.compare_versions(a, b)); - } - - let package = std::io::Cursor::new(chart("NGINX", "v2.1")); - let parsed = wasm.parse_package(&mut package.clone()).unwrap(); - assert_eq!(parsed, native.parse_package(&mut package.clone()).unwrap()); - assert_eq!( - wasm.manifest_summary(&parsed.manifest), - native.manifest_summary(&parsed.manifest) - ); - let mut garbage = std::io::Cursor::new(b"not a chart".to_vec()); - let mut garbage_too = garbage.clone(); - assert_eq!( - wasm.parse_package(&mut garbage), - native.parse_package(&mut garbage_too) - ); - - let mut multipart = request("POST", "/api/charts"); - multipart.headers = vec![( - "content-type".into(), - "multipart/form-data; boundary=x".into(), - )]; - for req in [ - request("GET", "/index.yaml"), - request("GET", "/charts/nginx/2.1.0/nginx-2.1.0.tgz"), - request("HEAD", "/api/charts/nginx"), - request("GET", "/api/charts/nginx/2.1.0"), - request("POST", "/api/charts"), - multipart, - request("DELETE", "/api/charts/nginx/2.1.0"), - request("GET", "/nope"), - ] { - assert_eq!( - wasm.handle(&repo(), &req), - native.handle(&repo(), &req), - "{req:?}" - ); - } - - let req = request("GET", "/api/charts/nginx"); - let data = [Loaded { - query: Query::Packages("nginx".into()), - answer: Answer::Packages(vec![PackageInfo { - original_id: "NGINX".into(), - version: "2.1.0".into(), - version_original: "v2.1".into(), - listed: true, - published: Some("2026-10-01T12:00:00Z".into()), - sha256: Some("ab".repeat(32)), - sha512: None, - size: Some(1234), - summary: native.manifest_summary(&parsed.manifest), - }]), - }]; - assert_eq!( - wasm.render(&repo(), &req, &data), - native.render(&repo(), &req, &data) - ); - - // The upstream side, through the component's `upstream` export: a - // proxy of another chart repository. - let index = b"apiVersion: v1\nentries:\n nginx:\n - apiVersion: v2\n name: nginx\n version: 2.1.0\n description: A web server\n created: '2026-05-01T10:00:00Z'\n urls: [charts/nginx-2.1.0.tgz]\n"; - let fetched = [forge_plugin::api::FetchResult { - fetch: forge_plugin::api::Fetch { - url: "https://charts.example/index.yaml".into(), - freshness: forge_plugin::api::Freshness::Metadata, - }, - outcome: forge_plugin::api::Fetched::Found(index.to_vec()), - }]; - let unresolved = forge_plugin::api::Remote { - url: "https://charts.example".into(), - index: None, - }; - let resolved = forge_plugin::api::Remote { - url: "https://charts.example".into(), - index: Some(r#"{"base":"https://charts.example"}"#.into()), - }; - let nginx = |version: &str| forge_plugin::api::PackageRef { - id: "nginx".into(), - version: version.into(), - }; - let wanted = forge_plugin::api::PackageVersions { - id: "nginx".into(), - versions: vec!["2.1.0".into()], - }; - for (remote, op) in [ - (&unresolved, forge_plugin::api::UpstreamOp::Index), - ( - &resolved, - forge_plugin::api::UpstreamOp::Versions("nginx".into()), - ), - ( - &resolved, - forge_plugin::api::UpstreamOp::Package(nginx("2.1.0")), - ), - ( - &resolved, - forge_plugin::api::UpstreamOp::Manifest(nginx("2.1.0")), - ), - (&resolved, forge_plugin::api::UpstreamOp::Dating(wanted)), - ( - &resolved, - forge_plugin::api::UpstreamOp::Search(forge_plugin::api::SearchQuery { - text: "ngi".into(), - skip: 0, - take: 0, - prerelease: true, - semver2: true, - upstream: true, - }), - ), - ] { - for fetched in [&fetched[..], &[]] { - assert_eq!( - wasm.upstream(remote, &op, fetched), - native.upstream(remote, &op, fetched), - "{op:?}" - ); - } - } -} diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 6d56b38b..1e6a5a7e 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -82,10 +82,7 @@ lettre = { workspace = true } figment = { version = "0.10", features = ["toml", "env"] } [dev-dependencies] -forge-plugin = { path = "../plugin", features = ["testing"] } -# The example format plugin the format-plugin tests serve repositories with -# (plugins/helm, not built in), and the chart archives they push to it. -forge-format-helm = { path = "../../plugins/helm" } +# Builds Helm chart archives for the registry tests. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "test-util"] } testcontainers = "0.27" diff --git a/crates/server/src/admin/formats.rs b/crates/server/src/admin/formats.rs index 319bc9ec..2f26119a 100644 --- a/crates/server/src/admin/formats.rs +++ b/crates/server/src/admin/formats.rs @@ -1,6 +1,5 @@ //! `GET /admin/formats` — the package formats this server can create -//! repositories of: the built-in format plugin (NuGet), Docker, raw files, -//! and every loaded WebAssembly plugin. +//! repositories of: the format plugins (NuGet), Docker and raw files. use axum::extract::State; use axum::response::{IntoResponse, Json, Response}; @@ -16,9 +15,6 @@ pub struct FormatView { pub id: String, /// Display name. pub name: String, - /// Whether the format is built into the server (Docker, raw and the - /// built-in plugins) rather than loaded as a WebAssembly plugin. - pub builtin: bool, /// The repository kinds it supports. pub kinds: Vec<&'static str>, /// Where its repositories are served: `{mount}/{repository}` (for @@ -30,9 +26,6 @@ pub struct FormatView { /// The plugin's description. #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - /// SHA-256 of the plugin's component (WebAssembly plugins only). - #[serde(skip_serializing_if = "Option::is_none")] - pub sha256: Option, /// The upload limit of its hosted repositories without one of their own /// (for docker, per blob). pub default_max_upload_bytes: u64, @@ -51,12 +44,10 @@ pub async fn list(State(state): State) -> Response { FormatView { id: plugin.id().to_string(), name: manifest.name.clone(), - builtin: plugin.is_builtin(), kinds: kinds(plugin.id()), mount: manifest.mount.clone(), version: Some(manifest.version.clone()), description: manifest.description.clone(), - sha256: plugin.sha256().map(str::to_string), default_max_upload_bytes: state.config.default_upload_limit(plugin.id()), max_upload_bytes_limit: ceiling, } @@ -68,12 +59,10 @@ pub async fn list(State(state): State) -> Response { formats.push(FormatView { id: DOCKER_FORMAT.to_string(), name: "Docker / OCI".into(), - builtin: true, kinds: kinds(DOCKER_FORMAT), mount: "/v2".into(), version: None, description: None, - sha256: None, default_max_upload_bytes: state.config.default_upload_limit(DOCKER_FORMAT), max_upload_bytes_limit: ceiling, }); @@ -81,12 +70,10 @@ pub async fn list(State(state): State) -> Response { others.push(FormatView { id: RAW_FORMAT.to_string(), name: "Raw files".into(), - builtin: true, kinds: kinds(RAW_FORMAT), mount: crate::raw::MOUNT.into(), version: None, description: Some("Arbitrary files under arbitrary paths".into()), - sha256: None, default_max_upload_bytes: state.config.default_upload_limit(RAW_FORMAT), max_upload_bytes_limit: ceiling, }); diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index 07f1285f..bcc42398 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -549,8 +549,8 @@ pub async fn create(State(state): State, Json(body): Json) "hosted" | "proxy" | "group" => body.kind.clone(), other => return bad_request(format!("invalid kind {other:?}")), }; - // The format must be installed: Docker, raw, the built-in format plugin - // (NuGet) or a loaded WebAssembly one (`GET /admin/formats`), and support + // The format must be installed: Docker, raw or a format plugin (NuGet; + // `GET /admin/formats`), and support // the kind. It is fixed for the repository's lifetime. let Some(kinds) = state.formats.kinds(&body.format) else { let available: Vec<&str> = state.formats.ids().collect(); diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 63164ae5..0dbe1d13 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -63,11 +63,6 @@ pub struct Config { /// proxied packages. #[serde(default)] pub nuget: NugetConfig, - /// WebAssembly format plugins: further package formats loaded at startup - /// (see `docs/format-plugins.md`). None are loaded by default; the - /// built-in plugin (NuGet) needs no configuration here. - #[serde(default)] - pub plugins: PluginsConfig, /// Docker / OCI registry (`/v2`): size limits, registry tokens, upload /// and blob-link expiry. #[serde(default)] @@ -1498,63 +1493,6 @@ fn default_nuget_max_nuspec_compression_ratio() -> u64 { forge_format_nuget::NupkgLimits::default().max_nuspec_compression_ratio } -/// `plugins.*` — WebAssembly format plugins. -/// -/// The format plugin built into the server (NuGet) is always installed. -/// Further ones are WebAssembly components, each listed explicitly with the -/// SHA-256 of its component: a directory under `dir` that is not listed is -/// never loaded, and a listed one whose component does not match its pin, or -/// whose id is taken by a built-in format, fails startup. Every instance must -/// therefore load the same plugins (the same configuration and files), since -/// a repository of a format an instance lacks is unavailable there. -/// -/// ```toml -/// [plugins] -/// dir = "/etc/forge/plugins" # holds /plugin.toml + its component -/// -/// [plugins.formats] -/// example = "sha256:3f1c…" # format id → pinned component digest -/// ``` -/// -/// As environment variables: `FORGE_PLUGINS__DIR`, -/// `FORGE_PLUGINS__FORMATS__EXAMPLE`. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PluginsConfig { - /// Directory with one subdirectory per plugin, named by its format id. - #[serde(default)] - pub dir: Option, - /// The plugins to load: format id → SHA-256 of its component (lowercase - /// hex, optionally prefixed `sha256:`). - #[serde(default)] - pub formats: std::collections::BTreeMap, -} - -impl PluginsConfig { - fn validate(&self) -> Result<(), ConfigError> { - if self.formats.is_empty() { - return Ok(()); - } - if self.dir.is_none() { - return Err(ConfigError::Invalid( - "plugins.formats lists plugins but plugins.dir is not set".into(), - )); - } - for (id, sha256) in &self.formats { - forge_plugin::manifest::validate_id(id) - .map_err(|e| ConfigError::Invalid(format!("plugins.formats.{id}: {e}")))?; - let hex = sha256.trim(); - let hex = hex.strip_prefix("sha256:").unwrap_or(hex); - if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(ConfigError::Invalid(format!( - "plugins.formats.{id} must be the component's SHA-256 (64 hex digits)" - ))); - } - } - Ok(()) - } -} - /// `web.*` — static SPA serving. /// /// When `enabled`, the server serves the built TanStack Start SPA from @@ -2255,7 +2193,6 @@ impl Config { self.validate_crypto()?; self.proxy.validate()?; self.nuget.validate()?; - self.plugins.validate()?; self.docker.validate()?; self.validate_workers()?; self.validate_insecure_defaults()?; @@ -4210,9 +4147,13 @@ mod tests { ); } - // `[plugins.max_upload_bytes]` is gone: limits are per repository. - let per_format = format!("{MINIMAL}\n[plugins.max_upload_bytes]\nnuget = 4096\n"); - assert!(Config::from_figment(fig_from_toml(&per_format)).is_err()); + // `[plugins]` went with WebAssembly plugins: a leftover section is + // ignored, like any other unknown top-level key. + let leftover = format!( + "{MINIMAL}\n[plugins]\ndir = \"/etc/forge/plugins\"\n\n[plugins.formats]\n\ + example = \"sha256:00\"\n" + ); + assert!(Config::from_figment(fig_from_toml(&leftover)).is_ok()); } #[test] diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 314160af..e958bed8 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -121,7 +121,6 @@ async fn main() -> anyhow::Result<()> { .context("ensure auth realms")?; let bind_addr = config.server.bind_addr().context("parse server.bind")?; - let plugins = forge_server::plugins::load(&config.plugins).context("load format plugins")?; // Registry (`/v2`) tokens are signed with one key shared by every // instance: docker.token_secret, or the install's generated key. let registry_keys = @@ -130,10 +129,7 @@ async fn main() -> anyhow::Result<()> { .context("load the registry token key")?; let state = AppState::new(pool, blob_store, cache, config) .with_envelope(envelope) - .with_registry_keys(registry_keys) - .with_plugins(plugins) - .map_err(anyhow::Error::msg) - .context("install format plugins")?; + .with_registry_keys(registry_keys); let layer = principal_layer(&state)?; let serve_options = forge_server::serve::ServeOptions::from_config(&state.config); diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index c0dc68ec..b9e7adb9 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -1,13 +1,12 @@ -//! Format plugins: the built-in ones, loading the WebAssembly ones at -//! startup, and serving the repositories of their formats under each -//! format's mount, `{mount}/{repo}/...` (`/repository` unless its -//! `plugin.toml` names another; see `docs/format-plugins.md`). +//! Format plugins: the built-in ones, and serving the repositories of their +//! formats under each format's mount, `{mount}/{repo}/...` (`/repository` +//! unless its `plugin.toml` names another; see `docs/format-plugins.md`). //! //! The dispatcher does what [`crate::dispatch`] does for the NuGet feed — //! resolve the repository, authorize the request's action (GET/HEAD read, //! PUT/POST write, DELETE delete), refuse an offline repository — and then //! lets the format's plugin decide what the request is -//! ([`FormatPlugin::handle`]), whether it is built in or a component. +//! ([`FormatPlugin::handle`]). //! Everything that touches data stays here: //! //! * **Queries** the plugin asks for run through the same read ops as every @@ -30,7 +29,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use crate::feed::{byte_range, ByteRange, PackageBytes}; -use anyhow::Context; use axum::body::Body; use axum::extract::{Request, State}; use axum::http::{header, HeaderValue, Method, StatusCode}; @@ -49,11 +47,10 @@ use forge_plugin::api::{ self as protocol, Action, Answer, ChangeRequest, FormatPlugin, Loaded, PackageInfo, PackageRef, PublishRequest, Query, Rendered, Upload, }; -use forge_plugin::{HostLimits, Plugin}; +use forge_plugin::Plugin; use futures_util::{StreamExt, TryStreamExt}; use tokio_util::io::StreamReader; -use crate::config::PluginsConfig; use crate::dispatch::{ action_for_method, authz_denied, offline_response, offline_serves_nothing, principal_allows_protocol_mutations, HostedWriteAdapter, @@ -108,39 +105,6 @@ pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec .expect("plugins/nuget/plugin.toml is valid")] } -/// Load the WebAssembly plugins `config` lists, each from `/`, -/// checked against its pinned digest. Any failure fails startup: a -/// configured format that is silently missing would leave its repositories -/// unavailable. -pub fn load(config: &PluginsConfig) -> anyhow::Result>> { - let Some(dir) = &config.dir else { - return Ok(Vec::new()); - }; - let limits = HostLimits::default(); - config - .formats - .iter() - .map(|(id, sha256)| { - let path = dir.join(id); - let plugin = Plugin::load(&path, sha256, &limits) - .with_context(|| format!("load format plugin {id:?} from {}", path.display()))?; - anyhow::ensure!( - plugin.id() == id, - "the plugin in {} declares format id {:?}, not {id:?}", - path.display(), - plugin.id() - ); - tracing::info!( - format = id, - version = %plugin.manifest().version, - sha256 = plugin.sha256().unwrap_or_default(), - "format plugin loaded" - ); - Ok(plugin) - }) - .collect() -} - /// Axum handler mounted at `/repository/{repo}` (and everything under it). pub async fn dispatch(State(state): State, req: Request) -> Response { let mut response = dispatch_request(state, req).await; @@ -308,8 +272,7 @@ struct Ctx<'a> { impl Ctx<'_> { /// Run a protocol call of the plugin off the async workers: rendering a - /// large index takes a while, and a component's call may run for as long - /// as its time limit. A plugin that panics answers 500. + /// large listing takes a while. A plugin that panics answers 500. async fn call( &self, f: impl FnOnce(&dyn FormatPlugin) -> R + Send + 'static, diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index 68d8c6da..8f7ed1fa 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -54,8 +54,8 @@ pub struct AppState { /// Authorization decision service: every read/write/admin gate routes /// through this. pub authz: Authz, - /// Package-format registry, keyed by `Format::id()`: the built-in format - /// plugin (NuGet) and any loaded WebAssembly plugins. + /// Package-format registry, keyed by `Format::id()`: the format plugins + /// (NuGet). pub formats: Arc, /// Parsed server configuration. pub config: Arc, @@ -139,10 +139,8 @@ impl Instance { } /// Format registry: maps a format id (the `repository.format` column) to its -/// [`Format`] and its plugin. The built-in format plugin -/// ([`crate::plugins::builtin`]: NuGet), Docker and raw are always -/// there; WebAssembly plugins are added at startup -/// ([`FormatRegistry::with_plugin`]). +/// [`Format`] and its plugin: the built-in format plugin +/// ([`crate::plugins::builtin`]: NuGet), Docker and raw. /// /// Docker ([`DOCKER_FORMAT`]) and raw ([`RAW_FORMAT`]) are installed but have /// no [`Format`]: their repositories hold images (manifests and blobs) or @@ -152,8 +150,7 @@ impl Instance { /// them. pub struct FormatRegistry { formats: BTreeMap>, - /// Every format's plugin (built in or WebAssembly), by format id: its - /// client protocol. + /// Every format's plugin, by format id: its client protocol. plugins: BTreeMap>, } @@ -179,8 +176,7 @@ impl FormatRegistry { pub fn with_format(mut self, format: Arc) -> Result { let id = format.id().to_string(); if self.is_installed(&id) { - let built_in = - Self::is_native(&id) || self.plugins.get(&id).is_some_and(|p| p.is_builtin()); + let built_in = Self::is_native(&id) || self.plugins.contains_key(&id); return Err(if built_in { format!("format {id:?} is built into this server") } else { @@ -192,7 +188,7 @@ impl FormatRegistry { } /// Add a format plugin: its [`Format`] and its client protocol. Its id - /// must be free: a WebAssembly plugin never replaces a built-in format. + /// must be free: a plugin never replaces another format. pub fn with_plugin(mut self, plugin: Arc) -> Result { let format = Arc::new(forge_plugin::PluginFormat::new(&plugin)); self = self.with_format(format)?; @@ -210,7 +206,7 @@ impl FormatRegistry { self.plugins.get(id).cloned() } - /// The plugin formats (built in and WebAssembly), by format id. + /// The plugin formats, by format id. pub fn plugins(&self) -> impl Iterator> { self.plugins.values() } @@ -456,9 +452,9 @@ impl AppState { self } - /// Install the loaded WebAssembly plugins (see [`crate::plugins::load`]) - /// next to the built-in formats. A plugin whose format id is taken - /// (a built-in plugin, or another plugin) is an error. + /// Install further format plugins next to the built-in formats (the + /// tests' own). A plugin whose format id is taken (a built-in plugin, or + /// another plugin) is an error. pub fn with_plugins(mut self, plugins: Vec>) -> Result { let mut registry = FormatRegistry::new(self.config.nuget.limits()); for plugin in plugins { diff --git a/crates/server/tests/admin.rs b/crates/server/tests/admin.rs index ceaedefa..797bc8d5 100644 --- a/crates/server/tests/admin.rs +++ b/crates/server/tests/admin.rs @@ -3284,7 +3284,7 @@ async fn blob_cache_statistics_name_their_instance() { format!("admin{}@example.com", common::unique()), "adminpw".into(), )), - plugins_block: Some(format!( + extra_toml: Some(format!( "[blob_cache]\npath = {:?}\n", dir.path().join("cache").display().to_string() )), diff --git a/crates/server/tests/common/mod.rs b/crates/server/tests/common/mod.rs index 8e19d163..5df2a2bb 100644 --- a/crates/server/tests/common/mod.rs +++ b/crates/server/tests/common/mod.rs @@ -205,7 +205,7 @@ fn config_toml(db_url: &str, s3_endpoint: &str, opts: &ServerOpts) -> String { let web_assets_dir = opts.web_assets_dir.as_deref(); let oidc_block = opts.oidc_block.as_deref(); let crypto_block = opts.crypto_block.as_deref(); - let plugins_block = opts.plugins_block.as_deref(); + let extra_toml = opts.extra_toml.as_deref(); let s3_credentials = opts .s3_credentials .clone() @@ -259,12 +259,12 @@ poll_interval_seconds = 1 {web} {oidc} {crypto} -{plugins} +{extra} "#, bucket = S3_BUCKET, oidc = oidc_block.unwrap_or(""), crypto = crypto_block.unwrap_or(""), - plugins = plugins_block.unwrap_or(""), + extra = extra_toml.unwrap_or(""), ) } @@ -314,9 +314,9 @@ pub struct ServerOpts { /// Extra raw TOML lines for the `[auth]` section (e.g. /// `realm_cache_ttl_seconds`). pub auth_extra: Option, - /// When set, this raw TOML is appended last (the `[plugins]` section, or - /// any other), and the format plugins it lists are loaded. - pub plugins_block: Option, + /// When set, this raw TOML is appended last (a section of its own, such + /// as `[docker]`). + pub extra_toml: Option, /// Format plugins installed next to the built-in ones, as if built in. pub extra_plugins: Vec>, /// When set, this raw TOML replaces the static keys at the end of the @@ -337,7 +337,7 @@ impl Default for ServerOpts { public_base_url: None, server_extra: None, auth_extra: None, - plugins_block: None, + extra_toml: None, extra_plugins: Vec::new(), s3_credentials: None, } @@ -398,8 +398,6 @@ pub async fn start_with(pool: PgPool, opts: ServerOpts) -> TestServer { .expect("seed auth realms"); let cache = Cache::l1_only(config.cache.l1_max_capacity); - let mut plugins = forge_server::plugins::load(&config.plugins).expect("load format plugins"); - plugins.extend(opts.extra_plugins.iter().cloned()); let registry_keys = forge_server::registry::load_token_keys(&config, &pool, envelope.as_deref()) .await @@ -407,7 +405,7 @@ pub async fn start_with(pool: PgPool, opts: ServerOpts) -> TestServer { let state = AppState::new(pool.clone(), store, cache, config) .with_envelope(envelope) .with_registry_keys(registry_keys) - .with_plugins(plugins) + .with_plugins(opts.extra_plugins.clone()) .expect("install format plugins"); let layer = principal_layer(&state).expect("principal layer"); diff --git a/crates/server/tests/group_proxy.rs b/crates/server/tests/group_proxy.rs index 5ab231c8..35b062aa 100644 --- a/crates/server/tests/group_proxy.rs +++ b/crates/server/tests/group_proxy.rs @@ -198,3 +198,66 @@ async fn downstream_proxy_does_not_cache_a_miss_while_a_group_member_is_failing( assert_eq!(get(path.clone()).await.unwrap().status(), 404, "{path}"); } } + +/// #283: a proxy whose upstream cannot be reached, with nothing cached for +/// the request, answers 503 with `Retry-After` (an upstream outage), not 500 +/// (a fault of Forge); so does a group whose member it is. What is answered +/// locally (NuGet's service index) is still served. +#[tokio::test] +async fn a_proxy_of_an_unreachable_upstream_answers_503() { + // Nothing listens on the port: connection refused. + let closed = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + let srv = common::start().await; + let (u, p) = srv.admin_basic(); + let creds = (srv.admin_user.clone().unwrap(), srv.admin_token().await); + let client = srv.client(); + let s = common::unique(); + let (nuget, group) = (format!("ndown{s}"), format!("gdown{s}")); + for body in [ + json!({ "name": nuget, "kind": "proxy", "remote_url": format!("http://{closed}/v3/index.json") }), + json!({ "name": group, "kind": "group" }), + ] { + let resp = client + .post(format!("{}/admin/repositories", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&body) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + } + let resp = client + .post(format!( + "{}/admin/repositories/{group}/members", + srv.base_url + )) + .basic_auth(&u, Some(&p)) + .json(&json!({ "member": nuget, "priority": 10 })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + + let get = |path: String| { + client + .get(format!("{}{path}", srv.base_url)) + .basic_auth(&creds.0, Some(&creds.1)) + .send() + }; + for path in [ + format!("/v3/{nuget}/flat/sample.lib/index.json"), + format!("/v3/{nuget}/registration/sample.lib/index.json"), + format!("/v3/{nuget}/flat/sample.lib/1.0.0/sample.lib.1.0.0.nupkg"), + format!("/v3/{group}/flat/sample.lib/index.json"), + format!("/v3/{group}/registration/sample.lib/index.json"), + ] { + let resp = get(path.clone()).await.unwrap(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE, "{path}"); + assert_eq!(resp.headers()["retry-after"], "30", "{path}"); + } + let resp = get(format!("/v3/{nuget}/index.json")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); +} diff --git a/crates/server/tests/plugin_contract.rs b/crates/server/tests/plugin_contract.rs index c5de24bf..25c25e06 100644 --- a/crates/server/tests/plugin_contract.rs +++ b/crates/server/tests/plugin_contract.rs @@ -198,7 +198,7 @@ async fn start_probe(extra_toml: Option) -> TestServer { ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), extra_plugins: vec![Plugin::builtin(PROBE_TOML, Probe).unwrap()], - plugins_block: extra_toml, + extra_toml, ..Default::default() }, ) @@ -349,6 +349,37 @@ async fn a_format_is_served_under_its_own_mount_and_kinds() { assert_eq!(nuget["mount"], "/v3"); } +/// A repository of a format the server does not have (its plugin removed +/// since) answers 503 under a plugin's mount, and so does an administrator's +/// delete of one of its versions: it is never read as another format. +#[tokio::test] +async fn a_repository_whose_format_is_missing_is_unavailable() { + let srv = start_probe(None).await; + let name = format!("orphan{}", unique()); + let mut row = forge_db::repository_dao::NewRepository::hosted(name.clone()); + row.format = "gone".into(); + forge_db::repository_dao::insert_repository(&srv.pool, row) + .await + .expect("insert repository"); + let (u, p) = srv.admin_basic(); + for (method, url) in [ + (reqwest::Method::GET, format!("/probe/{name}/nothing")), + ( + reqwest::Method::DELETE, + format!("/admin/repositories/{name}/packages/demo/1.0.0"), + ), + ] { + let resp = srv + .client() + .request(method, format!("{}{url}", srv.base_url)) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE, "{url}"); + } +} + #[tokio::test] async fn uploads_take_the_first_part_and_assets_their_names() { let srv = start_probe(None).await; diff --git a/crates/server/tests/plugins.rs b/crates/server/tests/plugins.rs deleted file mode 100644 index 30a9d400..00000000 --- a/crates/server/tests/plugins.rs +++ /dev/null @@ -1,1295 +0,0 @@ -//! Format plugins end to end, through the example plugin `plugins/helm` -//! (Helm chart repositories; Forge does not build it in): installed as if -//! built in, its repositories created through the admin API and used through -//! `/repository/{repo}` the way Helm clients use a chart repository — and the -//! same crate built as a WebAssembly component, loaded at startup. - -mod common; - -use std::io::Write; - -use common::{start_with, unique, ServerOpts, TestServer}; -use reqwest::StatusCode; -use serde_json::{json, Value}; -use sha2::Digest; - -/// The format id the Helm component is installed under (the Helm plugin -/// installed as if built in holds `helm`). -const WASM_HELM: &str = "helm-wasm"; - -/// The `[plugins]` block loading the Helm plugin built as a component, -/// served under a mount of its own (`/charts`). -fn wasm_helm_plugins_block() -> String { - let dir = forge_plugin::testing::example_plugin_with("helm", WASM_HELM, "mount = \"/charts\""); - let sha = forge_plugin::testing::component_sha256(&dir); - let parent = dir.parent().expect("plugin directory parent"); - format!("[plugins]\ndir = {parent:?}\n\n[plugins.formats]\n{WASM_HELM} = \"sha256:{sha}\"\n") -} - -/// The Helm plugin, to install as if built in. -fn helm_plugin() -> std::sync::Arc { - forge_plugin::Plugin::builtin(forge_format_helm::PLUGIN_TOML, forge_format_helm::Helm) - .expect("plugins/helm/plugin.toml is valid") -} - -/// A server with the Helm plugin installed as if built in. -async fn start_helm(mut opts: ServerOpts) -> TestServer { - opts.extra_plugins.push(helm_plugin()); - start_with( - common::pool().await, - ServerOpts { - bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - ..opts - }, - ) - .await -} - -/// A gzipped chart archive declaring `name` at `version`. -fn chart(name: &str, version: &str) -> Vec { - chart_with_values(name, version, "replicas: 1\n") -} - -/// [`chart`] with `values` as its `values.yaml`. -fn chart_with_values(name: &str, version: &str, values: &str) -> Vec { - chart_archive( - name, - &format!("apiVersion: v2\nname: {name}\nversion: {version}\ndescription: {name} chart\n"), - values, - ) -} - -/// A gzipped chart archive of the chart `name` with `chart_yaml` as its -/// Chart.yaml. -fn chart_archive(name: &str, chart_yaml: &str, values: &str) -> Vec { - tgz(&[ - (&format!("{name}/values.yaml"), values), - (&format!("{name}/Chart.yaml"), chart_yaml), - ]) -} - -/// A gzipped ustar archive of `(path, contents)` entries. -fn tgz(entries: &[(&str, &str)]) -> Vec { - let mut tar = Vec::new(); - for (path, data) in entries { - let mut header = [0u8; 512]; - header[..path.len()].copy_from_slice(path.as_bytes()); - header[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes()); - header[156] = b'0'; - header[257..263].copy_from_slice(b"ustar\0"); - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - tar.extend_from_slice(&header); - tar.extend_from_slice(data.as_bytes()); - tar.resize(tar.len().div_ceil(512) * 512, 0); - } - tar.extend_from_slice(&[0u8; 1024]); - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(&tar).unwrap(); - gz.finish().unwrap() -} - -async fn create_repo(srv: &TestServer, body: Value) -> reqwest::Response { - let (u, p) = srv.admin_basic(); - srv.client() - .post(format!("{}/admin/repositories", srv.base_url)) - .basic_auth(&u, Some(&p)) - .json(&body) - .send() - .await - .unwrap() -} - -/// Create a hosted Helm repository and return its name. -async fn helm_repo(srv: &TestServer, anonymous_read: bool) -> String { - hosted_repo(srv, "helm", anonymous_read).await -} - -/// Create a hosted repository of `format` and return its name. -async fn hosted_repo(srv: &TestServer, format: &str, anonymous_read: bool) -> String { - let name = format!("charts{}", unique()); - let resp = create_repo( - srv, - json!({ "name": name, "kind": "hosted", "format": format, "anonymous_read": anonymous_read }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - name -} - -/// The admin's username and an API token: what `helm repo add --username -/// --password` would be given. -async fn credentials(srv: &TestServer) -> (String, String) { - (srv.admin_user.clone().unwrap(), srv.admin_token().await) -} - -async fn push( - srv: &TestServer, - repo: &str, - creds: &(String, String), - body: Vec, -) -> reqwest::Response { - push_at(srv, "/repository", repo, creds, body).await -} - -/// [`push`] to a repository served under `mount`. -async fn push_at( - srv: &TestServer, - mount: &str, - repo: &str, - creds: &(String, String), - body: Vec, -) -> reqwest::Response { - srv.client() - .post(format!("{}{mount}/{repo}/api/charts", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .body(body) - .send() - .await - .unwrap() -} - -async fn get_index( - srv: &TestServer, - repo: &str, - creds: Option<&(String, String)>, -) -> reqwest::Response { - get_index_at(srv, "/repository", repo, creds).await -} - -/// [`get_index`] of a repository served under `mount`. -async fn get_index_at( - srv: &TestServer, - mount: &str, - repo: &str, - creds: Option<&(String, String)>, -) -> reqwest::Response { - let req = srv - .client() - .get(format!("{}{mount}/{repo}/index.yaml", srv.base_url)); - let req = match creds { - Some((u, p)) => req.basic_auth(u, Some(p)), - None => req, - }; - req.send().await.unwrap() -} - -#[tokio::test] -async fn a_helm_repository_round_trip() { - let srv = start_helm(ServerOpts::default()).await; - let repo = helm_repo(&srv, false).await; - let creds = credentials(&srv).await; - - // Publish: the raw body, and the multipart field `helm cm-push` sends. - let first = chart("Demo", "v1.0"); - let resp = push(&srv, &repo, &creds, first.clone()).await; - assert_eq!(resp.status(), StatusCode::CREATED); - assert_eq!( - resp.json::().await.unwrap(), - json!({ "saved": true }) - ); - let form = reqwest::multipart::Form::new() - .text("note", "skipped before the chart") - .part( - "chart", - reqwest::multipart::Part::bytes(chart("Demo", "1.1.0")).file_name("Demo-1.1.0.tgz"), - ); - let resp = srv - .client() - .post(format!("{}/repository/{repo}/api/charts", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .multipart(form) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - // The same version again (names match case-insensitively), and - // something that is not a chart. - let resp = push(&srv, &repo, &creds, chart("demo", "1.0.0")).await; - assert_eq!(resp.status(), StatusCode::CONFLICT); - let resp = push(&srv, &repo, &creds, b"not a chart".to_vec()).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp.text().await.unwrap().contains("not a chart archive")); - // A chart that Helm would install as another one: Helm loads the last - // Chart.yaml of an archive. - let twin = tgz(&[ - ( - "demo/Chart.yaml", - "apiVersion: v2\nname: demo\nversion: 3.0.0\n", - ), - ( - "evil/Chart.yaml", - "apiVersion: v2\nname: evil\nversion: 6.6.6\n", - ), - ]); - let resp = push(&srv, &repo, &creds, twin).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp - .text() - .await - .unwrap() - .contains("more than one top-level directory")); - - // The index lists both versions, newest first, as published. - let resp = get_index(&srv, &repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.headers()["x-content-type-options"], "nosniff"); - let index: Value = resp.json().await.unwrap(); - let entries = index["entries"]["Demo"].as_array().expect("Demo entries"); - let versions: Vec<&str> = entries - .iter() - .map(|e| e["version"].as_str().unwrap()) - .collect(); - assert_eq!(versions, ["1.1.0", "v1.0"]); - assert_eq!(entries[1]["description"], "Demo chart"); - assert_eq!( - entries[1]["digest"], - hex::encode(sha2::Sha256::digest(&first)), - "the digest is the stored package's SHA-256" - ); - - // The URL the index points at serves the chart, streamed by the host. - let url = entries[1]["urls"][0].as_str().unwrap(); - let path = url - .strip_prefix("http://test.local") - .expect("absolute URL on the public base"); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), first.as_slice()); - let resp = srv - .client() - .head(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.headers()["content-length"], first.len().to_string()); - - // The ChartMuseum listing of one chart. - let resp = srv - .client() - .get(format!( - "{}/repository/{repo}/api/charts/demo", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.json::() - .await - .unwrap() - .as_array() - .unwrap() - .len(), - 2 - ); - - // Delete a version: gone from the index and from downloads. - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/demo/1.0.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = get_index(&srv, &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["Demo"].as_array().unwrap().len(), 1); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/demo/1.0.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // The package detail of the web UI reads the plugin's summary. - let session = srv - .login( - &srv.admin_user.clone().unwrap(), - &srv.admin_pass.clone().unwrap(), - ) - .await; - let detail: Value = session - .apply( - srv.client() - .get(format!("{}/browse/packages/{repo}/demo", srv.base_url)), - ) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(detail["original_id"], "Demo"); - assert_eq!(detail["versions"].as_array().unwrap().len(), 1); -} - -/// #318: a holder of the built-in `writer` role uploads a new chart to a -/// hosted Helm repository and reads it back, but cannot delete a version. -#[tokio::test] -async fn a_writer_uploads_charts_but_deletes_none() { - let srv = start_helm(ServerOpts::default()).await; - let repo = helm_repo(&srv, false).await; - let (au, ap) = srv.admin_basic(); - let username = format!("charts-writer{}@example.com", unique()); - let user_id = srv - .create_user(&username, "pw12345678-charts", "reader") - .await; - let resp = srv - .client() - .put(format!("{}/admin/users/{user_id}/roles", srv.base_url)) - .basic_auth(&au, Some(&ap)) - .json(&json!({ "role": "writer" })) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - let token: Value = srv - .client() - .post(format!("{}/admin/users/{user_id}/tokens", srv.base_url)) - .basic_auth(&au, Some(&ap)) - .json(&json!({ "name": "ci" })) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - let creds = (username, token["token"].as_str().unwrap().to_string()); - - let resp = push(&srv, &repo, &creds, chart("Fresh", "0.1.0")).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let index: Value = get_index(&srv, &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["Fresh"].as_array().unwrap().len(), 1); - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/fresh/0.1.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::FORBIDDEN); -} - -#[tokio::test] -async fn plugin_repositories_are_authorized_like_feeds() { - let srv = start_with( - common::fresh_pool().await, - ServerOpts { - bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - extra_plugins: vec![helm_plugin()], - ..Default::default() - }, - ) - .await; - srv.set_auth_settings(serde_json::json!({"anonymous_mode": "all"})) - .await; - let private = helm_repo(&srv, false).await; - let public = helm_repo(&srv, true).await; - let creds = credentials(&srv).await; - let resp = push(&srv, &public, &creds, chart("web", "1.0.0")).await; - assert_eq!(resp.status(), StatusCode::CREATED); - - // Reading: anonymous only where anonymous reads are allowed. - let resp = get_index(&srv, &private, None).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - assert!(resp.headers().contains_key("www-authenticate")); - let resp = get_index(&srv, &public, None).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert!(index["entries"]["web"].is_array()); - // An unknown repository answers like one the caller may not read. - let resp = index_of_unknown(&srv).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - - // Writing: never anonymously, and never with a password (a browser - // credential), only with an API token. - let resp = srv - .client() - .post(format!("{}/repository/{public}/api/charts", srv.base_url)) - .body(chart("web", "2.0.0")) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - let (user, password) = srv.admin_password_basic(); - let resp = push(&srv, &public, &(user, password), chart("web", "2.0.0")).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - assert!(resp.text().await.unwrap().contains("API token")); -} - -async fn index_of_unknown(srv: &TestServer) -> reqwest::Response { - get_index(srv, &format!("missing{}", unique()), None).await -} - -#[tokio::test] -async fn formats_keep_to_their_own_mounts_and_kinds() { - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let charts = helm_repo(&srv, false).await; - let nuget = format!("nuget{}", unique()); - let resp = create_repo(&srv, json!({ "name": nuget, "kind": "hosted" })).await; - assert_eq!(resp.status(), StatusCode::CREATED); - - // A Helm repository is not a NuGet feed, and the other way around. - let resp = srv - .client() - .get(format!("{}/v3/{charts}/index.json", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let resp = get_index(&srv, &nuget, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // The admin API lists what can be created. - let (u, p) = srv.admin_basic(); - let formats: Value = srv - .client() - .get(format!("{}/admin/formats", srv.base_url)) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(formats[0]["id"], "nuget"); - assert_eq!(formats[1]["id"], "docker"); - assert_eq!(formats[2]["id"], "helm"); - assert_eq!(formats[2]["builtin"], true); - assert_eq!(formats[2]["kinds"], json!(["hosted", "proxy", "group"])); - assert_eq!(formats[2]["mount"], "/repository"); - assert!(formats[2].get("sha256").is_none()); - - // A proxy needs its upstream; unknown formats are refused. - let resp = create_repo( - &srv, - json!({ "name": format!("p{}", unique()), "kind": "proxy", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - let resp = create_repo( - &srv, - json!({ "name": format!("n{}", unique()), "kind": "hosted", "format": "npm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp.text().await.unwrap().contains("helm")); - - // A group serves one format: a NuGet member is refused, a Helm one is - // served through the group. - let group = format!("group{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": group, "kind": "group", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let add = |member: String, priority: i32| { - srv.client() - .post(format!( - "{}/admin/repositories/{group}/members", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .json(&json!({ "member": member, "priority": priority })) - .send() - }; - assert_eq!( - add(nuget.clone(), 1).await.unwrap().status(), - StatusCode::BAD_REQUEST - ); - assert_eq!( - add(charts.clone(), 1).await.unwrap().status(), - StatusCode::CREATED - ); - let resp = push(&srv, &charts, &creds, chart("api", "3.2.1")).await; - assert_eq!(resp.status(), StatusCode::CREATED); - let index: Value = get_index(&srv, &group, Some(&creds)) - .await - .json() - .await - .unwrap(); - let url = index["entries"]["api"][0]["urls"][0] - .as_str() - .unwrap() - .to_string(); - assert!( - url.contains(&format!("/repository/{group}/charts/api/3.2.1/")), - "{url}" - ); - let resp = srv - .client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - // A group publishes nothing. - let resp = push(&srv, &group, &creds, chart("api", "3.2.2")).await; - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); -} - -#[tokio::test] -async fn a_repository_has_an_upload_limit_of_its_own() { - let srv = start_helm(ServerOpts::default()).await; - let name = format!("charts{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": name, "kind": "hosted", "format": "helm", "max_upload_bytes": 1048576 }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let creds = credentials(&srv).await; - let resp = push(&srv, &name, &creds, chart("small", "1.0.0")).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let resp = push(&srv, &name, &creds, vec![0u8; 2 * 1024 * 1024]).await; - assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE); - assert_eq!( - resp.text().await.unwrap(), - "upload exceeds limit of 1048576 bytes" - ); -} - -#[tokio::test] -async fn a_repository_whose_plugin_is_missing_is_unavailable() { - // A server without the plugin, over a repository created with it, under - // a mount another plugin serves. - let srv = start_helm(ServerOpts::default()).await; - let name = format!("orphan{}", unique()); - let mut row = forge_db::repository_dao::NewRepository::hosted(name.clone()); - row.format = WASM_HELM.into(); - forge_db::repository_dao::insert_repository(&srv.pool, row) - .await - .expect("insert repository"); - let (u, p) = srv.admin_basic(); - let resp = get_index(&srv, &name, Some(&(u.clone(), p.clone()))).await; - assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); - let resp = srv - .client() - .delete(format!( - "{}/admin/repositories/{name}/packages/demo/1.0.0", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); -} - -/// The Helm crate built as a WebAssembly component serves its repositories -/// exactly as the built-in one does: one contract, two ways to run it. -#[tokio::test] -async fn a_webassembly_plugin_serves_like_a_builtin_one() { - let srv = start_helm(ServerOpts { - plugins_block: Some(wasm_helm_plugins_block()), - ..Default::default() - }) - .await; - let creds = credentials(&srv).await; - let (u, p) = srv.admin_basic(); - let formats: Value = srv - .client() - .get(format!("{}/admin/formats", srv.base_url)) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - let wasm = formats - .as_array() - .unwrap() - .iter() - .find(|f| f["id"] == WASM_HELM) - .expect("the component is listed"); - assert_eq!(wasm["builtin"], false); - assert_eq!(wasm["sha256"].as_str().map(str::len), Some(64)); - - assert_eq!(wasm["mount"], "/charts"); - - let repo = hosted_repo(&srv, WASM_HELM, false).await; - let package = chart("Demo", "v1.0"); - let resp = push_at(&srv, "/charts", &repo, &creds, package.clone()).await; - assert_eq!(resp.status(), StatusCode::CREATED); - let resp = push_at(&srv, "/charts", &repo, &creds, package.clone()).await; - assert_eq!(resp.status(), StatusCode::CONFLICT); - let resp = push_at(&srv, "/charts", &repo, &creds, b"not a chart".to_vec()).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - // Served under its own mount only. - let resp = get_index(&srv, &repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let index: Value = get_index_at(&srv, "/charts", &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - let entry = &index["entries"]["Demo"][0]; - assert_eq!(entry["version"], "v1.0"); - assert_eq!(entry["digest"], hex::encode(sha2::Sha256::digest(&package))); - let url = entry["urls"][0].as_str().unwrap(); - assert!(url.contains(&format!("/charts/{repo}/charts/")), "{url}"); - let resp = srv - .client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), package.as_slice()); -} - -/// A WebAssembly plugin never replaces a built-in format. -#[test] -fn a_component_cannot_take_a_builtin_format_id() { - let dir = forge_plugin::testing::example_plugin_as("helm", "nuget"); - let plugin = forge_plugin::Plugin::load( - &dir, - &forge_plugin::testing::component_sha256(&dir), - &forge_plugin::HostLimits::default(), - ) - .unwrap(); - let err = forge_server::state::FormatRegistry::new(Default::default()) - .with_plugin(plugin) - .err() - .expect("nuget is taken"); - assert!(err.contains("built into this server"), "{err}"); -} - -/// A chart repository upstream of a Helm proxy, serving `index.yaml` and -/// chart archives under `/stable`, and counting chart downloads. -struct ChartUpstream { - base: String, - charts: std::sync::Arc>>>, - downloads: std::sync::Arc, -} - -impl ChartUpstream { - async fn start(index: String) -> Self { - use axum::extract::{Path, State}; - use axum::response::IntoResponse; - use std::sync::atomic::Ordering; - - type Shared = ( - String, - std::sync::Arc>>>, - std::sync::Arc, - ); - let charts = std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); - let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let app = axum::Router::new() - .route( - "/stable/index.yaml", - axum::routing::get(|State((index, _, _)): State| async move { index }), - ) - .route( - "/stable/charts/{file}", - axum::routing::get( - |State((_, charts, downloads)): State, Path(file): Path| async move { - match charts.lock().unwrap().get(&file).cloned() { - Some(bytes) => { - downloads.fetch_add(1, Ordering::SeqCst); - bytes.into_response() - } - None => StatusCode::NOT_FOUND.into_response(), - } - }, - ), - ) - .with_state((index, charts.clone(), downloads.clone())); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base = format!("http://{}/stable", listener.local_addr().unwrap()); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - ChartUpstream { - base, - charts, - downloads, - } - } - - fn publish(&self, file: &str, bytes: Vec) { - self.charts.lock().unwrap().insert(file.into(), bytes); - } - - fn downloads(&self) -> usize { - self.downloads.load(std::sync::atomic::Ordering::SeqCst) - } -} - -/// Check `index` against the types Helm gives an index's fields (its -/// `repo.ChartVersion` and `chart.Metadata`). Helm reads a JSON index with -/// Go's `encoding/json` and refuses the whole index when one value has -/// another type; fields it does not know it ignores. -fn assert_helm_types(index: &Value) { - const STRINGS: &[&str] = &[ - "name", - "home", - "version", - "description", - "icon", - "apiVersion", - "condition", - "tags", - "appVersion", - "kubeVersion", - "type", - "created", - "digest", - ]; - let strings = |v: &Value| v.as_array().is_some_and(|a| a.iter().all(Value::is_string)); - let fields = |v: &Value, strs: &[&str], lists: &[&str], bools: &[&str], any: &[&str]| { - v.as_object().is_some_and(|o| { - o.iter().all(|(k, v)| { - (strs.contains(&k.as_str()) && v.is_string()) - || (lists.contains(&k.as_str()) && strings(v)) - || (bools.contains(&k.as_str()) && v.is_boolean()) - || (any.contains(&k.as_str()) && v.is_array()) - }) - }) - }; - assert!(index["apiVersion"].is_string()); - for (chart, versions) in index["entries"].as_object().expect("entries") { - for version in versions.as_array().expect("versions") { - for (key, value) in version.as_object().expect("a chart version") { - let ok = match key.as_str() { - key if STRINGS.contains(&key) => value.is_string(), - "sources" | "keywords" | "urls" => strings(value), - "deprecated" | "removed" => value.is_boolean(), - "annotations" => value - .as_object() - .is_some_and(|m| m.values().all(Value::is_string)), - "maintainers" => value.as_array().is_some_and(|a| { - a.iter() - .all(|m| fields(m, &["name", "email", "url"], &[], &[], &[])) - }), - "dependencies" => value.as_array().is_some_and(|a| { - a.iter().all(|d| { - fields( - d, - &["name", "version", "repository", "condition", "alias"], - &["tags"], - &["enabled"], - &["import-values"], - ) - }) - }), - _ => true, - }; - assert!(ok, "{chart}: Helm cannot read {key} = {value}"); - } - } - } -} - -/// A Chart.yaml (or an upstream's YAML index) may hold a number or a -/// boolean where Helm wants a string, as Helm's YAML reader accepts: the -/// index lists it as a string, so it does not break the index (#206). -#[tokio::test] -async fn an_index_lists_scalars_as_the_strings_helm_reads() { - const QUIRKS: &str = concat!( - "appVersion: 2\n", - "kubeVersion: 1.30\n", - "icon: 7\n", - "keywords: [web, 8, true]\n", - "annotations:\n", - " artifacthub.io/prerelease: false\n", - " revision: 3\n", - " nested: {a: b}\n", - "maintainers:\n", - " - name: 42\n", - " email: ops@example.com\n", - " extra: [x]\n", - "dependencies:\n", - " - name: redis\n", - " version: 18\n", - " repository: https://charts.example\n", - " enabled: true\n", - " tags: [cache, 1]\n", - " import-values: [data]\n", - "deprecated: false\n", - "home: {not: a string}\n", - "sources: just-one\n", - ); - let upstream_index = format!( - "apiVersion: v1\nentries:\n quirky:\n - apiVersion: v2\n name: quirky\n \ - version: 1.0.0\n urls: [charts/quirky-1.0.0.tgz]\n{}", - QUIRKS - .lines() - .map(|l| format!(" {l}\n")) - .collect::() - ); - let upstream = ChartUpstream::start(upstream_index).await; - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let hosted = helm_repo(&srv, false).await; - let archive = chart_archive( - "quirks", - &format!("apiVersion: v2\nname: quirks\nversion: 0.1.0\n{QUIRKS}"), - "replicas: 1\n", - ); - let resp = push(&srv, &hosted, &creds, archive).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let group = format!("group{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": group, "kind": "group", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let (u, p) = srv.admin_basic(); - for (member, priority) in [(&hosted, 10), (&proxy, 1)] { - let resp = srv - .client() - .post(format!( - "{}/admin/repositories/{group}/members", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .json(&json!({ "member": member, "priority": priority })) - .send() - .await - .unwrap(); - assert!(resp.status().is_success(), "{}", resp.text().await.unwrap()); - } - - for (repo, chart) in [ - (&hosted, "quirks"), - (&proxy, "quirky"), - (&group, "quirks"), - (&group, "quirky"), - ] { - let resp = get_index(&srv, repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert_helm_types(&index); - let entry = &index["entries"][chart][0]; - assert_eq!(entry["appVersion"], "2", "{repo}: {entry}"); - assert_eq!(entry["kubeVersion"], "1.30"); - assert_eq!(entry["keywords"], json!(["web", "8", "true"])); - assert_eq!(entry["annotations"]["artifacthub.io/prerelease"], "false"); - assert_eq!(entry["annotations"]["revision"], "3"); - assert!(entry["annotations"].get("nested").is_none()); - assert_eq!( - entry["maintainers"][0], - json!({"name": "42", "email": "ops@example.com"}) - ); - assert_eq!(entry["dependencies"][0]["version"], "18"); - assert_eq!(entry["dependencies"][0]["tags"], json!(["cache", "1"])); - assert_eq!(entry["dependencies"][0]["enabled"], true); - assert_eq!(entry["deprecated"], false); - assert!(entry.get("home").is_none() && entry.get("sources").is_none()); - } -} - -/// A Helm proxy lists its upstream's charts in its index, serves each from -/// the upstream once and from its cache after, and publishes nothing. -#[tokio::test] -async fn a_helm_proxy_serves_its_upstreams_charts() { - let index = "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n version: 1.0.0\n description: demo chart\n urls: [charts/demo-1.0.0.tgz]\n other:\n - apiVersion: v2\n name: other\n version: 2.0.0\n urls: [charts/other-2.0.0.tgz]\n twin:\n - apiVersion: v2\n name: twin\n version: 1.0.0\n urls: [charts/twin-1.0.0.tgz]\n".to_string(); - let upstream = ChartUpstream::start(index).await; - let demo = chart("demo", "1.0.0"); - upstream.publish("demo-1.0.0.tgz", demo.clone()); - // Helm would install it as `evil` 6.6.6, its last Chart.yaml. - upstream.publish( - "twin-1.0.0.tgz", - tgz(&[ - ( - "twin/Chart.yaml", - "apiVersion: v2\nname: twin\nversion: 1.0.0\n", - ), - ( - "twin/Chart.yaml", - "apiVersion: v2\nname: evil\nversion: 6.6.6\n", - ), - ]), - ); - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - // The index lists the upstream's charts, nothing downloaded yet. - let resp = get_index(&srv, &proxy, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert_eq!(index["entries"]["other"][0]["version"], "2.0.0"); - let url = index["entries"]["demo"][0]["urls"][0] - .as_str() - .unwrap() - .to_string(); - assert!( - url.ends_with(&format!( - "/repository/{proxy}/charts/demo/1.0.0/demo-1.0.0.tgz" - )), - "{url}" - ); - assert_eq!(index["entries"]["demo"][0]["description"], "demo chart"); - assert_eq!(upstream.downloads(), 0); - - // The chart comes from the upstream once, then from the cache. - let get = |url: String| { - srv.client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - }; - for _ in 0..2 { - let resp = get(url.clone()).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), demo.as_slice()); - } - assert_eq!(upstream.downloads(), 1); - - // A version the upstream does not have is not found. - let resp = get(format!( - "{}/repository/{proxy}/charts/other/2.0.0/other-2.0.0.tgz", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // A chart that is not what Helm would install is not cached. - let resp = get(format!( - "{}/repository/{proxy}/charts/twin/1.0.0/twin-1.0.0.tgz", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert_eq!(upstream.downloads(), 2); - - // The ChartMuseum view of one chart, from the upstream's index. - let resp = get(format!( - "{}/repository/{proxy}/api/charts/demo", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let versions: Value = resp.json().await.unwrap(); - assert_eq!(versions[0]["version"], "1.0.0"); - - // A proxy publishes nothing. - let resp = push(&srv, &proxy, &creds, chart("demo", "1.0.1")).await; - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); -} - -/// `len` lowercase letters and digits that PostgreSQL cannot compress, so -/// that an index entry holding them is as long as they are. -fn incompressible(len: usize) -> String { - const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; - let mut block = sha2::Sha256::digest(b"incompressible"); - let mut out = String::with_capacity(len); - while out.len() < len { - out.extend( - block - .iter() - .map(|b| char::from(ALPHABET[usize::from(*b) % ALPHABET.len()])) - .take(len - out.len()), - ); - block = sha2::Sha256::digest(&block[..]); - } - out -} - -/// #231: a chart version longer than 128 characters (an OCI tag's limit) -/// is refused on push with 400, not stored until its index entry fails the -/// insert (500); a URL naming one answers 404, in a hosted repository and in -/// a proxy, which neither records it nor asks its upstream; and a proxy -/// leaves an upstream's such version out of its listings. -#[tokio::test] -async fn chart_versions_longer_than_an_oci_tag_are_refused() { - let huge = format!("1.0.0-{}", incompressible(4096)); - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let get = |path: String| { - srv.client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - }; - let long_rows = |repo: String| { - let pool = srv.pool.clone(); - async move { - sqlx::query_scalar::<_, i64>( - "SELECT count(*) FROM proxy_cached_version v JOIN repository r \ - ON r.id = v.repository_id WHERE r.name = $1 AND length(v.version) > 128", - ) - .bind(repo) - .fetch_one(&pool) - .await - .unwrap() - } - }; - - let repo = helm_repo(&srv, false).await; - let resp = push(&srv, &repo, &creds, chart("demo", &huge)).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp - .text() - .await - .unwrap() - .contains("longer than 128 characters")); - let longest = format!("1.0.0-{}", "a".repeat(122)); - for (version, status) in [ - (format!("{longest}a"), StatusCode::BAD_REQUEST), - (longest.clone(), StatusCode::CREATED), - ] { - let resp = push(&srv, &repo, &creds, chart("demo", &version)).await; - assert_eq!(resp.status(), status, "{}", version.len()); - } - let resp = get(format!("/repository/{repo}/charts/demo/{longest}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let resp = get(format!("/repository/{repo}/charts/demo/{huge}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let index = format!( - "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n \ - version: 1.0.0\n urls: [charts/demo-1.0.0.tgz]\n - apiVersion: v2\n \ - name: demo\n version: {huge}\n urls: [charts/demo-huge.tgz]\n" - ); - let upstream = ChartUpstream::start(index).await; - upstream.publish("demo-huge.tgz", chart("demo", &huge)); - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let resp = get(format!("/repository/{proxy}/charts/demo/{huge}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - assert_eq!(upstream.downloads(), 0); - assert_eq!(long_rows(proxy.clone()).await, 0); - - // The upstream's longer version is left out of the proxy's listings. - let resp = get(format!("/repository/{proxy}/api/charts/demo")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let versions: Value = resp.json().await.unwrap(); - assert_eq!(versions.as_array().unwrap().len(), 1); - assert_eq!(versions[0]["version"], "1.0.0"); - let index: Value = get_index(&srv, &proxy, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["demo"].as_array().unwrap().len(), 1); - assert_eq!(long_rows(proxy).await, 0); -} - -/// A Helm index may list a chart on another host (a project's GitHub -/// releases): the proxy downloads it from there, a host its upstream's -/// index names (#233). -#[tokio::test] -async fn a_helm_proxy_downloads_a_chart_its_upstreams_index_lists_on_another_host() { - let releases = ChartUpstream::start(String::new()).await; - let demo = chart("demo", "1.0.0"); - releases.publish("demo-1.0.0.tgz", demo.clone()); - let index = format!( - "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n \ - version: 1.0.0\n urls:\n - {}/charts/demo-1.0.0.tgz\n", - releases.base - ); - let upstream = ChartUpstream::start(index).await; - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - let resp = srv - .client() - .get(format!( - "{}/repository/{proxy}/charts/demo/1.0.0/demo-1.0.0.tgz", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), demo.as_slice()); - assert_eq!((upstream.downloads(), releases.downloads()), (0, 1)); -} - -/// #283: a proxy whose upstream cannot be reached, with nothing cached for -/// the request, answers 503 with `Retry-After` (an upstream outage), not 500 -/// (a fault of Forge); so does a group whose member it is. What is answered -/// locally (NuGet's service index) is still served. -#[tokio::test] -async fn a_proxy_of_an_unreachable_upstream_answers_503() { - // Nothing listens on the port: connection refused. - let closed = std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap(); - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let s = unique(); - let (helm, nuget, group) = ( - format!("hdown{s}"), - format!("ndown{s}"), - format!("gdown{s}"), - ); - for body in [ - json!({ "name": helm, "kind": "proxy", "format": "helm", "remote_url": format!("http://{closed}") }), - json!({ "name": nuget, "kind": "proxy", "remote_url": format!("http://{closed}/v3/index.json") }), - json!({ "name": group, "kind": "group" }), - ] { - let resp = create_repo(&srv, body).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - } - let (u, p) = srv.admin_basic(); - let resp = srv - .client() - .post(format!( - "{}/admin/repositories/{group}/members", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .json(&json!({ "member": nuget, "priority": 10 })) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - let get = |path: String| { - srv.client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - }; - for path in [ - format!("/repository/{helm}/charts/demo/1.0.0/demo-1.0.0.tgz"), - format!("/v3/{nuget}/flat/sample.lib/index.json"), - format!("/v3/{nuget}/registration/sample.lib/index.json"), - format!("/v3/{nuget}/flat/sample.lib/1.0.0/sample.lib.1.0.0.nupkg"), - format!("/v3/{group}/flat/sample.lib/index.json"), - format!("/v3/{group}/registration/sample.lib/index.json"), - ] { - let resp = get(path.clone()).await.unwrap(); - assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE, "{path}"); - assert_eq!(resp.headers()["retry-after"], "30", "{path}"); - } - let resp = get(format!("/v3/{nuget}/index.json")).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); -} diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 206f5b15..221eb419 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -817,7 +817,7 @@ async fn chunked_uploads_cache_the_blob_not_its_chunks() { common::pool().await, common::ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - plugins_block: Some(format!( + extra_toml: Some(format!( "[blob_cache]\npath = {:?}\nmax_bytes = 1048576\nmin_free_bytes = 0\n", dir.path().join("cache").display().to_string() )), @@ -2560,7 +2560,7 @@ async fn admin_validates_docker_repositories() { .iter() .find(|f| f["id"] == "docker") .expect("docker is listed"); - assert_eq!(docker["builtin"], true); + assert!(docker.get("builtin").is_none(), "{docker}"); assert_eq!(docker["kinds"], json!(["hosted", "proxy", "group"])); assert_eq!(docker["mount"], "/v2"); @@ -2600,7 +2600,7 @@ async fn confirming_a_blob_records_the_use_of_its_link() { common::pool().await, common::ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - plugins_block: Some("[docker]\nunreferenced_blob_ttl_seconds = 3600\n".into()), + extra_toml: Some("[docker]\nunreferenced_blob_ttl_seconds = 3600\n".into()), ..Default::default() }, ) diff --git a/docker/licenses/about.toml b/docker/licenses/about.toml index e2f2103f..7875f4b4 100644 --- a/docker/licenses/about.toml +++ b/docker/licenses/about.toml @@ -12,10 +12,6 @@ # is before adding it. accepted = [ "Apache-2.0", - # Apache-2.0 plus an additional permission (the LLVM exception, waiving - # attribution for compiled-in code): Wasmtime and Cranelift, the runtime - # of format plugins. - "Apache-2.0 WITH LLVM-exception", "MIT", "ISC", "BSD-3-Clause", diff --git a/docs/changes/2.0.0/228.fixed.md b/docs/changes/2.0.0/228.fixed.md deleted file mode 100644 index 0dea4138..00000000 --- a/docs/changes/2.0.0/228.fixed.md +++ /dev/null @@ -1,4 +0,0 @@ -Format plugins: the host stores no manifest summary over 64 KiB from a -WebAssembly format plugin. Every listing hands each version's summary to -the plugin, so a plugin that summarized without a bound let a publisher make -every listing take gigabytes. diff --git a/docs/changes/2.0.0/229.security.md b/docs/changes/2.0.0/229.security.md deleted file mode 100644 index b6875d95..00000000 --- a/docs/changes/2.0.0/229.security.md +++ /dev/null @@ -1,10 +0,0 @@ -WebAssembly format plugins: the memory ceiling (`memory_mib`) did not bound -the host memory a component could take. It applied to each of up to 16 -linear memories on its own, left tables unbounded (a `table.grow` to -hundreds of millions of elements), and did not cover what a call returns: a -list whose elements all point at one large string was copied into the host -once per element. A buggy or malicious component could so make Forge run -out of memory on its first call. Now an instance's memories and tables -count against its ceiling together, and the strings and lists one call -returns, and the random bytes it asks WASI for, are bounded by it too; a -component that goes past it fails that call like any other trap. diff --git a/docs/changes/2.0.0/234.fixed.md b/docs/changes/2.0.0/234.fixed.md deleted file mode 100644 index 9e4c6abf..00000000 --- a/docs/changes/2.0.0/234.fixed.md +++ /dev/null @@ -1,5 +0,0 @@ -Format plugins: a plugin could take the mount `/images`, the web UI's image -pages, so a refresh, bookmark or shared link of an image page reached the -plugin instead of the web UI. `/images` and `/invite` are now reserved like -the other pages, and a test keeps the reserved mounts in step with the web -UI's routes. diff --git a/docs/deployment.md b/docs/deployment.md index b01cf242..070525c1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -889,11 +889,6 @@ notices within about 15 s, drops its in-memory cache and resubscribes. name ([format-plugins.md](./format-plugins.md#security-model)); to keep internal services out of reach even of an upstream that names them, block them on the outbound proxy or with network policy. -- **Format plugins.** Every instance must load the same WebAssembly plugins - (the same `[plugins]` section and component files): a repository whose - format is missing on an instance answers 503 there. The built-in plugin - (NuGet) comes with the image. See - [format-plugins.md](./format-plugins.md#loading-webassembly-plugins). ### Redis is a trust boundary diff --git a/docs/format-plugins.md b/docs/format-plugins.md index 41a3a976..95d13212 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -9,21 +9,10 @@ stays Forge's own. That covers storage, metadata, authentication, RBAC and package selectors, hosted, proxy and group repositories, retention, and the streaming of package bytes. -A plugin runs one of two ways, and the server cannot tell them apart: - -- **Built in.** The crate is linked into the server and called directly, like - any other Rust code. Built-in plugins need no configuration. Forge ships - one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). -- **As a WebAssembly component.** The same kind of crate, built for - `wasm32-wasip2`, loaded at startup from a digest the operator pinned and - run in a sandbox. This is how formats that are not part of Forge are added - without rebuilding it. - -[`plugins/helm`](../plugins/helm), classic Helm chart repositories -(`index.yaml`, ChartMuseum uploads), is the **example plugin**: the plugin -tests run it both ways, and it is the crate to start a new plugin from, but -Forge does not build it in. Helm charts are served by the registry -([docker.md](docker.md#helm-charts)). +Every plugin is built in: a crate under `plugins//` linked into the +server and called directly, like any other Rust code. Plugins need no +configuration. Forge ships one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). +Helm charts are served by the registry ([docker.md](docker.md#helm-charts)). > **Status: preview.** The plugin contract is `forge:format@0.3.0`. It may > change before 1.x of the contract; a plugin declares the contract version it @@ -31,29 +20,22 @@ Forge does not build it in. Helm charts are served by the registry ## Contents -- [One contract, two ways to run it](#one-contract-two-ways-to-run-it) +- [Layout](#layout) - [How it fits together](#how-it-fits-together) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) -- [Loading WebAssembly plugins](#loading-webassembly-plugins) - [Security model](#security-model) - [Writing a plugin](#writing-a-plugin) - [Limitations and roadmap](#limitations-and-roadmap) -- [Design notes: compared with artifact-keeper](#design-notes-compared-with-artifact-keeper) -## One contract, two ways to run it +## Layout -Every plugin, built in or not, has the same layout: +A plugin is a workspace crate with a `plugin.toml`: ``` -plugins/helm/ -├── Cargo.toml # a workspace crate; feature `component` builds it as a component -├── plugin.toml # id, name, version, contract version; component file and limits -└── src/lib.rs # impl FormatPlugin for Helm - plugins/nuget/ ├── Cargo.toml -├── plugin.toml # mount "/v3", kinds hosted/proxy/group +├── plugin.toml # id, name, version, contract version; mount "/v3", kinds hosted/proxy/group ├── src/lib.rs # impl FormatPlugin for Nuget ├── src/protocol.rs # the V3 protocol: handle and render ├── src/upstream.rs # proxies: reading another V3 feed (upstream) @@ -64,24 +46,17 @@ The pieces: | Crate | What it holds | | --- | --- | -| [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, the same contract as WIT ([`wit/format.wit`](../crates/plugin-api/wit/format.wit)), and, behind its `component` feature, the glue that exports a plugin as a component (`export_component!`). Depends on nothing but `std`. | +| [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, and, behind its `testing` feature, helpers that run a plugin's protocol against canned answers. Depends on nothing but `std`. | | [`plugins/`](../plugins) | One crate per plugin, implementing `FormatPlugin`. | -| [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin, built in or WebAssembly), the Wasmtime runtime whose `WasmPlugin` implements `FormatPlugin` by calling into a component, and `PluginFormat`, which adapts any plugin to Forge's `Format` seam. | -| [`forge-server`](../crates/server/src/plugins.rs) | The built-in plugins (`plugins::builtin`), loading the configured components, and the `{mount}/{repo}/...` dispatcher. | - -The trait mirrors the WIT world one to one: a component exports exactly the -trait's methods, and the host turns a loaded component back into a -`FormatPlugin`. So the dispatcher, the hosted write path, groups and -retention run the same code for both. The test -[`the_component_answers_like_the_builtin_plugin`](../crates/plugin/tests/component.rs) -runs the example plugin both ways and checks that every answer matches. +| [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin: its `plugin.toml` and its implementation) and `PluginFormat`, which adapts it to Forge's `Format` seam. | +| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`) and the `{mount}/{repo}/...` dispatcher. | ## How it fits together | | Forge (the host) | The plugin | | --- | --- | --- | | Identity | Calls the format for every id and version it sees | `canonical_id`, `validate_id`, `normalize_version`, `compare_versions` | -| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file (a component through a lent reader), returns id, version, manifest, dependencies | +| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file, returns id, version, manifest, dependencies | | Reads | Runs the queries the plugin asks for through the repository read path (hosted, proxy or group). Package selectors, group membership, prefer-hosted and reserved prefixes all apply | `handle` decides what a request is; `render` turns the loaded data into the response body | | Downloads | Streams the blob from S3, counts the download, answers `HEAD` without touching S3, and a single `Range` when the local blob cache is on | Names the version (`download`) | | Unlist, relist, delete | Hosted write path, with the repository's permissions and selectors; a package outside the caller's selector answers 404, like an unknown one | Names the version and the success response | @@ -116,8 +91,7 @@ in this order: hosted, and the caller must present an API token or a workload identity. -The contract is [`crates/plugin-api/src/lib.rs`](../crates/plugin-api/src/lib.rs), -and as WIT [`crates/plugin-api/wit/format.wit`](../crates/plugin-api/wit/format.wit). +The contract is [`crates/plugin-api/src/lib.rs`](../crates/plugin-api/src/lib.rs). Its queries are targeted: | Query | Answer | @@ -144,11 +118,7 @@ if present: `description`, and `dependency_groups` (the same shape as NuGet's). Every listing hands each version's summary to the plugin, so a summary -should stay small. The host stores none larger than 64 KiB from a -WebAssembly plugin, and logs a warning instead; a built-in plugin bounds its -own. The example plugin's summary, which holds a chart version's index -entry, takes at most 16 KiB: past that, its description is cut to 1 KiB, -then its largest fields are left out until it fits. +should stay small. ## Proxy repositories @@ -213,10 +183,9 @@ cached. ## Using a plugin `GET /admin/formats` lists every format the server can create repositories -of: the built-in plugin (`builtin: true`; NuGet), Docker, raw files, and the -loaded WebAssembly plugins (with the `sha256` of their component), each with -the repository kinds it supports and where its repositories are served. The -web UI's *New repository* form offers them. Plugins serve **hosted** and +of: the plugins (NuGet), Docker and raw files, each with the repository +kinds it supports and where its repositories are served. The web UI's *New +repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream side (NuGet does). Docker has proxies of its own. @@ -233,7 +202,11 @@ curl -u admin@example.com:$ADMIN_TOKEN -H 'content-type: application/json' \ ``` As with every format, the password is an **API token**. A password typed at -the web login is not a publishing credential. +the web login is not a publishing credential. The admin API refuses to +create a repository of a format the server does not have, and to put +repositories of different formats in one group. A repository whose format +the server no longer has stays in the database: it answers 503, is never +read as another format, and retention skips it (a failed run is recorded). The web UI's *Browse* screen filters repositories by format (`GET /browse/repos?format=`), and a repository's screen shows its @@ -249,70 +222,9 @@ a limit may be at most what a push can stage, `server.max_staging_bytes` less `nuget.max_nuspec_bytes` (`GET /admin/formats` reports it as `max_upload_bytes_limit`, with each format's `default_max_upload_bytes`). -## Loading WebAssembly plugins - -A WebAssembly plugin is a directory holding a `plugin.toml` and the component -it names: - -``` -/etc/forge/plugins/ -└── example/ - ├── plugin.toml - └── forge_format_example.wasm -``` - -The server loads only the plugins its configuration lists, each pinned to -the SHA-256 of its component: - -```toml -[plugins] -dir = "/etc/forge/plugins" - -[plugins.formats] -example = "sha256:c2053ffcbcc8cd38061813f5c42436464b7917c86fa3096eb849e653a0f1bf0a" -``` - -or, as environment variables: - -```bash -FORGE_PLUGINS__DIR=/etc/forge/plugins -FORGE_PLUGINS__FORMATS__EXAMPLE=sha256:c2053ffc… -``` - -- **Pinning is mandatory.** A component whose digest differs from its pin, - a `plugin.toml` whose `id` is not its directory name, a contract version - this server does not implement, or a component that does not instantiate - all fail startup. A directory under `dir` that is not listed is never - read. -- **Built-in formats come first.** A component cannot take the id of Docker, - raw or a built-in plugin (`nuget`); configuring one fails startup. -- **Every instance loads the same plugins.** A repository whose format is - not loaded on an instance answers 503 there; it is never read as another - format. The admin API refuses to create a repository of a format that is - not loaded, and to put repositories of different formats in one group. -- **Build them** with `cargo xtask plugins`. It builds every plugin in - `plugins/` as a component, installs it under - `target/plugins/installed//`, and prints the `[plugins]` section with - the digests. Built-in plugins are commented out there: their component - loads only under another id. A plugin whose `plugin.toml` names no - `component` is built in only, and skipped (NuGet, see - [NuGet as a plugin](#nuget-as-a-plugin)). In a container, bake the directory into an - image derived from Forge's, or mount it from a volume (a ConfigMap's 1 MiB - limit leaves little room for components). -- **Limits.** A plugin's `plugin.toml` may ask for a memory ceiling per - instance (`memory_mib`, default 64, at most 512; it covers the instance's - linear memories and tables together, and what one call returns to the - host), fuel per call (`fuel`, - default 5·10⁹, at most 5·10¹⁰) and a wall-clock budget per call - (`timeout_ms`, default 5,000, at most 30,000). Asking for more than the - maximum fails startup. Built-in plugins ignore `[limits]`. -- **Removing a plugin** leaves its repositories in the database. They answer - 503 until the plugin is back, and retention skips them (a failed run is - recorded). - ## Security model -The host enforces these for every plugin, built in or not: +The host enforces these for every plugin: - **No data beyond the caller's.** The plugin reads only through host queries, and those run with the caller's package selector and group @@ -328,9 +240,8 @@ The host enforces these for every plugin, built in or not: answers 400, a request naming one answers like one naming an unknown package (404), and a proxy leaves such versions of an upstream listing out. Ids and versions are keys of database indexes, whose entries - PostgreSQL limits to about 2,700 bytes. The plugins allow less: NuGet 100 - characters for an id and 64 for a version, the example plugin 250 for a - chart name and 128 for a version. + PostgreSQL limits to about 2,700 bytes. NuGet allows less: 100 characters + for an id and 64 for a version. - **Upstream requests go where the upstream points.** A proxy's plugin reaches the network only through the GETs the host sends for it, and the host sends them, and downloads packages, only from the origin of the @@ -347,45 +258,13 @@ The host enforces these for every plugin, built in or not: `Content-Disposition: attachment`). A plugin cannot set cookies, CORS headers, or redirect a client elsewhere. -A built-in plugin is part of the server: it is reviewed, tested and released -with it, and runs in-process like the rest of Forge's code. A WebAssembly -plugin is code from elsewhere, so it is also sandboxed: - -- **Integrity.** Only components pinned in the server's configuration are - loaded, by digest. There is no upload or install API: changing what runs - takes a configuration change and a restart, like any other code - deployment. -- **No ambient authority.** The component gets an empty WASI context: no - files, no sockets, no environment, no arguments; only clocks and - randomness. Its stdout and stderr go nowhere. It imports exactly one - host interface, the reader for the package being parsed. What it can make - the host request for a proxy is bounded by the rule above. -- **Resource limits.** Each call runs with a fresh fuel budget and an epoch - deadline (10 ms ticks), in an instance with a memory ceiling. The ceiling - bounds the host memory an instance can take: all its linear memories and - tables count against it together (each table element as a pointer), the - strings and lists one call returns are copied into the host only up to - it (a string that several list elements point at is copied once per - element, and counted so), and so are the random bytes it asks WASI for. A - trap (out of fuel, past the deadline, out of memory, a guest panic, a - result past the ceiling) fails that call with a 500 or 400 and discards - the instance. The sandbox tests include a gzip bomb, an instance whose - memories together exceed the ceiling, a `table.grow` past it, and a - result whose list elements all point at one string. -- **Isolation from the server's threads.** Guest code runs on dedicated - threads, never on the async runtime's workers. The quick rule calls - (`canonical-id`, `compare-versions`, …) and the slow calls (parsing a - package, rendering a response) have separate pools, so a long render - cannot hold up the rule calls every request makes. - -Plugins are still code you run: a plugin with a bug can serve wrong -metadata for its own repositories. Review a plugin like any other -dependency before you pin it. +A plugin is part of the server: it is reviewed, tested and released with +it, and runs in-process like the rest of Forge's code. The documents an upstream serves decide which hosts a proxy's plugin can reach. A public feed whose package metadata names an internal URL (a project URL or a chart's home page pointing at `http://10.0.0.5:9200/`) makes -that host reachable for a proxy of that feed, and a plugin can read what it +that host reachable for a proxy of that feed, and the plugin reads what it answers back into what the proxy serves. The rule bounds a plugin to the hosts its upstream points at, public or internal; where internal services must stay out of Forge's reach altogether, confine its egress with network @@ -393,9 +272,9 @@ policy (an outbound proxy, a Kubernetes `NetworkPolicy`). ## Writing a plugin -Start from the example plugin, [`plugins/helm`](../plugins/helm): a crate -in `plugins//` with a `plugin.toml`, a member of the workspace, -depending on `forge-plugin-api` and implementing its trait: +A plugin is a crate in `plugins//` with a `plugin.toml`, a member of the +workspace, depending on `forge-plugin-api` and implementing its trait +([`plugins/nuget`](../plugins/nuget) is a complete one): ```rust use forge_plugin_api::{Action, FormatPlugin, Loaded, Rendered, Repository, Request /* … */}; @@ -406,9 +285,6 @@ pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); #[derive(Default)] pub struct MyFormat; -#[cfg(feature = "component")] -forge_plugin_api::export_component!(MyFormat); - impl FormatPlugin for MyFormat { // identity: canonical_id, validate_id, normalize_version, compare_versions // packages: parse_package, manifest_summary @@ -417,12 +293,6 @@ impl FormatPlugin for MyFormat { } ``` -```toml -# Cargo.toml -[features] -component = ["forge-plugin-api/component"] -``` - ```toml # plugin.toml [plugin] @@ -430,7 +300,6 @@ id = "example" # the format id repositories are created with name = "Example packages" version = "0.1.0" api = "0.3.0" # the forge:format contract -component = "forge_format_example.wasm" # the component build's file mount = "/repository" # optional: served at {mount}/{repo}/… kinds = ["hosted", "group"] # optional: the repository kinds it serves # ("proxy" too when it implements `upstream`) @@ -440,9 +309,6 @@ package = "package" package_type = "application/octet-stream" manifest = "manifest" manifest_type = "application/octet-stream" - -[limits] # optional, as a component (see above) -timeout_ms = 2000 ``` A mount is `/` and one lowercase word; the server's own paths (`/admin`, @@ -450,24 +316,16 @@ A mount is `/` and one lowercase word; the server's own paths (`/admin`, may share a mount. List `proxy` among the kinds only when the plugin implements `upstream`. -Then either: - -- **build it in**: add the crate to `forge-server`'s dependencies and to - `plugins::builtin()` in - [`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs); or -- **ship it as a component**: `cargo xtask plugins` builds it (the crate as a - `cdylib` for `wasm32-wasip2` with the `component` feature) and prints the - configuration that loads it. - -A few rules hold either way: +Then build it in: add the crate to `forge-server`'s dependencies and to +`plugins::builtin()` in +[`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs). A few +rules hold: - **Keep methods pure.** The host may call any method from any thread, in - any order, any number of times; a component's instances are replaced at - will. + any order, any number of times. - **Read packages as a stream.** `parse_package` gets a `Read + Seek` over - the staged package (in a component, a reader the host lends with `size` and - `read-at`). Read only what you need, and bound everything you decompress: - a gzip bomb should fail fast, and a component runs under a fuel budget. + the staged package. Read only what you need, and bound everything you + decompress: a gzip bomb should fail fast. - **Ask for data, don't assume it.** `handle` returns `Action::Load(queries)`; `render` gets each query with its answer and may load more. Build absolute URLs from the repository's `base_url`. Groups and kinds are already @@ -477,17 +335,10 @@ A few rules hold either way: (`testing::serve`), as [`plugins/nuget/tests/protocol.rs`](../plugins/nuget/tests/protocol.rs) does; `testing::converse` runs `upstream` against canned upstream documents ([`plugins/nuget/tests/upstream.rs`](../plugins/nuget/tests/upstream.rs)). -- **Test both ways.** The crate's unit tests run with the workspace. - [`crates/plugin/tests/component.rs`](../crates/plugin/tests/component.rs) - loads the component and compares it with the crate built in, and - [`crates/server/tests/plugins.rs`](../crates/server/tests/plugins.rs) runs - a server with both; copy them for a new plugin. -- **Check it like CI does:** `cargo xtask plugins-check` lints the component - build and builds it. - -A component can also come from another language, as long as it implements -the `forge:format/format-plugin` world. The host provides WASI 0.2 imports -with no capabilities. +- **Test it in a server.** The crate's unit tests run with the workspace. + [`crates/server/tests/plugin_contract.rs`](../crates/server/tests/plugin_contract.rs) + runs a server with a small plugin of its own installed; copy it for a new + plugin. ## Limitations and roadmap @@ -517,8 +368,8 @@ to lift them: format with another rule (PEP 440's `1.0a1`) needs a contract function for it. - **Listings are bounded.** A response may load at most 2,048 answers, so - a listing of every package of a repository (the example plugin's - `index.yaml`) covers at most about 2,000 packages. + a listing of every package of a repository covers at most about 2,000 + packages. - **An upstream document is read whole.** A proxy reads each upstream document it needs (cached for the metadata TTL, and at most `proxy.max_metadata_bytes`, 32 MiB by default); the contract has no ranged @@ -539,11 +390,6 @@ index's resources, the flat container, the registration (bounded to the newest pages a request needs), catalog entries for the package delay of unlisted versions, and the search service. -One part is missing: **a component build.** The crate reads packages with -`zip`, whose `typed-path` dependency needs a nightly compiler for -`wasm32-wasip2`, so NuGet runs built in only (its `plugin.toml` names no -`component`). - ### Why Docker is not a plugin The contract models a repository as package versions, each one file with an @@ -568,34 +414,3 @@ several files in one form, always answer ranges and conditional requests, and their proxies follow redirects and re-validate with `If-None-Match`. They live in the server (`crates/server/src/raw`, served under `/raw`) and are registered like Docker, with selectors on paths. - -## Design notes: compared with artifact-keeper - -[artifact-keeper](https://github.com/artifact-keeper/artifact-keeper) also -runs format plugins as WebAssembly components under Wasmtime. Its native -formats are Rust handlers in `backend/src/formats/.rs` behind one -`FormatHandler` trait, and `formats/wasm.rs` wraps a loaded plugin in the -same trait, so services do not care which kind they hold. Forge follows that -shape: `FormatPlugin` is the one trait, the built-in plugins are its native -implementations, and `WasmPlugin` is the wrapper. Its WIT contract is the -`artifact-keeper:format@1.0.0` package (`backend/src/wit/format-plugin.wit`). -We studied the version of October 2026 and kept the idea while changing the -contract: - -| | artifact-keeper | Forge | -| --- | --- | --- | -| Native formats | Handlers in the backend crate; the WASM contract is a separate, narrower interface | Built-in plugins are crates implementing the same contract a component exports; any of them also builds as a component | -| Package bytes into the plugin | Whole artifact as `list` (uploads are read into memory) | A lent reader over the staged file; the plugin reads what it needs | -| Data for a request | Every artifact row of the repository, on every request | Targeted queries, bounded per request | -| Repository kinds | Hosted in practice (remote and virtual repositories see only their own rows) | Hosted, group, and proxy: the plugin reads the upstream through the host | -| Authorization of data | Repository visibility | Repository grants plus package selectors, group visibility and the dependency-confusion guards | -| Uploads | A separate generic upload endpoint; the plugin validates and parses | The plugin routes the publish; the host streams it through the hosted write path | -| Response headers | Passed through as the plugin sets them | Allowlist plus the feed's hardening headers | -| Execution limits | Fuel and memory | Fuel, memory and an epoch deadline; guest code off the async workers | -| Installing plugins | Admin API from git or a ZIP, Ed25519-signed when required | Operator configuration, pinned by SHA-256; no install API | -| Instances | A new store and instance per call | Instances reused per plugin, a fresh budget per call, recycled after a trap or 10,000 calls | - -The example plugin shows the difference in practice: a 64 MiB component serves -a repository index of any size the query bounds allow, and a chart is never -copied into the sandbox, which streams it through the lent reader and keeps -only its `Chart.yaml`. diff --git a/forge.toml.example b/forge.toml.example index f59a9373..b00e0ce9 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -159,18 +159,6 @@ max_total_uncompressed_bytes = 17179869184 # (default: 16 GiB) total declared s max_nuspec_bytes = 1048576 # (default: 1 MiB) also caps an upstream .nuspec a proxy verifies max_nuspec_compression_ratio = 200 # (default: 200) -# Optional: WebAssembly format plugins — further package formats loaded at -# startup (docs/format-plugins.md). The built-in one (NuGet) needs nothing here. -# Only the plugins listed under [plugins.formats] are loaded, each from -# // and pinned to its component's SHA-256; a mismatch, or an id a -# built-in format holds, fails startup. `cargo xtask plugins` builds the -# plugins of plugins/ as components and prints this section with their digests. -# [plugins] -# dir = "/etc/forge/plugins" -# -# [plugins.formats] -# example = "sha256:<64 hex digits>" - [docker] # The Docker / OCI registry under /v2 (see docs/docker.md). # max_blob_bytes = 1073741824 # (default: 1 GiB, or what an explicit server.max_staging_bytes holds) largest layer a proxy downloads; staged on disk (pushes follow the repository's upload limit) diff --git a/plugins/README.md b/plugins/README.md index ebef6cf4..482620bf 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,42 +1,24 @@ # Format plugins Package formats for Forge, one crate per directory. Each implements -`FormatPlugin` from [`forge-plugin-api`](../crates/plugin-api) and runs -either way: linked into the server (built in), or as a WebAssembly component -loaded at startup. See [docs/format-plugins.md](../docs/format-plugins.md) for -the design, the contract and how to operate them. +`FormatPlugin` from [`forge-plugin-api`](../crates/plugin-api) and is linked +into the server. See [docs/format-plugins.md](../docs/format-plugins.md) for +the design, the contract and how to write one. -| Plugin | Format id | Runs | What it serves | -| --- | --- | --- | --- | -| [`nuget`](nuget) | `nuget` | built in only | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | -| [`helm`](helm) | `helm` | the example plugin: not built in; the plugin tests run it built in and as a component | Classic Helm chart repositories: `index.yaml`, chart downloads, ChartMuseum-compatible push, delete and listing; hosted, proxy and group repositories | +| Plugin | Format id | What it serves | +| --- | --- | --- | +| [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | A plugin directory holds: -- `Cargo.toml`: a member of the workspace, with a `component` feature - (`component = ["forge-plugin-api/component"]`) unless it is built in only; +- `Cargo.toml`: a member of the workspace; - `plugin.toml`: the format id (also the directory's name), display name, - version, the `forge:format` contract version it targets (`api`), + version, the `forge:format` contract version it targets (`api`), and optionally its mount, repository kinds (`proxy` when it implements - `upstream`) and `[assets]` names, and, for the component build, the - component's file name and optional `[limits]`; -- `src/`: `impl FormatPlugin`, `pub const PLUGIN_TOML: &str = - forge_plugin_api::plugin_toml!();`, and - `#[cfg(feature = "component")] forge_plugin_api::export_component!(…);`. + `upstream`) and `[assets]` names; +- `src/`: `impl FormatPlugin`, and `pub const PLUGIN_TOML: &str = + forge_plugin_api::plugin_toml!();`. -The workspace builds, lints and tests the plugins natively like every other -crate. As components (needs the `wasm32-wasip2` target): - -```bash -rustup target add wasm32-wasip2 # once -cargo xtask plugins # build + install under target/plugins/installed/, print the [plugins] config -cargo xtask plugins-check # clippy of the component build, and the build, as CI runs them -``` - -Both skip a plugin whose `plugin.toml` names no `component`. NuGet is one: -its `.nupkg` reader (`zip`, through `typed-path`) needs a nightly compiler for -`wasm32-wasip2`. - -The tests that load a component build it through -`forge_plugin::testing::example_plugin`, so `cargo test --workspace` needs the -`wasm32-wasip2` target as well. +The server installs it in `plugins::builtin()` +([`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs)). The +workspace builds, lints and tests the plugins like every other crate. diff --git a/plugins/helm/Cargo.toml b/plugins/helm/Cargo.toml deleted file mode 100644 index 6c3570c1..00000000 --- a/plugins/helm/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "forge-format-helm" -# The plugin's own version: keep it equal to `version` in plugin.toml. -version = "0.1.0" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish.workspace = true -description = "Helm chart repositories for Forge, as a format plugin" - -# Built into the server; `--features component` builds it as a WebAssembly -# component instead (`cargo xtask plugins`). -[features] -component = ["forge-plugin-api/component"] - -[dependencies] -forge-plugin-api = { path = "../../crates/plugin-api" } -# .tgz charts: gzip with a pure-Rust backend (no C in a wasm build). -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } -# Chart.yaml, and the index.yaml it serves. -yaml-rust2 = { version = "0.13", default-features = false } -serde_json = { workspace = true } - -[dev-dependencies] -forge-plugin-api = { path = "../../crates/plugin-api", features = ["testing"] } diff --git a/plugins/helm/plugin.toml b/plugins/helm/plugin.toml deleted file mode 100644 index e389e057..00000000 --- a/plugins/helm/plugin.toml +++ /dev/null @@ -1,15 +0,0 @@ -# The example Forge format plugin: classic Helm chart repositories (see -# plugins/README.md). Forge does not build it in. -[plugin] -id = "helm" -name = "Helm charts" -version = "0.1.0" -api = "0.3.0" -component = "forge_format_helm.wasm" -description = "Helm chart repositories: `helm repo add` / `helm pull`, ChartMuseum-compatible push and delete; proxies of other chart repositories." -# Proxies mirror another chart repository through the plugin's upstream side. -kinds = ["hosted", "proxy", "group"] - -[limits] -memory_mib = 64 -timeout_ms = 5000 diff --git a/plugins/helm/src/chart.rs b/plugins/helm/src/chart.rs deleted file mode 100644 index f6110665..00000000 --- a/plugins/helm/src/chart.rs +++ /dev/null @@ -1,729 +0,0 @@ -//! Reading a chart archive: `/Chart.yaml` inside a gzipped tarball, -//! found the way Helm finds it. -//! -//! A chart's name and version decide where Forge stores it and whether a -//! publisher may push it, so Forge takes them from the Chart.yaml Helm loads -//! when it installs the chart, or refuses the archive. Helm -//! (`loader.LoadArchiveFiles`, `LoadFiles`) reads the whole archive, strips -//! the first component of every name ([`chart_path`]), and reads every file -//! it then calls `Chart.yaml`, then every `requirements.yaml`, into the -//! chart's metadata: the last one wins. So Forge reads the whole archive -//! too ([`tar`] reads it as Go does), and refuses one with more than one -//! top-level directory, more than one Chart.yaml or requirements.yaml, a -//! requirements.yaml that declares more than dependencies, or a Chart.yaml -//! key that Helm reads as another field ([`folded_field`]). - -use std::io::{self, Read}; - -use serde_json::{Map, Value}; -use yaml_rust2::Yaml; - -use crate::semver::Version; -use crate::tar; - -/// The largest Chart.yaml (or requirements.yaml) read. -const MAX_CHART_YAML: u64 = 1024 * 1024; - -/// How much of a chart is decompressed. The whole archive is read, so that -/// no Chart.yaml goes unseen; a chart that unpacks to more, or a gzip bomb, -/// is refused. (Helm itself loads at most 100 MiB of files.) -const MAX_UNPACKED: u64 = 256 * 1024 * 1024; - -/// The longest chart name. -const MAX_NAME: usize = 250; - -/// The UTF-8 byte order mark, which Helm strips from every file it loads. -const BOM: &[u8] = b"\xEF\xBB\xBF"; - -/// What a chart declares. -pub struct Chart { - /// `name`, as written. - pub name: String, - /// `version`, as written. - pub version: String, - /// The normalized version. - pub normalized: Version, - /// The raw Chart.yaml. - pub chart_yaml: Vec, - /// Chart.yaml as JSON. - pub metadata: Map, -} - -/// Whether `name` is a chart name: letters, digits, `.`, `_` and `-`, -/// starting with a letter or digit. -pub fn valid_name(name: &str) -> bool { - !name.is_empty() - && name.len() <= MAX_NAME - && name.starts_with(|c: char| c.is_ascii_alphanumeric()) - && name - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) - && !name.contains("..") -} - -/// Read the chart in a `.tgz`. -pub fn read(tgz: impl Read) -> Result { - read_unpacking(tgz, MAX_UNPACKED) -} - -/// [`read`], decompressing at most `max_unpacked` bytes. -fn read_unpacking(tgz: impl Read, max_unpacked: u64) -> Result { - // Go's gzip reader, which Helm uses, reads concatenated gzip members as - // one stream. - let unpacked = Bounded { - inner: flate2::read::MultiGzDecoder::new(tgz), - left: max_unpacked, - max: max_unpacked, - }; - let mut archive = tar::Archive::new(unpacked); - let lossy = |name: &[u8]| String::from_utf8_lossy(name).into_owned(); - let mut top_dir: Option> = None; - let (mut chart_yaml, mut requirements) = (None, None); - while let Some(entry) = archive - .next_entry() - .map_err(|e| format!("not a chart archive: {e}"))? - { - // Helm skips directories and global PAX headers. - if entry.is_dir() || entry.typeflag == b'g' { - continue; - } - let (top, path) = chart_path(&entry.name)?; - let dir = top_dir.get_or_insert_with(|| top.to_vec()); - if dir.as_slice() != top { - return Err(format!( - "the archive holds more than one top-level directory: {:?} and {:?}", - lossy(dir), - lossy(top) - )); - } - // Keep Chart.yaml and requirements.yaml; skip the rest. - let found = match path.as_slice() { - b"Chart.yaml" => &mut chart_yaml, - b"requirements.yaml" => &mut requirements, - _ => continue, - }; - if found.is_some() { - return Err(format!( - "the archive holds more than one /{}", - lossy(&path) - )); - } - let data = archive - .read_data(MAX_CHART_YAML) - .map_err(|e| format!("{}: {e}", lossy(&entry.name)))?; - *found = Some(strip_bom(data)); - } - let chart_yaml = - chart_yaml.ok_or_else(|| "the archive has no /Chart.yaml".to_string())?; - if let Some(requirements) = requirements { - check_requirements(&requirements)?; - } - parse_chart_yaml(chart_yaml) -} - -/// What Helm makes of an entry's name (`loader.LoadArchiveFiles`): the -/// top-level directory it strips, and the file's path in the chart, -/// cleaned. An error for a name Helm refuses. -fn chart_path(name: &[u8]) -> Result<(&[u8], Vec), String> { - // A name with a backslash is split at backslashes (a chart packaged on - // Windows), and its `/` are then part of a name. - let separator = if name.contains(&b'\\') { b'\\' } else { b'/' }; - let (top, rest) = match name.iter().position(|&b| b == separator) { - Some(i) => (&name[..i], &name[i + 1..]), - None => (name, &name[name.len()..]), - }; - let rest: Vec = rest - .iter() - .map(|&b| if b == separator { b'/' } else { b }) - .collect(); - let refused = |why: &str| format!("{why}: {:?}", String::from_utf8_lossy(name)); - if rest.starts_with(b"/") { - return Err(refused("the archive holds an absolute path")); - } - let path = clean(&rest); - if path == b"." { - return Err(refused("the archive holds a file outside a directory")); - } - if path.starts_with(b"..") { - return Err(refused("the archive holds a path to a parent directory")); - } - if let [drive, b':', b'/', ..] = path.as_slice() { - if drive.is_ascii_alphabetic() { - return Err(refused("the archive holds a path with a drive letter")); - } - } - if top == b"Chart.yaml" { - return Err(refused("the archive holds a directory named Chart.yaml")); - } - Ok((top, path)) -} - -/// Go's `path.Clean` of a relative path: no empty or `.` elements, a `..` -/// removing the element before it, and `.` for nothing at all. -fn clean(path: &[u8]) -> Vec { - let mut kept: Vec<&[u8]> = Vec::new(); - for element in path.split(|&b| b == b'/') { - match element { - b"" | b"." => {} - b".." if kept.last().is_some_and(|last| *last != b"..") => { - kept.pop(); - } - element => kept.push(element), - } - } - if kept.is_empty() { - b".".to_vec() - } else { - kept.join(&b'/') - } -} - -/// `data` without a leading byte order mark. -fn strip_bom(mut data: Vec) -> Vec { - if data.starts_with(BOM) { - data.drain(..BOM.len()); - } - data -} - -/// Refuse a `requirements.yaml` that declares more than dependencies. Helm -/// reads it (Helm 2's list of dependencies) into the chart's metadata after -/// Chart.yaml, so a `name` or a `version` in it would be the chart's. -fn check_requirements(yaml: &[u8]) -> Result<(), String> { - let text = std::str::from_utf8(yaml).map_err(|_| "requirements.yaml is not UTF-8")?; - let docs = crate::yaml::load(text).map_err(|e| format!("requirements.yaml: {e}"))?; - match docs.first() { - None | Some(Yaml::Null) => Ok(()), - Some(Yaml::Hash(map)) if map.keys().all(|k| k.as_str() == Some("dependencies")) => Ok(()), - _ => Err("requirements.yaml declares more than dependencies".into()), - } -} - -/// A reader of at most `max` bytes, which fails, rather than ending early, -/// when its source holds more. -struct Bounded { - /// The source. - inner: R, - /// The bytes it may still read. - left: u64, - /// The bytes it may read in all. - max: u64, -} - -impl Read for Bounded { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() { - return Ok(0); - } - if self.left == 0 { - return match self.inner.read(&mut [0u8])? { - 0 => Ok(0), - _ => Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("it unpacks to more than {} bytes", self.max), - )), - }; - } - let len = buf - .len() - .min(usize::try_from(self.left).unwrap_or(usize::MAX)); - let n = self.inner.read(&mut buf[..len])?; - self.left -= n as u64; - Ok(n) - } -} - -/// Parse a Chart.yaml. -pub fn parse_chart_yaml(chart_yaml: Vec) -> Result { - let text = std::str::from_utf8(&chart_yaml).map_err(|_| "Chart.yaml is not UTF-8")?; - let docs = crate::yaml::load(text).map_err(|e| format!("Chart.yaml: {e}"))?; - let Some(Value::Object(metadata)) = docs.first().map(to_json) else { - return Err("Chart.yaml is not a mapping".into()); - }; - if let Some((key, field)) = metadata - .keys() - .find_map(|key| Some((key, folded_field(key)?))) - { - return Err(format!( - "Chart.yaml has the key {key:?}, which Helm reads as {field:?}" - )); - } - let field = |key: &str| -> Result { - match metadata.get(key) { - Some(Value::String(s)) if !s.trim().is_empty() => Ok(s.trim().to_string()), - Some(Value::Number(n)) => Ok(n.to_string()), - _ => Err(format!("Chart.yaml has no {key}")), - } - }; - let name = field("name")?; - if !valid_name(&name) { - return Err(format!("invalid chart name {name:?}")); - } - let version = field("version")?; - let normalized = Version::parse(&version)?; - field("apiVersion")?; - Ok(Chart { - name, - version, - normalized, - chart_yaml, - metadata, - }) -} - -/// The field of Helm's chart metadata that `key` sets without being its -/// name: one that equals it only under Go's case folding. -/// -/// Helm reads Chart.yaml with `sigs.k8s.io/yaml`, which turns it into JSON, -/// keys sorted, and decodes that with Go's `encoding/json`. The decoder -/// matches a key to a field case-insensitively, by Unicode simple case -/// folding, and a later key overwrites an earlier one. So `verſion` (`ſ` -/// folds to `s`), which sorts after `version`, is the version Helm installs -/// whatever `version` says, and `apiversion` sorts after `apiVersion` (#422). -/// Only ASCII letters, U+017F (`ſ`, the long s, to `s`) and U+212A (the -/// Kelvin sign, to `k`) fold to ASCII letters, and every field name is ASCII. -fn folded_field(key: &str) -> Option<&'static str> { - if crate::METADATA_KEYS.contains(&key) { - return None; - } - let folded: String = key - .chars() - .map(|c| match c { - '\u{17f}' => Some('s'), - '\u{212a}' => Some('k'), - c if c.is_ascii() => Some(c.to_ascii_lowercase()), - _ => None, - }) - .collect::>()?; - crate::METADATA_KEYS - .iter() - .copied() - .find(|field| field.eq_ignore_ascii_case(&folded)) -} - -/// A YAML value as JSON. Scalars YAML would read as numbers keep their text -/// when JSON cannot hold them exactly (`version: 1.10` stays "1.10"). -pub fn to_json(yaml: &Yaml) -> Value { - match yaml { - Yaml::String(s) => Value::String(s.clone()), - Yaml::Integer(i) => Value::from(*i), - Yaml::Real(r) => Value::String(r.clone()), - Yaml::Boolean(b) => Value::Bool(*b), - Yaml::Array(items) => Value::Array(items.iter().map(to_json).collect()), - Yaml::Hash(map) => Value::Object( - map.iter() - .map(|(k, v)| { - let key = match k { - Yaml::String(s) | Yaml::Real(s) => s.clone(), - Yaml::Integer(i) => i.to_string(), - Yaml::Boolean(b) => b.to_string(), - _ => String::new(), - }; - (key, to_json(v)) - }) - .collect(), - ), - Yaml::Null | Yaml::BadValue | Yaml::Alias(_) => Value::Null, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tar::tests::{archive, entry, file, header, long, pax, seal, END}; - use std::io::Write; - - /// `tar`, gzipped. - fn gzip(tar: &[u8]) -> Vec { - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(tar).unwrap(); - gz.finish().unwrap() - } - - /// A gzipped archive of `(path, contents)` regular files. - pub(crate) fn tgz(entries: &[(&str, &[u8])]) -> Vec { - gzip(&archive(entries)) - } - - const GOOD: &[u8] = b"apiVersion: v2\nname: good\nversion: 1.0.0\n"; - const EVIL: &[u8] = b"apiVersion: v2\nname: evil\nversion: 6.6.6\n"; - - /// The name and version [`read`] finds in the archive `tar`, gzipped. - fn identity(tar: &[u8]) -> Result { - read(&gzip(tar)[..]).map(|chart| format!("{} {}", chart.name, chart.version)) - } - - /// The error [`read`] refuses the archive `tar` with. - fn refusal(tar: &[u8]) -> String { - identity(tar).unwrap_err() - } - - #[test] - fn reads_the_top_level_chart_yaml() { - let archive = tgz(&[ - ("demo/charts/sub/Chart.yaml", b"apiVersion: v2\nname: sub\nversion: 9.9.9\n"), - ( - "demo/Chart.yaml", - b"apiVersion: v2\nname: Demo\nversion: v1.10\nappVersion: 2.0\ndescription: A demo\n", - ), - ]); - let chart = read(&archive[..]).unwrap(); - assert_eq!(chart.name, "Demo"); - assert_eq!(chart.version, "v1.10"); - assert_eq!(chart.normalized.to_string(), "1.10.0"); - assert_eq!(chart.metadata["appVersion"], Value::String("2.0".into())); - assert_eq!( - chart.metadata["description"], - Value::String("A demo".into()) - ); - } - - #[test] - fn refuses_what_is_not_a_chart() { - assert!(read(&b"not gzip"[..]).is_err()); - assert!(read(&tgz(&[("demo/values.yaml", b"a: 1\n")])[..]).is_err()); - let no_version = tgz(&[("demo/Chart.yaml", b"apiVersion: v2\nname: demo\n")]); - assert!(read(&no_version[..]).is_err()); - let bad_name = tgz(&[( - "demo/Chart.yaml", - b"apiVersion: v2\nname: ../x\nversion: 1.0.0\n", - )]); - assert!(read(&bad_name[..]).is_err()); - } - - #[test] - fn refuses_versions_longer_than_an_oci_tag() { - let chart_yaml = |version: &str| { - format!("apiVersion: v2\nname: demo\nversion: {version}\n").into_bytes() - }; - let longest = format!( - "1.0.0-{}", - "a".repeat(crate::semver::MAX_VERSION_LENGTH - 6) - ); - let chart = parse_chart_yaml(chart_yaml(&longest)).unwrap(); - assert_eq!(chart.version, longest); - assert_eq!(chart.normalized.to_string(), longest); - for longer in [format!("{longest}a"), format!("1.0.0-{}", "x".repeat(4096))] { - let err = parse_chart_yaml(chart_yaml(&longer)).err().unwrap(); - assert!(err.contains("longer than 128 characters"), "{err}"); - } - } - - #[test] - fn chart_names() { - assert!(valid_name("nginx-ingress")); - assert!(valid_name("my_chart.v2")); - for bad in ["", "-x", ".x", "a/b", "a..b", "a b", "a%2f"] { - assert!(!valid_name(bad), "{bad:?}"); - } - } - - #[test] - fn reads_charts_as_tools_pack_them() { - // `helm package`: Chart.yaml first, a long name in a PAX header, no - // directories. - let long_name = format!("demo/templates/{}.yaml", "x".repeat(120)); - let tar = [ - &file("demo/Chart.yaml", GOOD)[..], - &file("demo/values.yaml", b"replicas: 1\n"), - &pax(b'x', &[("path", &long_name)]), - &file(&long_name[..99], b"kind: ConfigMap\n"), - &file("demo/charts/sub/Chart.yaml", EVIL), - &END, - ] - .concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // `git archive` (a global header first) and GNU tar (directories, - // GNU long names). - let tar = [ - &pax(b'g', &[("comment", "8b4c2f0")])[..], - &header(b"demo/", b'5', 0), - &header(b"demo/templates/", b'5', 0), - &long(b'L', &long_name), - &file(&long_name[..99], b""), - &file("demo/Chart.yaml", GOOD), - &END, - ] - .concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // Packed inside the chart's directory: `./Chart.yaml`. - let tar = archive(&[("./Chart.yaml", GOOD), ("./values.yaml", b"")]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // On Windows: backslashes. - let tar = archive(&[("demo\\templates\\a.yaml", b""), ("demo\\Chart.yaml", GOOD)]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // In two gzip members. - let first = gzip(&file("demo/values.yaml", b"")); - let second = gzip(&[&file("demo/Chart.yaml", GOOD)[..], &END].concat()); - let chart = read(&[first, second].concat()[..]).unwrap(); - assert_eq!(chart.name, "good"); - } - - #[test] - fn finds_the_chart_yaml_helm_finds() { - // Helm strips the first component of a name and cleans the rest; - // a name with a backslash is split at backslashes only. - for name in [ - "demo/./Chart.yaml", - "demo/templates/../Chart.yaml", - "demo/Chart.yaml/", - "demo/x\\Chart.yaml", - ] { - let tar = archive(&[(name, GOOD)]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0", "{name:?}"); - } - // `./demo/Chart.yaml` is the file `demo/Chart.yaml` of the - // directory `.` to Helm, not its Chart.yaml. - let tar = archive(&[("./demo/Chart.yaml", GOOD)]); - assert!(refusal(&tar).contains("no /Chart.yaml")); - // A directory by its mode is skipped like any directory, and a - // symlink has no data. - let mut dir = header(b"demo/Chart.yaml", b'0', EVIL.len() as u64); - dir[100..108].copy_from_slice(b"0040755\0"); - seal(&mut dir); - let tar = [&entry(dir, EVIL)[..], &file("demo/Chart.yaml", GOOD), &END].concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - let tar = [&header(b"demo/Chart.yaml", b'2', 0)[..], &END].concat(); - assert!(refusal(&tar).contains("not a mapping")); - // Helm strips a byte order mark. - let tar = archive(&[("demo/Chart.yaml", &[BOM, GOOD].concat())]); - assert_eq!(read(&gzip(&tar)[..]).unwrap().chart_yaml, GOOD); - } - - #[test] - fn refuses_more_than_one_top_level_directory() { - // The archive of the issue: Forge stored `good` 1.0.0, Helm installs - // `evil` 6.6.6. - let tar = archive(&[("a/Chart.yaml", GOOD), ("b/Chart.yaml", EVIL)]); - assert!(refusal(&tar).contains("more than one top-level directory")); - let tar = archive(&[("demo/Chart.yaml", GOOD), ("other/templates/a.yaml", b"")]); - assert!(refusal(&tar).contains("more than one top-level directory")); - let tar = archive(&[("demo/Chart.yaml", GOOD), ("./demo/values.yaml", b"")]); - assert!(refusal(&tar).contains("more than one top-level directory")); - } - - #[test] - fn refuses_more_than_one_chart_yaml() { - // Helm loads every one, the last one winning. - for second in [ - "demo/Chart.yaml", - "demo/./Chart.yaml", - "demo/x/../Chart.yaml", - "demo/Chart.yaml/", - "demo\\Chart.yaml", - ] { - let tar = archive(&[("demo/Chart.yaml", GOOD), (second, EVIL)]); - let err = refusal(&tar); - assert!( - err.contains("more than one /Chart.yaml"), - "{second:?}: {err}" - ); - } - } - - #[test] - fn sees_every_entry_helm_sees() { - let good = file("demo/Chart.yaml", GOOD); - let evil = file("demo/Chart.yaml", EVIL); - let refused = |tar: &[u8]| { - let err = refusal(tar); - assert!(err.contains("more than one /Chart.yaml"), "{err}"); - }; - // A PAX size of 0: what the header calls data is an entry. - refused( - &[ - &good[..], - &pax(b'x', &[("size", "0")]), - &entry(header(b"demo/values.yaml", b'0', evil.len() as u64), &evil), - &END, - ] - .concat(), - ); - // A symlink with a size has no data either. - refused( - &[ - &good[..], - &header(b"demo/link", b'2', evil.len() as u64), - &evil, - &END, - ] - .concat(), - ); - // A long link name does not take the long name before it. - refused( - &[ - &good[..], - &long(b'L', "demo/Chart.yaml"), - &long(b'K', "target"), - &file("demo/a", EVIL), - &END, - ] - .concat(), - ); - // A GNU header's times are not a prefix. - let mut gnu = header(b"demo/Chart.yaml", b'0', EVIL.len() as u64); - gnu[257..265].copy_from_slice(b"ustar \0"); - gnu[345..347].copy_from_slice(b"7\0"); - seal(&mut gnu); - refused(&[&good[..], &entry(gnu, EVIL), &END].concat()); - // A second gzip member. - let tgz = [gzip(&good), gzip(&[&evil[..], &END].concat())].concat(); - let err = read(&tgz[..]).err().unwrap(); - assert!(err.contains("more than one /Chart.yaml"), "{err}"); - // Not the end of the archive: a zero block, then more. - let tar = [&good[..], &[0; 512], &evil, &END].concat(); - assert!(refusal(&tar).contains("not a chart archive")); - } - - #[test] - fn refuses_names_helm_refuses() { - for name in [ - "demo//Chart.yaml", - "demo/../../etc/passwd", - "demo/..", - "Chart.yaml", - "README.md", - "Chart.yaml/values.yaml", - "demo/c:/x", - "demo\\c:\\x", - ] { - let tar = archive(&[(name, GOOD), ("demo/Chart.yaml", GOOD)]); - assert!(refusal(&tar).contains("the archive holds a"), "{name:?}"); - } - } - - #[test] - fn refuses_a_requirements_yaml_that_renames_the_chart() { - let dependencies = b"dependencies:\n - name: redis\n version: 17.0.0\n"; - for requirements in [&dependencies[..], b"", b"# none\n"] { - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", requirements), - ]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - } - // Helm reads it into the chart's metadata after Chart.yaml. - for requirements in [ - &b"name: evil\nversion: 6.6.6\n"[..], - b"dependencies: []\nVersion: 6.6.6\n", - b"- name: evil\n", - ] { - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", requirements), - ]); - assert!(refusal(&tar).contains("more than dependencies")); - } - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", dependencies), - ("demo/requirements.yaml", dependencies), - ]); - assert!(refusal(&tar).contains("more than one /requirements.yaml")); - // A subchart's is the subchart's. - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/charts/sub/requirements.yaml", b"name: evil\n"), - ]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - } - - #[test] - fn folds_keys_like_go() { - for (key, field) in [ - ("verſion", Some("version")), - ("VERSION", Some("version")), - ("Version", Some("version")), - ("apiversion", Some("apiVersion")), - ("APIVERSION", Some("apiVersion")), - ("\u{212a}ubeVersion", Some("kubeVersion")), - ("\u{212a}EYWORDſ", Some("keywords")), - ("Name", Some("name")), - // Exact names, and keys that are no field. - ("version", None), - ("apiVersion", None), - ("versions", None), - ("\u{212a}ind", None), - ("versión", None), - ("ver\u{0131}sion", None), - ("", None), - ] { - assert_eq!(folded_field(key), field, "{key:?}"); - } - } - - #[test] - fn refuses_keys_helm_reads_as_another_field() { - // What Helm 3.17 (sigs.k8s.io/yaml v1.4.0) reads, after `apiVersion: v2`: - // `good 6.6.6` for the first two, `evil 1.0.0` for the fourth, a v1 - // chart for `apiversion` and appVersion 9 for `appversion`. - for keys in [ - "name: good\nversion: 1.0.0\nverſion: 6.6.6\n", - "name: good\nverſion: 6.6.6\nversion: 1.0.0\n", - "name: good\nversion: 1.0.0\nVERSION: 6.6.6\n", - "NAME: evil\nversion: 1.0.0\n", - "name: good\nNAME: evil\nversion: 1.0.0\n", - "name: good\nversion: 1.0.0\napiversion: v1\n", - "name: good\nversion: 1.0.0\nappVersion: \"1\"\nappversion: \"9\"\n", - "name: good\nversion: 1.0.0\n\u{212a}ubeVersion: \">=1.0\"\n", - ] { - let chart_yaml = format!("apiVersion: v2\n{keys}"); - let err = parse_chart_yaml(chart_yaml.clone().into_bytes()).err(); - assert!( - err.as_deref() - .is_some_and(|e| e.contains("which Helm reads as")), - "{chart_yaml:?}: {err:?}" - ); - } - let err = parse_chart_yaml( - "apiVersion: v2\nname: good\nversion: 1.0.0\nverſion: 6.6.6\n" - .as_bytes() - .to_vec(), - ) - .err() - .unwrap(); - assert_eq!( - err, - r#"Chart.yaml has the key "verſion", which Helm reads as "version""# - ); - // In an archive, as a push or a proxy download reads it. - let tar = archive(&[( - "demo/Chart.yaml", - "apiVersion: v2\nname: good\nversion: 1.0.0\nverſion: 6.6.6\n".as_bytes(), - )]); - assert!(refusal(&tar).contains("which Helm reads as \"version\"")); - // Keys that are no field of Helm's stay accepted, as Helm ignores them. - let chart = parse_chart_yaml( - b"apiVersion: v2\nname: good\nversion: 1.0.0\nappVersion: \"1\"\nversions: 2\nKind: x\n" - .to_vec(), - ) - .unwrap(); - assert_eq!( - (chart.name.as_str(), chart.version.as_str()), - ("good", "1.0.0") - ); - } - - #[test] - fn refuses_an_archive_it_cannot_read_to_the_end() { - let tar = archive(&[("demo/Chart.yaml", GOOD), ("demo/big", &[7; 4096])]); - let len = tar.len() as u64; - assert!(read_unpacking(&gzip(&tar)[..], len).is_ok()); - let err = read_unpacking(&gzip(&tar)[..], len - 1).err().unwrap(); - assert!(err.contains("unpacks to more than"), "{err}"); - } - - #[test] - fn cleans_paths_like_go() { - for (path, clean_path) in [ - ("", "."), - ("a//b/./c/", "a/b/c"), - ("a/../..", ".."), - ("../a/../b", "../b"), - ("a/b/../../c", "c"), - (".", "."), - ] { - assert_eq!(clean(path.as_bytes()), clean_path.as_bytes(), "{path:?}"); - } - } -} diff --git a/plugins/helm/src/lib.rs b/plugins/helm/src/lib.rs deleted file mode 100644 index b3cd916f..00000000 --- a/plugins/helm/src/lib.rs +++ /dev/null @@ -1,998 +0,0 @@ -//! Helm chart repositories for Forge. -//! -//! Served under a Forge repository's base URL: -//! -//! * `GET /index.yaml` — the repository index (`helm repo add`, `helm repo -//! update`); every chart and version the caller may read; -//! * `GET /charts/{name}/{version}/{name}-{version}.tgz` — a chart (the URL -//! the index points at); -//! * `POST /api/charts` — upload a chart, as the request body -//! (`curl --data-binary @chart.tgz`) or as the multipart field `chart` -//! (`helm cm-push`); -//! * `DELETE /api/charts/{name}/{version}` — delete a version; -//! * `GET /api/charts`, `/api/charts/{name}`, `/api/charts/{name}/{version}` -//! — the ChartMuseum JSON listing. -//! -//! Chart names are matched case-insensitively (the canonical id is the -//! lowercased name); versions are SemVer, normalized (`v1.2` → `1.2.0`), and -//! at most 128 characters ([`semver::MAX_VERSION_LENGTH`]). -//! -//! A proxy repository mirrors another chart repository ([`upstream`]): its -//! index lists the upstream's charts, each downloaded and cached on first -//! use. -//! -//! This is the example plugin (`docs/format-plugins.md`): Forge does not -//! build it in, and serves Helm charts from its OCI registry instead. The -//! plugin tests install it as if built in ([`Helm`], [`PLUGIN_TOML`]); with -//! the `component` feature the same crate builds as a WebAssembly component -//! of the `forge:format` world (`cargo xtask plugins`). - -mod chart; -mod semver; -mod tar; -mod upstream; -mod yaml; - -use std::cmp::{Ordering, Reverse}; -use std::collections::BTreeMap; - -use forge_plugin_api::{ - Action, Answer, ChangeRequest, Dependency, DependencyGroup, FetchResult, FormatError, - FormatPlugin, Loaded, PackageInfo, PackageRef, PackageSource, ParsedPackage, PublishRequest, - Query, Remote, Rendered, Repository, RepositoryKind, Request, Response, SearchHit, SearchQuery, - Upload, UpstreamOp, UpstreamStep, -}; -use serde_json::{json, Map, Value}; - -use semver::Version; - -/// This plugin's `plugin.toml`: its format id, name and version. -pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); - -/// Searches page through the repository this many charts at a time. -const PAGE: u32 = 1000; - -/// The Chart.yaml keys an index entry carries: the JSON names of the fields -/// of Helm's chart metadata (`chart.Metadata`), every one of them. -const METADATA_KEYS: &[&str] = &[ - "apiVersion", - "name", - "version", - "kubeVersion", - "description", - "type", - "keywords", - "home", - "sources", - "dependencies", - "maintainers", - "icon", - "appVersion", - "deprecated", - "annotations", - "condition", - "tags", -]; - -/// The most a chart version's manifest summary takes, as JSON. Every -/// `index.yaml` request renders every version from its summary, so this -/// bounds what a version adds to the index: 300 versions make under 5 MB, -/// where a Chart.yaml of 1 MiB each made 300 MB. A typical chart's summary -/// takes a few hundred bytes to a few KiB. -const MAX_SUMMARY: usize = 16 * 1024; - -/// The longest description, in bytes, that a summary over [`MAX_SUMMARY`] -/// keeps. Helm asks for a single sentence, and `helm search` shows its first -/// 50 characters. -const MAX_DESCRIPTION: usize = 1024; - -/// The Chart.yaml keys a summary over [`MAX_SUMMARY`] gives up last: what -/// Helm needs to list, choose and install a version, each a short string or -/// a boolean. -const ESSENTIAL_KEYS: &[&str] = &[ - "apiVersion", - "name", - "version", - "appVersion", - "kubeVersion", - "type", - "deprecated", -]; - -/// The Helm chart format. -#[derive(Debug, Default, Clone, Copy)] -pub struct Helm; - -#[cfg(feature = "component")] -forge_plugin_api::export_component!(Helm); - -impl FormatPlugin for Helm { - fn canonical_id(&self, id: &str) -> String { - id.to_ascii_lowercase() - } - - fn validate_id(&self, id: &str) -> Result<(), FormatError> { - if chart::valid_name(id) { - Ok(()) - } else { - Err(FormatError::InvalidId(format!("invalid chart name {id:?}"))) - } - } - - fn normalize_version(&self, version: &str) -> Result { - Version::parse(version) - .map(|v| v.to_string()) - .map_err(FormatError::InvalidVersion) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - match (Version::parse(a), Version::parse(b)) { - (Ok(a), Ok(b)) => a.cmp(&b), - _ => a.cmp(b), - } - } - - fn parse_package(&self, package: &mut dyn PackageSource) -> Result { - let chart = chart::read(package).map_err(FormatError::InvalidPackage)?; - let dependencies = dependencies(&chart.metadata); - Ok(ParsedPackage { - id: chart.name.to_ascii_lowercase(), - original_id: chart.name, - version: chart.normalized.to_string(), - version_original: chart.version, - listed: true, - manifest: chart.chart_yaml, - dependencies: if dependencies.is_empty() { - Vec::new() - } else { - vec![DependencyGroup { - target: None, - dependencies, - }] - }, - }) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - let chart = chart::parse_chart_yaml(manifest.to_vec()).ok()?; - Some(summary(&chart.metadata)) - } - - fn handle(&self, repo: &Repository, req: &Request) -> Action { - let read = matches!(req.method.as_str(), "GET" | "HEAD"); - let segments: Vec<&str> = req.path.trim_start_matches('/').split('/').collect(); - match (req.method.as_str(), segments.as_slice()) { - (_, ["index.yaml"]) if read => Action::Load(vec![search_page(repo, 0)]), - (_, ["charts", name, version, file]) if read => match package_ref(name, version) { - Some(target) if file.ends_with(".tgz") => Action::Download(target), - _ => Action::Respond(not_found()), - }, - (_, ["api", "charts"]) if read => Action::Load(vec![search_page(repo, 0)]), - (_, ["api", "charts", name]) if read => match decode(name) { - Some(name) if chart::valid_name(&name) => { - Action::Load(vec![Query::Packages(name.to_ascii_lowercase())]) - } - _ => Action::Respond(not_found()), - }, - (_, ["api", "charts", name, version]) if read => match package_ref(name, version) { - Some(target) => Action::Load(vec![Query::Package(target)]), - None => Action::Respond(not_found()), - }, - ("POST", ["api", "charts"]) => { - let multipart = req - .header("content-type") - .is_some_and(|v| v.to_ascii_lowercase().starts_with("multipart/form-data")); - Action::Publish(PublishRequest { - upload: if multipart { - Upload::MultipartField("chart".into()) - } else { - Upload::Body - }, - done: json_response(201, &json!({ "saved": true })), - }) - } - ("DELETE", ["api", "charts", name, version]) => match package_ref(name, version) { - Some(target) => Action::Delete(ChangeRequest { - target, - done: json_response(200, &json!({ "deleted": true })), - }), - None => Action::Respond(not_found()), - }, - _ => Action::Respond(not_found()), - } - } - - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let segments: Vec<&str> = req.path.trim_start_matches('/').split('/').collect(); - match segments.as_slice() { - ["index.yaml"] | ["api", "charts"] => render_listing(repo, &segments, data), - ["api", "charts", _] => match data.first().map(|l| &l.answer) { - Some(Answer::Packages(versions)) if !versions.is_empty() => { - let mut versions: Vec<&PackageInfo> = versions.iter().collect(); - newest_first(&mut versions); - let entries: Vec = - versions.into_iter().map(|p| entry(repo, p)).collect(); - Rendered::Respond(json_response(200, &Value::Array(entries))) - } - _ => Rendered::Respond(not_found()), - }, - ["api", "charts", _, _] => match data.first().map(|l| &l.answer) { - Some(Answer::Package(Some(p))) => { - Rendered::Respond(json_response(200, &entry(repo, p))) - } - _ => Rendered::Respond(not_found()), - }, - _ => Rendered::Respond(not_found()), - } - } - - fn upstream( - &self, - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], - ) -> Result { - upstream::step(remote, op, fetched) - } -} - -/// The manifest summary of a chart's metadata (its Chart.yaml, or an index -/// entry): the chart metadata an index entry carries, and the description -/// and dependency groups Forge's web UI shows, within [`MAX_SUMMARY`]. -fn summary(metadata: &Map) -> String { - let deps: Vec = dependencies(metadata) - .into_iter() - .map(|d| json!({ "id": d.id, "range": d.range })) - .collect(); - let mut summary = Map::new(); - summary.insert("chart".into(), Value::Object(helm_metadata(metadata))); - summary.insert( - "description".into(), - metadata.get("description").cloned().unwrap_or(Value::Null), - ); - summary.insert( - "dependency_groups".into(), - if deps.is_empty() { - json!([]) - } else { - json!([{ "target_framework": null, "dependencies": deps }]) - }, - ); - bounded(&mut summary) -} - -/// `summary` (a manifest summary's `chart`, with its `description` and -/// `dependency_groups` when it has them) cut down to [`MAX_SUMMARY`], and -/// its JSON. -/// -/// A summary within the budget stays as it is. One over it has its -/// description cut to [`MAX_DESCRIPTION`] bytes and an ellipsis, then loses -/// whole fields, the largest first and the [`ESSENTIAL_KEYS`] last, until it -/// fits. So what is left of a field is all of it, never part of a list or a -/// map, and the same summary always comes out the same. -fn bounded(summary: &mut Map) -> String { - let text = serde_json::to_string(summary).unwrap_or_default(); - if text.len() <= MAX_SUMMARY { - return text; - } - // The description, in both of its places. - let mut description = None; - if let Some(Value::Object(chart)) = summary.get_mut("chart") { - if let Some(Value::String(long)) = chart.get_mut("description") { - let end = long.floor_char_boundary(MAX_DESCRIPTION); - if end < long.len() { - long.truncate(end); - long.push('…'); - } - } - description = chart.get("description").cloned(); - } - if summary.contains_key("description") { - summary.insert("description".into(), description.unwrap_or(Value::Null)); - } - // Then whole fields, the chart's and the summary's own, ordered by - // (essential, largest first, the summary's before the chart's, key). - let mut fields: Vec<(bool, Reverse, bool, String)> = Vec::new(); - for (key, value) in summary.iter() { - match value { - Value::Object(chart) if key == "chart" => { - fields.extend(chart.iter().map(|(key, value)| { - let essential = ESSENTIAL_KEYS.contains(&key.as_str()); - (essential, Reverse(field_len(key, value)), true, key.clone()) - })); - } - _ => fields.push((false, Reverse(field_len(key, value)), false, key.clone())), - } - } - fields.sort(); - let mut size = serde_json::to_string(summary).map_or(0, |text| text.len()); - for (_, Reverse(len), in_chart, key) in fields { - if size <= MAX_SUMMARY { - break; - } - let object = if in_chart { - match summary.get_mut("chart") { - Some(Value::Object(chart)) => chart, - _ => continue, - } - } else { - &mut *summary - }; - // The field, and the comma between it and its neighbour. - size = size.saturating_sub(len + usize::from(object.len() > 1)); - object.remove(&key); - } - serde_json::to_string(summary).unwrap_or_default() -} - -/// The length of `"key":value` in JSON. -fn field_len(key: &str, value: &Value) -> usize { - Value::from(key).to_string().len() + 1 + value.to_string().len() -} - -/// The [`METADATA_KEYS`] of `metadata`, each as the type Helm's -/// `chart.Metadata` declares for it. -/// -/// Helm reads an index that is valid JSON with Go's `encoding/json`, straight -/// into those types, and refuses the whole index when one value has another -/// type. A Chart.yaml (or a YAML index of an upstream) may hold -/// `appVersion: 2` or an annotation `false`: Helm's YAML reader turns such a -/// scalar into the string its field wants, so the chart is valid, but its -/// JSON number or boolean would break every chart of the index. So scalars -/// become strings where Helm wants strings, and a value that has no such -/// form (a mapping where a string belongs) is left out. -fn helm_metadata(metadata: &Map) -> Map { - metadata - .iter() - .filter_map(|(key, value)| { - let value = match key.as_str() { - "keywords" | "sources" => string_list(value), - "deprecated" => value.is_boolean().then(|| value.clone()), - "annotations" => string_map(value), - "maintainers" => objects(value, |maintainer| { - pick(maintainer, &["name", "email", "url"], &[], &[]) - }), - "dependencies" => objects(value, |dependency| { - let mut picked = pick( - dependency, - &["name", "version", "repository", "condition", "alias"], - &["tags"], - &["enabled"], - ); - // `[]interface{}`: any list. - if let Some(values @ Value::Array(_)) = dependency.get("import-values") { - picked.insert("import-values".into(), values.clone()); - } - picked - }), - key if METADATA_KEYS.contains(&key) => scalar_string(value), - _ => None, - }?; - Some((key.clone(), value)) - }) - .collect() -} - -/// A scalar as the string Helm's YAML reader makes of it. -fn scalar_string(value: &Value) -> Option { - match value { - Value::String(_) => Some(value.clone()), - Value::Number(n) => Some(Value::String(n.to_string())), - Value::Bool(b) => Some(Value::String(b.to_string())), - _ => None, - } -} - -/// A list of strings (`[]string`): its scalars as strings. -fn string_list(value: &Value) -> Option { - let items = value.as_array()?; - Some(Value::Array( - items.iter().filter_map(scalar_string).collect(), - )) -} - -/// A string map (`map[string]string`): its scalar values as strings. -fn string_map(value: &Value) -> Option { - let map = value.as_object()?; - Some(Value::Object( - map.iter() - .filter_map(|(k, v)| Some((k.clone(), scalar_string(v)?))) - .collect(), - )) -} - -/// A list of objects (`[]*Maintainer`, `[]*Dependency`), each made by `one`. -fn objects( - value: &Value, - one: impl Fn(&Map) -> Map, -) -> Option { - let items = value.as_array()?; - Some(Value::Array( - items - .iter() - .filter_map(|item| Some(Value::Object(one(item.as_object()?)))) - .collect(), - )) -} - -/// The `strings`, `lists` (of strings) and `bools` of `object`, as such. -fn pick( - object: &Map, - strings: &[&str], - lists: &[&str], - bools: &[&str], -) -> Map { - let mut picked = Map::new(); - for (key, value) in object { - let value = if strings.contains(&key.as_str()) { - scalar_string(value) - } else if lists.contains(&key.as_str()) { - string_list(value) - } else if bools.contains(&key.as_str()) { - value.is_boolean().then(|| value.clone()) - } else { - None - }; - if let Some(value) = value { - picked.insert(key.clone(), value); - } - } - picked -} - -/// Chart.yaml `dependencies` as Forge dependencies. -fn dependencies(metadata: &Map) -> Vec { - let Some(Value::Array(deps)) = metadata.get("dependencies") else { - return Vec::new(); - }; - deps.iter() - .filter_map(|d| { - let name = d.get("name")?.as_str()?; - Some(Dependency { - id: name.to_string(), - range: d.get("version").and_then(Value::as_str).map(str::to_string), - }) - }) - .collect() -} - -fn json_response(status: u16, value: &Value) -> Response { - Response::new(status, "application/json", value.to_string()) -} - -fn not_found() -> Response { - json_response(404, &json!({ "error": "not found" })) -} - -/// Percent-decode one path segment. -fn decode(segment: &str) -> Option { - let bytes = segment.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' { - let hex = segment.get(i + 1..i + 3)?; - out.push(u8::from_str_radix(hex, 16).ok()?); - i += 3; - } else { - out.push(bytes[i]); - i += 1; - } - } - String::from_utf8(out).ok() -} - -/// `(canonical id, normalized version)` of a name and version from a URL. -fn package_ref(name: &str, version: &str) -> Option { - let name = decode(name)?; - let version = Version::parse(&decode(version)?).ok()?; - chart::valid_name(&name).then(|| PackageRef { - id: name.to_ascii_lowercase(), - version: version.to_string(), - }) -} - -/// A page of every chart of `repo`: a proxy's (and a group's proxies') -/// upstream charts included, not only the ones it has cached. -fn search_page(repo: &Repository, skip: u32) -> Query { - Query::Search(SearchQuery { - text: String::new(), - skip, - take: PAGE, - prerelease: true, - semver2: true, - upstream: repo.kind != RepositoryKind::Hosted, - }) -} - -/// The index (`/index.yaml`) or the ChartMuseum listing (`/api/charts`): -/// page through the search, then load every chart's versions, then render. -fn render_listing(repo: &Repository, segments: &[&str], data: &[Loaded]) -> Rendered { - let mut total = 0u32; - let mut pages = 0u32; - let mut ids: Vec = Vec::new(); - let mut hits: Vec<&SearchHit> = Vec::new(); - let mut charts: Vec> = Vec::new(); - for loaded in data { - match (&loaded.query, &loaded.answer) { - (Query::Search(_), Answer::Search(result)) => { - total = total.max(result.total); - pages += 1; - ids.extend( - result - .hits - .iter() - .map(|h| h.original_id.to_ascii_lowercase()), - ); - hits.extend(result.hits.iter()); - } - // Unlisted versions stay downloadable but are not offered. - (Query::Packages(_), Answer::Packages(versions)) => { - let mut listed: Vec<&PackageInfo> = versions.iter().filter(|p| p.listed).collect(); - if !listed.is_empty() { - newest_first(&mut listed); - charts.push(listed); - } - } - _ => {} - } - } - // Search pages are all loaded in the second round, charts in the next. - let loaded_charts = data.iter().any(|l| matches!(l.query, Query::Packages(_))); - if !loaded_charts { - if pages * PAGE < total && pages == 1 { - let more = (1..total.div_ceil(PAGE)) - .map(|p| search_page(repo, p * PAGE)) - .collect(); - return Rendered::Load(more); - } - // A proxy's charts are its upstream's: the search lists each with - // its versions and the newest one's metadata, so the index comes - // from it rather than from reading the upstream once per chart. - if repo.kind == RepositoryKind::Proxy { - let mut entries = Map::new(); - for hit in hits { - entries.insert( - hit.original_id.clone(), - Value::Array(hit_entries(repo, hit)), - ); - } - return render_index(segments, entries, None); - } - if !ids.is_empty() { - ids.sort(); - ids.dedup(); - return Rendered::Load(ids.into_iter().map(Query::Packages).collect()); - } - } - - // A chart is listed under the name its newest version is published as - // (names match case-insensitively). - let mut by_name: BTreeMap> = BTreeMap::new(); - for versions in charts { - by_name - .entry(versions[0].original_id.clone()) - .or_default() - .extend(versions); - } - let mut entries = Map::new(); - let mut latest: Option<&str> = None; - for (name, versions) in by_name { - for p in &versions { - if let Some(published) = p.published.as_deref() { - if latest.is_none_or(|l| published > l) { - latest = Some(published); - } - } - } - let list: Vec = versions.into_iter().map(|p| entry(repo, p)).collect(); - entries.insert(name, Value::Array(list)); - } - render_index(segments, entries, latest) -} - -/// The index (`/index.yaml`) or the ChartMuseum listing (`/api/charts`) of -/// `entries`, generated as of `latest`. -fn render_index(segments: &[&str], entries: Map, latest: Option<&str>) -> Rendered { - if segments == ["api", "charts"] { - return Rendered::Respond(json_response(200, &Value::Object(entries))); - } - let mut index = Map::new(); - index.insert("apiVersion".into(), json!("v1")); - index.insert("entries".into(), Value::Object(entries)); - if let Some(latest) = latest { - index.insert("generated".into(), json!(latest)); - } - // JSON is YAML; Helm reads either. - Rendered::Respond( - Response::new(200, "application/x-yaml", Value::Object(index).to_string()) - .with_header("cache-control", "no-cache"), - ) -} - -/// The index entries of a chart a proxy's search found, newest first. The -/// newest version carries the chart's metadata; the others, which the -/// search does not describe, their name, version and URL. -fn hit_entries(repo: &Repository, hit: &SearchHit) -> Vec { - let chart = stored_chart(hit.summary.as_deref()); - let mut versions: Vec<_> = hit.versions.iter().collect(); - versions.sort_by( - |a, b| match (Version::parse(&a.version), Version::parse(&b.version)) { - (Ok(a), Ok(b)) => b.cmp(&a), - _ => b.version.cmp(&a.version), - }, - ); - versions - .into_iter() - .map(|v| { - let mut entry = if v.version == hit.latest_version { - chart.clone() - } else { - chart - .get("apiVersion") - .map(|api| Map::from_iter([("apiVersion".to_string(), api.clone())])) - .unwrap_or_default() - }; - entry.insert("name".into(), json!(hit.original_id)); - entry.insert("version".into(), json!(v.version_original)); - entry.insert( - "urls".into(), - json!([chart_url(repo, &hit.original_id, &v.version)]), - ); - Value::Object(entry) - }) - .collect() -} - -/// Where a chart version is downloaded from this repository. -fn chart_url(repo: &Repository, name: &str, version: &str) -> String { - format!( - "{}/charts/{}/{version}/{name}-{version}.tgz", - repo.base_url, - name.to_ascii_lowercase(), - ) -} - -fn newest_first(versions: &mut [&PackageInfo]) { - versions.sort_by( - |a, b| match (Version::parse(&a.version), Version::parse(&b.version)) { - (Ok(a), Ok(b)) => b.cmp(&a), - _ => b.version.cmp(&a.version), - }, - ); -} - -/// The chart metadata of a stored manifest summary, typed for Helm again (a -/// summary stored before may not be). A summary this plugin makes fits in -/// [`MAX_SUMMARY`]; one over it, stored before summaries were bounded, has -/// its chart cut down the same way ([`bounded`]). -fn stored_chart(summary: Option<&str>) -> Map { - let Some(text) = summary else { - return Map::new(); - }; - let chart = serde_json::from_str::(text) - .ok() - .and_then(|s| match s.get("chart") { - Some(Value::Object(map)) => Some(helm_metadata(map)), - _ => None, - }) - .unwrap_or_default(); - if text.len() <= MAX_SUMMARY { - return chart; - } - let mut summary = Map::from_iter([("chart".to_string(), Value::Object(chart))]); - bounded(&mut summary); - match summary.remove("chart") { - Some(Value::Object(chart)) => chart, - _ => Map::new(), - } -} - -/// One chart version as the index and the ChartMuseum API list it. -fn entry(repo: &Repository, p: &PackageInfo) -> Value { - let mut chart = stored_chart(p.summary.as_deref()); - chart.insert("name".into(), json!(p.original_id)); - chart.insert("version".into(), json!(p.version_original)); - chart.insert( - "urls".into(), - json!([chart_url(repo, &p.original_id, &p.version)]), - ); - if let Some(published) = &p.published { - chart.insert("created".into(), json!(published)); - } - if let Some(sha256) = &p.sha256 { - chart.insert("digest".into(), json!(sha256)); - } - Value::Object(chart) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn index_metadata_has_the_types_helm_reads() { - let chart = chart::parse_chart_yaml( - b"apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: 2\ndeprecated: true\n\ - annotations: {a: false, b: 1.5, c: [x]}\nkeywords: [k, 3]\nhome: [x]\nextra: 1\n" - .to_vec(), - ) - .unwrap(); - let metadata = helm_metadata(&chart.metadata); - assert_eq!( - Value::Object(metadata), - json!({ - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2", - "deprecated": true, - "annotations": {"a": "false", "b": "1.5"}, - "keywords": ["k", "3"], - }) - ); - } - - fn summary_of(chart_yaml: &str) -> Value { - let chart = chart::parse_chart_yaml(chart_yaml.as_bytes().to_vec()).unwrap(); - let text = summary(&chart.metadata); - assert!(text.len() <= MAX_SUMMARY, "{} bytes", text.len()); - serde_json::from_str(&text).unwrap() - } - - fn package_info(summary: &str) -> PackageInfo { - PackageInfo { - original_id: "demo".into(), - version: "1.0.0".into(), - version_original: "1.0.0".into(), - listed: true, - published: None, - sha256: None, - sha512: None, - size: None, - summary: Some(summary.into()), - } - } - - fn hosted() -> Repository { - forge_plugin_api::testing::repository(RepositoryKind::Hosted, "/repository") - } - - #[test] - fn a_typical_charts_summary_is_all_of_its_metadata() { - let summary = summary_of( - "apiVersion: v2\nname: web\nversion: 1.2.0\nappVersion: \"1.27\"\n\ - kubeVersion: \">=1.25.0-0\"\ndescription: A web server\ntype: application\n\ - keywords: [web, http]\nhome: https://example.com/web\n\ - sources: [https://github.com/example/web]\nicon: https://example.com/web.png\n\ - maintainers:\n - name: Ada\n email: ada@example.com\n\ - dependencies:\n - name: redis\n version: \">=17\"\n \ - repository: https://charts.example.com\n condition: redis.enabled\n\ - annotations:\n artifacthub.io/license: Apache-2.0\n", - ); - let chart = json!({ - "apiVersion": "v2", - "name": "web", - "version": "1.2.0", - "appVersion": "1.27", - "kubeVersion": ">=1.25.0-0", - "description": "A web server", - "type": "application", - "keywords": ["web", "http"], - "home": "https://example.com/web", - "sources": ["https://github.com/example/web"], - "icon": "https://example.com/web.png", - "maintainers": [{"name": "Ada", "email": "ada@example.com"}], - "dependencies": [{ - "name": "redis", - "version": ">=17", - "repository": "https://charts.example.com", - "condition": "redis.enabled", - }], - "annotations": {"artifacthub.io/license": "Apache-2.0"}, - }); - assert_eq!( - summary, - json!({ - "chart": chart, - "description": "A web server", - "dependency_groups": [{ - "target_framework": null, - "dependencies": [{"id": "redis", "range": ">=17"}], - }], - }) - ); - // The index entry carries all of it (and the stored version's name - // and version). - let entry = entry(&hosted(), &package_info(&summary.to_string())); - for (key, value) in chart.as_object().unwrap() { - if key != "name" && key != "version" { - assert_eq!(&entry[key], value, "{key}"); - } - } - } - - #[test] - fn a_huge_description_is_cut_short() { - // 990 kB of a three-byte character: the cut falls inside one. - let description = "€".repeat(330_000); - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: \"2.0\"\n\ - keywords: [a, b]\ndescription: {description}\n" - )); - let cut = format!("{}…", "€".repeat(MAX_DESCRIPTION / 3)); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2.0", - "keywords": ["a", "b"], - "description": cut, - }, - "description": cut, - "dependency_groups": [], - }) - ); - let entry = entry(&hosted(), &package_info(&summary.to_string())); - assert!(entry.to_string().len() <= MAX_SUMMARY); - assert_eq!(entry["description"], json!(cut)); - } - - #[test] - fn the_largest_fields_go_first_and_the_essential_ones_last() { - let big = "x".repeat(20_000); - // Long annotations go, and nothing else. - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\ndescription: A demo\n\ - keywords: [a]\nannotations:\n changes: {big}\n license: MIT\n" - )); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "description": "A demo", - "keywords": ["a"], - }, - "description": "A demo", - "dependency_groups": [], - }) - ); - // A long essential field goes only after every other one. - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: {big}\n\ - description: A demo\nkeywords: [a]\ntype: application\n" - )); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "type": "application", - }, - }) - ); - } - - #[test] - fn an_oversized_stored_summary_renders_a_bounded_entry() { - // As stored before summaries were bounded, and not typed for Helm. - let big = "x".repeat(1 << 20); - let stored = json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": 2, - "description": big, - "keywords": ["a"], - "annotations": {"changes": big}, - }, - "description": big, - "dependency_groups": [], - }) - .to_string(); - let expected = json!({ - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2", - "description": format!("{}…", &big[..MAX_DESCRIPTION]), - "keywords": ["a"], - }); - let check = |mut entry: Value| { - assert!(entry.to_string().len() <= MAX_SUMMARY); - let entry = entry.as_object_mut().unwrap(); - for key in ["urls", "created", "digest"] { - entry.remove(key); - } - assert_eq!(Value::Object(entry.clone()), expected); - }; - check(entry(&hosted(), &package_info(&stored))); - let hit = SearchHit { - original_id: "demo".into(), - latest_version: "1.0.0".into(), - versions: vec![forge_plugin_api::VersionInfo { - original_id: "demo".into(), - version: "1.0.0".into(), - version_original: "1.0.0".into(), - listed: true, - }], - summary: Some(stored), - total_downloads: None, - }; - let proxy = forge_plugin_api::testing::repository(RepositoryKind::Proxy, "/repository"); - let entries = hit_entries(&proxy, &hit); - check(entries.into_iter().next().unwrap()); - } - - #[test] - fn versions_longer_than_an_oci_tag_are_refused() { - let longest = format!("1.0.0-{}", "a".repeat(semver::MAX_VERSION_LENGTH - 6)); - let longer = format!("{longest}a"); - assert_eq!(Helm.normalize_version(&longest).unwrap(), longest); - assert!(matches!( - Helm.normalize_version(&longer), - Err(FormatError::InvalidVersion(_)) - )); - assert_eq!( - package_ref("Demo", &longest), - Some(PackageRef { - id: "demo".into(), - version: longest.clone(), - }) - ); - assert_eq!(package_ref("demo", &longer), None); - - // A URL naming a longer version is not found: nothing is loaded, - // downloaded or deleted. - let repo = Repository { - name: "charts".into(), - kind: RepositoryKind::Proxy, - base_url: "https://forge.example.com/repository/charts".into(), - }; - let download = Helm.handle( - &repo, - &Request::new("GET", &format!("/charts/demo/{longest}/demo.tgz")), - ); - assert!(matches!(download, Action::Download(_)), "{download:?}"); - let huge = format!("1.0.0-{}", "x".repeat(4096)); - for (method, path) in [ - ("GET", format!("/charts/demo/{longer}/demo.tgz")), - ("GET", format!("/charts/demo/{huge}/demo.tgz")), - ("GET", format!("/api/charts/demo/{huge}")), - ("DELETE", format!("/api/charts/demo/{huge}")), - ] { - let action = Helm.handle(&repo, &Request::new(method, &path)); - assert!( - matches!(&action, Action::Respond(r) if r.status == 404), - "{method} {path}: {action:?}" - ); - } - } - - #[test] - fn plugin_toml_names_this_crate() { - let value = |key: &str| { - PLUGIN_TOML - .lines() - .filter_map(|line| line.split_once('=')) - .find(|(k, _)| k.trim() == key) - .map(|(_, v)| v.trim().trim_matches('"').to_string()) - }; - assert_eq!(value("id").as_deref(), Some("helm")); - assert_eq!(value("version").as_deref(), Some(env!("CARGO_PKG_VERSION"))); - assert_eq!(value("api").as_deref(), Some(forge_plugin_api::API)); - assert_eq!( - value("component"), - Some(format!("{}.wasm", env!("CARGO_CRATE_NAME"))) - ); - } -} diff --git a/plugins/helm/src/semver.rs b/plugins/helm/src/semver.rs deleted file mode 100644 index 4faf1c3e..00000000 --- a/plugins/helm/src/semver.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! SemVer 2.0.0, as Helm reads chart versions: a leading `v` and missing -//! minor/patch components are accepted and normalized away. - -use std::cmp::Ordering; - -/// The longest chart version, as written and normalized. Helm pushes a chart -/// to an OCI registry under its version as the tag (with `+` written `_`), -/// and a tag is at most 128 characters (the OCI distribution spec's -/// `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}`), so no chart a Helm user can push to -/// an OCI registry has a longer version. Without a limit a multi-kilobyte -/// prerelease passed parsing and failed the `(repository_id, name, version)` -/// index insert in PostgreSQL, so a push answered 500 instead of 400. A -/// version is ASCII, so this counts bytes. -pub const MAX_VERSION_LENGTH: usize = 128; - -/// A parsed version. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Version { - major: u64, - minor: u64, - patch: u64, - pre: Vec, - build: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum Ident { - Num(u64), - Alpha(String), -} - -impl Version { - /// Parse `raw`, or say why it is not a version. A version longer than - /// [`MAX_VERSION_LENGTH`], as written or normalized, is not one. - pub fn parse(raw: &str) -> Result { - let written = raw.trim(); - // Before parsing, so an oversized version is not copied around first. - if written.len() > MAX_VERSION_LENGTH { - return Err(too_long(written)); - } - let s = written.strip_prefix(['v', 'V']).unwrap_or(written); - let (rest, build) = match s.split_once('+') { - Some((rest, build)) => (rest, Some(build)), - None => (s, None), - }; - let (core, pre) = match rest.split_once('-') { - Some((core, pre)) => (core, Some(pre)), - None => (rest, None), - }; - let mut parts = core.split('.'); - let mut number = |required: bool| -> Result { - match parts.next() { - Some(p) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => p - .parse() - .map_err(|_| format!("version {raw:?}: {p} is too large")), - Some(p) => Err(format!("version {raw:?}: {p:?} is not a number")), - None if required => Err(format!("version {raw:?} has no major version")), - None => Ok(0), - } - }; - let major = number(true)?; - let minor = number(false)?; - let patch = number(false)?; - if parts.next().is_some() { - return Err(format!("version {raw:?} has more than three numbers")); - } - let identifiers = |text: &str| -> Result<(), String> { - let ok = text.split('.').all(|i| { - !i.is_empty() && i.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') - }); - if ok { - Ok(()) - } else { - Err(format!("version {raw:?}: invalid identifiers {text:?}")) - } - }; - let pre = match pre { - Some(pre) => { - identifiers(pre)?; - pre.split('.') - .map(|i| { - // A number with a leading zero is not a numeric - // identifier; keep it as text rather than refuse it. - match i.parse::() { - Ok(n) if !(i.len() > 1 && i.starts_with('0')) => Ident::Num(n), - _ => Ident::Alpha(i.to_string()), - } - }) - .collect() - } - None => Vec::new(), - }; - let build = match build { - Some(build) => { - identifiers(build)?; - build.to_string() - } - None => String::new(), - }; - let version = Version { - major, - minor, - patch, - pre, - build, - }; - // Normalizing can lengthen a version (`1-x` → `1.0.0-x`). - let normalized = version.to_string(); - if normalized.len() > MAX_VERSION_LENGTH { - return Err(format!( - "version {written:?} normalizes to {normalized:?}, longer than \ - {MAX_VERSION_LENGTH} characters" - )); - } - Ok(version) - } -} - -/// Why `written` is not a version: it is longer than [`MAX_VERSION_LENGTH`]. -/// The reason names only its start. -fn too_long(written: &str) -> String { - let start: String = written.chars().take(MAX_VERSION_LENGTH).collect(); - format!( - "version {:?} is longer than {MAX_VERSION_LENGTH} characters", - format!("{start}…") - ) -} - -impl std::fmt::Display for Version { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}.{}.{}", self.major, self.minor, self.patch)?; - for (i, ident) in self.pre.iter().enumerate() { - f.write_str(if i == 0 { "-" } else { "." })?; - match ident { - Ident::Num(n) => write!(f, "{n}")?, - Ident::Alpha(s) => f.write_str(s)?, - } - } - if !self.build.is_empty() { - write!(f, "+{}", self.build)?; - } - Ok(()) - } -} - -impl Ord for Version { - /// SemVer precedence; build metadata, which precedence ignores, breaks - /// ties so that distinct versions never compare equal. - fn cmp(&self, other: &Self) -> Ordering { - (self.major, self.minor, self.patch) - .cmp(&(other.major, other.minor, other.patch)) - .then_with(|| match (self.pre.is_empty(), other.pre.is_empty()) { - (true, true) => Ordering::Equal, - (true, false) => Ordering::Greater, - (false, true) => Ordering::Less, - (false, false) => { - for (a, b) in self.pre.iter().zip(&other.pre) { - let order = match (a, b) { - (Ident::Num(a), Ident::Num(b)) => a.cmp(b), - (Ident::Num(_), Ident::Alpha(_)) => Ordering::Less, - (Ident::Alpha(_), Ident::Num(_)) => Ordering::Greater, - (Ident::Alpha(a), Ident::Alpha(b)) => a.cmp(b), - }; - if order != Ordering::Equal { - return order; - } - } - self.pre.len().cmp(&other.pre.len()) - } - }) - .then_with(|| self.build.cmp(&other.build)) - } -} - -impl PartialOrd for Version { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn v(s: &str) -> Version { - Version::parse(s).unwrap() - } - - #[test] - fn normalizes() { - assert_eq!(v("v1.2.3").to_string(), "1.2.3"); - assert_eq!(v("1.2").to_string(), "1.2.0"); - assert_eq!(v("1").to_string(), "1.0.0"); - assert_eq!(v("01.002.3").to_string(), "1.2.3"); - assert_eq!(v("1.0.0-rc.01+build.5").to_string(), "1.0.0-rc.01+build.5"); - for bad in [ - "", - "x", - "1.2.3.4", - "1..2", - "1.0.0-", - "1.0.0+", - "1.0.0-a..b", - "1.0.0-a_b", - ] { - assert!(Version::parse(bad).is_err(), "{bad:?}"); - } - } - - #[test] - fn refuses_versions_longer_than_an_oci_tag() { - let longest = format!("1.0.0-{}", "a".repeat(MAX_VERSION_LENGTH - 6)); - assert_eq!(longest.len(), MAX_VERSION_LENGTH); - assert_eq!(v(&longest).to_string(), longest); - // Surrounding whitespace is not part of the version. - assert_eq!(v(&format!(" {longest}\n")).to_string(), longest); - let err = Version::parse(&format!("{longest}a")).unwrap_err(); - assert!(err.contains("longer than 128 characters"), "{err}"); - // A multi-kilobyte prerelease, named only by its start. - let err = Version::parse(&format!("1.0.0-{}", "x".repeat(4096))).unwrap_err(); - assert!(err.len() < 200, "{err}"); - // Normalized, `1-…` gains `.0.0`: the written form fits, not the - // normalized one. - let short = format!("1-{}", "a".repeat(MAX_VERSION_LENGTH - 2)); - assert_eq!(short.len(), MAX_VERSION_LENGTH); - let err = Version::parse(&short).unwrap_err(); - assert!(err.contains("normalizes to"), "{err}"); - assert!(Version::parse(&format!("1-{}", "a".repeat(MAX_VERSION_LENGTH - 6))).is_ok()); - } - - #[test] - fn orders_by_precedence() { - let ordered = [ - "1.0.0-alpha", - "1.0.0-alpha.1", - "1.0.0-alpha.beta", - "1.0.0-beta", - "1.0.0-beta.2", - "1.0.0-beta.11", - "1.0.0-rc.1", - "1.0.0", - "1.0.0+build", - "1.9.0", - "1.10.0", - "2.0.0", - ]; - for pair in ordered.windows(2) { - assert!(v(pair[0]) < v(pair[1]), "{} < {}", pair[0], pair[1]); - } - // Forge's rule: a normalized version is a prerelease when it has a `-`. - assert!(v("1.0.0-beta").to_string().contains('-')); - assert!(!v("v1.0+build").to_string().contains('-')); - } -} diff --git a/plugins/helm/src/tar.rs b/plugins/helm/src/tar.rs deleted file mode 100644 index a60df243..00000000 --- a/plugins/helm/src/tar.rs +++ /dev/null @@ -1,914 +0,0 @@ -//! A tar reader that sees a chart archive the way Helm does. -//! -//! Helm reads a chart with Go's `archive/tar`, so an entry's name, its type -//! and where its data ends are whatever that reader makes of the headers. -//! Forge must make the same of them, or the two disagree about which file -//! is the chart's `Chart.yaml`. [`Archive`] follows Go's `Reader.Next` for -//! every archive it accepts: ustar, PAX, GNU and STAR headers and their -//! checksums, PAX `path` and `size` records (of the last extended header -//! before an entry), GNU long names and long link names, global PAX headers -//! as entries of their own, entry types without data, and Go's end of -//! archive. It refuses what it does not model: sparse files, entry types -//! charts do not use, and a few archives Go reads (a stream that ends inside -//! padding, an extension header over [`MAX_HEADER_DATA`], a negative size). - -use std::io::{self, Read}; - -/// The size of a header, and the unit data is padded to. -const BLOCK: usize = 512; - -/// The longest PAX header or GNU long name read. Go reads up to 1 MiB; the -/// names in a chart are far shorter. -const MAX_HEADER_DATA: u64 = 64 * 1024; - -/// An entry of an archive, as Go's `tar.Reader.Next` returns it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Entry { - /// Its name (`Header.Name`): the header's own, joined to a ustar prefix, - /// or taken from a PAX `path` record or a GNU long name. Bytes, as in a - /// Go string. - pub name: Vec, - /// Its type flag. An old-style regular entry (`\0`) reads as `0`, or as - /// a directory (`5`) when its name ends with `/`, as in Go. - pub typeflag: u8, - /// Its mode field (`Header.Mode`). - pub mode: i64, - /// The length of its data: 0 for the types that have none (links, - /// devices, directories, FIFOs), whatever the header says. - pub size: u64, -} - -impl Entry { - /// Whether Go's `Header.FileInfo().IsDir()` holds: a directory by its - /// type, or by the file type bits of its mode, whatever its type. - pub fn is_dir(&self) -> bool { - // Go converts the mode to a 32-bit `fs.FileMode`. - self.typeflag == b'5' || (self.mode as u32) & !0o7777 == 0o040000 - } -} - -/// The entries of a tar stream, one at a time. -pub struct Archive { - reader: R, - /// The current entry's data not read yet. - unread: u64, - /// The padding after the current entry's data. - padding: u64, - /// Whether the end of the archive was reached. - ended: bool, -} - -impl Archive { - /// Read the archive in `reader`. - pub fn new(reader: R) -> Self { - Self { - reader, - unread: 0, - padding: 0, - ended: false, - } - } - - /// The next entry, after skipping what is left of the current one; - /// `None` at the end of the archive. Extended headers and GNU long names - /// apply to the entry after them and are not entries; a global PAX - /// header (`g`) is one, as in Go, and ends what came before it. - pub fn next_entry(&mut self) -> io::Result> { - if self.ended { - return Ok(None); - } - skip(&mut self.reader, self.unread + self.padding)?; - self.unread = 0; - self.padding = 0; - // What extension headers say about the entry after them. Go keeps - // the last extended header, and the last long name. - let mut pax: Option = None; - let mut long_name: Vec = Vec::new(); - loop { - let Some(header) = self.read_header()? else { - self.ended = true; - return Ok(None); - }; - match header.typeflag { - b'x' | b'g' => { - let records = Pax::parse(&self.special_data(&header)?)?; - if header.typeflag == b'g' { - let path = records.path.filter(|p| !p.is_empty()); - return Ok(Some(Entry { - name: path.unwrap_or(header.name), - typeflag: b'g', - mode: 0, - size: 0, - })); - } - pax = Some(records); - } - b'L' | b'K' => { - let data = self.special_data(&header)?; - // A long link name only names a link's target. - if header.typeflag == b'L' { - long_name = c_string(&data).to_vec(); - } - } - b'S' => return Err(invalid("the archive holds a GNU sparse file")), - typeflag => { - let mut name = header.name; - let mut size = header.size; - if let Some(pax) = pax { - pax.check()?; - if let Some(path) = pax.path.filter(|p| !p.is_empty()) { - name = path; - } - if let Some(value) = pax.size.filter(|s| !s.is_empty()) { - size = decimal(&value).ok_or_else(|| invalid("bad PAX size record"))?; - } - } - // A GNU long name wins over a PAX path, in either order. - if !long_name.is_empty() { - name = long_name; - } - let typeflag = match typeflag { - 0 if name.ends_with(b"/") => b'5', - 0 => b'0', - other => other, - }; - if !matches!(typeflag, b'0'..=b'7') { - return Err(invalid(&format!( - "the archive holds an entry of type {:?}", - char::from(typeflag) - ))); - } - let size = u64::try_from(size).map_err(|_| invalid("negative tar size"))?; - // Links, devices, directories and FIFOs have no data, - // whatever their size says. - let size = if matches!(typeflag, b'1'..=b'6') { - 0 - } else { - size - }; - self.unread = size; - self.padding = padding(size); - return Ok(Some(Entry { - name, - typeflag, - mode: header.mode, - size, - })); - } - } - } - } - - /// The current entry's data. An entry larger than `max` bytes is an - /// error. - pub fn read_data(&mut self, max: u64) -> io::Result> { - if self.unread > max { - return Err(invalid(&format!("larger than {max} bytes"))); - } - let data = read_exact(&mut self.reader, self.unread)?; - self.unread = 0; - Ok(data) - } - - /// The next header; `None` at the end of the archive: the end of the - /// stream before a header or after one zero block, or two zero blocks. - /// A zero block followed by anything else is an error, as in Go. - fn read_header(&mut self) -> io::Result> { - let mut block = [0u8; BLOCK]; - if !read_block(&mut self.reader, &mut block)? { - return Ok(None); - } - if block.iter().all(|&b| b == 0) { - if !read_block(&mut self.reader, &mut block)? || block.iter().all(|&b| b == 0) { - return Ok(None); - } - return Err(invalid("a zero block in the middle of the archive")); - } - let header = Header::parse(&block)?; - if header.size < 0 { - return Err(invalid("negative tar size")); - } - Ok(Some(header)) - } - - /// The data of an extension header, and its padding skipped. - fn special_data(&mut self, header: &Header) -> io::Result> { - let size = header.size as u64; - if size > MAX_HEADER_DATA { - return Err(invalid("oversized tar extension header")); - } - let data = read_exact(&mut self.reader, size)?; - skip(&mut self.reader, padding(size))?; - Ok(data) - } -} - -/// The fields of a header block that Go's reader uses. -struct Header { - /// The name, joined to the prefix of the header's format. - name: Vec, - /// The type flag, as written. - typeflag: u8, - /// The size field, which an extended header may replace. - size: i64, - /// The mode field. - mode: i64, -} - -/// The header formats Go tells apart by their magic. -#[derive(PartialEq)] -enum Format { - /// No magic: no prefix, no user or device fields. - V7, - /// `ustar\0`, which PAX archives use too: a 155-byte prefix. - Ustar, - /// `ustar \0`: times where ustar has its prefix. - Gnu, - /// `ustar\0` and `tar\0` at the end: a 131-byte prefix, then times. - Star, -} - -impl Header { - /// Parse a header block as Go's `Reader.readHeader` does. - fn parse(block: &[u8; BLOCK]) -> io::Result { - let format = format(block).ok_or_else(|| invalid("bad tar header checksum"))?; - let mut fields = Numbers::default(); - let mode = fields.numeric(&block[100..108]); - fields.numeric(&block[108..116]); // uid - fields.numeric(&block[116..124]); // gid - let size = fields.numeric(&block[124..136]); - fields.numeric(&block[136..148]); // mtime - if format != Format::V7 { - fields.numeric(&block[329..337]); // devmajor - fields.numeric(&block[337..345]); // devminor - } - let prefix = match format { - Format::V7 => &[], - Format::Ustar => c_string(&block[345..500]), - Format::Star => { - fields.numeric(&block[476..488]); // atime - fields.numeric(&block[488..500]); // ctime - c_string(&block[345..476]) - } - // A GNU header has times where ustar has its prefix. Go reads - // them as a prefix after all when they are no times (old Go - // versions wrote GNU headers with a prefix) and it is ASCII. - Format::Gnu => { - let mut times = Numbers::default(); - if block[345] != 0 { - times.numeric(&block[345..357]); - } - if block[357] != 0 { - times.numeric(&block[357..369]); - } - let prefix = c_string(&block[345..500]); - if times.failed && prefix.is_ascii() { - prefix - } else { - &[] - } - } - }; - if fields.failed { - return Err(invalid("bad tar header field")); - } - let name = c_string(&block[0..100]); - Ok(Self { - name: if prefix.is_empty() { - name.to_vec() - } else { - [prefix, b"/", name].concat() - }, - typeflag: block[156], - size, - mode, - }) - } -} - -/// The format of a header block; `None` when its checksum is wrong. Go -/// accepts the unsigned sum of the bytes and the signed one (Sun tar). -fn format(block: &[u8; BLOCK]) -> Option { - let mut fields = Numbers::default(); - let stored = fields.octal(&block[148..156]); - let (mut unsigned, mut signed) = (0i64, 0i64); - for (i, &b) in block.iter().enumerate() { - // The checksum field counts as spaces. - let b = if (148..156).contains(&i) { b' ' } else { b }; - unsigned += i64::from(b); - signed += i64::from(b as i8); - } - if fields.failed || (stored != unsigned && stored != signed) { - return None; - } - let (magic, version, trailer) = (&block[257..263], &block[263..265], &block[508..512]); - Some(if magic == b"ustar\0" && trailer == b"tar\0" { - Format::Star - } else if magic == b"ustar\0" { - Format::Ustar - } else if magic == b"ustar " && version == b" \0" { - Format::Gnu - } else { - Format::V7 - }) -} - -/// Go's reader of numeric header fields; `failed` is set by any field it -/// cannot read, like Go's `parser.err`. -#[derive(Default)] -struct Numbers { - failed: bool, -} - -impl Numbers { - /// An octal field, or GNU base-256 (two's complement) when its first - /// bit is set (Go's `parseNumeric`). - fn numeric(&mut self, field: &[u8]) -> i64 { - if field[0] & 0x80 == 0 { - return self.octal(field); - } - let inverse = if field[0] & 0x40 != 0 { 0xff } else { 0 }; - let mut n: u64 = 0; - for (i, &b) in field.iter().enumerate() { - let mut b = b ^ inverse; - if i == 0 { - b &= 0x7f; - } - if n >> 56 > 0 { - self.failed = true; - return 0; - } - n = n << 8 | u64::from(b); - } - if n >> 63 > 0 { - self.failed = true; - return 0; - } - if inverse == 0xff { - !(n as i64) - } else { - n as i64 - } - } - - /// An octal field: NULs and spaces trimmed from both ends, then the - /// digits up to the first NUL (Go's `parseOctal`). - fn octal(&mut self, field: &[u8]) -> i64 { - let padding = |b: &u8| *b == b' ' || *b == 0; - let start = field.iter().position(|b| !padding(b)); - let end = field.iter().rposition(|b| !padding(b)); - let (Some(start), Some(end)) = (start, end) else { - return 0; - }; - let mut n: u64 = 0; - for &b in c_string(&field[start..=end]) { - let next = match b { - b'0'..=b'7' => n - .checked_mul(8) - .and_then(|n| n.checked_add(u64::from(b - b'0'))), - _ => None, - }; - let Some(next) = next else { - self.failed = true; - return 0; - }; - n = next; - } - n as i64 - } -} - -/// The records of a PAX extended header that bear on an entry. A key given -/// twice keeps its last value, as in Go. -#[derive(Default)] -struct Pax { - /// `path`: the entry's name. - path: Option>, - /// `size`: the length of the entry's data. - size: Option>, - /// `uid`. - uid: Option>, - /// `gid`. - gid: Option>, - /// `atime`, `mtime` and `ctime`. - times: [Option>; 3], -} - -impl Pax { - /// Parse the records of a PAX header (`" =\n"`, - /// Go's `parsePAX`). GNU sparse records, which describe a sparse file's - /// layout and may rename it, are refused. - fn parse(mut data: &[u8]) -> io::Result { - let bad = || invalid("bad PAX record"); - let mut pax = Self::default(); - while !data.is_empty() { - // The length, of the whole record, ends at the first space. - let space = data.iter().position(|&b| b == b' ').ok_or_else(bad)?; - let length = decimal(&data[..space]).ok_or_else(bad)?; - if length < 5 || length > data.len() as i64 { - return Err(bad()); - } - let length = length as usize; - if length <= space + 1 || data[length - 1] != b'\n' { - return Err(bad()); - } - let record = &data[space + 1..length - 1]; - data = &data[length..]; - let equals = record.iter().position(|&b| b == b'=').ok_or_else(bad)?; - let (key, value) = (&record[..equals], record[equals + 1..].to_vec()); - let field = match key { - b"" => return Err(bad()), - b"path" | b"linkpath" | b"uname" | b"gname" if value.contains(&0) => { - return Err(bad()) - } - _ if key.contains(&0) => return Err(bad()), - _ if key.starts_with(b"GNU.sparse.") => { - return Err(invalid("the archive holds a GNU sparse file")) - } - b"path" => &mut pax.path, - b"size" => &mut pax.size, - b"uid" => &mut pax.uid, - b"gid" => &mut pax.gid, - b"atime" => &mut pax.times[0], - b"mtime" => &mut pax.times[1], - b"ctime" => &mut pax.times[2], - _ => continue, - }; - *field = Some(value); - } - Ok(pax) - } - - /// Refuse the records Go cannot apply to an entry (`mergePAX`). Empty - /// values are ignored, as in Go. - fn check(&self) -> io::Result<()> { - let bad = || invalid("bad PAX record"); - for id in [&self.uid, &self.gid].into_iter().flatten() { - if !id.is_empty() { - decimal(id).ok_or_else(bad)?; - } - } - // Seconds, and an optional fraction. - for time in self.times.iter().flatten().filter(|t| !t.is_empty()) { - let (seconds, fraction) = match time.iter().position(|&b| b == b'.') { - Some(dot) => (&time[..dot], &time[dot + 1..]), - None => (&time[..], &[][..]), - }; - decimal(seconds).ok_or_else(bad)?; - if !fraction.iter().all(u8::is_ascii_digit) { - return Err(bad()); - } - } - Ok(()) - } -} - -/// A decimal integer with an optional sign, as Go's `strconv.ParseInt(s, -/// 10, 64)` reads it. -fn decimal(text: &[u8]) -> Option { - let (negative, digits) = match text.split_first() { - Some((b'-', rest)) => (true, rest), - Some((b'+', rest)) => (false, rest), - _ => (false, text), - }; - if digits.is_empty() { - return None; - } - let mut n: i64 = 0; - for &b in digits { - if !b.is_ascii_digit() { - return None; - } - let digit = i64::from(b - b'0'); - n = n.checked_mul(10)?; - n = if negative { - n.checked_sub(digit)? - } else { - n.checked_add(digit)? - }; - } - Some(n) -} - -/// An error for an archive Forge does not read. -fn invalid(msg: &str) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, msg.to_string()) -} - -/// The padding after `size` bytes of data, to the end of their last block. -fn padding(size: u64) -> u64 { - size.next_multiple_of(BLOCK as u64) - size -} - -/// Fill `block`; `false` at a clean end of the stream. -fn read_block(r: &mut impl Read, block: &mut [u8; BLOCK]) -> io::Result { - let mut filled = 0; - while filled < BLOCK { - match r.read(&mut block[filled..])? { - 0 if filled == 0 => return Ok(false), - 0 => return Err(invalid("truncated tar header")), - n => filled += n, - } - } - Ok(true) -} - -/// The next `len` bytes; an error when the stream ends first. -fn read_exact(r: &mut impl Read, len: u64) -> io::Result> { - let mut data = Vec::new(); - r.take(len).read_to_end(&mut data)?; - if data.len() as u64 != len { - return Err(invalid("truncated tar entry")); - } - Ok(data) -} - -/// Skip `len` bytes; an error when the stream ends first. -fn skip(r: &mut impl Read, len: u64) -> io::Result<()> { - let skipped = io::copy(&mut r.take(len), &mut io::sink())?; - if skipped != len { - return Err(invalid("truncated tar entry")); - } - Ok(()) -} - -/// The bytes up to the first NUL (Go's `parseString`). -fn c_string(bytes: &[u8]) -> &[u8] { - let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); - &bytes[..end] -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - - /// A ustar header (mode 0644) with a valid checksum. - pub(crate) fn header(name: &[u8], typeflag: u8, size: u64) -> [u8; BLOCK] { - let mut header = [0u8; BLOCK]; - header[..name.len()].copy_from_slice(name); - header[100..108].copy_from_slice(b"0000644\0"); - header[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); - header[156] = typeflag; - header[257..263].copy_from_slice(b"ustar\0"); - header[263..265].copy_from_slice(b"00"); - seal(&mut header); - header - } - - /// Write the checksum of `header`, after changing it. - pub(crate) fn seal(header: &mut [u8; BLOCK]) { - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - } - - /// `header` followed by `data`, padded to whole blocks. - pub(crate) fn entry(header: [u8; BLOCK], data: &[u8]) -> Vec { - let mut out = header.to_vec(); - out.extend_from_slice(data); - out.resize(out.len().next_multiple_of(BLOCK), 0); - out - } - - /// A regular file. - pub(crate) fn file(path: &str, data: &[u8]) -> Vec { - entry(header(path.as_bytes(), b'0', data.len() as u64), data) - } - - /// A GNU long name (`L`) or long link name (`K`). - pub(crate) fn long(typeflag: u8, name: &str) -> Vec { - let data = [name.as_bytes(), b"\0"].concat(); - entry(header(b"././@LongLink", typeflag, data.len() as u64), &data) - } - - /// A PAX header of type `typeflag` (`x` or `g`) holding `records`. - pub(crate) fn pax(typeflag: u8, records: &[(&str, &str)]) -> Vec { - let mut data = Vec::new(); - for (key, value) in records { - let body = format!(" {key}={value}\n"); - // The length counts its own digits. - let mut length = body.len(); - while length != length.to_string().len() + body.len() { - length = length.to_string().len() + body.len(); - } - data.extend_from_slice(format!("{length}{body}").as_bytes()); - } - entry( - header(b"PaxHeaders.0/x", typeflag, data.len() as u64), - &data, - ) - } - - /// The end of an archive: two zero blocks. - pub(crate) const END: [u8; 2 * BLOCK] = [0; 2 * BLOCK]; - - /// A minimal ustar archive of `(path, contents)` entries. - pub(crate) fn archive(entries: &[(&str, &[u8])]) -> Vec { - let mut out: Vec = entries - .iter() - .flat_map(|(path, data)| file(path, data)) - .collect(); - out.extend_from_slice(&END); - out - } - - /// Every entry of `tar`: its name, type and data. - fn entries(tar: &[u8]) -> io::Result)>> { - let mut archive = Archive::new(tar); - let mut out = Vec::new(); - while let Some(entry) = archive.next_entry()? { - let data = archive.read_data(u64::MAX)?; - assert_eq!(data.len() as u64, entry.size); - let name = String::from_utf8_lossy(&entry.name).into_owned(); - out.push((name, entry.typeflag, data)); - } - Ok(out) - } - - /// The names of the entries of `tar`. - fn names(tar: &[u8]) -> Vec { - entries(tar) - .unwrap() - .into_iter() - .map(|(n, _, _)| n) - .collect() - } - - #[test] - fn reads_entries_and_their_data() { - let tar = archive(&[ - ("demo/values.yaml", b"replicas: 1\n"), - ("demo/Chart.yaml", b"name: demo\n"), - ]); - assert_eq!( - entries(&tar).unwrap(), - [ - ("demo/values.yaml".into(), b'0', b"replicas: 1\n".to_vec()), - ("demo/Chart.yaml".into(), b'0', b"name: demo\n".to_vec()), - ] - ); - // Data not read is skipped. - let mut archive = Archive::new(&tar[..]); - archive.next_entry().unwrap(); - assert!(archive.read_data(4).is_err()); - let second = archive.next_entry().unwrap().unwrap(); - assert_eq!(second.name, b"demo/Chart.yaml"); - assert_eq!(archive.next_entry().unwrap(), None); - assert_eq!(archive.next_entry().unwrap(), None); - // Truncated: in the data, in the padding, in a header. - assert!(entries(&tar[..520]).is_err()); - assert!(entries(&tar[..700]).is_err()); - assert!(entries(&tar[..1030]).is_err()); - } - - #[test] - fn ends_like_go() { - let one = file("demo/a", b"a"); - // The stream ends before a header, or after one zero block. - assert_eq!(names(&one), ["demo/a"]); - assert_eq!(names(&[&one[..], &[0; BLOCK]].concat()), ["demo/a"]); - // Two zero blocks end the archive; what follows is not read. - let after = [&one[..], &END, &file("demo/b", b"b")].concat(); - assert_eq!(names(&after), ["demo/a"]); - // A zero block followed by a header is an error, not the end. - let hidden = [&one[..], &[0; BLOCK], &file("demo/b", b"b")].concat(); - assert!(entries(&hidden).is_err()); - } - - #[test] - fn checks_header_checksums() { - let mut bad = header(b"demo/a", b'0', 0); - bad[0] = b'x'; - assert!(entries(&[&bad[..], &END].concat()).is_err()); - // Go accepts the signed sum too (Sun tar). - let mut signed = header("demo/é".as_bytes(), b'0', 0); - let sum: i64 = signed - .iter() - .enumerate() - .map(|(i, &b)| if (148..156).contains(&i) { b' ' } else { b }) - .map(|b| i64::from(b as i8)) - .sum(); - signed[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - assert_eq!(names(&[&signed[..], &END].concat()), ["demo/é"]); - } - - #[test] - fn reads_numeric_fields_like_go() { - // Leading NULs and spaces are padding, not the end of the number. - let mut padded = header(b"demo/a", b'0', 0); - padded[124..136].copy_from_slice(b"\0 0000000003"); - seal(&mut padded); - let tar = [&entry(padded, b"abc")[..], &END].concat(); - assert_eq!(entries(&tar).unwrap()[0].2, b"abc"); - // GNU base-256. - let mut binary = header(b"demo/a", b'0', 0); - binary[124..136].copy_from_slice(&[0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3]); - seal(&mut binary); - let tar = [&entry(binary, b"abc")[..], &END].concat(); - assert_eq!(entries(&tar).unwrap()[0].2, b"abc"); - // What Go cannot read: a sign, a digit that is not octal, a mode - // in hexadecimal. - for (field, value) in [ - (124..136, &b"+0000000003\0"[..]), - (124..136, b"0000000009\0\0"), - (100..108, b"0x644\0\0\0"), - ] { - let mut bad = header(b"demo/a", b'0', 3); - bad[field].copy_from_slice(value); - seal(&mut bad); - assert!(entries(&[&entry(bad, b"abc")[..], &END].concat()).is_err()); - } - } - - #[test] - fn joins_a_ustar_prefix_but_not_gnu_times() { - let mut ustar = header(b"Chart.yaml", b'0', 0); - ustar[345..349].copy_from_slice(b"demo"); - seal(&mut ustar); - assert_eq!(names(&[&ustar[..], &END].concat()), ["demo/Chart.yaml"]); - - // A GNU header has times where ustar has its prefix. - let gnu = |times: &[u8]| { - let mut gnu = header(b"demo/Chart.yaml", b'0', 0); - gnu[257..265].copy_from_slice(b"ustar \0"); - gnu[345..345 + times.len()].copy_from_slice(times); - seal(&mut gnu); - names(&[&gnu[..], &END].concat()) - }; - assert_eq!(gnu(b"14567012345\0"), ["demo/Chart.yaml"]); - // Junk that reads as octal times stays times. - assert_eq!(gnu(b"7\0junk"), ["demo/Chart.yaml"]); - // Junk that does not is a prefix to Go (old Go versions wrote one), - // when it is ASCII. - assert_eq!(gnu(b"evil"), ["evil/demo/Chart.yaml"]); - assert_eq!(gnu("évil".as_bytes()), ["demo/Chart.yaml"]); - - // STAR: a shorter prefix, then times. - let mut star = header(b"Chart.yaml", b'0', 0); - star[345..349].copy_from_slice(b"demo"); - star[508..512].copy_from_slice(b"tar\0"); - seal(&mut star); - assert_eq!(names(&[&star[..], &END].concat()), ["demo/Chart.yaml"]); - - // V7, without a magic: no prefix. - let mut v7 = header(b"demo/Chart.yaml", b'0', 0); - v7[257..265].fill(0); - v7[345..349].copy_from_slice(b"evil"); - seal(&mut v7); - assert_eq!(names(&[&v7[..], &END].concat()), ["demo/Chart.yaml"]); - } - - #[test] - fn applies_pax_records_like_go() { - // A PAX path renames the next entry only. - let tar = [ - &pax(b'x', &[("mtime", "1.5"), ("path", "demo/values.yaml")])[..], - &file("x", b"a: 1\n"), - &file("demo/b", b""), - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/values.yaml", "demo/b"]); - // A key given twice keeps its last value; an empty one keeps the - // header's name. - let twice = [ - &pax(b'x', &[("path", "demo/a"), ("path", "demo/b")])[..], - &file("x", b""), - &pax(b'x', &[("path", "")]), - &file("demo/c", b""), - &END, - ] - .concat(); - assert_eq!(names(&twice), ["demo/b", "demo/c"]); - // Only the last extended header before an entry counts. - let last = [ - &pax(b'x', &[("path", "demo/a")])[..], - &pax(b'x', &[("mtime", "1")]), - &file("demo/b", b""), - &END, - ] - .concat(); - assert_eq!(names(&last), ["demo/b"]); - - // A PAX size moves the end of the entry's data: what its header - // calls data is the next entry. - let chart = file("demo/Chart.yaml", b"name: demo\n"); - let sized = [ - &pax(b'x', &[("size", "0")])[..], - &entry(header(b"demo/a", b'0', chart.len() as u64), &chart), - &END, - ] - .concat(); - assert_eq!(names(&sized), ["demo/a", "demo/Chart.yaml"]); - - // Records Go cannot apply, and sparse files. - for records in [ - &[("size", "x")][..], - &[("uid", "1x")], - &[("mtime", "1.x")], - &[("GNU.sparse.major", "1"), ("GNU.sparse.minor", "0")], - &[("GNU.sparse.name", "demo/Chart.yaml")], - ] { - let tar = [&pax(b'x', records)[..], &file("demo/a", b""), &END].concat(); - assert!(entries(&tar).is_err(), "{records:?}"); - } - // A record longer than the header. - let malformed = [ - &entry(header(b"x", b'x', 11), b"13 path=ab\n")[..], - &file("demo/a", b""), - &END, - ] - .concat(); - assert!(entries(&malformed).is_err()); - } - - #[test] - fn applies_gnu_long_names_like_go() { - // A long name renames the next entry; a long link name does not, - // and does not take the long name before it. - let tar = [ - &long(b'L', "demo/values.yaml")[..], - &file("x", b""), - &long(b'L', "demo/Chart.yaml"), - &long(b'K', "demo/target"), - &file("demo/a", b""), - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/values.yaml", "demo/Chart.yaml"]); - // A long name wins over a PAX path, before it or after it. - for (first, second) in [ - (long(b'L', "demo/long"), pax(b'x', &[("path", "demo/pax")])), - (pax(b'x', &[("path", "demo/pax")]), long(b'L', "demo/long")), - ] { - let tar = [&first[..], &second, &file("x", b""), &END].concat(); - assert_eq!(names(&tar), ["demo/long"]); - } - } - - #[test] - fn a_global_header_is_an_entry_of_its_own() { - // It ends what came before it: the long name applies to no entry. - let tar = [ - &long(b'L', "demo/Chart.yaml")[..], - &pax(b'g', &[("comment", "abc")]), - &file("demo/a", b""), - &END, - ] - .concat(); - let entries = entries(&tar).unwrap(); - assert_eq!(entries[0].1, b'g'); - assert_eq!(entries[1].0, "demo/a"); - } - - #[test] - fn types_without_data_have_none() { - // A symlink's size does not make the next header its data. - let chart = file("demo/Chart.yaml", b"name: demo\n"); - for typeflag in *b"123456" { - let tar = [ - &header(b"demo/link", typeflag, chart.len() as u64)[..], - &chart, - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/link", "demo/Chart.yaml"]); - } - // An old-style regular entry named with a `/` is a directory. - let tar = [ - &header(b"demo/", 0, chart.len() as u64)[..], - &entry(header(b"demo/a", 0, 1), b"a"), - &chart, - &END, - ] - .concat(); - let entries = entries(&tar).unwrap(); - assert_eq!((entries[0].1, entries[1].1), (b'5', b'0')); - assert_eq!(entries[2].0, "demo/Chart.yaml"); - } - - #[test] - fn a_directory_by_its_mode() { - let mut dir = header(b"demo/Chart.yaml", b'0', 0); - dir[100..108].copy_from_slice(b"0040755\0"); - seal(&mut dir); - let tar = [&dir[..], &END].concat(); - let mut archive = Archive::new(&tar[..]); - assert!(archive.next_entry().unwrap().unwrap().is_dir()); - let tar = [&header(b"demo/a", b'0', 0)[..], &END].concat(); - let mut archive = Archive::new(&tar[..]); - assert!(!archive.next_entry().unwrap().unwrap().is_dir()); - } - - #[test] - fn refuses_what_it_does_not_model() { - for typeflag in *b"SDMVA" { - let tar = [&header(b"demo/a", typeflag, 0)[..], &END].concat(); - assert!(entries(&tar).is_err(), "{}", char::from(typeflag)); - } - let mut negative = header(b"demo/a", b'0', 0); - negative[124..136].copy_from_slice(&[0xff; 12]); - seal(&mut negative); - assert!(entries(&[&negative[..], &END].concat()).is_err()); - } -} diff --git a/plugins/helm/src/upstream.rs b/plugins/helm/src/upstream.rs deleted file mode 100644 index 8d6be726..00000000 --- a/plugins/helm/src/upstream.rs +++ /dev/null @@ -1,709 +0,0 @@ -//! The upstream side of Helm proxies: another chart repository, read through -//! the host ([`FormatPlugin::upstream`](forge_plugin_api::FormatPlugin::upstream)). -//! -//! A chart repository is one document, its `index.yaml`: every chart, every -//! version, and where each version's archive is. Every operation reads it -//! (the host caches it for the repository's metadata TTL). The upstream's -//! index ([`Remote::index`]) is just its base URL. -//! -//! Parsing a large index takes long (half a second and some 200 MB for a -//! 10 MB one), and a proxy's chart listing runs several operations per -//! chart, so the built-in plugin keeps the parsed index while the upstream -//! serves the same document, and parses one index at a time -//! ([`parsed_index`]). - -use std::collections::BTreeMap; -use std::sync::Arc; - -use forge_plugin_api::{ - Fetch, FetchResult, Fetched, FormatError, Freshness, PackageMetadata, PackageRef, - PackageVersions, Remote, SearchQuery, UpstreamAnswer, UpstreamHit, UpstreamOp, UpstreamSearch, - UpstreamStep, VersionMetadata, -}; -use serde_json::{json, Map, Value}; - -use crate::chart::{self, to_json}; -use crate::semver::Version; - -/// The default page of an upstream search. -const DEFAULT_TAKE: usize = 20; - -/// One chart version an upstream index lists. Compact, since parsed -/// indexes are kept ([`parsed_index`]): the entry itself stays JSON text, -/// read back only for the versions an operation describes. -struct Entry { - /// The chart name, as written. - name: String, - version: Version, - /// Its `created`. - created: Option, - /// The first of its `urls`. - url: Option, - /// The index entry (Chart.yaml's metadata, `urls`, `created`, `digest`), - /// as JSON. - metadata: Box, -} - -impl Entry { - /// The index entry ([`Entry::metadata`]). - fn metadata(&self) -> Map { - serde_json::from_str(&self.metadata).unwrap_or_default() - } -} - -/// An upstream index: each chart's versions by canonical name, ascending. -type Index = BTreeMap>; - -/// One step of the conversation for `op` ([`crate::Helm`]'s -/// `FormatPlugin::upstream`). -pub(crate) fn step( - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], -) -> Result { - let base = match op { - UpstreamOp::Index => remote.url.trim_end_matches('/').to_string(), - _ => match remote - .index - .as_deref() - .and_then(|i| serde_json::from_str::(i).ok()) - .and_then(|i| i.get("base")?.as_str().map(str::to_string)) - { - Some(base) => base, - None => return Ok(failed("the upstream's index is not resolved")), - }, - }; - // Nothing to read: a chart repository has no manifest without its - // chart, so the host takes the manifest from the package. - if let UpstreamOp::Manifest(_) = op { - return Ok(done(UpstreamAnswer::Manifest(None))); - } - let url = format!("{base}/index.yaml"); - let index = match fetched - .iter() - .find(|f| f.fetch.url == url) - .map(|f| &f.outcome) - { - None => { - return Ok(UpstreamStep::Fetch(vec![Fetch { - url, - freshness: Freshness::Metadata, - }])) - } - Some(Fetched::Found(body)) => match parsed_index(&url, body) { - Ok(index) => index, - Err(e) => return Ok(failed(format!("upstream index {url}: {e}"))), - }, - Some(Fetched::NotFound) => return Ok(failed(format!("upstream index {url} returned 404"))), - Some(Fetched::Failed(reason)) => return Ok(failed(reason.clone())), - }; - Ok(match op { - UpstreamOp::Index => done(UpstreamAnswer::Index(json!({ "base": base }).to_string())), - UpstreamOp::Versions(id) => { - done(UpstreamAnswer::Versions(index.get(id).map(|versions| { - versions.iter().map(|e| e.version.to_string()).collect() - }))) - } - UpstreamOp::Package(target) => { - done(UpstreamAnswer::Package(package_url(&base, &index, target))) - } - UpstreamOp::Manifest(_) => unreachable!("answered above"), - UpstreamOp::Metadata(wanted) => metadata(&index, wanted, false), - UpstreamOp::Dating(wanted) => metadata(&index, wanted, true), - UpstreamOp::Search(query) => done(UpstreamAnswer::Search(Some(search(&index, query)))), - }) -} - -fn done(answer: UpstreamAnswer) -> UpstreamStep { - UpstreamStep::Done(answer) -} - -fn failed(reason: impl Into) -> UpstreamStep { - done(UpstreamAnswer::Failed(reason.into())) -} - -/// The parsed `index.yaml` at `url` whose bytes are `body`. -/// -/// The built-in plugin keeps what it parsed, one index per URL, as many as -/// fit in [`parsed::BUDGET`] (one budget for every Helm proxy of the -/// process), and reuses one while the upstream serves the same bytes (the -/// host hands every operation the document it caches for the metadata TTL): -/// comparing the bytes takes a millisecond where parsing takes half a second. -/// Since the result depends on the bytes alone, it is the same for every -/// repository that fetches them. Parses run one at a time: concurrent -/// operations on a document nobody has parsed yet wait for the first parse -/// instead of each running their own, and memory holds one parse at most. -/// -/// A group asks its members in turn for every chart, so the indexes of all -/// its upstreams are in use at once. Making room by evicting the least -/// recently used would evict the very index the group asks for next, and -/// every chart would parse every member's index again. So only an index -/// nobody used for [`parsed::IDLE`] makes room for a new one; while every -/// kept index is in use, a new one that does not fit is served without -/// being kept. Past the budget, a group keeps the indexes that fit and -/// parses only the others for each chart. -/// -/// A WebAssembly build parses for every operation: its instances are -/// recycled and their memory is bounded, so it keeps nothing. -fn parsed_index(url: &str, body: &[u8]) -> Result, String> { - #[cfg(not(target_family = "wasm"))] - { - parsed::get_or_parse(url, body) - } - #[cfg(target_family = "wasm")] - { - let _ = url; - parse_index(body).map(Arc::new) - } -} - -#[cfg(not(target_family = "wasm"))] -mod parsed { - use std::sync::{Arc, Mutex, PoisonError}; - use std::time::{Duration, Instant}; - - use super::{parse_index, Index}; - - /// What the kept indexes may hold together ([`retained`]), about twice - /// the size of their documents, so about 60 MB of `index.yaml`: the - /// indexes of a few dozen typical chart repositories, or of a handful of - /// the largest public ones. A new index that does not fit takes the - /// place of kept ones that went unused for [`IDLE`], or is not kept. One - /// that alone is larger is kept while nothing else is. - pub(super) const BUDGET: usize = 128 * 1024 * 1024; - - /// How long a kept index must go unused before a new one may take its - /// place. A group's listing uses each of its members' indexes for every - /// chart, so the ones it keeps stay in place until it is done. - pub(super) const IDLE: Duration = Duration::from_secs(60); - - /// A parsed index and the bytes it was parsed from. - pub(super) struct Parsed { - url: String, - body: Arc<[u8]>, - index: Arc, - /// What it holds ([`retained`]). - bytes: usize, - /// When it was last used. - used: Instant, - } - - impl Parsed { - pub(super) fn new(url: &str, body: &[u8], index: Arc, now: Instant) -> Self { - Self { - url: url.to_string(), - bytes: retained(body, &index), - body: body.into(), - index, - used: now, - } - } - } - - pub(super) struct State { - parsed: Vec, - } - - impl State { - pub(super) const fn new() -> Self { - Self { parsed: Vec::new() } - } - - /// The bytes and the index kept for `url`. - pub(super) fn find(&self, url: &str) -> Option<(Arc<[u8]>, Arc)> { - let p = self.parsed.iter().find(|p| p.url == url)?; - Some((Arc::clone(&p.body), Arc::clone(&p.index))) - } - - /// Marks `index`, kept for `url`, as used at `now`. - pub(super) fn touch(&mut self, url: &str, index: &Arc, now: Instant) { - if let Some(p) = self - .parsed - .iter_mut() - .find(|p| p.url == url && Arc::ptr_eq(&p.index, index)) - { - p.used = now; - } - } - - /// Keeps `parsed` as the index of its URL, if it fits in `budget` - /// once the others unused for [`IDLE`] since it was parsed gave way, - /// the least recently used first, or nothing else is kept. Returns - /// the indexes it does not keep, `parsed` among them when it does - /// not fit. - pub(super) fn keep(&mut self, parsed: Parsed, budget: usize) -> Vec { - let now = parsed.used; - let mut removed = Vec::new(); - if let Some(i) = self.parsed.iter().position(|p| p.url == parsed.url) { - removed.push(self.parsed.swap_remove(i)); - } - let mut total: usize = self.parsed.iter().map(|p| p.bytes).sum(); - while total + parsed.bytes > budget { - let Some(oldest) = (0..self.parsed.len()) - .filter(|&i| now.saturating_duration_since(self.parsed[i].used) >= IDLE) - .min_by_key(|&i| self.parsed[i].used) - else { - break; - }; - let oldest = self.parsed.swap_remove(oldest); - total -= oldest.bytes; - removed.push(oldest); - } - if total + parsed.bytes > budget && !self.parsed.is_empty() { - // The kept ones are in use: keeping this one would push out - // what the next operation asks for. - removed.push(parsed); - } else { - self.parsed.push(parsed); - } - removed - } - } - - static STATE: Mutex = Mutex::new(State::new()); - - /// Held while an index is parsed. - static PARSING: Mutex<()> = Mutex::new(()); - - pub(super) fn get_or_parse(url: &str, body: &[u8]) -> Result, String> { - get_or_parse_in(&STATE, url, body, Instant::now, BUDGET) - } - - /// [`get_or_parse`] with the indexes kept in `state`, the time from - /// `clock`, and `budget`. - pub(super) fn get_or_parse_in( - state: &Mutex, - url: &str, - body: &[u8], - clock: impl Fn() -> Instant, - budget: usize, - ) -> Result, String> { - if let Some(index) = lookup(state, url, body, clock()) { - return Ok(index); - } - let _parsing = PARSING.lock().unwrap_or_else(PoisonError::into_inner); - // Parsed while this one waited. - if let Some(index) = lookup(state, url, body, clock()) { - return Ok(index); - } - let index = Arc::new(parse_index(body)?); - // Sized, copied and dropped outside the lock: a large index takes a - // moment. - let parsed = Parsed::new(url, body, Arc::clone(&index), clock()); - let removed = state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .keep(parsed, budget); - drop(removed); - Ok(index) - } - - /// The index kept in `state` for `url`, when it was parsed from `body`. - fn lookup(state: &Mutex, url: &str, body: &[u8], now: Instant) -> Option> { - let (kept, index) = state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .find(url)?; - // Compared outside the lock: a large index takes a moment. - if *kept != *body { - return None; - } - state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .touch(url, &index, now); - Some(index) - } - - /// About what `index`, parsed from `body`, holds while kept: the - /// document and the entries. - pub(super) fn retained(body: &[u8], index: &Index) -> usize { - let entries: usize = index - .iter() - .map(|(name, versions)| { - name.len() - + versions - .iter() - .map(|e| { - std::mem::size_of_val(e) - + e.name.len() - + e.created.as_ref().map_or(0, String::len) - + e.url.as_ref().map_or(0, String::len) - + e.metadata.len() - }) - .sum::() - }) - .sum(); - body.len() + entries - } -} - -/// Read an `index.yaml` (or a JSON one: JSON is YAML). Entries that are not -/// a valid chart version (name, version) are left out. -fn parse_index(body: &[u8]) -> Result { - let text = std::str::from_utf8(body).map_err(|_| "not UTF-8".to_string())?; - let docs = crate::yaml::load(text).map_err(|e| format!("not YAML: {e}"))?; - let doc = docs.first().map(to_json).unwrap_or(Value::Null); - let Some(entries) = doc.get("entries").and_then(Value::as_object) else { - // An empty repository may write `entries: {}` or leave it out, but - // a document without `apiVersion` is no chart repository index. - return match doc.get("apiVersion") { - Some(_) => Ok(Index::new()), - None => Err("not a chart repository index (no apiVersion)".into()), - }; - }; - let mut index = Index::new(); - for versions in entries.values() { - let Some(versions) = versions.as_array() else { - continue; - }; - for entry in versions { - let Some(metadata) = entry.as_object() else { - continue; - }; - let (Some(name), Some(version)) = ( - metadata.get("name").and_then(Value::as_str), - metadata.get("version").and_then(|v| match v { - Value::String(s) => Some(s.clone()), - Value::Number(n) => Some(n.to_string()), - _ => None, - }), - ) else { - continue; - }; - let (true, Ok(version)) = (chart::valid_name(name), Version::parse(&version)) else { - continue; - }; - index - .entry(name.to_ascii_lowercase()) - .or_default() - .push(Entry { - name: name.to_string(), - version, - created: metadata - .get("created") - .and_then(Value::as_str) - .map(str::to_string), - url: metadata - .get("urls") - .and_then(Value::as_array) - .and_then(|urls| urls.first()) - .and_then(Value::as_str) - .map(str::to_string), - metadata: serde_json::to_string(metadata).unwrap_or_default().into(), - }); - } - } - for versions in index.values_mut() { - versions.sort_by(|a, b| a.version.cmp(&b.version)); - versions.dedup_by(|a, b| a.version == b.version); - } - Ok(index) -} - -/// Where a version's archive is: the first of its `urls`, resolved against -/// the repository (`charts/x-1.0.0.tgz`, `/x-1.0.0.tgz`, or absolute). A -/// version the index does not list gets the conventional -/// `{base}/{name}-{version}.tgz`, which an upstream without it answers with -/// a 404. -fn package_url(base: &str, index: &Index, target: &PackageRef) -> String { - let listed = index.get(&target.id).and_then(|versions| { - versions - .iter() - .find(|e| e.version.to_string() == target.version) - }); - let url = listed.and_then(|e| e.url.clone()); - match url { - Some(url) => resolve(base, &url), - None => format!("{base}/{}-{}.tgz", target.id, target.version), - } -} - -/// `url` resolved against the repository's base URL. -fn resolve(base: &str, url: &str) -> String { - if url.starts_with("https://") || url.starts_with("http://") { - return url.to_string(); - } - if let Some(path) = url.strip_prefix('/') { - // Relative to the host: keep the base's scheme and authority. - let authority_end = base - .find("://") - .map(|i| i + 3) - .and_then(|start| base[start..].find('/').map(|end| start + end)) - .unwrap_or(base.len()); - return format!("{}/{path}", &base[..authority_end]); - } - format!("{base}/{}", url.trim_start_matches("./")) -} - -/// The index's metadata of `wanted.versions`: every chart version is -/// listed, published and created when its entry says (`created`), and -/// summarized like its Chart.yaml. -fn metadata(index: &Index, wanted: &PackageVersions, dating: bool) -> UpstreamStep { - let Some(versions) = index.get(&wanted.id) else { - return done(UpstreamAnswer::Metadata(PackageMetadata::default())); - }; - done(UpstreamAnswer::Metadata(PackageMetadata { - // The name the newest version is published under. - original_id: versions.last().map(|e| e.name.clone()), - versions: versions - .iter() - .filter(|e| wanted.versions.contains(&e.version.to_string())) - .map(|e| VersionMetadata { - version: e.version.to_string(), - original_id: Some(e.name.clone()), - listed: true, - published: e.created.clone(), - created: if dating { e.created.clone() } else { None }, - summary: Some(crate::summary(&e.metadata())), - }) - .collect(), - })) -} - -/// A page of the charts whose name matches `query`: each term must match, -/// `id:x` and `packageid:x` exactly, any other as a part of the name. -fn search(index: &Index, query: &SearchQuery) -> UpstreamSearch { - let terms: Vec = query - .text - .split_whitespace() - .map(str::to_ascii_lowercase) - .collect(); - let matches = |name: &str| { - terms.iter().all(|term| { - match term - .strip_prefix("packageid:") - .or_else(|| term.strip_prefix("id:")) - { - Some(exact) => name == exact, - None => name.contains(term.as_str()), - } - }) - }; - let matched: Vec<&Vec> = index - .iter() - .filter(|(name, _)| matches(name)) - .map(|(_, versions)| versions) - .filter(|versions| !versions.is_empty()) - .collect(); - let take = match query.take { - 0 => DEFAULT_TAKE, - n => usize::try_from(n).unwrap_or(usize::MAX), - }; - let hits = matched - .iter() - .skip(usize::try_from(query.skip).unwrap_or(usize::MAX)) - .take(take) - .map(|versions| { - let newest = versions.last().expect("non-empty"); - UpstreamHit { - id: newest.name.clone(), - versions: versions.iter().map(|e| e.version.to_string()).collect(), - summary: Some(crate::summary(&newest.metadata())), - total_downloads: None, - } - }) - .collect(); - UpstreamSearch { - total: matched.len() as u64, - hits, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn urls_resolve_against_the_repository() { - let base = "https://charts.example/stable"; - assert_eq!( - resolve(base, "charts/x-1.0.0.tgz"), - "https://charts.example/stable/charts/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "./x-1.0.0.tgz"), - "https://charts.example/stable/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "/dl/x-1.0.0.tgz"), - "https://charts.example/dl/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "https://cdn.example/x-1.0.0.tgz"), - "https://cdn.example/x-1.0.0.tgz" - ); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_parsed_index_is_kept_while_the_upstream_serves_the_same_document() { - let url = "https://kept.example/stable/index.yaml"; - let body = "apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - // Concurrent operations on a new document share one parse. - let first: Vec> = std::thread::scope(|s| { - let parses: Vec<_> = (0..8) - .map(|_| s.spawn(|| parsed_index(url, body.as_bytes()).unwrap())) - .collect(); - parses.into_iter().map(|p| p.join().unwrap()).collect() - }); - assert!(first.iter().all(|p| Arc::ptr_eq(p, &first[0]))); - assert!(Arc::ptr_eq( - &parsed_index(url, body.as_bytes()).unwrap(), - &first[0] - )); - - // A changed document is parsed again. - let changed = format!("{body} - name: demo\n version: 2.0.0\n"); - let again = parsed_index(url, changed.as_bytes()).unwrap(); - assert!(!Arc::ptr_eq(&again, &first[0])); - assert_eq!(again["demo"].len(), 2); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_group_of_many_upstreams_parses_each_index_once() { - // A group asks its members in turn for every chart: each member's - // index is parsed for the first chart and kept for the others. - let body = "apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let urls: Vec = (0..12) - .map(|n| format!("https://member{n}.example/index.yaml")) - .collect(); - let first: Vec> = urls - .iter() - .map(|url| parsed_index(url, body.as_bytes()).unwrap()) - .collect(); - for _chart in 0..3 { - for (url, first) in urls.iter().zip(&first) { - assert!(Arc::ptr_eq( - &parsed_index(url, body.as_bytes()).unwrap(), - first - )); - } - } - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn kept_indexes_fit_in_the_budget() { - use std::time::{Duration, Instant}; - - let body = b"apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let index = Arc::new(parse_index(body).unwrap()); - let one = parsed::retained(body, &index); - assert!(one > body.len()); - let kept = |state: &parsed::State| -> Vec<&str> { - ["a", "b", "c", "d", "e"] - .into_iter() - .filter(|url| state.find(url).is_some()) - .collect() - }; - let start = Instant::now(); - let at = |secs| start + Duration::from_secs(secs); - let parsed = |url, secs| parsed::Parsed::new(url, body, Arc::clone(&index), at(secs)); - - let mut state = parsed::State::new(); - for (secs, url) in [(0, "a"), (1, "b"), (2, "c")] { - state.keep(parsed(url, secs), 3 * one); - } - assert_eq!(kept(&state), ["a", "b", "c"]); - // While the kept ones are in use, one that does not fit is not kept. - state.touch("a", &index, at(30)); - state.keep(parsed("d", 31), 3 * one); - assert_eq!(kept(&state), ["a", "b", "c"]); - // One unused for IDLE gives way, the least recently used first. - let later = 2 + parsed::IDLE.as_secs(); - state.keep(parsed("d", later), 3 * one); - assert_eq!(kept(&state), ["a", "c", "d"]); - // A document parsed again replaces the index of its URL. - state.keep(parsed("c", later), 3 * one); - assert_eq!(kept(&state), ["a", "c", "d"]); - - // An index larger than the budget is kept while nothing else is. - let mut state = parsed::State::new(); - state.keep(parsed("e", 0), one - 1); - assert_eq!(kept(&state), ["e"]); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_group_past_the_budget_parses_only_the_indexes_that_do_not_fit() { - use std::cell::Cell; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - let body = b"apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let one = parsed::retained(body, &parse_index(body).unwrap()); - let state = Mutex::new(parsed::State::new()); - let start = Instant::now(); - let elapsed = Cell::new(Duration::ZERO); - let clock = || start + elapsed.get(); - let members: Vec = (0..6) - .map(|n| format!("https://member{n}.example/index.yaml")) - .collect(); - let mut last: Vec>> = vec![None; members.len()]; - // Asks the members from `first` on for `charts` charts in turn, - // 10 ms apart, as a group's listing does; how many indexes that - // parsed. - let mut list = |first: usize, charts: usize| { - let mut parses = 0; - for _ in 0..charts { - for (m, url) in members.iter().enumerate().skip(first) { - elapsed.set(elapsed.get() + Duration::from_millis(10)); - let index = parsed::get_or_parse_in(&state, url, body, clock, 4 * one).unwrap(); - if !last[m].as_ref().is_some_and(|l| Arc::ptr_eq(l, &index)) { - parses += 1; - } - last[m] = Some(index); - } - } - parses - }; - - // Four of the six indexes fit: those are parsed once, the other two - // for every chart. - assert_eq!(list(0, 50), 4 + 2 * 50); - // Once members 0 and 1 go unused, 4 and 5 take their places. - elapsed.set(elapsed.get() + parsed::IDLE); - assert_eq!(list(2, 50), 2); - } - - #[test] - fn an_index_keeps_valid_chart_versions() { - let index = parse_index( - b"apiVersion: v1\nentries:\n Demo:\n - name: Demo\n version: v1.2\n - name: Demo\n version: 1.0.0\n - name: Demo\n version: not-a-version\n ../evil:\n - name: ../evil\n version: 1.0.0\n", - ) - .unwrap(); - assert_eq!(index.keys().collect::>(), ["demo"]); - let versions: Vec = index["demo"] - .iter() - .map(|e| e.version.to_string()) - .collect(); - assert_eq!(versions, ["1.0.0", "1.2.0"]); - assert!(parse_index(b"apiVersion: v1\n").unwrap().is_empty()); - assert!(parse_index(b"portal").is_err()); - } - - #[test] - fn an_index_leaves_out_versions_longer_than_an_oci_tag() { - let longest = format!( - "1.0.0-{}", - "a".repeat(crate::semver::MAX_VERSION_LENGTH - 6) - ); - let mut body = String::from("apiVersion: v1\nentries:\n demo:\n"); - for version in [ - "1.0.0".to_string(), - longest.clone(), - format!("{longest}a"), - format!("2.0.0-{}", "x".repeat(4096)), - ] { - body.push_str(&format!(" - name: demo\n version: {version}\n")); - } - let index = parse_index(body.as_bytes()).unwrap(); - let versions: Vec = index["demo"] - .iter() - .map(|e| e.version.to_string()) - .collect(); - assert_eq!(versions, [longest, "1.0.0".to_string()]); - } -} diff --git a/plugins/helm/src/yaml.rs b/plugins/helm/src/yaml.rs deleted file mode 100644 index bc23a7dd..00000000 --- a/plugins/helm/src/yaml.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Loading the YAML a chart repository serves and stores (`Chart.yaml`, -//! `index.yaml`) within bounds. -//! -//! `yaml-rust2` builds the document tree by cloning an anchored node for -//! every alias that names it, so a few hundred bytes of nested anchors -//! expand to gigabytes. Charts and chart indexes have no use for that, but -//! both are written by others (a pusher, an upstream), so a document is -//! first walked as parser events, which costs no more than its size, and -//! refused when its aliases would expand it by more than -//! [`MAX_ALIAS_NODES`] nodes or when it nests deeper than [`MAX_DEPTH`]. - -use std::collections::HashMap; -use yaml_rust2::parser::{Event, EventReceiver, Parser}; -use yaml_rust2::{Yaml, YamlLoader}; - -/// The most nodes that aliases may add to a document, all aliases -/// together. Anchors in a chart or an index are rare and small. -pub const MAX_ALIAS_NODES: u64 = 100_000; - -/// The deepest nesting of sequences and mappings a document may have. A -/// chart index nests about six levels deep. -pub const MAX_DEPTH: usize = 64; - -/// Load every document of `text`, refusing one that aliases would expand -/// past [`MAX_ALIAS_NODES`] or that nests past [`MAX_DEPTH`]. -pub fn load(text: &str) -> Result, String> { - let mut budget = Budget::default(); - Parser::new_from_str(text) - .load(&mut budget, true) - .map_err(|e| e.to_string())?; - if budget.deepest > MAX_DEPTH { - return Err(format!("nested deeper than {MAX_DEPTH} levels")); - } - if budget.alias_nodes > MAX_ALIAS_NODES { - return Err(format!( - "its aliases expand to more than {MAX_ALIAS_NODES} nodes" - )); - } - YamlLoader::load_from_str(text).map_err(|e| e.to_string()) -} - -/// Counts what loading a document would build, without building it. -#[derive(Default)] -struct Budget { - /// Nodes built so far, aliases expanded. - nodes: u64, - /// Nodes the aliases added. - alias_nodes: u64, - /// Expanded size of each anchored node, by anchor id. - anchors: HashMap, - /// Open sequences and mappings: their anchor id and `nodes` before them. - open: Vec<(usize, u64)>, - deepest: usize, -} - -impl EventReceiver for Budget { - fn on_event(&mut self, event: Event) { - match event { - Event::Scalar(_, _, anchor, _) => { - self.nodes = self.nodes.saturating_add(1); - if anchor > 0 { - self.anchors.insert(anchor, 1); - } - } - Event::SequenceStart(anchor, _) | Event::MappingStart(anchor, _) => { - self.open.push((anchor, self.nodes)); - self.deepest = self.deepest.max(self.open.len()); - self.nodes = self.nodes.saturating_add(1); - } - Event::SequenceEnd | Event::MappingEnd => { - if let Some((anchor, before)) = self.open.pop() { - if anchor > 0 { - self.anchors.insert(anchor, self.nodes - before); - } - } - } - Event::Alias(anchor) => { - let size = self.anchors.get(&anchor).copied().unwrap_or(1); - self.nodes = self.nodes.saturating_add(size); - self.alias_nodes = self.alias_nodes.saturating_add(size); - } - Event::Nothing - | Event::StreamStart - | Event::StreamEnd - | Event::DocumentStart - | Event::DocumentEnd => {} - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `levels` anchors, each a sequence of ten aliases of the previous one. - fn laughs(levels: usize) -> String { - let mut doc = - String::from("name: demo\nversion: 1.0.0\na0: &a0 [x, x, x, x, x, x, x, x, x, x]\n"); - for i in 1..=levels { - let prev = format!("*a{}", i - 1); - doc.push_str(&format!("a{i}: &a{i} [{}]\n", vec![prev; 10].join(", "))); - } - doc - } - - #[test] - fn plain_documents_and_small_anchors_load() { - let docs = load("name: demo\nversion: 1.0.0\nkeywords: [a, b]\n").unwrap(); - assert_eq!(docs[0]["name"].as_str(), Some("demo")); - // A shared block reused a few times is fine. - let docs = load(&laughs(2)).unwrap(); - assert_eq!(docs[0]["a2"][9][9][9].as_str(), Some("x")); - } - - #[test] - fn alias_bombs_are_refused_before_they_expand() { - // 6 levels expand to over a million nodes (gigabytes once built); - // the check costs what the 500 bytes cost. - let err = load(&laughs(6)).unwrap_err(); - assert!(err.contains("aliases"), "{err}"); - let err = load(&laughs(30)).unwrap_err(); - assert!(err.contains("aliases"), "{err}"); - } - - #[test] - fn deep_nesting_is_refused() { - let deep = format!("{}{}", "[".repeat(200), "]".repeat(200)); - assert!(load(&deep).unwrap_err().contains("deeper")); - let mut block = String::new(); - for depth in 0..100 { - block.push_str(&format!("{}k{depth}:\n", " ".repeat(depth))); - } - assert!(load(&block).unwrap_err().contains("deeper")); - assert!(load(&format!("{}{}", "[".repeat(10), "]".repeat(10))).is_ok()); - } -} diff --git a/plugins/helm/tests/upstream.rs b/plugins/helm/tests/upstream.rs deleted file mode 100644 index e99d9bf7..00000000 --- a/plugins/helm/tests/upstream.rs +++ /dev/null @@ -1,255 +0,0 @@ -//! Helm proxies without a server: the upstream side run through -//! `forge_plugin_api::testing::converse` against a canned `index.yaml`, and a -//! proxy's index rendered through `testing::serve` from the search the host -//! answers with the upstream's charts. - -use std::collections::BTreeMap; - -use forge_format_helm::Helm; -use forge_plugin_api::testing::{converse, repository, serve, Conversed}; -use forge_plugin_api::{ - Answer, Fetch, Fetched, Freshness, PackageRef, PackageVersions, Query, Remote, RepositoryKind, - Request, SearchHit, SearchQuery, SearchResult, UpstreamAnswer, UpstreamOp, VersionInfo, -}; -use serde_json::{json, Value}; - -const BASE: &str = "https://charts.example/stable"; - -const INDEX: &str = r#"apiVersion: v1 -entries: - Nginx: - - apiVersion: v2 - name: Nginx - version: 2.1.0 - appVersion: "1.25" - description: A web server - created: "2026-05-01T10:00:00.123456789Z" - digest: abc - urls: - - charts/Nginx-2.1.0.tgz - - apiVersion: v2 - name: Nginx - version: v1.0 - description: An old web server - created: "2024-01-01T00:00:00Z" - urls: - - https://cdn.example/Nginx-1.0.0.tgz - redis: - - apiVersion: v2 - name: redis - version: 7.0.0 - urls: [redis-7.0.0.tgz] -generated: "2026-05-01T10:00:00Z" -"#; - -fn resolved() -> Remote { - Remote { - url: format!("{BASE}/"), - index: Some(json!({ "base": BASE }).to_string()), - } -} - -fn run(remote: &Remote, op: UpstreamOp, docs: &BTreeMap) -> Conversed { - converse(&Helm, remote, &op, |fetch: &Fetch| { - docs.get(&fetch.url).cloned().unwrap_or(Fetched::NotFound) - }) - .expect("the conversation ends") -} - -fn upstream() -> BTreeMap { - BTreeMap::from([( - format!("{BASE}/index.yaml"), - Fetched::Found(INDEX.as_bytes().to_vec()), - )]) -} - -fn package(id: &str, version: &str) -> PackageRef { - PackageRef { - id: id.into(), - version: version.into(), - } -} - -#[test] -fn the_index_is_the_repository_base() { - let unresolved = Remote { - url: format!("{BASE}/"), - index: None, - }; - let done = run(&unresolved, UpstreamOp::Index, &upstream()); - assert_eq!( - done.fetches, - [Fetch { - url: format!("{BASE}/index.yaml"), - freshness: Freshness::Metadata, - }] - ); - let UpstreamAnswer::Index(index) = done.answer else { - panic!("{:?}", done.answer) - }; - assert_eq!( - serde_json::from_str::(&index).unwrap(), - json!({ "base": BASE }) - ); - - // Not a chart repository: a failure, never an empty one. - let portal = BTreeMap::from([( - format!("{BASE}/index.yaml"), - Fetched::Found(b"sign in".to_vec()), - )]); - assert!(matches!( - run(&unresolved, UpstreamOp::Index, &portal).answer, - UpstreamAnswer::Failed(_) - )); - assert!(matches!( - run(&unresolved, UpstreamOp::Index, &BTreeMap::new()).answer, - UpstreamAnswer::Failed(m) if m.contains("404") - )); -} - -#[test] -fn versions_and_packages_come_from_the_index() { - let remote = resolved(); - assert_eq!( - run(&remote, UpstreamOp::Versions("nginx".into()), &upstream()).answer, - UpstreamAnswer::Versions(Some(vec!["1.0.0".into(), "2.1.0".into()])) - ); - assert_eq!( - run(&remote, UpstreamOp::Versions("mysql".into()), &upstream()).answer, - UpstreamAnswer::Versions(None) - ); - // Relative and absolute URLs; a version the index does not list gets - // the conventional location. - for (target, url) in [ - ( - package("nginx", "2.1.0"), - format!("{BASE}/charts/Nginx-2.1.0.tgz"), - ), - ( - package("nginx", "1.0.0"), - "https://cdn.example/Nginx-1.0.0.tgz".to_string(), - ), - (package("redis", "7.0.0"), format!("{BASE}/redis-7.0.0.tgz")), - (package("redis", "6.0.0"), format!("{BASE}/redis-6.0.0.tgz")), - ] { - assert_eq!( - run(&remote, UpstreamOp::Package(target), &upstream()).answer, - UpstreamAnswer::Package(url) - ); - } - // A chart repository has no manifest without its chart. - let manifest = run( - &remote, - UpstreamOp::Manifest(package("nginx", "2.1.0")), - &upstream(), - ); - assert!(manifest.fetches.is_empty()); - assert_eq!(manifest.answer, UpstreamAnswer::Manifest(None)); -} - -#[test] -fn metadata_and_dating_read_the_entries() { - let wanted = PackageVersions { - id: "nginx".into(), - versions: vec!["1.0.0".into(), "2.1.0".into(), "9.9.9".into()], - }; - let UpstreamAnswer::Metadata(metadata) = - run(&resolved(), UpstreamOp::Dating(wanted.clone()), &upstream()).answer - else { - panic!("metadata") - }; - assert_eq!(metadata.original_id.as_deref(), Some("Nginx")); - assert_eq!(metadata.versions.len(), 2); - let newest = &metadata.versions[1]; - assert_eq!(newest.version, "2.1.0"); - assert!(newest.listed); - assert_eq!( - newest.created.as_deref(), - Some("2026-05-01T10:00:00.123456789Z") - ); - let summary: Value = serde_json::from_str(newest.summary.as_deref().unwrap()).unwrap(); - assert_eq!(summary["description"], "A web server"); - assert_eq!(summary["chart"]["appVersion"], "1.25"); - - let UpstreamAnswer::Metadata(plain) = - run(&resolved(), UpstreamOp::Metadata(wanted), &upstream()).answer - else { - panic!("metadata") - }; - assert!(plain.versions.iter().all(|v| v.created.is_none())); -} - -#[test] -fn search_matches_chart_names() { - let query = |text: &str, skip: u32, take: u32| SearchQuery { - text: text.into(), - skip, - take, - prerelease: true, - semver2: true, - upstream: true, - }; - let search = |q: SearchQuery| match run(&resolved(), UpstreamOp::Search(q), &upstream()).answer - { - UpstreamAnswer::Search(Some(page)) => page, - other => panic!("{other:?}"), - }; - let all = search(query("", 0, 0)); - assert_eq!(all.total, 2); - assert_eq!(all.hits[0].id, "Nginx"); - assert_eq!(all.hits[0].versions, ["1.0.0", "2.1.0"]); - assert_eq!(all.hits[1].id, "redis"); - assert_eq!(search(query("ngi", 0, 0)).total, 1); - assert_eq!(search(query("id:ngi", 0, 0)).total, 0); - assert_eq!(search(query("packageid:NGINX", 0, 0)).hits[0].id, "Nginx"); - let second = search(query("", 1, 1)); - assert_eq!((second.total, second.hits.len()), (2, 1)); - assert_eq!(second.hits[0].id, "redis"); -} - -#[test] -fn a_proxys_index_lists_what_its_upstream_search_found() { - let repo = repository(RepositoryKind::Proxy, "/repository"); - let summary = json!({ "chart": { - "apiVersion": "v2", "name": "Nginx", "version": "2.1.0", "appVersion": "1.25" - }, "description": "A web server", "dependency_groups": [] }) - .to_string(); - let version = |v: &str| VersionInfo { - original_id: "Nginx".into(), - version: v.into(), - version_original: v.into(), - listed: true, - }; - let served = serve(&Helm, &repo, &Request::new("GET", "/index.yaml"), |query| { - match query { - // The upstream's charts too, not only the cached ones. - Query::Search(q) if q.upstream => Answer::Search(SearchResult { - total: 1, - hits: vec![SearchHit { - original_id: "Nginx".into(), - latest_version: "2.1.0".into(), - versions: vec![version("1.0.0"), version("2.1.0")], - summary: Some(summary.clone()), - total_downloads: None, - }], - }), - other => panic!("unexpected {other:?}"), - } - }); - // One search, no per-chart reads of the upstream. - assert_eq!(served.queries.len(), 1); - let index: Value = serde_json::from_str(&served.text()).unwrap(); - let entries = index["entries"]["Nginx"].as_array().unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0]["version"], "2.1.0"); - assert_eq!(entries[0]["appVersion"], "1.25"); - assert_eq!( - entries[0]["urls"][0], - "https://forge.example.com/repository/repo/charts/nginx/2.1.0/Nginx-2.1.0.tgz" - ); - // An older version: name, version and URL (the search describes the - // newest only). - assert_eq!(entries[1]["version"], "1.0.0"); - assert_eq!(entries[1]["apiVersion"], "v2"); - assert!(entries[1].get("appVersion").is_none()); -} diff --git a/plugins/nuget/Cargo.toml b/plugins/nuget/Cargo.toml index 4a9c272c..5a3ae9a8 100644 --- a/plugins/nuget/Cargo.toml +++ b/plugins/nuget/Cargo.toml @@ -8,10 +8,6 @@ license.workspace = true publish.workspace = true description = "NuGet V3 feeds for Forge, as a format plugin" -# Built into the server only, for now: the `zip` reader's `typed-path` -# dependency needs nightly Rust on wasm32-wasip2, so there is no `component` -# feature (and no `component` in plugin.toml) yet. - [dependencies] forge-plugin-api = { path = "../../crates/plugin-api" } diff --git a/plugins/nuget/src/lib.rs b/plugins/nuget/src/lib.rs index c5dd472d..0b6c3a1d 100644 --- a/plugins/nuget/src/lib.rs +++ b/plugins/nuget/src/lib.rs @@ -17,9 +17,7 @@ //! Ids are matched case-insensitively (`ToLowerInvariant`); versions are //! NuGet versions, normalized (`1.0` → `1.0.0`). //! -//! Forge links this plugin in ([`Nuget`], [`PLUGIN_TOML`]). It is built in -//! only for now: the `zip` reader it parses packages with does not build for -//! `wasm32-wasip2` on stable Rust, so it has no component build yet. +//! Forge links this plugin in ([`Nuget`], [`PLUGIN_TOML`]). pub mod model; pub mod nuspec; @@ -115,8 +113,6 @@ mod tests { assert_eq!(value("version").as_deref(), Some(env!("CARGO_PKG_VERSION"))); assert_eq!(value("api").as_deref(), Some(forge_plugin_api::API)); assert_eq!(value("mount").as_deref(), Some("/v3")); - // Built in only (see the crate docs). - assert_eq!(value("component"), None); } #[test] diff --git a/web/src/api/repositories.ts b/web/src/api/repositories.ts index 57af6f08..4d4c1cd7 100644 --- a/web/src/api/repositories.ts +++ b/web/src/api/repositories.ts @@ -183,16 +183,13 @@ export interface UpdateRepo { /** * An installed package format (`FormatView` of `GET /admin/formats`): the - * built-in format plugin (NuGet), Docker, raw and every loaded WebAssembly - * plugin. + * format plugins (NuGet), Docker and raw. */ export interface FormatView { /** The format id (`RepoView.format`). */ id: string /** Display name. */ name: string - /** Built into the server rather than a WebAssembly plugin. */ - builtin: boolean /** The repository kinds it supports (proxy only with a host-side upstream, e.g. NuGet). */ kinds: RepoKind[] /** Where its repositories are served: `{mount}/{repository}`. */ @@ -201,8 +198,6 @@ export interface FormatView { version?: string /** Plugin description. */ description?: string - /** SHA-256 of a WebAssembly plugin's component. */ - sha256?: string /** The upload limit of its hosted repositories without one of their own. */ default_max_upload_bytes?: number /** The largest upload limit a repository may be given; `null`: unbounded. */ @@ -220,7 +215,7 @@ export interface InvalidateProxyCache { /** What a proxy cache invalidation removed (`Invalidated`). */ export interface ProxyCacheInvalidated { - /** Remembered upstream "not found" answers removed (NuGet, plugins). */ + /** Remembered upstream "not found" answers removed (NuGet). */ not_found_cleared: number /** Cached tags due for re-validation against the upstream (docker). */ tags_expired: number diff --git a/web/src/components/PackageConnect.tsx b/web/src/components/PackageConnect.tsx index 16c67105..9dd15787 100644 --- a/web/src/components/PackageConnect.tsx +++ b/web/src/components/PackageConnect.tsx @@ -13,7 +13,7 @@ interface Snippet { * The client commands for a repository of `format` at `url` (its * `/browse/repos` address), named `repo` in the client's configuration. * Push commands are left out unless `canPush`. A format without commands of - * its own (a WebAssembly plugin's) gets the repository URL. + * its own gets the repository URL. */ export function connectSnippets( format: string, diff --git a/web/src/components/RepoFormatBadge.tsx b/web/src/components/RepoFormatBadge.tsx index b9970a95..fe7b9ce6 100644 --- a/web/src/components/RepoFormatBadge.tsx +++ b/web/src/components/RepoFormatBadge.tsx @@ -4,7 +4,7 @@ export { formatLabel } /** * The repository format (`nuget` | `docker` | `raw`) with its display - * name and a generic glyph. Other formats (WebAssembly plugins) show their + * name and a generic glyph. Other formats (other plugins) show their * raw id, like `RepoKindBadge`. */ export function RepoFormatBadge({ format }: { format: string }) { diff --git a/web/src/components/RepositoryForm.test.tsx b/web/src/components/RepositoryForm.test.tsx index 8ce41414..f0afded7 100644 --- a/web/src/components/RepositoryForm.test.tsx +++ b/web/src/components/RepositoryForm.test.tsx @@ -33,7 +33,6 @@ import { const example: FormatView = { id: 'example', name: 'Example packages', - builtin: false, kinds: ['hosted', 'group'], mount: '/repository', version: '0.1.0', diff --git a/web/src/components/RepositoryForm.tsx b/web/src/components/RepositoryForm.tsx index ac91609e..caa45be7 100644 --- a/web/src/components/RepositoryForm.tsx +++ b/web/src/components/RepositoryForm.tsx @@ -133,7 +133,6 @@ function originOf(url: string | null | undefined): string | null { export const NUGET_FORMAT: FormatView = { id: 'nuget', name: 'NuGet', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3', } @@ -145,7 +144,6 @@ export const NUGET_FORMAT: FormatView = { export const DOCKER_FORMAT: FormatView = { id: 'docker', name: 'Docker / OCI', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2', } @@ -158,7 +156,6 @@ export const DOCKER_FORMAT: FormatView = { export const RAW_FORMAT: FormatView = { id: 'raw', name: 'Raw files', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/raw', } @@ -412,7 +409,6 @@ export function RepositoryForm({ [NUGET_FORMAT, DOCKER_FORMAT, RAW_FORMAT].find((f) => f.id === values.format) ?? { id: values.format, name: values.format, - builtin: false, kinds: ['hosted', 'group'], mount: '/repository', } @@ -485,7 +481,7 @@ export function RepositoryForm({ > {available.map((f) => ( ))} @@ -493,7 +489,7 @@ export function RepositoryForm({ { mockApi({ repositories: [], formats: [ - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, - { id: 'acme', name: 'Acme packages', builtin: false, kinds: ['hosted', 'group'], mount: '/repository' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'acme', name: 'Acme packages', kinds: ['hosted', 'group'], mount: '/repository' }, ], onCreate: (body) => { posted = body @@ -244,8 +244,7 @@ describe('admin repositories', () => { await renderAt('/manage/repositories/new') await user.type(await screen.findByLabelText('Name'), 'pkgs') - // A WebAssembly plugin is named with its id. - await screen.findByRole('option', { name: 'Acme packages (plugin acme)' }) + await screen.findByRole('option', { name: 'Acme packages' }) await user.selectOptions(screen.getByLabelText('Format'), 'acme') await user.click(screen.getByRole('button', { name: 'Create repository' })) @@ -277,8 +276,8 @@ describe('admin repositories', () => { mockApi({ repositories: [], formats: [ - { id: 'docker', name: 'Docker / OCI', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'docker', name: 'Docker / OCI', kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ], onCreate: (body) => { posted = body as Record @@ -317,8 +316,8 @@ describe('admin repositories', () => { mockApi({ repositories: [], formats: [ - { id: 'docker', name: 'Docker / OCI', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'docker', name: 'Docker / OCI', kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ], onCreate: () => new Response('another docker repository already answers on this host', { diff --git a/web/src/test/fakeRbacApi.ts b/web/src/test/fakeRbacApi.ts index cb758717..b55061cf 100644 --- a/web/src/test/fakeRbacApi.ts +++ b/web/src/test/fakeRbacApi.ts @@ -203,7 +203,7 @@ export class FakeRbacApi { if (path === '/admin/me') return json(s.me) if (path === '/admin/formats') { return json([ - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ]) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 60977781..acb14c61 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -76,9 +76,6 @@ fn main() -> ExitCode { "prepare" => prepare(), "prepare-check" => prepare_check(), "clean" => clean(), - // Format plugins (WebAssembly components in plugins/) - "plugins" => plugins(), - "plugins-check" => plugins_check(), // Docker image (build it; run it against the dev infra) "docker-build" => docker_build(), "docker-run" => docker_run(), @@ -148,13 +145,6 @@ Rust build & quality: prepare-check Migrate, then verify the committed .sqlx/ cache is in sync (CI's drift guard). clean Remove build artifacts (cargo clean). -Format plugins as WebAssembly components (plugins//; the workspace builds -them natively, these need the wasm32-wasip2 target: `rustup target add wasm32-wasip2`): - plugins Build every plugin as a component and install it under - target/plugins/installed//, printing the [plugins] - configuration that loads them (with digests). - plugins-check clippy -D warnings of every plugin's component build, and the build. - Docker image (single container: web SPA + Rust server): docker-build Build the application image from ./Dockerfile (multi-stage: pnpm builds the SPA, cargo builds forge-server release). @@ -734,158 +724,6 @@ fn clean() -> Result<(), String> { Cmd::new("cargo").arg("clean").run() } -// ── Format plugins ─────────────────────────────────────────────────────────── - -/// Format plugins: one workspace crate per directory (`plugins//`). -const PLUGINS: &str = "plugins"; - -/// Where the components are built and installed (apart from `target/`, so a -/// wasm build never invalidates the host's). -const PLUGINS_TARGET: &str = "target/plugins"; - -/// The plugin directories with a component build (`plugins//` whose -/// `plugin.toml` names a `component`), sorted. A plugin without one is built -/// in only. -fn plugin_dirs() -> Result, String> { - let mut dirs: Vec<_> = std::fs::read_dir(PLUGINS) - .map_err(|e| format!("read {PLUGINS}/: {e}"))? - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|dir| dir.join("plugin.toml").is_file() && plugin_component(dir).is_ok()) - .collect(); - dirs.sort(); - Ok(dirs) -} - -/// The `component = "…"` of a plugin.toml (a plain line scan: xtask has no -/// TOML parser, and the key is a single string). -fn plugin_component(dir: &std::path::Path) -> Result { - let manifest = dir.join("plugin.toml"); - let text = std::fs::read_to_string(&manifest) - .map_err(|e| format!("read {}: {e}", manifest.display()))?; - text.lines() - .filter_map(|line| line.trim().strip_prefix("component")) - .filter_map(|rest| rest.trim().strip_prefix('=')) - .map(|value| value.trim().trim_matches('"').to_string()) - .next() - .ok_or_else(|| format!("{} has no component", manifest.display())) -} - -/// Whether the server links the plugin in `dir` (a path dependency of -/// forge-server, not only of its tests): its id is then taken by the -/// built-in plugin. -fn plugin_is_builtin(dir: &std::path::Path) -> bool { - let server = std::fs::read_to_string("crates/server/Cargo.toml").unwrap_or_default(); - let dependencies = server - .split("[dev-dependencies]") - .next() - .unwrap_or_default(); - let path = format!("\"../../{}\"", dir.display()).replace('\\', "/"); - dependencies.contains(&path) -} - -/// The SHA-256 of `file` (`sha256sum`, or `shasum -a 256` on macOS). -fn sha256_of(file: &std::path::Path) -> Option { - let tools: [(&str, &[&str]); 2] = [("sha256sum", &[]), ("shasum", &["-a", "256"])]; - tools.iter().find_map(|(tool, args)| { - let out = std::process::Command::new(tool) - .args(*args) - .arg(file) - .output() - .ok()?; - let text = String::from_utf8(out.stdout).ok()?; - out.status - .success() - .then(|| text.split_whitespace().next().map(str::to_string))? - }) -} - -/// `cargo ` of the plugin crate in `dir` for wasm32-wasip2 with its -/// `component` feature, into [`PLUGINS_TARGET`]. -fn plugin_cargo(sub: &str, dir: &std::path::Path) -> Cmd { - Cmd::new("cargo") - .args([sub, "--locked", "--target", "wasm32-wasip2"]) - .args(["--features", "component", "--manifest-path"]) - .arg(dir.join("Cargo.toml")) - .args(["--target-dir", PLUGINS_TARGET]) -} - -/// Build the component of the plugin in `dir` (the crate as a `cdylib`, with -/// the `plugin` profile); returns its path. -fn build_component(dir: &std::path::Path) -> Result { - plugin_cargo("rustc", dir) - .args(["--lib", "--crate-type", "cdylib", "--profile", "plugin"]) - .run()?; - Ok(std::path::Path::new(PLUGINS_TARGET) - .join("wasm32-wasip2/plugin") - .join(plugin_component(dir)?)) -} - -/// Build every plugin as a component and install it as an operator would -/// (plugin.toml plus component under `target/plugins/installed//`). -fn plugins() -> Result<(), String> { - let mut pins = Vec::new(); - for dir in plugin_dirs()? { - let id = dir - .file_name() - .and_then(|n| n.to_str()) - .ok_or("plugin directory name is not UTF-8")? - .to_string(); - let component = plugin_component(&dir)?; - let built = build_component(&dir)?; - let installed = std::path::Path::new(PLUGINS_TARGET) - .join("installed") - .join(&id); - std::fs::create_dir_all(&installed) - .map_err(|e| format!("create {}: {e}", installed.display()))?; - std::fs::copy(dir.join("plugin.toml"), installed.join("plugin.toml")) - .map_err(|e| format!("copy plugin.toml: {e}"))?; - std::fs::copy(&built, installed.join(&component)) - .map_err(|e| format!("copy {}: {e}", built.display()))?; - let pin = sha256_of(&installed.join(&component)).map_or_else( - || "".to_string(), - |sha| format!("sha256:{sha}"), - ); - pins.push((id, pin, plugin_is_builtin(&dir))); - } - let dir = std::env::current_dir() - .map_err(|e| e.to_string())? - .join(PLUGINS_TARGET) - .join("installed"); - println!( - "\nInstalled into {}. To load them, add to forge.toml:\n", - dir.display() - ); - println!( - "[plugins]\ndir = {:?}\n\n[plugins.formats]", - dir.display().to_string() - ); - for (id, pin, builtin) in pins { - if builtin { - println!( - "# {id} is built into the server: its component loads only under another id\n\ - # (rename its directory, `id` in its plugin.toml and the key here).\n\ - # {id} = \"{pin}\"" - ); - } else { - println!("{id} = \"{pin}\""); - } - } - Ok(()) -} - -/// What CI checks of every plugin beyond the workspace's own fmt, clippy and -/// tests: that it lints clean and builds as a WebAssembly component. -fn plugins_check() -> Result<(), String> { - for dir in plugin_dirs()? { - plugin_cargo("clippy", &dir) - .args(["--", "-D", "warnings"]) - .run()?; - build_component(&dir)?; - } - Ok(()) -} - // ── Docker image (single-container web SPA + Rust server) ──────────────────── /// Build the single application image from the root Dockerfile (multi-stage: