From 3f5213b2930fb00aced9545b13374c9671e5ac04 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:36:20 +0000 Subject: [PATCH] Remove WebAssembly format plugins and Wasmtime: built-in plugins only Format plugins loaded as WebAssembly components were too big an API to keep: the forge:format WIT contract, the component glue, the Wasmtime sandbox and its limits, and the configuration around them. Forge now supports built-in plugins only, crates under plugins/ that the server links in, like NuGet. Gone: forge-plugin's wasm runtime (Plugin::load, HostLimits, the digest pin), wasmtime and wasmtime-wasi, forge-plugin-api's component feature, export_component! and wit/, the component and limits keys of plugin.toml, the example plugin plugins/helm and its tests, `cargo xtask plugins` and `plugins-check`, the CI `plugins` job (and its place in the release gate), the wasm32-wasip2 target, and builtin/sha256 in GET /admin/formats. A config that still has a [plugins] section fails startup and says what changed. The 64 KiB summary bound applied to WebAssembly plugins only, so it goes too. The host contract stays covered by the native probe plugin, which now also checks that a repository whose format is missing answers 503. The changelog fragments that described WebAssembly-only fixes (228, 229, 234) are dropped. Fixes #437 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JpCggxduZYrAYTnobHTzhY --- .env.example | 9 - .github/dependabot.yml | 6 - .github/scripts/ci-changes.sh | 2 +- .github/workflows/ci.yml | 26 - .github/workflows/release.yml | 2 +- .gitignore | 2 - Cargo.lock | 976 +------------ Cargo.toml | 28 - DEVELOPMENT.md | 44 +- README.md | 15 +- crates/core/src/net/origins.rs | 11 +- crates/plugin-api/Cargo.toml | 11 +- crates/plugin-api/src/component.rs | 159 --- crates/plugin-api/src/lib.rs | 233 +--- crates/plugin-api/src/types.rs | 3 +- crates/plugin-api/wit/format.wit | 482 ------- crates/plugin/Cargo.toml | 21 - crates/plugin/src/format.rs | 116 +- crates/plugin/src/lib.rs | 124 +- crates/plugin/src/manifest.rs | 93 +- crates/plugin/src/testing.rs | 109 -- crates/plugin/src/wasm/bindings.rs | 12 - crates/plugin/src/wasm/executor.rs | 94 -- crates/plugin/src/wasm/host.rs | 241 ---- crates/plugin/src/wasm/mod.rs | 826 ----------- crates/plugin/tests/component.rs | 522 ------- crates/server/Cargo.toml | 5 +- crates/server/src/admin/formats.rs | 15 +- crates/server/src/admin/repositories.rs | 4 +- crates/server/src/config.rs | 89 +- crates/server/src/main.rs | 6 +- crates/server/src/plugins.rs | 49 +- crates/server/src/state.rs | 26 +- crates/server/tests/admin.rs | 2 +- crates/server/tests/common/mod.rs | 18 +- crates/server/tests/plugin_contract.rs | 33 +- crates/server/tests/plugins.rs | 1222 ----------------- crates/server/tests/registry.rs | 6 +- docker/licenses/about.toml | 4 - docs/changes/2.0.0/228.fixed.md | 4 - docs/changes/2.0.0/229.security.md | 10 - docs/changes/2.0.0/234.fixed.md | 5 - docs/deployment.md | 5 - docs/format-plugins.md | 273 +--- forge.toml.example | 12 - plugins/README.md | 46 +- plugins/helm/Cargo.toml | 25 - plugins/helm/plugin.toml | 15 - plugins/helm/src/chart.rs | 614 --------- plugins/helm/src/lib.rs | 997 -------------- plugins/helm/src/semver.rs | 254 ---- plugins/helm/src/tar.rs | 914 ------------ plugins/helm/src/upstream.rs | 709 ---------- plugins/helm/src/yaml.rs | 136 -- plugins/helm/tests/upstream.rs | 255 ---- plugins/nuget/Cargo.toml | 4 - plugins/nuget/src/lib.rs | 6 +- web/src/api/repositories.ts | 9 +- web/src/components/PackageConnect.tsx | 2 +- web/src/components/RepoFormatBadge.tsx | 2 +- web/src/components/RepositoryForm.test.tsx | 1 - web/src/components/RepositoryForm.tsx | 8 +- .../admin/repositories/repositories.test.tsx | 15 +- web/src/test/fakeRbacApi.ts | 2 +- xtask/src/main.rs | 162 --- 65 files changed, 231 insertions(+), 9900 deletions(-) delete mode 100644 crates/plugin-api/src/component.rs delete mode 100644 crates/plugin-api/wit/format.wit delete mode 100644 crates/plugin/src/testing.rs delete mode 100644 crates/plugin/src/wasm/bindings.rs delete mode 100644 crates/plugin/src/wasm/executor.rs delete mode 100644 crates/plugin/src/wasm/host.rs delete mode 100644 crates/plugin/src/wasm/mod.rs delete mode 100644 crates/plugin/tests/component.rs delete mode 100644 crates/server/tests/plugins.rs delete mode 100644 docs/changes/2.0.0/228.fixed.md delete mode 100644 docs/changes/2.0.0/229.security.md delete mode 100644 docs/changes/2.0.0/234.fixed.md delete mode 100644 plugins/helm/Cargo.toml delete mode 100644 plugins/helm/plugin.toml delete mode 100644 plugins/helm/src/chart.rs delete mode 100644 plugins/helm/src/lib.rs delete mode 100644 plugins/helm/src/semver.rs delete mode 100644 plugins/helm/src/tar.rs delete mode 100644 plugins/helm/src/upstream.rs delete mode 100644 plugins/helm/src/yaml.rs delete mode 100644 plugins/helm/tests/upstream.rs diff --git a/.env.example b/.env.example index 4486869a..3d6fe0b0 100644 --- a/.env.example +++ b/.env.example @@ -151,15 +151,6 @@ FORGE_AUTH__BOOTSTRAP_ADMIN__PASSWORD=changeme # FORGE_DOCKER__UNREFERENCED_BLOB_TTL_SECONDS=86400 # FORGE_DOCKER__SWEEP_INTERVAL_SECONDS=300 -# --------------------------------------------------------------------------- -# WebAssembly format plugins (optional; docs/format-plugins.md) -# Load further package formats from //, each pinned to the SHA-256 of -# its component (a mismatch fails startup). The built-in format (NuGet) needs -# nothing here. `cargo xtask plugins` builds components and prints digests. -# --------------------------------------------------------------------------- -# FORGE_PLUGINS__DIR=/etc/forge/plugins -# FORGE_PLUGINS__FORMATS__EXAMPLE=sha256:<64 hex digits> - # --------------------------------------------------------------------------- # Cache # --------------------------------------------------------------------------- diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3c39fb19..44e6b013 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,12 +12,6 @@ updates: interval: weekly open-pull-requests-limit: 10 groups: - # wasmtime-wasi of a major builds only against the wasmtime of the same - # major, so a major update of one alone fails to compile. Every update - # of the Wasmtime crates comes as one pull request; listed first, since - # a dependency joins the first group it matches. - wasmtime: - patterns: ["wasmtime", "wasmtime-*"] cargo-minor-and-patch: update-types: [minor, patch] diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index dd245d4c..19313b0e 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -40,7 +40,7 @@ COMMON=(.github/scripts/ci-changes.sh) # jobs would hide a change from the jobs using it; ci.yml has none.) CI=.github/workflows/ci.yml declare -A CI_JOB_GROUP=( - [fmt]=changelog [clippy]=rust [audit]=audit [plugins]=rust [test]=rust + [fmt]=changelog [clippy]=rust [audit]=audit [test]=rust [docker]=image [kubernetes]=kubernetes [web]=web [oci-e2e]=rust [raw-e2e]=rust [e2e]=rust [web-e2e]=web ) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d71f88a..03957c1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,29 +231,6 @@ jobs: - name: cargo audit run: cargo audit - # ── Format plugins as WebAssembly components (plugins//) ────────────── - # The plugins are workspace crates (fmt, clippy and their tests run with the - # workspace); this checks that each one also lints clean and builds as a - # component for wasm32-wasip2. The tests that load a component run in `test`. - plugins: - name: plugins - runs-on: ubuntu-latest - needs: changes - if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install Rust toolchain (rust-toolchain.toml) (with clippy, wasm32-wasip2) - uses: dtolnay/rust-toolchain@d124f86f28ff42855784ff3319f48ac182c248f8 # 1.99.1 - with: - components: clippy - targets: wasm32-wasip2 - - name: Cache cargo registry + dependency builds - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - save-if: ${{ github.event_name == 'push' }} - - name: cargo xtask plugins-check - run: cargo xtask plugins-check - # ── Unit + integration tests ─────────────────────────────────────────────── # The integration tests start their own Postgres, versitygw and Redis with # testcontainers (the runner has a Docker daemon). There is no separate @@ -281,9 +258,6 @@ jobs: df -h / - name: Install Rust toolchain (rust-toolchain.toml) uses: dtolnay/rust-toolchain@d124f86f28ff42855784ff3319f48ac182c248f8 # 1.99.1 - with: - # The format-plugin tests build plugins/ as WebAssembly components. - targets: wasm32-wasip2 - name: Cache cargo registry + dependency builds uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00940c39..03fd5236 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The `name:` of every ci.yml job that runs on the default branch. The # required status checks in DEVELOPMENT.md (merge queue) list the same. - REQUIRED_JOBS: fmt clippy audit plugins test docker kubernetes web oci-e2e raw-e2e e2e web-e2e + REQUIRED_JOBS: fmt clippy audit test docker kubernetes web oci-e2e raw-e2e e2e web-e2e steps: - name: Wait for the commit's CI run and check its jobs shell: bash # -eo pipefail: a failing gh call fails the step diff --git a/.gitignore b/.gitignore index b8101b80..186985ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ /target -# Format plugins build in their own workspaces (`cargo xtask plugins`). -/plugins/*/target **/*.rs.bk Cargo.lock.orig .env diff --git a/Cargo.lock b/Cargo.lock index c02ebab0..47c0d88a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aead" version = "0.6.1" @@ -69,12 +54,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "ambient-authority" -version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" - [[package]] name = "android_system_properties" version = "0.1.6" @@ -90,12 +69,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" - [[package]] name = "arc-swap" version = "1.9.2" @@ -123,12 +96,6 @@ dependencies = [ "password-hash", ] -[[package]] -name = "arraydeque" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - [[package]] name = "asn1-rs" version = "0.7.2" @@ -980,9 +947,6 @@ name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -dependencies = [ - "allocator-api2", -] [[package]] name = "bytemuck" @@ -1012,24 +976,6 @@ dependencies = [ "either", ] -[[package]] -name = "cap-primitives" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18" -dependencies = [ - "ambient-authority", - "fs-set-times", - "io-extras", - "io-lifetimes 3.0.1", - "ipnet", - "maybe-owned", - "rustix", - "rustix-linux-procfs", - "windows-sys 0.61.2", - "winx", -] - [[package]] name = "cc" version = "1.4.7" @@ -1118,15 +1064,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "cobs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" -dependencies = [ - "thiserror", -] - [[package]] name = "combine" version = "4.6.8" @@ -1185,15 +1122,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" -[[package]] -name = "cpp_demangle" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" -dependencies = [ - "cfg-if", -] - [[package]] name = "cpubits" version = "0.1.1" @@ -1218,152 +1146,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cranelift-assembler-x64" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c587da2696fbaa58e7a2fa2e5bfdd365d0928219f36992304ac1f15e5bc254" -dependencies = [ - "cranelift-assembler-x64-meta", -] - -[[package]] -name = "cranelift-assembler-x64-meta" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a60027b7556b680b051651291053ec381edf4ffbc01784997ae51a64269bb" -dependencies = [ - "cranelift-srcgen", -] - -[[package]] -name = "cranelift-bforest" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bfdb846acebfd591d601994ea283c98d525ac0bc9a84ca3f2c36bcf5b69ff1" -dependencies = [ - "cranelift-entity", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-bitset" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9f8cc1080739d5f72b4ee7e00aff7d589f1284beaf0a19472c07878be7f299" -dependencies = [ - "serde", - "serde_derive", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-codegen" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85417d493ecec833b5134546b4a252c78cc0bd5be11729a35d25dcff9e4e70b" -dependencies = [ - "bumpalo", - "cranelift-assembler-x64", - "cranelift-bforest", - "cranelift-bitset", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-control", - "cranelift-entity", - "cranelift-isle", - "gimli", - "hashbrown 0.17.1", - "libm", - "log", - "postcard", - "pulley-interpreter", - "regalloc2", - "rustc-hash", - "serde", - "serde_derive", - "sha2 0.10.9", - "smallvec", - "target-lexicon", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-codegen-meta" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01c8ed0098360bae69ad6b82952d4247516799ce8050d45ad168d6689f4eae6" -dependencies = [ - "cranelift-assembler-x64-meta", - "cranelift-codegen-shared", - "cranelift-srcgen", - "heck", - "pulley-interpreter", -] - -[[package]] -name = "cranelift-codegen-shared" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f9b86d809cec9c4e88b201f1b3ed3b0347674c9ccfbb8c355228669e34be81" - -[[package]] -name = "cranelift-control" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce182038fb10b719841e65fdf5deeee258c684f63439f5be680c2be059274b58" -dependencies = [ - "arbitrary", -] - -[[package]] -name = "cranelift-entity" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bca812b03a7b942796391f25bc02aa70a1a007fee3ce2756490c788429e9c39" -dependencies = [ - "cranelift-bitset", - "serde", - "serde_derive", - "wasmtime-internal-core", -] - -[[package]] -name = "cranelift-frontend" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c30563d2aef95e8d1a00a920199c99117b350339599f0e9487a7a1cf5d59de" -dependencies = [ - "cranelift-codegen", - "hashbrown 0.17.1", - "log", - "smallvec", - "target-lexicon", -] - -[[package]] -name = "cranelift-isle" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c54184333a275698edb1d3bdff226131a25d47133d792aa716e47e32b8b7f05" - -[[package]] -name = "cranelift-native" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb67a4c8a3180b5f1d77dffcd6317c05386ced42e5a31c7004523a9d99dc85c1" -dependencies = [ - "cranelift-codegen", - "libc", - "target-lexicon", -] - -[[package]] -name = "cranelift-srcgen" -version = "0.136.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b533db1679b4b728568bd490da6533cf50974c48a4d7534e78f1e0e02bc39769" - [[package]] name = "crc" version = "3.4.0" @@ -1407,16 +1189,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.21" @@ -1738,18 +1510,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - [[package]] name = "encoding_rs" version = "0.8.41" @@ -1866,8 +1626,6 @@ version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ - "crc32fast", - "miniz_oxide", "zlib-rs", ] @@ -2035,16 +1793,6 @@ dependencies = [ "zip", ] -[[package]] -name = "forge-format-helm" -version = "0.1.0" -dependencies = [ - "flate2", - "forge-plugin-api", - "serde_json", - "yaml-rust2", -] - [[package]] name = "forge-format-nuget" version = "1.0.0" @@ -2093,30 +1841,18 @@ dependencies = [ name = "forge-plugin" version = "1.1.0" dependencies = [ - "flate2", "forge-core", - "forge-format-helm", - "forge-plugin", "forge-plugin-api", - "hex", "serde", "serde_json", - "sha2 0.11.1", - "tempfile", "thiserror", "toml 1.1.8+spec-1.1.0", "tracing", - "wasm-encoder 0.258.3", - "wasmtime", - "wasmtime-wasi", ] [[package]] name = "forge-plugin-api" version = "1.1.0" -dependencies = [ - "wit-bindgen 0.63.0", -] [[package]] name = "forge-server" @@ -2136,7 +1872,6 @@ dependencies = [ "forge-core", "forge-crypto", "forge-db", - "forge-format-helm", "forge-format-nuget", "forge-oci", "forge-plugin", @@ -2217,17 +1952,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-set-times" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" -dependencies = [ - "io-lifetimes 2.0.4", - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -2393,18 +2117,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "gimli" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" -dependencies = [ - "fnv", - "hashbrown 0.16.1", - "indexmap 2.14.2", - "stable_deref_trait", -] - [[package]] name = "globset" version = "0.4.20" @@ -2474,8 +2186,6 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash", - "serde", - "serde_core", ] [[package]] @@ -2487,15 +2197,6 @@ dependencies = [ "hashbrown 0.16.1", ] -[[package]] -name = "hashlink" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "heck" version = "0.5.0" @@ -2834,12 +2535,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -2905,28 +2600,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "io-extras" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" -dependencies = [ - "io-lifetimes 3.0.1", - "windows-sys 0.52.0", -] - -[[package]] -name = "io-lifetimes" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" - -[[package]] -name = "io-lifetimes" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" - [[package]] name = "ipnet" version = "2.12.2" @@ -3099,12 +2772,6 @@ dependencies = [ "spin 0.9.9", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "lettre" version = "0.11.23" @@ -3197,12 +2864,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" -[[package]] -name = "mach2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" - [[package]] name = "matchers" version = "0.2.0" @@ -3218,12 +2879,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "maybe-owned" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" - [[package]] name = "md-5" version = "0.11.0" @@ -3240,15 +2895,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "memfd" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" -dependencies = [ - "rustix", -] - [[package]] name = "mime" version = "0.3.17" @@ -3271,16 +2917,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.3" @@ -3511,18 +3147,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.40.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" -dependencies = [ - "crc32fast", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "memchr", -] - [[package]] name = "oid-registry" version = "0.8.1" @@ -3791,18 +3415,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "postcard" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" -dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "serde", -] - [[package]] name = "potential_utf" version = "0.1.6" @@ -3827,16 +3439,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" -dependencies = [ - "proc-macro2", - "syn 3.0.6", -] - [[package]] name = "primeorder" version = "0.13.6" @@ -3901,42 +3503,19 @@ dependencies = [ ] [[package]] -name = "pulley-interpreter" -version = "49.0.4" +name = "quick-xml" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd3de43f4f4cff536de8902097edbdf749321b97dda548f36a37918eb85bba3" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ - "cranelift-bitset", - "log", - "pulley-macros", - "wasmtime-internal-core", + "memchr", ] [[package]] -name = "pulley-macros" -version = "49.0.4" +name = "quinn" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8319a8fa5269b4895957f8795861c42822bd26218e495f70060a963323d94847" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "quick-xml" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" -dependencies = [ - "memchr", -] - -[[package]] -name = "quinn" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -4100,26 +3679,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "rcgen" version = "0.14.10" @@ -4194,21 +3753,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "regalloc2" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" -dependencies = [ - "allocator-api2", - "bumpalo", - "hashbrown 0.17.1", - "log", - "rustc-hash", - "serde", - "smallvec", -] - [[package]] name = "regex" version = "1.13.1" @@ -4331,12 +3875,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustc-demangle" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" - [[package]] name = "rustc-hash" version = "2.1.3" @@ -4374,16 +3912,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustix-linux-procfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" -dependencies = [ - "once_cell", - "rustix", -] - [[package]] name = "rustls" version = "0.23.46" @@ -4572,10 +4100,6 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "serde" @@ -4906,7 +4430,7 @@ dependencies = [ "futures-io", "futures-util", "hashbrown 0.16.1", - "hashlink 0.11.1", + "hashlink", "indexmap 2.14.2", "log", "memchr", @@ -5185,12 +4709,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "target-lexicon" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" - [[package]] name = "tempfile" version = "3.27.0" @@ -5204,15 +4722,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "testcontainers" version = "0.27.3" @@ -5854,7 +5363,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", + "wit-bindgen", ] [[package]] @@ -5912,50 +5421,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64765e43c000fae008a48db8b288e175e237fc9ce20a26586836878c1bee89f" -dependencies = [ - "leb128fmt", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasm-encoder" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a" -dependencies = [ - "leb128fmt", - "wasmparser 0.262.0", -] - -[[package]] -name = "wasm-metadata" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e10808eb27efde692f1b1706089fe19296728bf872291f5b4e0d50fba88f2f" -dependencies = [ - "anyhow", - "indexmap 2.14.2", - "wasm-encoder 0.258.3", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasm-metadata" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c86ff6668190d9fb70372b306b4f9fe8930f0e2f57ce72e4922bf2f117900949" -dependencies = [ - "anyhow", - "indexmap 2.14.2", - "wasm-encoder 0.262.0", - "wasmparser 0.262.0", -] - [[package]] name = "wasm-streams" version = "0.5.0" @@ -5969,282 +5434,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9269dd817e0e7021bf534ccd9761bc2d64778b6708e41228b8439d75eaa0b1" -dependencies = [ - "bitflags 2.13.2", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "semver", - "serde", -] - -[[package]] -name = "wasmparser" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b" -dependencies = [ - "bitflags 2.13.2", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "semver", -] - -[[package]] -name = "wasmprinter" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a457ba8670a3db3d357c1aeac3ed4fe378a76054a221789f0df0ea507e292b" -dependencies = [ - "anyhow", - "termcolor", - "wasmparser 0.258.3", -] - -[[package]] -name = "wasmtime" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36abb25de5518707e5ba96b586366b09db168d2d4976cb9dac57265a317497ea" -dependencies = [ - "addr2line", - "async-trait", - "bitflags 2.13.2", - "bumpalo", - "cc", - "encoding_rs", - "futures", - "libc", - "log", - "mach2", - "memfd", - "object", - "once_cell", - "postcard", - "pulley-interpreter", - "rayon", - "rustix", - "semver", - "serde", - "serde_derive", - "smallvec", - "target-lexicon", - "wasmparser 0.258.3", - "wasmtime-environ", - "wasmtime-internal-component-macro", - "wasmtime-internal-component-util", - "wasmtime-internal-core", - "wasmtime-internal-cranelift", - "wasmtime-internal-fiber", - "wasmtime-internal-jit-debug", - "wasmtime-internal-jit-icache-coherence", - "wasmtime-internal-unwinder", - "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-environ" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92f16a4d09fb1fbf2dcfeddb9818e1517a258e13b505b3bfdadcf47a19bf31ca" -dependencies = [ - "anyhow", - "cpp_demangle", - "cranelift-bforest", - "cranelift-bitset", - "cranelift-entity", - "gimli", - "hashbrown 0.17.1", - "indexmap 2.14.2", - "log", - "object", - "postcard", - "rustc-demangle", - "semver", - "serde", - "serde_derive", - "sha2 0.10.9", - "smallvec", - "target-lexicon", - "wasm-encoder 0.258.3", - "wasmparser 0.258.3", - "wasmprinter", - "wasmtime-internal-component-util", - "wasmtime-internal-core", -] - -[[package]] -name = "wasmtime-internal-component-macro" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fca03bd4425856574c0807d1dee16fad43f947feaf4e9ae1d2129596ee4de4" -dependencies = [ - "anyhow", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasmtime-internal-component-util", - "wasmtime-internal-wit-bindgen", - "wit-parser 0.258.3", -] - -[[package]] -name = "wasmtime-internal-component-util" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4006009fc3598af4e895ab18c6c33d2c6b7c657b1bd0a755ca7ee1abef6f0c16" - -[[package]] -name = "wasmtime-internal-core" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947a732ad659f42c10ec639706fc8cb1f068dd9a55eebde8a1f24dfd06dfa525" -dependencies = [ - "hashbrown 0.17.1", - "libm", - "serde", -] - -[[package]] -name = "wasmtime-internal-cranelift" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfb8b455d524939e03df42e3e749f8e26e26ba2acd8b08fa00b3bda55686750" -dependencies = [ - "cranelift-codegen", - "cranelift-control", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "gimli", - "itertools", - "log", - "object", - "pulley-interpreter", - "smallvec", - "target-lexicon", - "thiserror", - "wasmparser 0.258.3", - "wasmtime-environ", - "wasmtime-internal-core", - "wasmtime-internal-unwinder", - "wasmtime-internal-versioned-export-macros", -] - -[[package]] -name = "wasmtime-internal-fiber" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8aa8711f68b2cdd94c2b85ba5c8317edaa1ac4ee0a0534b957123680239307" -dependencies = [ - "cc", - "libc", - "rustix", - "wasmtime-environ", - "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-internal-jit-debug" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e873e4504faf50aa8ce255d6ea7210d87b39d5f379c2486782e2c5f179c70ca8" -dependencies = [ - "cc", - "wasmtime-internal-versioned-export-macros", -] - -[[package]] -name = "wasmtime-internal-jit-icache-coherence" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197ab1fda5da958a5c0b8a97eb241beafc48ac568b82ef4f12f38d6866c74173" -dependencies = [ - "libc", - "wasmtime-internal-core", - "windows-sys 0.61.2", -] - -[[package]] -name = "wasmtime-internal-unwinder" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5577f3a5a979a7c3b1b23c1e9f13e70a8583acc8828471ec56f282117fc4952b" -dependencies = [ - "cranelift-codegen", - "log", - "object", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-internal-versioned-export-macros" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6399c9c4050ed431cf66e21c05fadd6f9731db44476f452f8c7142b4597b883" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "wasmtime-internal-wit-bindgen" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eaf45d4da7681d234bb6efd846b4a528b08a75384a7fc23f125d0145364b06" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "heck", - "indexmap 2.14.2", - "wit-component 0.258.3", - "wit-parser 0.258.3", -] - -[[package]] -name = "wasmtime-wasi" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "821e45765c35555d3b0bdf9277cb99478f7d1f29a17b3315ba76e01220f02a38" -dependencies = [ - "async-trait", - "bitflags 2.13.2", - "bytes", - "cap-primitives", - "futures", - "rand 0.10.3", - "rustix", - "rustix-linux-procfs", - "thiserror", - "tokio", - "tracing", - "url", - "wasmtime", - "wasmtime-wasi-io", - "windows-sys 0.61.2", - "winx", -] - -[[package]] -name = "wasmtime-wasi-io" -version = "49.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdff38d1c730bdaf4fcfbe0eedf3265fb9c7a4deb119deecbfc1c1778950716" -dependencies = [ - "async-trait", - "bytes", - "futures", - "tracing", - "wasmtime", -] - [[package]] name = "web-sys" version = "0.3.105" @@ -6478,16 +5667,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" -[[package]] -name = "winx" -version = "0.36.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" -dependencies = [ - "bitflags 2.13.2", - "windows-sys 0.52.0", -] - [[package]] name = "wiremock" version = "0.6.5" @@ -6517,133 +5696,6 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85a987ab61d365382d25c7d47e26edd48941b3bf6e3fbf76eff918bc71dab06" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e122e390603abfacd548027016cd0113a6e2bf2f8ce6bdd3658b013456ea8" -dependencies = [ - "anyhow", - "heck", - "wit-parser 0.262.0", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84e26e13b9d972026778d2ffb7985736de116026fdc8d9464f966e12df2eb240" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.2", - "prettyplease", - "syn 3.0.6", - "wasm-metadata 0.262.0", - "wit-bindgen-core", - "wit-component 0.262.0", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a008a0d6b4840677326beb4edbeb210a2f1b7d0183fb7cc33f466b90fd7fb6" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 3.0.6", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fca88950af0c1f42bc77e17315e48798b8d68340171b6d204f62bc19a878a6" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "indexmap 2.14.2", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder 0.258.3", - "wasm-metadata 0.258.3", - "wasmparser 0.258.3", - "wit-parser 0.258.3", -] - -[[package]] -name = "wit-component" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078def49fdb49cab405c3006e05793b3fdaf7069126ba6b9c97d01ec8c98fe4c" -dependencies = [ - "anyhow", - "bitflags 2.13.2", - "indexmap 2.14.2", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder 0.262.0", - "wasm-metadata 0.262.0", - "wasmparser 0.262.0", - "wit-parser 0.262.0", -] - -[[package]] -name = "wit-parser" -version = "0.258.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2127663dcf3ee16614fb0e3c33d5c6c66fec9dad317a8b0fdd3869487a533" -dependencies = [ - "anyhow", - "hashbrown 0.17.1", - "id-arena", - "indexmap 2.14.2", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-ident", - "wasmparser 0.258.3", -] - -[[package]] -name = "wit-parser" -version = "0.262.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f0dc051466ffe46787721fca3d088536f158ae400be044f1b1c386ba9397ad" -dependencies = [ - "anyhow", - "hashbrown 0.17.1", - "id-arena", - "indexmap 2.14.2", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-ident", - "wasmparser 0.262.0", -] - [[package]] name = "writeable" version = "0.6.4" @@ -6694,16 +5746,6 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" -[[package]] -name = "yaml-rust2" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e5b818a27a4cd30884ea380857a5e56f7ec3ba24a3990a3cc0b95af3238e18" -dependencies = [ - "arraydeque", - "hashlink 0.12.2", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 4d3b432f..63ebd495 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ members = [ "crates/format-oci", "crates/server", "crates/e2e", - "plugins/helm", "plugins/nuget", "xtask", ] @@ -184,33 +183,6 @@ tempfile = { version = "3" } # Date/time (shared with forge-db's model types) chrono = { version = "0.4", default-features = false, features = ["clock"] } -# WebAssembly runtime for format plugins (forge-plugin). 49 is a regular -# Wasmtime release: it gets security fixes for two months, an LTS release -# (every 12th, like 48) for two years, so the next major is due within two -# months. wasmtime-wasi of a major builds only against the wasmtime of the same -# major: bump both together (Dependabot groups them). Only what the host uses: -# the component model compiled by Cranelift, no WAT text parser, no GC -# proposals, no profiling or debugging. -wasmtime = { version = "49.0.4", default-features = false, features = [ - "runtime", - "cranelift", - "component-model", - "std", - "parallel-compilation", -] } -# WASI 0.2 for plugin components (a Rust guest's std imports it); the host -# grants no filesystem, network or environment. -wasmtime-wasi = { version = "49.0.4", default-features = false, features = ["p2"] } - -# Format plugins built as WebAssembly components (`cargo xtask plugins`): -# small, self-contained `.wasm` files. -[profile.plugin] -inherits = "release" -opt-level = "s" -lto = true -codegen-units = 1 -strip = true - # Tests build unoptimized, yet the mock IdPs generate RSA keys and every test # server hashes its bootstrap admin password: ~30x (RSA-2048 keygen, ~3 s) and # ~15x (Argon2 default, ~0.3 s) slower than optimized. Optimize just those diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3772bf4a..14b321a0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -9,9 +9,7 @@ Locally it runs as a single `forge-server` process backed by **PostgreSQL** ## Prerequisites - **Docker** (for Postgres + versitygw, and for the test suites). -- **Rust** stable toolchain (`rustup`, pinned in `rust-toolchain.toml`), with - the `wasm32-wasip2` target for the format plugins' component builds and the - tests that load them (`rustup target add wasm32-wasip2`). +- **Rust** stable toolchain (`rustup`, pinned in `rust-toolchain.toml`). - **Node 26 + pnpm 11** (for the web UI in `web/`; `corepack enable` activates the pinned `pnpm@11.5.0` from `web/package.json`. Node 25+ no longer bundles corepack: `npm install -g corepack` first). @@ -251,22 +249,13 @@ groups. See [docs/raw.md](docs/raw.md). ### Repositories of other formats (format plugins) -Formats other than NuGet come from format plugins, crates in -[`plugins/`](./plugins) (see [docs/format-plugins.md](./docs/format-plugins.md)). -NuGet is the one built in. `plugins/helm`, classic Helm chart repositories, is -the example plugin the plugin tests run; it is not built in (Helm charts are -pushed to docker repositories, see [docs/docker.md](docs/docker.md#helm-charts)). -To run a plugin as a WebAssembly component: - -```bash -cargo xtask plugins # builds plugins/* as components, prints the [plugins] section for forge.toml -``` - -A repository of a plugin format is created with `"format": ""` (hosted, -proxy or group; a plugin format has proxies when it implements the upstream -side) and served under `/repository//` unless the plugin names another -mount. A built-in plugin's component loads only under another id (rename its -installed directory and the `id` in its `plugin.toml`). +Package formats come from format plugins, crates in [`plugins/`](./plugins) +linked into the server (see [docs/format-plugins.md](./docs/format-plugins.md)); +NuGet is the one there is. A repository of a plugin format is created with +`"format": ""` (hosted, proxy or group; a plugin format has proxies when +it implements the upstream side) and served under `/repository//` +unless the plugin names another mount (NuGet's is `/v3`). Helm charts are +pushed to docker repositories, see [docs/docker.md](docs/docker.md#helm-charts). ## Configuration @@ -608,14 +597,8 @@ way (`mirror.gcr.io/library/` for an official image). cargo xtask test # unit + integration (cargo test --workspace) cargo xtask e2e # end-to-end suite (cargo test -p forge-e2e --features forge-e2e) cargo xtask check # cargo fmt --check + clippy -D warnings -cargo xtask plugins-check # the format plugins' component builds: clippy and build ``` -The format-plugin tests (`crates/plugin/tests`, `crates/server/tests/plugins.rs`) -run the example plugin, `plugins/helm`: built in for the test, and built as a -component themselves into `target/plugins/`; they need the `wasm32-wasip2` -target. - The e2e suite includes an optional `dotnet` smoke test that is skipped when the .NET SDK isn't installed. @@ -729,9 +712,6 @@ server hashes its bootstrap admin password. `clippy` · `build` · `prepare` (regenerate sqlx offline data) · `psql` · `logs` · `ps` · `clean`. -Format plugins as components: `plugins` (build + install, prints the config) · -`plugins-check` (clippy and build). - Web UI: `web` (build) · `web-install` · `web-dev` (Vite HMR) · `web-test` (Vitest) · `web-e2e` (Playwright). @@ -819,7 +799,7 @@ backups, upgrades) follow [docs/deployment.md](docs/deployment.md). ## CI and the merge queue CI (`.github/workflows/ci.yml`) has the jobs `fmt`, `clippy`, `audit`, -`plugins`, `test`, `docker`, `kubernetes`, `web` and `oci-e2e`, and the slow +`test`, `docker`, `kubernetes`, `web`, `oci-e2e` and `raw-e2e`, and the slow `e2e` and `web-e2e`, which pull requests skip: those two run in the merge queue and on every push to `main`. It runs on pull requests that target `main` only; a stacked one waits (see [Stacked pull @@ -853,7 +833,7 @@ It sorts the change into groups of paths: A change to `ci-changes.sh` counts for every group. A change to `ci.yml` counts for the groups whose jobs it changes (#365): `fmt` is in Changelog; -`clippy`, `plugins`, `test`, `oci-e2e`, `raw-e2e` and `e2e` in Rust; `audit` +`clippy`, `test`, `oci-e2e`, `raw-e2e` and `e2e` in Rust; `audit` in Audit; `web` and `web-e2e` in Web; `docker` in Image and `kubernetes` in Kubernetes. A change outside the jobs (the triggers, the settings, the `changes` job) or to a job missing from `CI_JOB_GROUP` in `ci-changes.sh` @@ -896,8 +876,8 @@ To turn it on, edit the ruleset (or branch protection rule) of `main`: 1. Enable **Require merge queue**. Keep its **status check timeout** above the run time of `e2e` and `web-e2e` (60 minutes by default). 2. Under **Require status checks to pass**, require `fmt`, `clippy`, `audit`, - `plugins`, `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, - `e2e` and `web-e2e` from GitHub Actions: every job of `ci.yml` but + `test`, `docker`, `kubernetes`, `web`, `oci-e2e`, `raw-e2e`, `e2e` and + `web-e2e` from GitHub Actions: every job of `ci.yml` but `changes`, the jobs the release workflow requires of the tagged commit (`REQUIRED_JOBS` in `.github/workflows/release.yml`). A job left out here can fail on a pull request that merges anyway, and the release then diff --git a/README.md b/README.md index 32a78f6f..860e9c11 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ without stored API keys through Forge's **direct-token OIDC trusted-publishing** flow for GitHub Actions, GitLab CI, and compatible workload issuers; its bearer token, cache, and audit limitations are documented below. A **retention** engine reaps cold versions and garbage-collects dereferenced -blobs. **Format plugins** (preview) add further package formats that reuse all -of the above, either built into the server or loaded as sandboxed WebAssembly -components ([docs/format-plugins.md](docs/format-plugins.md)). The admin + browse UI is +blobs. **Format plugins** (preview), crates built into the server, add +further package formats that reuse all of the above +([docs/format-plugins.md](docs/format-plugins.md)). The admin + browse UI is a **TanStack Router SPA** served by the same Rust process on a single origin — there is no Node server in production. @@ -51,12 +51,12 @@ A Cargo workspace of small crates plus the web SPA: | `cache` | `forge-cache` | In-process L1 + optional Redis L2 with cross-instance invalidation (no Postgres advisory locks). | | `core` | `forge-core` | Domain logic: repo dispatch, auth chain, authz/RBAC, sessions, OIDC, retention. | | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | -| `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait, its WIT twin and the component glue. | -| `plugin` | `forge-plugin` | The host side of format plugins: built-in and WebAssembly plugins (Wasmtime) behind one trait. | +| `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | +| `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | | `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`), the Docker registry, raw file repositories (`/raw`), `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | -| `plugins/` | `forge-format-*` | Format plugins, built in or as WebAssembly components: `nuget` (built in only), and `helm`, the example plugin the plugin tests run (not built in). | +| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | **Single-origin serving.** `forge-server` mounts each format plugin's feed @@ -206,8 +206,7 @@ curl -u admin@example.com: http://localhost:8421/raw/binaries/tools/1.0/t workload identity for GitHub Actions, GitLab CI, and compatible OIDC issuers, and exchange a job's OIDC token for a short-lived API key (`POST /api/v2/token`). - **[Format plugins](./docs/format-plugins.md)** — further package formats, - built in or as sandboxed WebAssembly components: design, operation, and - writing one. + built into the server: design, operation, and writing one. - **[Importing from Nexus](./docs/nexus-import.md)** — copy a Nexus hosted repository (NuGet, Docker) into Forge, with its publish dates: dry run, conflicts, resuming, and the cutover. diff --git a/crates/core/src/net/origins.rs b/crates/core/src/net/origins.rs index 9b8f0496..5f016f1b 100644 --- a/crates/core/src/net/origins.rs +++ b/crates/core/src/net/origins.rs @@ -2,12 +2,11 @@ //! //! A proxy repository's format says which documents to fetch from its //! upstream, and where a version's package is -//! ([`crate::repo::proxy::ProxyRepo`]). The format may be a WebAssembly -//! plugin, code from elsewhere with no network access of its own: these -//! requests are its only reach beyond its sandbox, the host sends them from -//! Forge's network position, and the plugin reads the answers back into what -//! the proxy serves. So they go only where the upstream itself points -//! ([`UpstreamOrigins`]): +//! ([`crate::repo::proxy::ProxyRepo`]). The host sends these requests from +//! Forge's network position, and the format reads the answers back into what +//! the proxy serves, so a document an upstream serves must not be able to +//! steer them at an internal service. They go only where the upstream itself +//! points ([`UpstreamOrigins`]): //! //! * the origin (scheme, host, port) of the repository's `remote_url`; //! * an origin that an absolute `http`/`https` URL names in a document the diff --git a/crates/plugin-api/Cargo.toml b/crates/plugin-api/Cargo.toml index 53ccd221..17d55565 100644 --- a/crates/plugin-api/Cargo.toml +++ b/crates/plugin-api/Cargo.toml @@ -5,16 +5,9 @@ edition.workspace = true rust-version.workspace = true license.workspace = true publish.workspace = true -description = "The contract of a Forge format plugin: built in, or a WebAssembly component" +description = "The contract of a Forge format plugin, built into the server" -# Nothing but std: a plugin crate builds against this for the server and for -# `wasm32-wasip2` alike. +# Nothing but std. [features] -# `export_component!`: build the plugin crate as a WebAssembly component of -# the `format-plugin` world (`wit/format.wit`). -component = ["dep:wit-bindgen"] # `testing`: run a plugin's protocol against canned answers, without a server. testing = [] - -[dependencies] -wit-bindgen = { version = "0.63", optional = true, default-features = false, features = ["macros", "realloc", "std"] } diff --git a/crates/plugin-api/src/component.rs b/crates/plugin-api/src/component.rs deleted file mode 100644 index 680ee751..00000000 --- a/crates/plugin-api/src/component.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Building a plugin crate as a WebAssembly component (feature `component`). -//! -//! ```ignore -//! #[cfg(feature = "component")] -//! forge_plugin_api::export_component!(Helm); -//! ``` -//! -//! and build the crate as a `cdylib` for `wasm32-wasip2` with the feature on -//! (`cargo xtask plugins` does): the component exports the `format-plugin` -//! world of `wit/format.wit`, each function calling the plugin's -//! [`FormatPlugin`] method of the same name. - -use std::io::{self, Read, Seek, SeekFrom}; - -use crate::FormatPlugin; - -/// The guest bindings of `wit/format.wit`. -#[doc(hidden)] -pub mod bindings { - wit_bindgen::generate!({ - world: "format-plugin", - path: "wit", - pub_export_macro: true, - default_bindings_module: "forge_plugin_api::component::bindings", - }); -} - -crate::__wit_conversions!(crate::component::bindings); - -use bindings::exports::forge::format::{format, protocol, upstream}; -use bindings::forge::format::package_source::PackageReader; -use bindings::forge::format::types; - -/// The plugin a component exports (implemented by [`export_component!`]). -#[doc(hidden)] -pub trait Exported { - fn plugin() -> &'static dyn FormatPlugin; -} - -/// Export `$plugin` (an expression of a type implementing [`FormatPlugin`], -/// built once on the first call) as this component's `format-plugin` world. -#[macro_export] -macro_rules! export_component { - ($plugin:expr) => { - #[doc(hidden)] - pub struct __ForgeFormatComponent; - - impl $crate::component::Exported for __ForgeFormatComponent { - fn plugin() -> &'static dyn $crate::FormatPlugin { - static PLUGIN: ::std::sync::OnceLock<::std::boxed::Box> = - ::std::sync::OnceLock::new(); - PLUGIN - .get_or_init(|| ::std::boxed::Box::new($plugin)) - .as_ref() - } - } - - $crate::component::bindings::export!( - __ForgeFormatComponent with_types_in $crate::component::bindings - ); - }; -} - -/// The package reader the host lends, as [`Read`] and [`Seek`]. -struct Lent<'a> { - reader: &'a PackageReader, - size: u64, - offset: u64, -} - -impl Read for Lent<'_> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let len = u32::try_from(buf.len()).unwrap_or(u32::MAX); - let chunk = self - .reader - .read_at(self.offset, len) - .map_err(io::Error::other)?; - let n = chunk.len().min(buf.len()); - buf[..n].copy_from_slice(&chunk[..n]); - self.offset += n as u64; - Ok(n) - } -} - -impl Seek for Lent<'_> { - fn seek(&mut self, pos: SeekFrom) -> io::Result { - let target = match pos { - SeekFrom::Start(n) => Some(n), - SeekFrom::End(d) => self.size.checked_add_signed(d), - SeekFrom::Current(d) => self.offset.checked_add_signed(d), - }; - self.offset = target - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "seek before the start"))?; - Ok(self.offset) - } -} - -impl format::Guest for T { - fn canonical_id(id: String) -> String { - T::plugin().canonical_id(&id) - } - - fn validate_id(id: String) -> Result<(), types::FormatError> { - T::plugin().validate_id(&id).map_err(Into::into) - } - - fn normalize_version(version: String) -> Result { - T::plugin().normalize_version(&version).map_err(Into::into) - } - - fn compare_versions(a: String, b: String) -> i8 { - T::plugin().compare_versions(&a, &b) as i8 - } - - fn parse_package(source: &PackageReader) -> Result { - let mut lent = Lent { - reader: source, - size: source.size(), - offset: 0, - }; - T::plugin() - .parse_package(&mut lent) - .map(Into::into) - .map_err(Into::into) - } - - fn manifest_summary(manifest: Vec) -> Option { - T::plugin().manifest_summary(&manifest) - } -} - -impl protocol::Guest for T { - fn handle(repo: protocol::Repository, req: protocol::Request) -> protocol::Action { - T::plugin().handle(&repo.into(), &req.into()).into() - } - - fn render( - repo: protocol::Repository, - req: protocol::Request, - data: Vec, - ) -> protocol::Rendered { - let data: Vec = data.into_iter().map(Into::into).collect(); - T::plugin().render(&repo.into(), &req.into(), &data).into() - } -} - -impl upstream::Guest for T { - fn upstream( - remote: upstream::Remote, - op: upstream::UpstreamOp, - fetched: Vec, - ) -> Result { - let fetched: Vec = fetched.into_iter().map(Into::into).collect(); - T::plugin() - .upstream(&remote.into(), &op.into(), &fetched) - .map(Into::into) - .map_err(Into::into) - } -} diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 5e57f4f1..68c77a72 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -8,22 +8,11 @@ //! storage key, or a whole package in memory. //! //! A plugin is a crate under `plugins//` implementing [`FormatPlugin`], -//! with a `plugin.toml` naming it. The same crate runs either way: -//! -//! * **built in** — linked into the server and called directly (NuGet); -//! * **as a WebAssembly component** — built for `wasm32-wasip2` with this -//! crate's `component` feature and `export_component!`, then loaded at -//! startup into a sandbox, from a digest the operator pinned. -//! -//! The trait and its types mirror the `forge:format` WIT package -//! (`wit/format.wit`) one to one: a component exports exactly these -//! functions, and the host adapts a loaded component back into a -//! [`FormatPlugin`], so the server serves both kinds through one code path. +//! with a `plugin.toml` naming it, linked into the server and called +//! directly (NuGet). mod types; -#[cfg(feature = "component")] -pub mod component; #[cfg(feature = "testing")] pub mod testing; @@ -33,8 +22,8 @@ use std::path::Path; pub use types::*; -/// The `forge:format` WIT package version this crate implements: what a -/// plugin built against it declares as `api` in its `plugin.toml`. +/// The `forge:format` contract version this crate implements: what a plugin +/// built against it declares as `api` in its `plugin.toml`. pub const API: &str = "0.3.0"; /// The `plugin.toml` of the crate this is expanded in: how a built-in plugin @@ -91,8 +80,7 @@ pub trait FormatPlugin: Send + Sync + 'static { /// A JSON object summarizing `manifest`, stored with the version so that /// rendering never re-reads the manifest ([`PackageInfo::summary`]). /// `None` when there is nothing worth keeping. Every listing hands each - /// version's summary back, so keep it small: the host stores none larger - /// than 64 KiB from a WebAssembly plugin. + /// version's summary back, so keep it small. fn manifest_summary(&self, manifest: &[u8]) -> Option; /// What a request is: a response right away, data to load first (then @@ -130,214 +118,3 @@ pub trait FormatPlugin: Send + Sync + 'static { )) } } - -/// Conversions between the types of this crate and the bindings of -/// `wit/format.wit`, for both sides of the component boundary: the -/// `wit-bindgen` guest bindings (module `component`) and the host's Wasmtime -/// bindings. Both generators name every type, field and case the same, so -/// one definition serves both. Expanded once per bindings module: -/// `forge_plugin_api::__wit_conversions!(crate::bindings);`. -#[doc(hidden)] -#[macro_export] -macro_rules! __wit_conversions { - ($($root:tt)*) => { - const _: () = { - use $crate as api; - use $crate::__private::{into, opt, vec}; - use $($root)*::forge::format::types as t; - use $($root)*::exports::forge::format::format as f; - use $($root)*::exports::forge::format::protocol as p; - use $($root)*::exports::forge::format::upstream as u; - - $crate::__record!(t, Dependency { id: into, range: into }); - $crate::__record!(t, DependencyGroup { target: into, dependencies: vec }); - $crate::__variant!(t, FormatError {} { - InvalidPackage: into, - MissingManifest: into, - MalformedManifest: into, - InvalidVersion: into, - InvalidId: into, - Internal: into, - }); - $crate::__record!(f, ParsedPackage { - id: into, - original_id: into, - version: into, - version_original: into, - listed: into, - manifest: into, - dependencies: vec, - }); - $crate::__variant!(p, RepositoryKind { Hosted, Proxy, Group } {}); - $crate::__record!(p, Repository { name: into, kind: into, base_url: into }); - $crate::__record!(p, Request { method: into, path: into, query: into, headers: into }); - $crate::__record!(p, Response { status: into, headers: into, body: into }); - $crate::__record!(p, PackageRef { id: into, version: into }); - $crate::__record!(p, PackageVersions { id: into, versions: into }); - $crate::__record!(p, SearchQuery { - text: into, - skip: into, - take: into, - prerelease: into, - semver2: into, - upstream: into, - }); - $crate::__variant!(p, Query {} { - Versions: into, - Packages: into, - Package: into, - PackagesOf: into, - Manifest: into, - Search: into, - }); - $crate::__record!(p, VersionInfo { - original_id: into, - version: into, - version_original: into, - listed: into, - }); - $crate::__record!(p, PackageInfo { - original_id: into, - version: into, - version_original: into, - listed: into, - published: into, - sha256: into, - sha512: into, - size: into, - summary: into, - }); - $crate::__record!(p, SearchHit { - original_id: into, - latest_version: into, - versions: vec, - summary: into, - total_downloads: into, - }); - $crate::__record!(p, SearchResult { total: into, hits: vec }); - $crate::__variant!(p, Answer {} { - Versions: vec, - Packages: vec, - Package: opt, - Manifest: into, - Search: into, - }); - $crate::__record!(p, Loaded { query: into, answer: into }); - $crate::__variant!(p, Upload { Body, FirstPart } { MultipartField: into }); - $crate::__record!(p, PublishRequest { upload: into, done: into }); - $crate::__record!(p, ChangeRequest { target: into, done: into }); - $crate::__variant!(p, Action {} { - Respond: into, - Load: vec, - Download: into, - Publish: into, - Unlist: into, - Relist: into, - Delete: into, - }); - $crate::__variant!(p, Rendered {} { Respond: into, Load: vec }); - $crate::__record!(u, Remote { url: into, index: into }); - $crate::__variant!(u, UpstreamOp { Index } { - Versions: into, - Package: into, - Manifest: into, - Metadata: into, - Dating: into, - Search: into, - }); - $crate::__variant!(u, Freshness { Metadata, Immutable } {}); - $crate::__record!(u, Fetch { url: into, freshness: into }); - $crate::__variant!(u, Fetched { NotFound } { Found: into, Failed: into }); - $crate::__record!(u, FetchResult { fetch: into, outcome: into }); - $crate::__record!(u, VersionMetadata { - version: into, - original_id: into, - listed: into, - published: into, - created: into, - summary: into, - }); - $crate::__record!(u, PackageMetadata { original_id: into, versions: vec }); - $crate::__record!(u, UpstreamHit { - id: into, - versions: into, - summary: into, - total_downloads: into, - }); - $crate::__record!(u, UpstreamSearch { total: into, hits: vec }); - $crate::__variant!(u, UpstreamAnswer {} { - Index: into, - Versions: into, - Package: into, - Manifest: into, - Metadata: into, - Search: opt, - Failed: into, - }); - $crate::__variant!(u, UpstreamStep {} { Fetch: vec, Done: into }); - }; - }; -} - -/// `From` both ways between `api::$ty` and `$m::$ty`, a record. -#[doc(hidden)] -#[macro_export] -macro_rules! __record { - ($m:ident, $ty:ident { $($field:ident: $how:ident),* $(,)? }) => { - impl From for $m::$ty { - fn from(x: api::$ty) -> Self { - $m::$ty { $($field: $how(x.$field)),* } - } - } - impl From<$m::$ty> for api::$ty { - fn from(x: $m::$ty) -> Self { - api::$ty { $($field: $how(x.$field)),* } - } - } - }; -} - -/// `From` both ways between `api::$ty` and `$m::$ty`, a variant: its cases -/// without a payload, then its cases with one. -#[doc(hidden)] -#[macro_export] -macro_rules! __variant { - ($m:ident, $ty:ident { $($unit:ident),* $(,)? } { $($case:ident: $how:ident),* $(,)? }) => { - impl From for $m::$ty { - fn from(x: api::$ty) -> Self { - match x { - $(api::$ty::$unit => $m::$ty::$unit,)* - $(api::$ty::$case(v) => $m::$ty::$case($how(v)),)* - } - } - } - impl From<$m::$ty> for api::$ty { - fn from(x: $m::$ty) -> Self { - match x { - $($m::$ty::$unit => api::$ty::$unit,)* - $($m::$ty::$case(v) => api::$ty::$case($how(v)),)* - } - } - } - }; -} - -#[doc(hidden)] -pub mod __private { - //! What [`__wit_conversions!`](crate::__wit_conversions) expands to. - - /// A field or payload converted as a whole. - pub fn into>(a: A) -> B { - B::from(a) - } - - /// A list converted element by element. - pub fn vec>(v: Vec) -> Vec { - v.into_iter().map(B::from).collect() - } - - /// An option converted inside. - pub fn opt>(o: Option) -> Option { - o.map(B::from) - } -} diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 437fb43c..30c8d5af 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -1,5 +1,4 @@ -//! The data a plugin and the host exchange: the records and variants of -//! `wit/format.wit`, in Rust. +//! The data a plugin and the host exchange. use std::fmt; diff --git a/crates/plugin-api/wit/format.wit b/crates/plugin-api/wit/format.wit deleted file mode 100644 index 95f9b46c..00000000 --- a/crates/plugin-api/wit/format.wit +++ /dev/null @@ -1,482 +0,0 @@ -// The contract between Forge and a format plugin: what teaches Forge one more -// package format. -// -// A plugin is a Rust crate implementing `forge_plugin_api::FormatPlugin`, the -// same contract in Rust (crates/plugin-api/src/lib.rs). Forge links its -// built-in plugins natively; any plugin can also be built as a WebAssembly -// component of the `format-plugin` world below and loaded at startup. -// -// The division of labor is fixed: the host owns everything a repository is — -// storage (the content-addressed store), metadata (Postgres), authentication, -// authorization and package selectors, hosted/group semantics, retention, -// streaming of package bytes in both directions. The plugin owns what is -// specific to its format: the identity rules (ids, versions), reading a -// package's manifest, and the client protocol (which URL answers what, and how -// a response looks). -// -// A plugin never sees credentials, never touches storage, and never holds a -// whole package in memory: the host stages uploads on its disk and lends the -// plugin a reader, and the host streams downloads itself. -// -// Versioning: a plugin is built against one version of this package and -// declares it in its plugin.toml (`api`). A host supports a fixed set of -// versions; changing anything in this file means a new package version. -package forge:format@0.3.0; - -/// Types shared by the interfaces below. -interface types { - /// A dependency on another package of the same format. - record dependency { - /// The package id, as written. - id: string, - /// The version range in the format's syntax; none for any version. - range: option, - } - - /// The dependencies that apply to one target (a framework, a platform); - /// a format without targets uses a single group without one. - record dependency-group { - target: option, - dependencies: list, - } - - /// Why a package, an id or a version was refused. The text is shown to - /// the client that sent it. - variant format-error { - invalid-package(string), - missing-manifest(string), - malformed-manifest(string), - invalid-version(string), - invalid-id(string), - /// The plugin itself failed (a bug, an exhausted resource): the - /// client gets a server error, not this text. - internal(string), - } -} - -/// Read access to a package the host staged on its local disk. -interface package-source { - resource package-reader { - /// The package's size in bytes. - size: func() -> u64; - /// Up to `len` bytes starting at `offset`: fewer only at the end of - /// the package, none at or past it. `len` is clamped to 1 MiB. - read-at: func(offset: u64, len: u32) -> result, string>; - } -} - -/// The format's identity rules and package parsing. Every function is pure: -/// the host may call them in any order, on any instance, any number of times. -interface format { - use types.{dependency-group, format-error}; - use package-source.{package-reader}; - - /// What a package declares about itself. - record parsed-package { - /// The canonical id: what `canonical-id` returns for `original-id`. - id: string, - /// The id exactly as the package writes it. - original-id: string, - /// The normalized version (what `normalize-version` returns). - version: string, - /// The version exactly as the package writes it. - version-original: string, - /// Whether the version shows up in searches and as "latest". - listed: bool, - /// The manifest (e.g. Chart.yaml, package.json), stored next to the - /// package and handed back through the `manifest` query. Keep it - /// small: it is held in memory. - manifest: list, - dependencies: list, - } - - /// The canonical form of a package id, used for lookups and uniqueness - /// (e.g. lowercased). - canonical-id: func(id: string) -> string; - - /// Whether `id` (canonical) is a well-formed id. The host treats a query - /// or an action naming any other id like one naming an unknown package - /// (an empty answer, a 404). - validate-id: func(id: string) -> result<_, format-error>; - - /// The normalized form of a version, used for lookups and uniqueness. - /// Prerelease versions must contain a `-` (the host filters on it). - normalize-version: func(version: string) -> result; - - /// The order of two normalized versions: negative when `a` sorts before - /// `b`, zero when equal, positive when after. - compare-versions: func(a: string, b: string) -> s8; - - /// Read a package's identity, manifest and dependencies. - parse-package: func(source: borrow) -> result; - - /// A JSON object summarizing `manifest`, stored with the version so that - /// rendering never re-reads the manifest (the `summary` of package-info). - /// None when there is nothing worth keeping. - manifest-summary: func(manifest: list) -> option; -} - -/// The client protocol of the format. -/// -/// The host has authenticated the caller and checked that it may perform the -/// request's action on the repository (GET/HEAD read, PUT/POST write, DELETE -/// delete) before `handle` is called. `handle` decides what the request is: -/// a response it can give right away, data it needs first (then `render` is -/// called with that data), or an action only the host can perform (serve a -/// package, publish one, unlist, relist or delete a version). -interface protocol { - /// How the repository stores its packages. - enum repository-kind { - hosted, - proxy, - group, - } - - /// The repository a request is for. - record repository { - name: string, - kind: repository-kind, - /// The absolute URL the repository is served at, without a trailing - /// slash (e.g. https://forge.example.com/repository/charts). - base-url: string, - } - - /// An HTTP request from a package client. - record request { - /// The method, upper case. - method: string, - /// The path below `base-url`, starting with "/", still - /// percent-encoded as the client sent it. - path: string, - /// The query string without its "?", empty when there is none. - query: string, - /// The request headers the host passes on (lowercase names): accept, - /// accept-language, content-type, if-match, if-modified-since, - /// if-none-match, user-agent. Never credentials or cookies. - headers: list>, - } - - /// An HTTP response. The host keeps only the headers it allows (content - /// type, caching and validators, content-disposition, location) and adds - /// its own security headers. - record response { - status: u16, - headers: list>, - body: list, - } - - /// One version of one package (canonical id, normalized version). - record package-ref { - id: string, - version: string, - } - - /// Some versions of one package. - record package-versions { - id: string, - versions: list, - } - - record search-query { - /// Free text; empty matches every package. `packageid:x` and `id:x` - /// match an id exactly; several terms must all match. - text: string, - skip: u32, - /// The page size: 0 means the default (20), more than 1000 is 1000. - take: u32, - /// Include prerelease versions. - prerelease: bool, - /// Include SemVer 2.0.0 versions (dotted prerelease labels, build - /// metadata). - semver2: bool, - /// Let a proxy repository forward the search to its upstream, so - /// packages it has not cached yet are found too. - upstream: bool, - } - - /// Data the plugin can ask for. Package ids are canonical, versions - /// normalized. Everything the caller may not read (package selectors, - /// group members it cannot see) is left out, exactly as for an unknown - /// package. - variant query { - /// Every version of a package, ascending. - versions(string), - /// Every version of a package with its metadata, ascending. - packages(string), - /// One version with its metadata. - %package(package-ref), - /// Some versions of a package with their metadata, ascending; - /// versions it does not have are left out. - packages-of(package-versions), - /// The manifest stored with one version. - manifest(package-ref), - /// The packages matching a search. - search(search-query), - } - - record version-info { - /// The id as the package writes it. - original-id: string, - version: string, - version-original: string, - listed: bool, - } - - record package-info { - original-id: string, - version: string, - version-original: string, - listed: bool, - /// When the version was published into the repository: RFC 3339 in - /// UTC with milliseconds (2026-10-21T12:00:00.000+00:00). - published: option, - /// The package's SHA-256, lowercase hex. - sha256: option, - /// The package's SHA-512, base64. - sha512: option, - /// The package's size in bytes. - size: option, - /// What `manifest-summary` returned at publish. - summary: option, - } - - record search-hit { - original-id: string, - latest-version: string, - /// Ascending. - versions: list, - /// The summary of the latest version. - summary: option, - /// Downloads of the package, all versions, when the repository counts - /// them. - total-downloads: option, - } - - record search-result { - /// Matching packages before paging. - total: u32, - hits: list, - } - - variant answer { - versions(list), - /// The answer to `packages` and to `packages-of`. - packages(list), - %package(option), - manifest(option>), - search(search-result), - } - - /// A query with its answer. - record loaded { - query: query, - answer: answer, - } - - /// Where a publish request carries the package. - variant upload { - /// The request body is the package. - body, - /// The package is the multipart/form-data part of this name. - multipart-field(string), - /// The package is the first part of a multipart/form-data body, - /// whatever its name. - first-part, - } - - /// A publish, unlist, relist or delete, and the response when it succeeds. - /// When it fails the host answers with its own status (400 invalid - /// package, 403 not allowed, 404 unknown version, 409 version exists, - /// 413 too large). - record publish-request { - upload: upload, - done: response, - } - - record change-request { - target: package-ref, - done: response, - } - - variant action { - /// Answer with this response. - respond(response), - /// Load this data, then call `render` with it. - load(list), - /// Serve the stored package of this version, or 404 when the caller - /// cannot read it. The host streams it and counts the download. - download(package-ref), - /// Publish the package the request carries (hosted repositories, - /// write permission). The host parses it with `parse-package`. - publish(publish-request), - /// Hide a version from searches and "latest" (delete permission). - unlist(change-request), - /// Show an unlisted version again (write permission). - relist(change-request), - /// Remove a version and its bytes (delete permission). - delete(change-request), - } - - variant rendered { - /// Answer with this response. - respond(response), - /// Load more data, then call `render` again with everything loaded - /// so far (the host bounds the rounds and the number of queries). - load(list), - } - - handle: func(repo: repository, req: request) -> action; - - render: func(repo: repository, req: request, data: list) -> rendered; -} - -/// The upstream side of the format's proxy repositories. -/// -/// The host performs every request: it sends the repository's credentials -/// only to the upstream's own origin, bounds each response's size and time, -/// inflates gzip-encoded bodies, refuses link-local and cloud-metadata -/// targets, and caches what it fetched as the plugin says. It also keeps -/// everything else a proxy is: downloading and verifying packages (with -/// `parse-package`), the package delay, the not-found markers, the offline -/// mode. The plugin says what to fetch and reads what came back, so the -/// conversation is stateless: every call gets everything fetched so far. -interface upstream { - use types.{format-error}; - use protocol.{package-ref, package-versions, search-query}; - - /// The upstream a proxy repository mirrors. - record remote { - /// The upstream's URL as the repository is configured with it. - url: string, - /// What `index` answered for it (a JSON object of the plugin's own - /// making); none while the index is being resolved. - index: option, - } - - /// What the host wants to know about the upstream. - variant upstream-op { - /// Resolve the upstream into the index every other operation gets. - /// The host keeps it, refreshes it once per metadata TTL and falls - /// back to the last good one when a refresh fails. - index, - /// The versions the upstream lists of a package (canonical id). - versions(string), - /// The URL of a version's package. The host downloads it, checks it - /// with `parse-package` against the version asked for, and stores it. - %package(package-ref), - /// A version's manifest without its package, verified to be that - /// version's; none sends the host to the package instead. - manifest(package-ref), - /// The upstream's metadata of some versions (listing state, publish - /// time, manifest summary), best effort. - metadata(package-versions), - /// `metadata`, plus when each version was created upstream: the - /// package delay counts from it. It may cost more requests. - dating(package-versions), - /// A page of the upstream's search results. - search(search-query), - } - - /// How long the host may reuse a fetched document. - enum freshness { - /// For the repository's metadata TTL: listings, indexes, search pages. - metadata, - /// For a day: a document that never changes once published. - immutable, - } - - /// A GET the plugin asks the host to send to the upstream. - record fetch { - url: string, - freshness: freshness, - } - - variant fetched { - /// A 2xx, with its body. - found(list), - /// A 404. - not-found, - /// Anything else: a transport error, another status, a body over the - /// size limit. - failed(string), - } - - record fetch-result { - fetch: fetch, - outcome: fetched, - } - - record version-metadata { - /// Normalized. - version: string, - /// The id as this version spells it, when the upstream says. - original-id: option, - listed: bool, - /// When the upstream published the version: RFC 3339, shown to - /// clients. - published: option, - /// When the version was created upstream, RFC 3339 (`dating` only). - created: option, - /// A manifest summary for a version the proxy has not downloaded. - summary: option, - } - - record package-metadata { - /// The id as the upstream spells the package (its newest version's). - original-id: option, - versions: list, - } - - record upstream-hit { - /// The id as the upstream spells it. - id: string, - /// Normalized, ascending. - versions: list, - /// A manifest summary of the latest version. - summary: option, - total-downloads: option, - } - - record upstream-search { - /// Matching packages before paging, as the upstream counts them. - total: u64, - hits: list, - } - - variant upstream-answer { - /// The answer to `index`: a JSON object. - index(string), - /// None: the upstream does not know the package. - versions(option>), - /// The package's URL. - %package(string), - manifest(option>), - /// The answer to `metadata` and `dating`. - metadata(package-metadata), - /// None: the upstream offers no search. - search(option), - /// The upstream answered something unusable: the host serves what - /// it has cached. - failed(string), - } - - variant upstream-step { - /// Fetch these, then call `upstream` again with everything fetched - /// so far (the host bounds the rounds and the number of fetches). - fetch(list), - done(upstream-answer), - } - - upstream: func(remote: remote, op: upstream-op, fetched: list) -> result; -} - -/// What a format plugin is: it exports the format, its protocol and its -/// upstream side, and imports nothing but the package reader (and WASI, -/// which the host provides without any file system, network, environment or -/// arguments). -world format-plugin { - import package-source; - export format; - export protocol; - export upstream; -} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index 1cb7be01..087ad2da 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -6,11 +6,6 @@ rust-version.workspace = true license.workspace = true publish.workspace = true -[features] -# `testing::example_plugin`: builds a plugin of `plugins/` as a component for -# tests. -testing = [] - [dependencies] forge-core = { path = "../core" } forge-plugin-api = { path = "../plugin-api" } @@ -19,20 +14,4 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } -sha2 = { workspace = true } -hex = { workspace = true } toml = "1.1" -# Spools a package handed over as a reader into a file the plugin can be lent. -tempfile = { workspace = true } - -wasmtime = { workspace = true } -wasmtime-wasi = { workspace = true } - -[dev-dependencies] -forge-plugin = { path = ".", features = ["testing"] } -# The Helm example plugin built in, to compare with its component. -forge-format-helm = { path = "../../plugins/helm" } -flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } -# Hand-built components for the sandbox tests: the version Wasmtime itself -# depends on, so bump it with `wasmtime`. -wasm-encoder = "0.258.3" diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index 350cc930..f4483cc2 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -19,38 +19,22 @@ use crate::Plugin; /// larger than about 2,700 bytes: a plugin without a limit of its own let a /// multi-kilobyte version through to that insert, so a push answered 500 /// instead of 400, and so did a proxy asked for such a version. 256 is above -/// the plugins' own limits (NuGet's 64 characters, Helm's 128) and, +/// NuGet's own limit (64 characters) and, /// with [`MAX_ID_LENGTH`], keeps an entry far below PostgreSQL's. const MAX_VERSION_LENGTH: usize = 256; /// The longest package id the host takes from a format plugin, in bytes: /// an id to validate, and a parsed package's id as written and canonical. -/// Ids share the indexes of [`MAX_VERSION_LENGTH`]. 512 is above the -/// plugins' own limits (Helm's 250 ASCII characters, NuGet's 100 -/// characters, at most 400 bytes in UTF-8). +/// Ids share the indexes of [`MAX_VERSION_LENGTH`]. 512 is above NuGet's +/// own limit (100 characters, at most 400 bytes in UTF-8). const MAX_ID_LENGTH: usize = 512; -/// The largest manifest summary, as JSON text, stored from a WebAssembly -/// plugin. Every listing hands each version's summary to the plugin, so a -/// plugin that summarizes without a bound lets a publisher make every -/// listing take gigabytes. Several times the 16 KiB a Helm chart's summary -/// is cut down to, and many times a typical summary. -/// -/// A built-in plugin is Forge's own code and bounds its own summaries, or -/// needs them whole: NuGet's carries a package's dependency groups, and a -/// version served without them would install without its dependencies. -const MAX_SUMMARY: usize = 64 * 1024; - -/// A format plugin, built in or WebAssembly, as a [`Format`]: what the -/// hosted write path, the read ops, the proxy read path and retention work -/// with. +/// A format plugin as a [`Format`]: what the hosted write path, the read +/// ops, the proxy read path and retention work with. pub struct PluginFormat { id: &'static str, plugin: Arc, assets: Assets, - /// The largest manifest summary stored: [`MAX_SUMMARY`] for a - /// WebAssembly plugin, none for a built-in one. - max_summary: Option, } impl PluginFormat { @@ -60,7 +44,6 @@ impl PluginFormat { id: plugin.id(), plugin: plugin.implementation().clone(), assets: plugin.manifest().assets.clone(), - max_summary: (!plugin.is_builtin()).then_some(MAX_SUMMARY), } } } @@ -215,15 +198,6 @@ impl Format for PluginFormat { fn manifest_summary(&self, manifest: &[u8]) -> Option { let text = self.plugin.manifest_summary(manifest)?; - if let Some(max) = self.max_summary.filter(|max| text.len() > *max) { - tracing::warn!( - plugin = self.id, - bytes = text.len(), - max, - "manifest-summary returned more than the host keeps; not stored" - ); - return None; - } match serde_json::from_str::(&text) { Ok(summary @ serde_json::Value::Object(_)) => Some(summary), _ => { @@ -240,7 +214,6 @@ impl Format for PluginFormat { #[cfg(test)] mod tests { use super::*; - use crate::{Manifest, Origin}; /// A format without limits of its own: every id is valid, every version /// normalizes to `normalized` (or to itself), and every package parses @@ -386,83 +359,4 @@ mod tests { assert!(matches!(parse(parsed), Err(FormatError::InvalidPackage(_)))); } } - - /// A plugin whose summary of a manifest is `{"description": }`. - struct Describes; - - impl FormatPlugin for Describes { - fn canonical_id(&self, id: &str) -> String { - id.to_string() - } - - fn validate_id(&self, _id: &str) -> Result<(), api::FormatError> { - Ok(()) - } - - fn normalize_version(&self, version: &str) -> Result { - Ok(version.to_string()) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - a.cmp(b) - } - - fn parse_package( - &self, - _package: &mut dyn api::PackageSource, - ) -> Result { - Err(api::FormatError::InvalidPackage("not a package".into())) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - Some( - serde_json::json!({ "description": String::from_utf8_lossy(manifest) }).to_string(), - ) - } - - fn handle(&self, _repo: &api::Repository, _req: &api::Request) -> api::Action { - api::Action::Respond(api::Response::text(404, "not found")) - } - - fn render( - &self, - _repo: &api::Repository, - _req: &api::Request, - _data: &[api::Loaded], - ) -> api::Rendered { - api::Rendered::Respond(api::Response::text(404, "not found")) - } - } - - fn plugin_toml() -> String { - format!( - "[plugin]\nid = \"describes\"\nname = \"Describes\"\nversion = \"1.0.0\"\n\ - api = \"{}\"\ncomponent = \"describes.wasm\"\n", - api::API - ) - } - - #[test] - fn a_webassembly_plugins_oversized_summary_is_not_stored() { - let manifest = Manifest::parse(&plugin_toml()).unwrap(); - let wasm = Plugin::new( - manifest, - Origin::Wasm { - sha256: "0".repeat(64), - }, - Arc::new(Describes), - ); - let format = PluginFormat::new(&wasm); - // `{"description":"…"}` takes 18 bytes more than the description. - let fits = "x".repeat(MAX_SUMMARY - 18); - let summary = format.manifest_summary(fits.as_bytes()).unwrap(); - assert_eq!(summary["description"], serde_json::json!(fits)); - let over = "x".repeat(MAX_SUMMARY - 17); - assert_eq!(format.manifest_summary(over.as_bytes()), None); - - // A built-in plugin bounds its own. - let builtin = Plugin::builtin(&plugin_toml(), Describes).unwrap(); - let format = PluginFormat::new(&builtin); - assert!(format.manifest_summary(over.as_bytes()).is_some()); - } } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index b5a31cf2..5e67b9b4 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -6,70 +6,28 @@ //! storage, metadata, authentication and authorization, hosted/group //! semantics, retention, the streaming of package bytes. //! -//! A [`Plugin`] is one installed plugin, of either kind: -//! -//! * **built in** ([`Plugin::builtin`]): a crate under `plugins//` -//! linked into the server, called directly; -//! * **WebAssembly** ([`Plugin::load`]): the same kind of crate built as a -//! component, loaded from a plugin directory against the SHA-256 the -//! operator pinned, and run in a sandbox ([`WasmPlugin`], which implements -//! [`FormatPlugin`] by calling into the component). -//! -//! Either way the server holds an `Arc` and cannot tell the -//! two apart. [`PluginFormat`] adapts a plugin to [`forge_core::Format`], so -//! hosted publishing, reads, groups and retention work for it unchanged. +//! A [`Plugin`] is one installed plugin: a crate under `plugins//` linked +//! into the server ([`Plugin::builtin`]) and called directly. The server holds +//! it as an `Arc`; [`PluginFormat`] adapts it to +//! [`forge_core::Format`], so hosted publishing, reads, groups and retention +//! work for it unchanged. mod format; pub mod manifest; -#[cfg(feature = "testing")] -pub mod testing; -pub mod wasm; -use std::path::Path; use std::sync::Arc; pub use forge_plugin_api as api; pub use forge_plugin_api::FormatPlugin; pub use format::PluginFormat; pub use manifest::Manifest; -pub use wasm::{HostLimits, Limits, WasmPlugin}; -/// Errors loading a plugin. +/// Errors installing a plugin. #[derive(Debug, thiserror::Error)] pub enum PluginError { - /// The plugin directory or component could not be read. - #[error("{0}")] - Io(String), /// `plugin.toml` is missing a key, malformed, or asks for too much. #[error("invalid plugin.toml: {0}")] Manifest(String), - /// The component's SHA-256 is not the pinned one. - #[error("component digest {actual} does not match the pinned {expected}")] - Digest { - /// The pinned digest. - expected: String, - /// The component's digest. - actual: String, - }, - /// The component does not compile or does not implement the world. - #[error("invalid component: {0}")] - Component(String), - /// A call into a component trapped, ran out of fuel or time, or its - /// instance could not be created. - #[error("plugin call failed: {0}")] - Trap(String), -} - -/// Where a plugin's code comes from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Origin { - /// Linked into the server. - Builtin, - /// A WebAssembly component. - Wasm { - /// The component's SHA-256 (lowercase hex). - sha256: String, - }, } /// An installed format plugin: its `plugin.toml` and its implementation. @@ -78,7 +36,6 @@ pub struct Plugin { /// The format id, leaked once per plugin so [`forge_core::Format::id`] /// can hand out a `&'static str`. id: &'static str, - origin: Origin, implementation: Arc, } @@ -87,7 +44,6 @@ impl std::fmt::Debug for Plugin { f.debug_struct("Plugin") .field("id", &self.id) .field("version", &self.manifest.version) - .field("origin", &self.origin) .finish_non_exhaustive() } } @@ -101,55 +57,11 @@ impl Plugin { implementation: impl FormatPlugin, ) -> Result, PluginError> { let manifest = Manifest::parse(plugin_toml)?; - Ok(Self::new( - manifest, - Origin::Builtin, - Arc::new(implementation), - )) - } - - /// Load the WebAssembly plugin in `dir`: read `plugin.toml`, check the - /// component against `expected_sha256` (hex, optionally `sha256:`-prefixed; - /// an operator pins every plugin), resolve its limits against `host`, and - /// compile it. - pub fn load( - dir: &Path, - expected_sha256: &str, - host: &HostLimits, - ) -> Result, PluginError> { - let manifest = Manifest::load(dir)?; - let Some(component) = &manifest.component else { - return Err(PluginError::Manifest( - "a WebAssembly plugin names its `component`".into(), - )); - }; - let path = dir.join(component); - let bytes = std::fs::read(&path) - .map_err(|e| PluginError::Io(format!("read {}: {e}", path.display())))?; - Self::from_parts(manifest, &bytes, expected_sha256, host) - } - - /// [`Self::load`] from a manifest and the component's bytes. - pub fn from_parts( - manifest: Manifest, - component: &[u8], - expected_sha256: &str, - host: &HostLimits, - ) -> Result, PluginError> { - let wasm = WasmPlugin::new(&manifest, component, expected_sha256, host)?; - let origin = Origin::Wasm { - sha256: wasm.sha256().to_string(), - }; - Ok(Self::new(manifest, origin, Arc::new(wasm))) - } - - fn new(manifest: Manifest, origin: Origin, implementation: Arc) -> Arc { - Arc::new(Plugin { + Ok(Arc::new(Plugin { id: Box::leak(manifest.id.clone().into_boxed_str()), manifest, - origin, - implementation, - }) + implementation: Arc::new(implementation), + })) } /// The format id. @@ -162,24 +74,6 @@ impl Plugin { &self.manifest } - /// Where the plugin's code comes from. - pub fn origin(&self) -> &Origin { - &self.origin - } - - /// Whether the plugin is linked into the server. - pub fn is_builtin(&self) -> bool { - self.origin == Origin::Builtin - } - - /// The component's SHA-256 (WebAssembly plugins only). - pub fn sha256(&self) -> Option<&str> { - match &self.origin { - Origin::Builtin => None, - Origin::Wasm { sha256 } => Some(sha256), - } - } - /// The format, as the plugin implements it. pub fn implementation(&self) -> &Arc { &self.implementation diff --git a/crates/plugin/src/manifest.rs b/crates/plugin/src/manifest.rs index dc161c5d..b77d8217 100644 --- a/crates/plugin/src/manifest.rs +++ b/crates/plugin/src/manifest.rs @@ -1,16 +1,13 @@ //! `plugin.toml`: what a plugin declares about itself, next to its crate's -//! `Cargo.toml` (`plugins//plugin.toml`) and in an installed plugin -//! directory. +//! `Cargo.toml` (`plugins//plugin.toml`). //! //! ```toml //! [plugin] -//! id = "helm" # the format id repositories are created with -//! name = "Helm charts" -//! version = "0.1.0" # the plugin's own version (informational) +//! id = "nuget" # the format id repositories are created with +//! name = "NuGet" +//! version = "1.0.0" # the plugin's own version (informational) //! api = "0.3.0" # the forge:format contract it is built against -//! component = "helm.wasm" # built as a component: its file, relative to -//! # the directory (unused when built in) -//! mount = "/repository" # optional: served at {mount}/{repository}/… +//! mount = "/v3" # optional: served at {mount}/{repository}/… //! kinds = ["hosted", "group"] # optional: the repository kinds it supports //! # ("proxy" too, when it implements `upstream`) //! @@ -19,15 +16,8 @@ //! package_type = "application/octet-stream" //! manifest = "manifest" //! manifest_type = "application/octet-stream" -//! -//! [limits] # as a component; each capped by the host's maximum -//! memory_mib = 64 -//! fuel = 2000000000 -//! timeout_ms = 2000 //! ``` -use std::path::{Component, Path, PathBuf}; - use forge_plugin_api::RepositoryKind; use serde::Deserialize; @@ -92,9 +82,6 @@ pub struct Manifest { pub api: String, /// Optional one-line description. pub description: Option, - /// Path of the component, relative to the plugin directory (WebAssembly - /// plugins). - pub component: Option, /// Where repositories of the format are served: `{mount}/{repository}/…`. pub mount: String, /// The repository kinds the format supports. `proxy` means the plugin @@ -102,8 +89,6 @@ pub struct Manifest { pub kinds: Vec, /// How a version's files are stored. pub assets: Assets, - /// Requested limits as a component (`None` → the host default). - pub limits: RequestedLimits, } /// The `[assets]` table: the names (within a version) and content types of @@ -128,27 +113,12 @@ impl Default for Assets { } } -/// The `[limits]` table. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RequestedLimits { - /// Memory ceiling per instance, in MiB: its linear memories and tables - /// together, and the most one call returns to the host. - pub memory_mib: Option, - /// Fuel (roughly, WebAssembly instructions) per call. - pub fuel: Option, - /// Wall-clock budget per call, in milliseconds. - pub timeout_ms: Option, -} - #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawManifest { plugin: RawPlugin, #[serde(default)] assets: Assets, - #[serde(default)] - limits: RequestedLimits, } #[derive(Deserialize)] @@ -158,7 +128,6 @@ struct RawPlugin { name: String, version: String, api: String, - component: Option, description: Option, mount: Option, kinds: Option>, @@ -177,18 +146,6 @@ impl Manifest { p.api ))); } - // The component is a file inside the plugin directory: a plain - // relative path, never `..`, absolute, or a drive/root prefix. - let component = p.component.map(|raw| { - let path = PathBuf::from(&raw); - if raw.is_empty() || !path.components().all(|c| matches!(c, Component::Normal(_))) { - return Err(PluginError::Manifest(format!( - "component {raw:?} must be a relative path inside the plugin directory" - ))); - } - Ok(path) - }); - let component = component.transpose()?; if p.name.trim().is_empty() { return Err(PluginError::Manifest("name must not be empty".into())); } @@ -205,21 +162,11 @@ impl Manifest { version: p.version, api: p.api, description: p.description, - component, mount, kinds, assets: raw.assets, - limits: raw.limits, }) } - - /// Read and parse `/plugin.toml`. - pub fn load(dir: &Path) -> Result { - let path = dir.join("plugin.toml"); - let text = std::fs::read_to_string(&path) - .map_err(|e| PluginError::Io(format!("read {}: {e}", path.display())))?; - Self::parse(&text) - } } /// A mount: `/` and one path segment of 2–32 lowercase letters, digits and @@ -341,19 +288,14 @@ mod tests { name = "Helm charts" version = "0.1.0" api = "0.3.0" - component = "helm.wasm" - - [limits] - memory_mib = 32 "#; #[test] fn parses_a_manifest() { let m = Manifest::parse(GOOD).unwrap(); assert_eq!(m.id, "helm"); - assert_eq!(m.component, Some(PathBuf::from("helm.wasm"))); - assert_eq!(m.limits.memory_mib, Some(32)); - assert_eq!(m.limits.fuel, None); + assert_eq!(m.name, "Helm charts"); + assert_eq!(m.description, None); } #[test] @@ -376,19 +318,6 @@ mod tests { assert!(validate_id("go-modules").is_ok()); } - #[test] - fn refuses_components_outside_the_directory() { - for path in ["../helm.wasm", "/etc/passwd", "a/../../b.wasm", ""] { - let text = GOOD.replace("\"helm.wasm\"", &format!("{path:?}")); - assert!(Manifest::parse(&text).is_err(), "{path:?} accepted"); - } - let text = GOOD.replace("\"helm.wasm\"", "\"lib/helm.wasm\""); - assert!(Manifest::parse(&text).is_ok()); - // A built-in plugin needs none. - let text = GOOD.replace("component = \"helm.wasm\"", ""); - assert_eq!(Manifest::parse(&text).unwrap().component, None); - } - #[test] fn mounts_kinds_and_assets_default_and_validate() { let m = Manifest::parse(GOOD).unwrap(); @@ -398,8 +327,8 @@ mod tests { let with = |plugin_keys: &str, tables: &str| { let text = GOOD.replace( - "component = \"helm.wasm\"", - &format!("component = \"helm.wasm\"\n{plugin_keys}"), + "api = \"0.3.0\"", + &format!("api = \"0.3.0\"\n{plugin_keys}"), ); Manifest::parse(&format!("{text}\n{tables}")) }; @@ -476,6 +405,8 @@ mod tests { // 0.1.0 was a preview that no release shipped. assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"0.2.0\"")).is_err()); assert!(Manifest::parse(&format!("{GOOD}\nunknown = 1\n")).is_err()); - assert!(Manifest::parse(&GOOD.replace("memory_mib", "memory_gib")).is_err()); + // What a WebAssembly component named is gone with it. + assert!(Manifest::parse(&format!("{GOOD}component = \"helm.wasm\"\n")).is_err()); + assert!(Manifest::parse(&format!("{GOOD}\n[limits]\nmemory_mib = 32\n")).is_err()); } } diff --git a/crates/plugin/src/testing.rs b/crates/plugin/src/testing.rs deleted file mode 100644 index d4a6d5ef..00000000 --- a/crates/plugin/src/testing.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Test support: build the plugins of `plugins/` as WebAssembly components -//! (feature `testing`). - -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::Mutex; - -/// Build `plugins/` as a component (once per process) and install it as -/// an operator would: a plugin directory holding its `plugin.toml` and the -/// component. -/// -/// Needs the `wasm32-wasip2` target (`rustup target add wasm32-wasip2`). -pub fn example_plugin(name: &str) -> PathBuf { - let source = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../plugins") - .join(name); - let id = crate::Manifest::load(&source) - .expect("example plugin.toml") - .id; - example_plugin_as(name, &id) -} - -/// [`example_plugin`], installed under format id `id` (its `plugin.toml` -/// rewritten): a component of a plugin that is also built in, under an id of -/// its own. -pub fn example_plugin_as(name: &str, id: &str) -> PathBuf { - example_plugin_with(name, id, "") -} - -/// [`example_plugin_as`], with `plugin_keys` (TOML lines such as -/// `mount = "/charts"`) added to its `[plugin]` table. -pub fn example_plugin_with(name: &str, id: &str, plugin_keys: &str) -> PathBuf { - static BUILT: Mutex> = Mutex::new(Vec::new()); - let mut built = BUILT.lock().unwrap_or_else(|p| p.into_inner()); - if let Some((_, _, dir)) = built - .iter() - .find(|(i, keys, _)| i == id && keys == plugin_keys) - { - return dir.clone(); - } - let workspace = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); - let source = workspace.join("plugins").join(name); - let target = workspace.join("target/plugins"); - let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); - let status = Command::new(cargo) - .args([ - "rustc", - "--locked", - "--lib", - "--crate-type", - "cdylib", - "--profile", - "plugin", - "--target", - "wasm32-wasip2", - "--features", - "component", - ]) - .arg("--manifest-path") - .arg(source.join("Cargo.toml")) - .arg("--target-dir") - .arg(&target) - // The outer build's flags are for the host, not for wasm. - .env_remove("RUSTFLAGS") - .env_remove("CARGO_ENCODED_RUSTFLAGS") - .env_remove("CARGO_BUILD_TARGET") - .status() - .expect("run cargo to build the example plugin"); - assert!( - status.success(), - "building plugins/{name} failed (is the wasm32-wasip2 target installed? \ - `rustup target add wasm32-wasip2`)" - ); - let text = std::fs::read_to_string(source.join("plugin.toml")).expect("read plugin.toml"); - let text: String = text - .lines() - .map(|line| { - let key = line.split('=').next().unwrap_or("").trim(); - if key == "id" { - format!("id = {id:?}\n") - } else if line.trim() == "[plugin]" { - format!("{line}\n{plugin_keys}\n") - } else { - format!("{line}\n") - } - }) - .collect(); - let manifest = crate::Manifest::parse(&text).expect("example plugin.toml"); - let component = manifest - .component - .expect("the example plugin.toml names its component"); - let dir = target.join("installed").join(id); - std::fs::create_dir_all(&dir).expect("create the plugin directory"); - std::fs::write(dir.join("plugin.toml"), text).expect("write plugin.toml"); - let wasm = target.join("wasm32-wasip2/plugin").join(&component); - std::fs::copy(&wasm, dir.join(&component)).expect("copy the component"); - built.push((id.to_string(), plugin_keys.to_string(), dir.clone())); - dir -} - -/// The SHA-256 (lowercase hex) of the component in plugin directory `dir`, -/// what an operator pins. -pub fn component_sha256(dir: &Path) -> String { - use sha2::Digest; - let manifest = crate::Manifest::load(dir).expect("plugin.toml"); - let component = manifest.component.expect("plugin.toml names a component"); - let bytes = std::fs::read(dir.join(component)).expect("read the component"); - hex::encode(sha2::Sha256::digest(bytes)) -} diff --git a/crates/plugin/src/wasm/bindings.rs b/crates/plugin/src/wasm/bindings.rs deleted file mode 100644 index abbae03e..00000000 --- a/crates/plugin/src/wasm/bindings.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Host bindings generated from `crates/plugin-api/wit/format.wit`. - -wasmtime::component::bindgen!({ - world: "format-plugin", - path: "../plugin-api/wit", - // The package reader the host lends to `parse-package` is a staged file. - with: { - "forge:format/package-source.package-reader": crate::wasm::host::StagedPackage, - }, - // A host function may trap the guest (an unknown resource handle). - imports: { default: trappable }, -}); diff --git a/crates/plugin/src/wasm/executor.rs b/crates/plugin/src/wasm/executor.rs deleted file mode 100644 index c2636624..00000000 --- a/crates/plugin/src/wasm/executor.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! The threads guest code runs on. -//! -//! Guest code never runs on a tokio worker: the WASI bindings answer a -//! guest's stdout/stderr writes (a panic message, say) by blocking on a tokio -//! runtime, which panics inside an asynchronous context. Every call into a -//! component is therefore handed to a dedicated thread, and the caller -//! blocks until the result is there. -//! -//! There are two pools, so that the calls a request makes synchronously — the -//! [`forge_core::Format`] rules, microseconds each, waited for on a tokio -//! worker — never queue behind long ones (rendering a large index, parsing a -//! package) and stall the server's workers while they wait. - -use std::sync::mpsc::{self, Receiver, Sender}; -use std::sync::{Arc, Mutex, OnceLock}; - -use crate::PluginError; - -type Job = Box; - -/// Which pool a call runs on. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Pool { - /// The format's rules (ids, versions, summaries): short calls a request - /// waits for synchronously. - Quick, - /// Parsing packages and the client protocol: calls that may take as long - /// as the plugin's time limit. - Slow, -} - -/// At most this many threads per pool (calls queue beyond it). -const MAX_THREADS: usize = 8; - -struct Executor { - jobs: Sender, -} - -fn executor(pool: Pool) -> &'static Executor { - static QUICK: OnceLock = OnceLock::new(); - static SLOW: OnceLock = OnceLock::new(); - let (cell, name) = match pool { - Pool::Quick => (&QUICK, "quick"), - Pool::Slow => (&SLOW, "slow"), - }; - cell.get_or_init(|| { - let (jobs, queue) = mpsc::channel::(); - let queue = Arc::new(Mutex::new(queue)); - let threads = std::thread::available_parallelism() - .map_or(2, |n| n.get()) - .clamp(2, MAX_THREADS); - for i in 0..threads { - let queue = Arc::clone(&queue); - std::thread::Builder::new() - .name(format!("forge-plugin-{name}-{i}")) - .spawn(move || worker(&queue)) - .expect("spawn a plugin thread"); - } - Executor { jobs } - }) -} - -fn worker(queue: &Mutex>) { - loop { - // Hold the lock only to take a job, not while running it. - let job = match queue.lock() { - Ok(queue) => queue.recv(), - Err(poisoned) => poisoned.into_inner().recv(), - }; - let Ok(job) = job else { - return; - }; - // A panicking job (a host bug) must not take the thread with it; the - // caller sees its result channel closed. - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(job)); - } -} - -/// Run `f` on a thread of `pool` and block until it returns. -pub(crate) fn run_blocking(pool: Pool, f: F) -> Result -where - R: Send + 'static, - F: FnOnce() -> R + Send + 'static, -{ - let (tx, rx) = mpsc::sync_channel(1); - executor(pool) - .jobs - .send(Box::new(move || { - let _ = tx.send(f()); - })) - .map_err(|_| PluginError::Trap("the plugin threads are gone".into()))?; - rx.recv() - .map_err(|_| PluginError::Trap("the plugin call panicked".into())) -} diff --git a/crates/plugin/src/wasm/host.rs b/crates/plugin/src/wasm/host.rs deleted file mode 100644 index 78784626..00000000 --- a/crates/plugin/src/wasm/host.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! The per-instance host state and the functions a plugin imports. - -use std::fs::File; -use std::io::{Read, Seek, SeekFrom}; - -use wasmtime::component::{Resource, ResourceTable}; -use wasmtime::ResourceLimiter; -use wasmtime_wasi::{WasiCtx, WasiCtxView, WasiView}; - -use super::bindings::forge::format::{package_source, types}; - -/// The most `read-at` returns in one call. -const MAX_READ: u32 = 1024 * 1024; - -/// What a plugin instance's store holds: an empty WASI context (no files, -/// network, environment or arguments; stdout and stderr discard), the -/// resource table, and the memory budget. -pub(crate) struct HostState { - wasi: WasiCtx, - table: ResourceTable, - pub(crate) budget: Budget, -} - -impl HostState { - /// The state of an instance whose memories and tables may hold - /// `memory_bytes` together. WASI's random bytes are capped the same way: - /// they are the one thing a guest can ask the host to allocate for it. - pub(crate) fn new(memory_bytes: usize) -> Self { - let wasi = WasiCtx::builder() - .allow_tcp(false) - .allow_udp(false) - .allow_ip_name_lookup(false) - .max_random_size(u64::try_from(memory_bytes).unwrap_or(u64::MAX)) - .build(); - HostState { - wasi, - table: ResourceTable::new(), - budget: Budget::new(memory_bytes), - } - } - - /// Lend `package` to the guest for one call. - pub(crate) fn lend( - &mut self, - package: StagedPackage, - ) -> wasmtime::Result> { - Ok(self.table.push(package)?) - } - - /// Take a lent package back after the call. - pub(crate) fn reclaim(&mut self, package: Resource) { - let _ = self.table.delete(package); - } -} - -impl WasiView for HostState { - fn ctx(&mut self) -> WasiCtxView<'_> { - WasiCtxView { - ctx: &mut self.wasi, - table: &mut self.table, - } - } -} - -/// Core instances a store may create: a Rust component instantiates a few -/// (its module, WASI adapters and shims). -const MAX_INSTANCES: usize = 64; - -/// Tables a store may create. -const MAX_TABLES: usize = 64; - -/// Linear memories a store may create. They share one [`Budget`], so this -/// bounds bookkeeping, not bytes. -const MAX_MEMORIES: usize = 16; - -/// What one table element holds on the host: a pointer, in Wasmtime. -const TABLE_ELEMENT_BYTES: usize = std::mem::size_of::(); - -/// One memory ceiling for everything an instance allocates on its behalf: -/// all its linear memories and all its tables together, counted in bytes. -/// -/// Wasmtime's own limits apply per memory and leave tables unbounded, so an -/// instance with several memories, or a `table.grow` to hundreds of millions -/// of elements, would take far more host memory than its ceiling. A growth -/// past the budget traps the guest, which fails the call and discards the -/// instance. -pub(crate) struct Budget { - /// What the memories and tables may hold together, in bytes. - limit: usize, - /// What they hold now. - used: usize, - /// What the last growth allowed added, given back when that growth then - /// fails (past a memory's declared maximum, say). - pending: usize, -} - -impl Budget { - fn new(limit: usize) -> Self { - Budget { - limit, - used: 0, - pending: 0, - } - } - - /// Take `bytes` more out of the budget, or refuse with a trap. - fn grow(&mut self, bytes: usize, what: &str) -> wasmtime::Result { - match self.used.checked_add(bytes) { - Some(used) if used <= self.limit => { - self.used = used; - self.pending = bytes; - Ok(true) - } - _ => Err(wasmtime::format_err!( - "growing a {what} by {bytes} bytes exceeds the plugin's memory limit of {} \ - bytes ({} in use)", - self.limit, - self.used - )), - } - } - - /// Give back what the growth that just failed took. - fn grow_failed(&mut self) { - self.used -= std::mem::take(&mut self.pending); - } -} - -// Wasmtime asks before it creates a memory or table (`current` 0) as well as -// before every growth, so the budget sees every byte. -impl ResourceLimiter for Budget { - fn memory_growing( - &mut self, - current: usize, - desired: usize, - _maximum: Option, - ) -> wasmtime::Result { - self.grow(desired.saturating_sub(current), "linear memory") - } - - fn memory_grow_failed(&mut self, _error: wasmtime::Error) -> wasmtime::Result<()> { - self.grow_failed(); - Ok(()) - } - - fn table_growing( - &mut self, - current: usize, - desired: usize, - _maximum: Option, - ) -> wasmtime::Result { - let elements = desired.saturating_sub(current); - self.grow(elements.saturating_mul(TABLE_ELEMENT_BYTES), "table") - } - - fn table_grow_failed(&mut self, _error: wasmtime::Error) -> wasmtime::Result<()> { - self.grow_failed(); - Ok(()) - } - - fn instances(&self) -> usize { - MAX_INSTANCES - } - - fn tables(&self) -> usize { - MAX_TABLES - } - - fn memories(&self) -> usize { - MAX_MEMORIES - } -} - -/// A package the host staged on disk, lent to `parse-package`. -pub struct StagedPackage { - file: File, - size: u64, -} - -impl StagedPackage { - /// Wrap an open package file. - pub(crate) fn new(file: File) -> std::io::Result { - let size = file.metadata()?.len(); - Ok(StagedPackage { file, size }) - } - - fn read_at(&mut self, offset: u64, len: u32) -> std::io::Result> { - if offset >= self.size { - return Ok(Vec::new()); - } - let len = u64::from(len.min(MAX_READ)).min(self.size - offset); - self.file.seek(SeekFrom::Start(offset))?; - let mut buf = Vec::with_capacity(len as usize); - (&mut self.file).take(len).read_to_end(&mut buf)?; - Ok(buf) - } -} - -// Both interfaces are imported for their types; only the reader has functions. -impl types::Host for HostState {} - -impl package_source::Host for HostState {} - -impl package_source::HostPackageReader for HostState { - fn size(&mut self, package: Resource) -> wasmtime::Result { - Ok(self.table.get(&package)?.size) - } - - fn read_at( - &mut self, - package: Resource, - offset: u64, - len: u32, - ) -> wasmtime::Result, String>> { - let package = self.table.get_mut(&package)?; - Ok(package.read_at(offset, len).map_err(|e| e.to_string())) - } - - fn drop(&mut self, package: Resource) -> wasmtime::Result<()> { - self.table.delete(package)?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[test] - fn reads_are_bounded_by_the_file_and_the_chunk_limit() { - let mut file = tempfile::tempfile().unwrap(); - file.write_all(b"0123456789").unwrap(); - let mut staged = StagedPackage::new(file).unwrap(); - assert_eq!(staged.size, 10); - assert_eq!(staged.read_at(0, 4).unwrap(), b"0123"); - assert_eq!(staged.read_at(8, 100).unwrap(), b"89"); - assert!(staged.read_at(10, 1).unwrap().is_empty()); - assert!(staged.read_at(u64::MAX, u32::MAX).unwrap().is_empty()); - } -} diff --git a/crates/plugin/src/wasm/mod.rs b/crates/plugin/src/wasm/mod.rs deleted file mode 100644 index baef11b3..00000000 --- a/crates/plugin/src/wasm/mod.rs +++ /dev/null @@ -1,826 +0,0 @@ -//! WebAssembly format plugins: a [component] implementing the `forge:format` -//! world (`crates/plugin-api/wit/format.wit`), run by Wasmtime. -//! -//! [`WasmPlugin`] implements [`FormatPlugin`] by calling into the component, -//! so a loaded component serves exactly like a built-in plugin. -//! -//! ## Sandbox -//! Every call runs in an instance with a memory ceiling, a fuel budget and a -//! wall-clock deadline (epoch interruption), on a dedicated thread (see -//! `executor`). The memory ceiling covers the instance's linear memories and -//! tables together, and what one call may hand back to the host (see -//! [`store`]). A trap — out of fuel, past the deadline, out of memory, a -//! guest panic — fails that call and discards the instance. The WASI context -//! is empty: no files, no sockets, no environment, no arguments; stdout and -//! stderr go nowhere. A component reaches the network only through the GETs -//! the host sends for a proxy's upstream conversation, and the host sends -//! them only to the origins that upstream points at -//! (`forge_core::net::origins`). -//! -//! A failed call cannot fail the infallible trait methods: they log it and -//! answer as for an unknown input — [`FormatPlugin::canonical_id`] the id -//! unchanged, [`FormatPlugin::compare_versions`] byte order, -//! [`FormatPlugin::manifest_summary`] none — and the protocol methods answer -//! with a 500 response. The fallible ones return -//! [`FormatError::Internal`]. -//! -//! [component]: https://component-model.bytecodealliance.org/ - -mod bindings; -mod executor; -mod host; - -use std::cmp::Ordering; -use std::collections::HashMap; -use std::io::SeekFrom; -use std::path::Path; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::Duration; - -use forge_plugin_api::{ - Action, FetchResult, FormatError, FormatPlugin, Loaded, PackageSource, ParsedPackage, Remote, - Rendered, Repository, Request, Response, UpstreamOp, UpstreamStep, -}; -use sha2::{Digest, Sha256}; -use wasmtime::component::{Component, Linker}; -use wasmtime::{Config, Engine, Store, Trap}; - -use crate::{manifest, Manifest, PluginError}; -use bindings::{FormatPlugin as Bindings, FormatPluginPre}; -use executor::Pool; -use host::{HostState, StagedPackage}; - -forge_plugin_api::__wit_conversions!(crate::wasm::bindings); - -/// The most a plugin may ask for, and what it gets when it asks for nothing. -#[derive(Debug, Clone)] -pub struct HostLimits { - /// Memory ceiling per instance when the plugin names none (MiB). - pub default_memory_mib: u64, - /// The largest memory ceiling a plugin may request (MiB). - pub max_memory_mib: u64, - /// Fuel per call when the plugin names none. - pub default_fuel: u64, - /// The most fuel per call a plugin may request. - pub max_fuel: u64, - /// Wall-clock budget per call when the plugin names none. - pub default_timeout: Duration, - /// The longest wall-clock budget a plugin may request. - pub max_timeout: Duration, -} - -impl Default for HostLimits { - fn default() -> Self { - HostLimits { - default_memory_mib: 64, - max_memory_mib: 512, - default_fuel: 5_000_000_000, - max_fuel: 50_000_000_000, - default_timeout: Duration::from_secs(5), - max_timeout: Duration::from_secs(30), - } - } -} - -/// The limits one plugin runs under. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Limits { - /// Memory ceiling per instance, in bytes: its linear memories and tables - /// together, and the most one call hands back to the host. - pub memory_bytes: usize, - /// Fuel per call. - pub fuel: u64, - /// Wall-clock budget per call. - pub timeout: Duration, -} - -impl HostLimits { - /// What `requested` resolves to, or why it is refused. - pub fn resolve(&self, requested: &manifest::RequestedLimits) -> Result { - let memory_mib = requested.memory_mib.unwrap_or(self.default_memory_mib); - if memory_mib == 0 || memory_mib > self.max_memory_mib { - return Err(PluginError::Manifest(format!( - "limits.memory_mib {memory_mib} is outside 1..={}", - self.max_memory_mib - ))); - } - let fuel = requested.fuel.unwrap_or(self.default_fuel); - if fuel == 0 || fuel > self.max_fuel { - return Err(PluginError::Manifest(format!( - "limits.fuel {fuel} is outside 1..={}", - self.max_fuel - ))); - } - let timeout = requested - .timeout_ms - .map_or(self.default_timeout, Duration::from_millis); - if timeout.is_zero() || timeout > self.max_timeout { - return Err(PluginError::Manifest(format!( - "limits.timeout_ms {} is outside 1..={}", - timeout.as_millis(), - self.max_timeout.as_millis() - ))); - } - let memory_bytes = usize::try_from(memory_mib * 1024 * 1024) - .map_err(|_| PluginError::Manifest("limits.memory_mib is too large".into()))?; - Ok(Limits { - memory_bytes, - fuel, - timeout, - }) - } -} - -/// How often the engine's epoch advances: the granularity of call deadlines. -const EPOCH_TICK: Duration = Duration::from_millis(10); - -/// An instance is replaced after this many calls, so whatever a guest leaks -/// (allocator fragmentation, a growing cache) is bounded. -const MAX_CALLS_PER_INSTANCE: u32 = 10_000; - -/// Idle instances kept per plugin. -const MAX_IDLE_INSTANCES: usize = 16; - -/// The engine every plugin is compiled for, and the thread advancing its -/// epoch. -fn engine() -> Result<&'static Engine, PluginError> { - static ENGINE: OnceLock> = OnceLock::new(); - ENGINE - .get_or_init(|| { - let mut config = Config::new(); - config - .wasm_component_model(true) - .consume_fuel(true) - .epoch_interruption(true) - .max_wasm_stack(1024 * 1024); - let engine = Engine::new(&config).map_err(|e| e.to_string())?; - let ticker = engine.clone(); - std::thread::Builder::new() - .name("forge-plugin-epoch".into()) - .spawn(move || loop { - std::thread::sleep(EPOCH_TICK); - ticker.increment_epoch(); - }) - .map_err(|e| e.to_string())?; - Ok(engine) - }) - .as_ref() - .map_err(|e| PluginError::Component(format!("engine: {e}"))) -} - -/// Compile `bytes` (whose SHA-256 is `sha256`), or reuse the compilation of -/// the same component: loading a plugin again (other limits, a reload) does -/// not pay for Cranelift twice. -fn compile(engine: &Engine, sha256: &str, bytes: &[u8]) -> Result { - static COMPILED: Mutex> = Mutex::new(Vec::new()); - let mut compiled = COMPILED.lock().unwrap_or_else(|p| p.into_inner()); - if let Some((_, component)) = compiled.iter().find(|(sha, _)| sha == sha256) { - return Ok(component.clone()); - } - let component = - Component::new(engine, bytes).map_err(|e| PluginError::Component(format!("{e:#}")))?; - compiled.push((sha256.to_string(), component.clone())); - Ok(component) -} - -/// A store for one instance of a plugin running under `limits`. -/// -/// `memory_bytes` bounds what the instance takes from the host: -/// -/// * its linear memories and tables, all together ([`host::Budget`]); -/// * what one call hands back: Wasmtime copies every string and list of a -/// result into the host, once per reference, so a result whose list -/// elements all point at the same large string would otherwise cost the -/// host its length times their number. Wasmtime charges each copy against -/// the store's hostcall fuel and fails the call once it is spent; -/// * the random bytes it may ask WASI for at once. -fn store(engine: &Engine, limits: &Limits) -> Store { - let mut store = Store::new(engine, HostState::new(limits.memory_bytes)); - store.limiter(|state| &mut state.budget); - store.set_hostcall_fuel(limits.memory_bytes); - store -} - -/// Comparisons remembered per plugin: sorting a package's versions asks the -/// plugin once per pair, and the same packages are listed over and over. -const MAX_REMEMBERED_COMPARISONS: usize = 65_536; - -/// A format plugin running as a WebAssembly component. -pub struct WasmPlugin { - inner: Arc, - comparisons: Mutex>, -} - -/// What the plugin threads share of a [`WasmPlugin`]. -struct Inner { - /// The format id, for logs. - id: String, - sha256: String, - limits: Limits, - pre: FormatPluginPre, - idle: Mutex>, -} - -/// An instantiated plugin, reused across calls. -struct Warm { - store: Store, - plugin: Bindings, - calls: u32, -} - -impl std::fmt::Debug for WasmPlugin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WasmPlugin") - .field("id", &self.inner.id) - .field("sha256", &self.inner.sha256) - .field("limits", &self.inner.limits) - .finish_non_exhaustive() - } -} - -impl WasmPlugin { - /// Check `component` against `expected_sha256` (hex, optionally - /// `sha256:`-prefixed), resolve the limits `manifest` asks for against - /// `host`, compile it, and instantiate it once. - pub fn new( - manifest: &Manifest, - component: &[u8], - expected_sha256: &str, - host: &HostLimits, - ) -> Result { - let actual = hex::encode(Sha256::digest(component)); - let expected = expected_sha256.trim().to_ascii_lowercase(); - let expected = expected.strip_prefix("sha256:").unwrap_or(&expected); - if actual != expected { - return Err(PluginError::Digest { - expected: expected.to_string(), - actual, - }); - } - let limits = host.resolve(&manifest.limits)?; - - let engine = engine()?; - let component = compile(engine, &actual, component)?; - let mut linker = Linker::::new(engine); - wasmtime_wasi::p2::add_to_linker_sync(&mut linker) - .map_err(|e| PluginError::Component(format!("link WASI: {e:#}")))?; - Bindings::add_to_linker::<_, wasmtime::component::HasSelf>( - &mut linker, - |state| state, - ) - .map_err(|e| PluginError::Component(format!("link host functions: {e:#}")))?; - let pre = linker - .instantiate_pre(&component) - .map_err(|e| PluginError::Component(format!("{e:#}")))?; - let pre = - FormatPluginPre::new(pre).map_err(|e| PluginError::Component(format!("{e:#}")))?; - - let inner = Arc::new(Inner { - id: manifest.id.clone(), - sha256: actual, - limits, - pre, - idle: Mutex::new(Vec::new()), - }); - // Instantiate once now, so a component that cannot start fails the - // load instead of the first request (on a plugin thread, like every - // run of guest code). - let starting = Arc::clone(&inner); - let warm = executor::run_blocking(Pool::Slow, move || starting.instantiate())??; - inner.park(warm); - Ok(WasmPlugin { - inner, - comparisons: Mutex::new(HashMap::new()), - }) - } - - /// The component's SHA-256 (lowercase hex). - pub fn sha256(&self) -> &str { - &self.inner.sha256 - } - - /// The limits every call runs under. - pub fn limits(&self) -> Limits { - self.inner.limits - } - - /// Run `f` against an instance on a thread of `pool`, blocking the - /// caller. - fn call(&self, pool: Pool, what: &'static str, f: F) -> Result - where - R: Send + 'static, - F: FnOnce(&mut Store, &Bindings) -> wasmtime::Result + Send + 'static, - { - let inner = Arc::clone(&self.inner); - executor::run_blocking(pool, move || inner.call(what, f))? - } -} - -impl Inner { - fn instantiate(&self) -> Result { - let mut store = store(self.pre.engine(), &self.limits); - self.arm(&mut store)?; - let plugin = self - .pre - .instantiate(&mut store) - .map_err(|e| PluginError::Trap(format!("instantiate: {}", describe(&e))))?; - Ok(Warm { - store, - plugin, - calls: 0, - }) - } - - /// Give `store` a fresh fuel budget and deadline for one call. - fn arm(&self, store: &mut Store) -> Result<(), PluginError> { - store - .set_fuel(self.limits.fuel) - .map_err(|e| PluginError::Trap(e.to_string()))?; - let ticks = self - .limits - .timeout - .as_millis() - .div_ceil(EPOCH_TICK.as_millis()) - + 1; - store.set_epoch_deadline(u64::try_from(ticks).unwrap_or(u64::MAX)); - store.epoch_deadline_trap(); - Ok(()) - } - - fn park(&self, warm: Warm) { - if warm.calls >= MAX_CALLS_PER_INSTANCE { - return; - } - let mut idle = self.idle.lock().unwrap_or_else(|p| p.into_inner()); - if idle.len() < MAX_IDLE_INSTANCES { - idle.push(warm); - } - } - - /// Run `f` against an instance, on the calling thread (which must be a - /// plugin thread, see `executor`). A failed call discards the instance. - fn call( - &self, - what: &str, - f: impl FnOnce(&mut Store, &Bindings) -> wasmtime::Result, - ) -> Result { - let idle = self.idle.lock().unwrap_or_else(|p| p.into_inner()).pop(); - let mut warm = match idle { - Some(warm) => warm, - None => self.instantiate()?, - }; - self.arm(&mut warm.store)?; - match f(&mut warm.store, &warm.plugin) { - Ok(value) => { - warm.calls += 1; - self.park(warm); - Ok(value) - } - Err(e) => { - let reason = describe(&e); - tracing::warn!(plugin = %self.id, call = what, %reason, "format plugin call failed"); - Err(PluginError::Trap(format!("{what}: {reason}"))) - } - } - } -} - -fn internal(e: PluginError) -> FormatError { - FormatError::Internal(e.to_string()) -} - -/// What a protocol call that failed answers. -fn failed() -> Response { - Response::new(500, "text/plain; charset=utf-8", "format plugin failed") -} - -impl FormatPlugin for WasmPlugin { - fn canonical_id(&self, id: &str) -> String { - let raw = id.to_string(); - self.call(Pool::Quick, "canonical-id", move |store, plugin| { - plugin.forge_format_format().call_canonical_id(store, &raw) - }) - .unwrap_or_else(|_| id.to_string()) - } - - fn validate_id(&self, id: &str) -> Result<(), FormatError> { - let id = id.to_string(); - self.call(Pool::Quick, "validate-id", move |store, plugin| { - plugin.forge_format_format().call_validate_id(store, &id) - }) - .map_err(internal)? - .map_err(Into::into) - } - - fn normalize_version(&self, version: &str) -> Result { - let version = version.to_string(); - self.call(Pool::Quick, "normalize-version", move |store, plugin| { - plugin - .forge_format_format() - .call_normalize_version(store, &version) - }) - .map_err(internal)? - .map_err(Into::into) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - let key = (a.to_string(), b.to_string()); - if let Some(order) = self - .comparisons - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get(&key) - { - return *order; - } - let (x, y) = key.clone(); - let order = match self.call(Pool::Quick, "compare-versions", move |store, plugin| { - plugin - .forge_format_format() - .call_compare_versions(store, &x, &y) - }) { - Ok(n) => n.cmp(&0), - Err(_) => return a.cmp(b), - }; - let mut remembered = self.comparisons.lock().unwrap_or_else(|p| p.into_inner()); - if remembered.len() >= MAX_REMEMBERED_COMPARISONS { - remembered.clear(); - } - remembered.insert(key, order); - order - } - - fn parse_package(&self, package: &mut dyn PackageSource) -> Result { - // The component reads a file it is lent; spool a reader into one. - let spool = |e: std::io::Error| FormatError::Internal(format!("spool the package: {e}")); - let mut spooled = tempfile::NamedTempFile::new().map_err(spool)?; - package.seek(SeekFrom::Start(0)).map_err(spool)?; - std::io::copy(package, &mut spooled).map_err(spool)?; - self.parse_package_file(spooled.path()) - } - - fn parse_package_file(&self, path: &Path) -> Result { - let file = std::fs::File::open(path) - .map_err(|e| FormatError::Internal(format!("open the staged package: {e}")))?; - let package = StagedPackage::new(file) - .map_err(|e| FormatError::Internal(format!("open the staged package: {e}")))?; - self.call(Pool::Slow, "parse-package", move |store, plugin| { - let lent = store.data_mut().lend(package)?; - let borrowed = wasmtime::component::Resource::new_borrow(lent.rep()); - let result = plugin - .forge_format_format() - .call_parse_package(&mut *store, borrowed); - store.data_mut().reclaim(lent); - result - }) - .map_err(internal)? - .map(Into::into) - .map_err(Into::into) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - let manifest = manifest.to_vec(); - self.call(Pool::Quick, "manifest-summary", move |store, plugin| { - plugin - .forge_format_format() - .call_manifest_summary(store, &manifest) - }) - .ok()? - } - - fn handle(&self, repo: &Repository, req: &Request) -> Action { - let repo = repo.clone().into(); - let req = req.clone().into(); - match self.call(Pool::Slow, "handle", move |store, plugin| { - plugin - .forge_format_protocol() - .call_handle(store, &repo, &req) - }) { - Ok(action) => action.into(), - Err(_) => Action::Respond(failed()), - } - } - - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let repo = repo.clone().into(); - let req = req.clone().into(); - let data: Vec = - data.iter().cloned().map(Into::into).collect(); - match self.call(Pool::Slow, "render", move |store, plugin| { - plugin - .forge_format_protocol() - .call_render(store, &repo, &req, &data) - }) { - Ok(rendered) => rendered.into(), - Err(_) => Rendered::Respond(failed()), - } - } - - fn upstream( - &self, - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], - ) -> Result { - let remote = remote.clone().into(); - let op = op.clone().into(); - let fetched: Vec = - fetched.iter().cloned().map(Into::into).collect(); - self.call(Pool::Slow, "upstream", move |store, plugin| { - plugin - .forge_format_upstream() - .call_upstream(store, &remote, &op, &fetched) - }) - .map_err(internal)? - .map(Into::into) - .map_err(Into::into) - } -} - -/// A readable reason for a failed call. -fn describe(e: &wasmtime::Error) -> String { - match e.downcast_ref::() { - Some(Trap::OutOfFuel) => "ran out of fuel".into(), - Some(Trap::Interrupt) => "exceeded its time limit".into(), - Some(trap) => format!("trapped: {trap}"), - None => format!("{e:#}"), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use manifest::RequestedLimits; - - #[test] - fn limits_default_and_are_capped() { - let host = HostLimits::default(); - let l = host.resolve(&RequestedLimits::default()).unwrap(); - assert_eq!(l.memory_bytes, 64 * 1024 * 1024); - assert_eq!(l.fuel, host.default_fuel); - assert_eq!(l.timeout, host.default_timeout); - - let asks = |memory_mib, fuel, timeout_ms| RequestedLimits { - memory_mib, - fuel, - timeout_ms, - }; - assert!(host.resolve(&asks(Some(513), None, None)).is_err()); - assert!(host.resolve(&asks(Some(0), None, None)).is_err()); - assert!(host.resolve(&asks(None, Some(0), None)).is_err()); - assert!(host.resolve(&asks(None, Some(u64::MAX), None)).is_err()); - assert!(host.resolve(&asks(None, None, Some(30_001))).is_err()); - assert!(host.resolve(&asks(None, None, Some(0))).is_err()); - let l = host - .resolve(&asks(Some(128), Some(1_000), Some(250))) - .unwrap(); - assert_eq!(l.memory_bytes, 128 * 1024 * 1024); - assert_eq!(l.fuel, 1_000); - assert_eq!(l.timeout, Duration::from_millis(250)); - } - - #[test] - fn a_component_with_another_digest_is_refused() { - let manifest = Manifest::parse( - r#" - [plugin] - id = "demo" - name = "Demo" - version = "0.1.0" - api = "0.3.0" - component = "demo.wasm" - "#, - ) - .unwrap(); - let err = WasmPlugin::new( - &manifest, - b"not a component", - &"0".repeat(64), - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Digest { .. }), "{err}"); - } - - /// Hand-built components that test the store's limits ([`store`]), each - /// against a memory ceiling of 1 MiB. - mod sandbox { - use super::*; - use wasm_encoder::{ - CanonicalOption, CodeSection, ComponentBuilder, ComponentExportKind, ComponentValType, - ConstExpr, DataSection, ExportKind, ExportSection, Function, FunctionSection, HeapType, - MemArg, MemorySection, MemoryType, Module, ModuleArg, PrimitiveValType, RefType, - TableSection, TableType, TypeSection, ValType, - }; - use wasmtime::component::Instance; - - /// The limits every test component runs under. - const CEILING: Limits = Limits { - memory_bytes: 1024 * 1024, - fuel: 1_000_000_000, - timeout: Duration::from_secs(10), - }; - - /// A store for [`CEILING`], armed for calls. - fn armed_store() -> Store { - let mut store = store(engine().unwrap(), &CEILING); - store.set_fuel(CEILING.fuel).unwrap(); - store.set_epoch_deadline(1_000); - store - } - - /// Instantiate `component` (its bytes) in `store`. - fn instantiate( - store: &mut Store, - component: &[u8], - ) -> wasmtime::Result { - let engine = engine().unwrap(); - let component = Component::new(engine, component).unwrap(); - Linker::new(engine).instantiate(store, &component) - } - - /// A memory section of one memory of `pages` 64 KiB pages. - fn memories(pages: u64) -> MemorySection { - let mut memories = MemorySection::new(); - memories.memory(MemoryType { - minimum: pages, - maximum: None, - memory64: false, - shared: false, - page_size_log2: None, - }); - memories - } - - /// A core module with one memory of `pages` 64 KiB pages. - fn memory_module(pages: u64) -> Module { - let mut module = Module::new(); - module.section(&memories(pages)); - module - } - - /// A component instantiating each of `modules` once, exporting - /// nothing. - fn component_of(modules: &[Module]) -> Vec { - let mut component = ComponentBuilder::default(); - for module in modules { - let index = component.core_module(None, module); - component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - } - component.finish() - } - - #[test] - fn memories_share_the_ceiling() { - // 768 KiB fits, twice does not: each memory alone is below the - // ceiling. - let one = component_of(&[memory_module(12)]); - assert!(instantiate(&mut armed_store(), &one).is_ok()); - let two = component_of(&[memory_module(12), memory_module(12)]); - let err = instantiate(&mut armed_store(), &two).unwrap_err(); - assert!(format!("{err:#}").contains("memory limit"), "{err:#}"); - } - - #[test] - fn tables_count_against_the_ceiling() { - // `grow: func(n: u32) -> s32`: `table.grow` by `n` null elements. - let mut types = TypeSection::new(); - types.ty().function([ValType::I32], [ValType::I32]); - let mut functions = FunctionSection::new(); - functions.function(0); - let mut tables = TableSection::new(); - tables.table(TableType { - element_type: RefType::FUNCREF, - table64: false, - minimum: 0, - maximum: None, - shared: false, - }); - let mut exports = ExportSection::new(); - exports.export("grow", ExportKind::Func, 0); - let mut grow = Function::new([]); - grow.instructions() - .ref_null(HeapType::FUNC) - .local_get(0) - .table_grow(0) - .end(); - let mut code = CodeSection::new(); - code.function(&grow); - let mut module = Module::new(); - module - .section(&types) - .section(&functions) - .section(&tables) - .section(&exports) - .section(&code); - - let mut component = ComponentBuilder::default(); - let index = component.core_module(None, &module); - let instance = component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - let core = component.core_alias_export(None, instance, "grow", ExportKind::Func); - let (ty, mut encoder) = component.type_function(None); - encoder - .async_(false) - .params([("n", PrimitiveValType::U32)]) - .result(Some(PrimitiveValType::S32.into())); - let lifted = component.lift_func(None, core, ty, []); - component.export("grow", ComponentExportKind::Func, lifted, None); - let component = component.finish(); - - let mut store = armed_store(); - let instance = instantiate(&mut store, &component).unwrap(); - let grow = instance - .get_typed_func::<(u32,), (i32,)>(&mut store, "grow") - .unwrap(); - // 1,000 elements fit; 10 million would take 80 MB of the host's. - assert_eq!(grow.call(&mut store, (1_000,)).unwrap(), (0,)); - let err = grow.call(&mut store, (10_000_000,)).unwrap_err(); - assert!(format!("{err:#}").contains("memory limit"), "{err:#}"); - } - - #[test] - fn results_are_bounded_by_the_ceiling() { - // `strings: func(n: u32) -> list`: the first `n` of 1,000 - // list elements that all point at the same 16 KiB string. - const ELEMENTS: u32 = 1_000; - const STRING: u32 = 16 * 1024; - let list = 8; - let string = list + 8 * ELEMENTS; - let mut data = Vec::new(); - data.extend_from_slice(&list.to_le_bytes()); - data.extend_from_slice(&0u32.to_le_bytes()); - for _ in 0..ELEMENTS { - data.extend_from_slice(&string.to_le_bytes()); - data.extend_from_slice(&STRING.to_le_bytes()); - } - data.resize(data.len() + STRING as usize, b'x'); - - let mut types = TypeSection::new(); - types.ty().function([ValType::I32], [ValType::I32]); - let mut functions = FunctionSection::new(); - functions.function(0); - let mut exports = ExportSection::new(); - exports.export("strings", ExportKind::Func, 0); - exports.export("memory", ExportKind::Memory, 0); - // Store `n` as the list's length, return the return area. - let mut strings = Function::new([]); - strings - .instructions() - .i32_const(4) - .local_get(0) - .i32_store(MemArg { - offset: 0, - align: 2, - memory_index: 0, - }) - .i32_const(0) - .end(); - let mut code = CodeSection::new(); - code.function(&strings); - let mut segments = DataSection::new(); - segments.active(0, &ConstExpr::i32_const(0), data); - let mut module = Module::new(); - module - .section(&types) - .section(&functions) - .section(&memories(1)) - .section(&exports) - .section(&code) - .section(&segments); - - let mut component = ComponentBuilder::default(); - let index = component.core_module(None, &module); - let instance = component.core_instantiate(None, index, [] as [(&str, ModuleArg); 0]); - let core = component.core_alias_export(None, instance, "strings", ExportKind::Func); - let memory = component.core_alias_export(None, instance, "memory", ExportKind::Memory); - let (list_of_strings, encoder) = component.type_defined(None); - encoder.list(PrimitiveValType::String); - let (ty, mut encoder) = component.type_function(None); - encoder - .async_(false) - .params([("n", PrimitiveValType::U32)]) - .result(Some(ComponentValType::Type(list_of_strings))); - let lifted = component.lift_func( - None, - core, - ty, - [CanonicalOption::Memory(memory), CanonicalOption::UTF8], - ); - component.export("strings", ComponentExportKind::Func, lifted, None); - let component = component.finish(); - - let mut store = armed_store(); - let instance = instantiate(&mut store, &component).unwrap(); - let strings = instance - .get_typed_func::<(u32,), (Vec,)>(&mut store, "strings") - .unwrap(); - // 10 copies (160 KiB) fit the ceiling. - let (ten,) = strings.call(&mut store, (10,)).unwrap(); - assert_eq!(ten.len(), 10); - assert!(ten.iter().all(|s| s.len() == STRING as usize)); - // 1,000 copies (16 MiB) do not, though the guest holds 16 KiB. - let err = strings.call(&mut store, (ELEMENTS,)).unwrap_err(); - assert!(format!("{err:#}").contains("too much data"), "{err:#}"); - } - } -} diff --git a/crates/plugin/tests/component.rs b/crates/plugin/tests/component.rs deleted file mode 100644 index 133fc7e8..00000000 --- a/crates/plugin/tests/component.rs +++ /dev/null @@ -1,522 +0,0 @@ -//! The Helm plugin (`plugins/helm`) built as a WebAssembly component and -//! loaded into the host: its format rules, its package parser, its protocol, -//! the sandbox around it — and that it answers exactly like the same crate -//! built in. - -use std::cmp::Ordering; -use std::io::Write; -use std::path::PathBuf; -use std::sync::Arc; - -use forge_core::Format; -use forge_plugin::api::{ - Action, Answer, FormatPlugin, Loaded, PackageInfo, Query, Rendered, Repository, RepositoryKind, - Request, SearchHit, SearchResult, Upload, VersionInfo, -}; -use forge_plugin::testing::{component_sha256, example_plugin}; -use forge_plugin::{HostLimits, Manifest, Origin, Plugin, PluginError, PluginFormat}; - -fn helm() -> (PathBuf, Arc) { - let dir = example_plugin("helm"); - let plugin = Plugin::load(&dir, &component_sha256(&dir), &HostLimits::default()) - .expect("load the helm plugin"); - (dir, plugin) -} - -fn builtin() -> Arc { - Plugin::builtin(forge_format_helm::PLUGIN_TOML, forge_format_helm::Helm).unwrap() -} - -/// A ustar header of a regular file, with its checksum. -fn tar_header(path: &str, size: u64) -> [u8; 512] { - let mut header = [0u8; 512]; - header[..path.len()].copy_from_slice(path.as_bytes()); - header[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); - header[156] = b'0'; - header[257..263].copy_from_slice(b"ustar\0"); - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - header -} - -/// A gzipped ustar archive of `(path, contents)` entries. -fn tgz(entries: &[(&str, &[u8])]) -> Vec { - let mut tar = Vec::new(); - for (path, data) in entries { - tar.extend_from_slice(&tar_header(path, data.len() as u64)); - tar.extend_from_slice(data); - tar.resize(tar.len().div_ceil(512) * 512, 0); - } - tar.extend_from_slice(&[0u8; 1024]); - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(&tar).unwrap(); - gz.finish().unwrap() -} - -fn chart(name: &str, version: &str) -> Vec { - let yaml = format!( - "apiVersion: v2\nname: {name}\nversion: {version}\nappVersion: \"1.27\"\n\ - description: The {name} chart\ndependencies:\n - name: redis\n version: \">=17\"\n" - ); - tgz(&[ - (&format!("{name}/values.yaml"), b"replicas: 1\n"), - (&format!("{name}/Chart.yaml"), yaml.as_bytes()), - ]) -} - -fn repo() -> Repository { - Repository { - name: "charts".into(), - kind: RepositoryKind::Hosted, - base_url: "https://forge.example.com/repository/charts".into(), - } -} - -fn request(method: &str, path: &str) -> Request { - Request { - method: method.into(), - path: path.into(), - query: String::new(), - headers: Vec::new(), - } -} - -#[test] -fn loads_only_the_pinned_component() { - let (dir, plugin) = helm(); - assert_eq!(plugin.id(), "helm"); - assert!(!plugin.is_builtin()); - assert_eq!( - plugin.origin(), - &Origin::Wasm { - sha256: component_sha256(&dir) - } - ); - let err = Plugin::load(&dir, &"0".repeat(64), &HostLimits::default()).unwrap_err(); - assert!(matches!(err, PluginError::Digest { .. }), "{err}"); - // The pin may carry its algorithm. - let pinned = format!("sha256:{}", component_sha256(&dir).to_uppercase()); - assert!(Plugin::load(&dir, &pinned, &HostLimits::default()).is_ok()); -} - -#[test] -fn the_format_rules_come_from_the_plugin() { - let (_, plugin) = helm(); - let format = PluginFormat::new(&plugin); - assert_eq!(format.id(), "helm"); - assert_eq!(format.lower_id("Ingress-NGINX"), "ingress-nginx"); - assert!(format.validate_id("ingress-nginx").is_ok()); - assert!(format.validate_id("../etc").is_err()); - assert_eq!(format.normalize_version("v1.2").unwrap(), "1.2.0"); - assert!(format.normalize_version("one").is_err()); - assert_eq!( - format.compare_versions("1.10.0", "1.9.0"), - Ordering::Greater - ); - assert_eq!( - format.compare_versions("1.0.0-rc.1", "1.0.0"), - Ordering::Less - ); - // Remembered comparisons answer the same. - assert_eq!( - format.compare_versions("1.10.0", "1.9.0"), - Ordering::Greater - ); - assert_eq!(format.package_asset_path(), "package"); -} - -#[test] -fn parses_a_chart_through_the_lent_reader() { - let (_, plugin) = helm(); - let format = PluginFormat::new(&plugin); - let mut staged = tempfile::NamedTempFile::new().unwrap(); - staged.write_all(&chart("NGINX", "v2.1")).unwrap(); - - let parsed = format.parse_upload_file(staged.path()).unwrap(); - assert_eq!(parsed.id, "nginx"); - assert_eq!(parsed.original_id, "NGINX"); - assert_eq!(parsed.version, "2.1.0"); - assert_eq!(parsed.version_original, "v2.1"); - assert!(parsed.listed); - assert!(String::from_utf8_lossy(&parsed.nuspec_bytes).contains("name: NGINX")); - assert_eq!(parsed.deps.len(), 1); - assert_eq!(parsed.deps[0].dependencies[0].id, "redis"); - - // The reader-based entry point spools into a file first. - let mut cursor = std::io::Cursor::new(chart("demo", "1.0.0")); - assert_eq!(format.parse_upload(&mut cursor).unwrap().id, "demo"); - - let summary = format.manifest_summary(&parsed.nuspec_bytes).unwrap(); - assert_eq!(summary["chart"]["appVersion"], "1.27"); - assert_eq!(summary["description"], "The NGINX chart"); - assert_eq!( - summary["dependency_groups"][0]["dependencies"][0]["id"], - "redis" - ); - - let mut not_a_chart = tempfile::NamedTempFile::new().unwrap(); - not_a_chart.write_all(b"definitely not gzip").unwrap(); - let err = format.parse_upload_file(not_a_chart.path()).unwrap_err(); - assert!(err.to_string().contains("not a chart archive"), "{err}"); -} - -#[test] -fn routes_requests_to_host_actions() { - let (_, plugin) = helm(); - let plugin = plugin.implementation(); - - let action = plugin.handle( - &repo(), - &request("GET", "/charts/nginx/v2.1/nginx-2.1.0.tgz"), - ); - let Action::Download(target) = action else { - panic!("expected a download, got {action:?}"); - }; - assert_eq!( - (target.id.as_str(), target.version.as_str()), - ("nginx", "2.1.0") - ); - - let action = plugin.handle(&repo(), &request("POST", "/api/charts")); - assert!( - matches!(&action, Action::Publish(p) if matches!(p.upload, Upload::Body)), - "{action:?}" - ); - let mut multipart = request("POST", "/api/charts"); - multipart.headers = vec![( - "content-type".into(), - "multipart/form-data; boundary=x".into(), - )]; - let action = plugin.handle(&repo(), &multipart); - assert!( - matches!(&action, Action::Publish(p) if matches!(&p.upload, Upload::MultipartField(f) if f == "chart")), - "{action:?}" - ); - - let action = plugin.handle(&repo(), &request("DELETE", "/api/charts/NGINX/2.1.0")); - assert!( - matches!(&action, Action::Delete(c) if c.target.id == "nginx"), - "{action:?}" - ); - - for (method, path) in [ - ("GET", "/nope"), - ("PUT", "/index.yaml"), - ("GET", "/charts/..%2Fx/1.0.0/x.tgz"), - ] { - let action = plugin.handle(&repo(), &request(method, path)); - assert!( - matches!(&action, Action::Respond(r) if r.status == 404), - "{method} {path}: {action:?}" - ); - } -} - -#[test] -fn renders_the_index_from_what_it_loads() { - let (_, plugin) = helm(); - let plugin = plugin.implementation(); - let req = request("GET", "/index.yaml"); - let Action::Load(first) = plugin.handle(&repo(), &req) else { - panic!("the index loads data"); - }; - assert!(matches!(&first[..], [Query::Search(s)] if s.skip == 0 && s.prerelease)); - - let version = |v: &str| VersionInfo { - original_id: "NGINX".into(), - version: v.into(), - version_original: v.into(), - listed: true, - }; - let mut data = vec![Loaded { - query: first[0].clone(), - answer: Answer::Search(SearchResult { - total: 1, - hits: vec![SearchHit { - original_id: "NGINX".into(), - latest_version: "2.1.0".into(), - versions: vec![version("1.0.0"), version("2.1.0")], - summary: None, - total_downloads: None, - }], - }), - }]; - let Rendered::Load(second) = plugin.render(&repo(), &req, &data) else { - panic!("the index loads each chart's versions"); - }; - assert!( - matches!(&second[..], [Query::Packages(id)] if id == "nginx"), - "{second:?}" - ); - - let package = |v: &str, listed: bool| PackageInfo { - original_id: "NGINX".into(), - version: v.into(), - version_original: v.into(), - listed, - published: Some("2026-10-01T12:00:00+00:00".into()), - sha256: Some("ab".repeat(32)), - sha512: None, - size: Some(1234), - summary: Some(r#"{"chart":{"apiVersion":"v2","appVersion":"1.27"}}"#.into()), - }; - data.push(Loaded { - query: second[0].clone(), - answer: Answer::Packages(vec![ - package("1.0.0", true), - package("1.5.0", false), - package("2.1.0", true), - ]), - }); - let Rendered::Respond(response) = plugin.render(&repo(), &req, &data) else { - panic!("the index renders once everything is loaded"); - }; - assert_eq!(response.status, 200); - let index: serde_json::Value = serde_json::from_slice(&response.body).unwrap(); - let entries = index["entries"]["NGINX"].as_array().unwrap(); - // Newest first; unlisted versions are not offered. - let versions: Vec<&str> = entries - .iter() - .map(|e| e["version"].as_str().unwrap()) - .collect(); - assert_eq!(versions, ["2.1.0", "1.0.0"]); - assert_eq!( - entries[0]["urls"][0], - "https://forge.example.com/repository/charts/charts/nginx/2.1.0/NGINX-2.1.0.tgz" - ); - assert_eq!(entries[0]["digest"], "ab".repeat(32)); - assert_eq!(entries[0]["appVersion"], "1.27"); - assert_eq!(index["apiVersion"], "v1"); -} - -#[test] -fn runs_inside_its_limits() { - let dir = example_plugin("helm"); - let sha = component_sha256(&dir); - let manifest = |limits: &str| { - let text = std::fs::read_to_string(dir.join("plugin.toml")).unwrap(); - let text = text.split("[limits]").next().unwrap().to_string() + "[limits]\n" + limits; - Manifest::parse(&text).unwrap() - }; - let component = std::fs::read(dir.join("forge_format_helm.wasm")).unwrap(); - - // Not enough fuel to even start. - let err = Plugin::from_parts( - manifest("fuel = 1\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(err.to_string().contains("ran out of fuel"), "{err}"); - - // Not enough memory for the guest's initial heap. - let err = Plugin::from_parts( - manifest("memory_mib = 1\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Trap(_)), "{err}"); - - // More than the host allows. - let err = Plugin::from_parts( - manifest("memory_mib = 4096\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap_err(); - assert!(matches!(err, PluginError::Manifest(_)), "{err}"); - - // A decompression bomb runs out of time or fuel instead of hanging the - // host, and the plugin keeps serving afterwards. - let plugin = Plugin::from_parts( - manifest("fuel = 20000000\ntimeout_ms = 2000\n"), - &component, - &sha, - &HostLimits::default(), - ) - .unwrap(); - let format = PluginFormat::new(&plugin); - let bomb = { - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - // A tar entry claiming 32 MiB of zeros before any Chart.yaml. - gz.write_all(&tar_header("bomb/zeros", 32 << 20)).unwrap(); - let zeros = vec![0u8; 1 << 20]; - for _ in 0..32 { - gz.write_all(&zeros).unwrap(); - } - gz.finish().unwrap() - }; - let mut staged = tempfile::NamedTempFile::new().unwrap(); - staged.write_all(&bomb).unwrap(); - let err = format.parse_upload_file(staged.path()).unwrap_err(); - assert!( - err.to_string().contains("fuel") || err.to_string().contains("time limit"), - "{err}" - ); - assert_eq!(format.normalize_version("1.0").unwrap(), "1.0.0"); -} - -#[test] -fn a_builtin_plugin_has_no_component() { - let plugin = builtin(); - assert_eq!(plugin.id(), "helm"); - assert!(plugin.is_builtin()); - assert_eq!(plugin.sha256(), None); - assert_eq!(plugin.manifest().name, "Helm charts"); - let format = PluginFormat::new(&plugin); - assert_eq!(format.lower_id("NGINX"), "nginx"); -} - -/// One contract, two ways to run it: the component and the built-in crate -/// answer every call the same. -#[test] -fn the_component_answers_like_the_builtin_plugin() { - let (_, component) = helm(); - let builtin = builtin(); - let (wasm, native): (&dyn FormatPlugin, &dyn FormatPlugin) = ( - component.implementation().as_ref(), - builtin.implementation().as_ref(), - ); - - for id in ["Ingress-NGINX", "nginx", "../etc", ""] { - assert_eq!(wasm.canonical_id(id), native.canonical_id(id), "{id:?}"); - assert_eq!(wasm.validate_id(id), native.validate_id(id), "{id:?}"); - } - for version in ["v1.2", "1.0.0-rc.1+build.5", "one", ""] { - assert_eq!( - wasm.normalize_version(version), - native.normalize_version(version), - "{version:?}" - ); - } - for (a, b) in [ - ("1.10.0", "1.9.0"), - ("1.0.0-rc.1", "1.0.0"), - ("2.0.0", "2.0.0"), - ] { - assert_eq!(wasm.compare_versions(a, b), native.compare_versions(a, b)); - } - - let package = std::io::Cursor::new(chart("NGINX", "v2.1")); - let parsed = wasm.parse_package(&mut package.clone()).unwrap(); - assert_eq!(parsed, native.parse_package(&mut package.clone()).unwrap()); - assert_eq!( - wasm.manifest_summary(&parsed.manifest), - native.manifest_summary(&parsed.manifest) - ); - let mut garbage = std::io::Cursor::new(b"not a chart".to_vec()); - let mut garbage_too = garbage.clone(); - assert_eq!( - wasm.parse_package(&mut garbage), - native.parse_package(&mut garbage_too) - ); - - let mut multipart = request("POST", "/api/charts"); - multipart.headers = vec![( - "content-type".into(), - "multipart/form-data; boundary=x".into(), - )]; - for req in [ - request("GET", "/index.yaml"), - request("GET", "/charts/nginx/2.1.0/nginx-2.1.0.tgz"), - request("HEAD", "/api/charts/nginx"), - request("GET", "/api/charts/nginx/2.1.0"), - request("POST", "/api/charts"), - multipart, - request("DELETE", "/api/charts/nginx/2.1.0"), - request("GET", "/nope"), - ] { - assert_eq!( - wasm.handle(&repo(), &req), - native.handle(&repo(), &req), - "{req:?}" - ); - } - - let req = request("GET", "/api/charts/nginx"); - let data = [Loaded { - query: Query::Packages("nginx".into()), - answer: Answer::Packages(vec![PackageInfo { - original_id: "NGINX".into(), - version: "2.1.0".into(), - version_original: "v2.1".into(), - listed: true, - published: Some("2026-10-01T12:00:00Z".into()), - sha256: Some("ab".repeat(32)), - sha512: None, - size: Some(1234), - summary: native.manifest_summary(&parsed.manifest), - }]), - }]; - assert_eq!( - wasm.render(&repo(), &req, &data), - native.render(&repo(), &req, &data) - ); - - // The upstream side, through the component's `upstream` export: a - // proxy of another chart repository. - let index = b"apiVersion: v1\nentries:\n nginx:\n - apiVersion: v2\n name: nginx\n version: 2.1.0\n description: A web server\n created: '2026-05-01T10:00:00Z'\n urls: [charts/nginx-2.1.0.tgz]\n"; - let fetched = [forge_plugin::api::FetchResult { - fetch: forge_plugin::api::Fetch { - url: "https://charts.example/index.yaml".into(), - freshness: forge_plugin::api::Freshness::Metadata, - }, - outcome: forge_plugin::api::Fetched::Found(index.to_vec()), - }]; - let unresolved = forge_plugin::api::Remote { - url: "https://charts.example".into(), - index: None, - }; - let resolved = forge_plugin::api::Remote { - url: "https://charts.example".into(), - index: Some(r#"{"base":"https://charts.example"}"#.into()), - }; - let nginx = |version: &str| forge_plugin::api::PackageRef { - id: "nginx".into(), - version: version.into(), - }; - let wanted = forge_plugin::api::PackageVersions { - id: "nginx".into(), - versions: vec!["2.1.0".into()], - }; - for (remote, op) in [ - (&unresolved, forge_plugin::api::UpstreamOp::Index), - ( - &resolved, - forge_plugin::api::UpstreamOp::Versions("nginx".into()), - ), - ( - &resolved, - forge_plugin::api::UpstreamOp::Package(nginx("2.1.0")), - ), - ( - &resolved, - forge_plugin::api::UpstreamOp::Manifest(nginx("2.1.0")), - ), - (&resolved, forge_plugin::api::UpstreamOp::Dating(wanted)), - ( - &resolved, - forge_plugin::api::UpstreamOp::Search(forge_plugin::api::SearchQuery { - text: "ngi".into(), - skip: 0, - take: 0, - prerelease: true, - semver2: true, - upstream: true, - }), - ), - ] { - for fetched in [&fetched[..], &[]] { - assert_eq!( - wasm.upstream(remote, &op, fetched), - native.upstream(remote, &op, fetched), - "{op:?}" - ); - } - } -} diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 6d56b38b..1e6a5a7e 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -82,10 +82,7 @@ lettre = { workspace = true } figment = { version = "0.10", features = ["toml", "env"] } [dev-dependencies] -forge-plugin = { path = "../plugin", features = ["testing"] } -# The example format plugin the format-plugin tests serve repositories with -# (plugins/helm, not built in), and the chart archives they push to it. -forge-format-helm = { path = "../../plugins/helm" } +# Builds Helm chart archives for the registry tests. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "test-util"] } testcontainers = "0.27" diff --git a/crates/server/src/admin/formats.rs b/crates/server/src/admin/formats.rs index 319bc9ec..2f26119a 100644 --- a/crates/server/src/admin/formats.rs +++ b/crates/server/src/admin/formats.rs @@ -1,6 +1,5 @@ //! `GET /admin/formats` — the package formats this server can create -//! repositories of: the built-in format plugin (NuGet), Docker, raw files, -//! and every loaded WebAssembly plugin. +//! repositories of: the format plugins (NuGet), Docker and raw files. use axum::extract::State; use axum::response::{IntoResponse, Json, Response}; @@ -16,9 +15,6 @@ pub struct FormatView { pub id: String, /// Display name. pub name: String, - /// Whether the format is built into the server (Docker, raw and the - /// built-in plugins) rather than loaded as a WebAssembly plugin. - pub builtin: bool, /// The repository kinds it supports. pub kinds: Vec<&'static str>, /// Where its repositories are served: `{mount}/{repository}` (for @@ -30,9 +26,6 @@ pub struct FormatView { /// The plugin's description. #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, - /// SHA-256 of the plugin's component (WebAssembly plugins only). - #[serde(skip_serializing_if = "Option::is_none")] - pub sha256: Option, /// The upload limit of its hosted repositories without one of their own /// (for docker, per blob). pub default_max_upload_bytes: u64, @@ -51,12 +44,10 @@ pub async fn list(State(state): State) -> Response { FormatView { id: plugin.id().to_string(), name: manifest.name.clone(), - builtin: plugin.is_builtin(), kinds: kinds(plugin.id()), mount: manifest.mount.clone(), version: Some(manifest.version.clone()), description: manifest.description.clone(), - sha256: plugin.sha256().map(str::to_string), default_max_upload_bytes: state.config.default_upload_limit(plugin.id()), max_upload_bytes_limit: ceiling, } @@ -68,12 +59,10 @@ pub async fn list(State(state): State) -> Response { formats.push(FormatView { id: DOCKER_FORMAT.to_string(), name: "Docker / OCI".into(), - builtin: true, kinds: kinds(DOCKER_FORMAT), mount: "/v2".into(), version: None, description: None, - sha256: None, default_max_upload_bytes: state.config.default_upload_limit(DOCKER_FORMAT), max_upload_bytes_limit: ceiling, }); @@ -81,12 +70,10 @@ pub async fn list(State(state): State) -> Response { others.push(FormatView { id: RAW_FORMAT.to_string(), name: "Raw files".into(), - builtin: true, kinds: kinds(RAW_FORMAT), mount: crate::raw::MOUNT.into(), version: None, description: Some("Arbitrary files under arbitrary paths".into()), - sha256: None, default_max_upload_bytes: state.config.default_upload_limit(RAW_FORMAT), max_upload_bytes_limit: ceiling, }); diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index 07f1285f..bcc42398 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -549,8 +549,8 @@ pub async fn create(State(state): State, Json(body): Json) "hosted" | "proxy" | "group" => body.kind.clone(), other => return bad_request(format!("invalid kind {other:?}")), }; - // The format must be installed: Docker, raw, the built-in format plugin - // (NuGet) or a loaded WebAssembly one (`GET /admin/formats`), and support + // The format must be installed: Docker, raw or a format plugin (NuGet; + // `GET /admin/formats`), and support // the kind. It is fixed for the repository's lifetime. let Some(kinds) = state.formats.kinds(&body.format) else { let available: Vec<&str> = state.formats.ids().collect(); diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 80d1a06b..0f9d52d6 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -63,11 +63,11 @@ pub struct Config { /// proxied packages. #[serde(default)] pub nuget: NugetConfig, - /// WebAssembly format plugins: further package formats loaded at startup - /// (see `docs/format-plugins.md`). None are loaded by default; the - /// built-in plugin (NuGet) needs no configuration here. - #[serde(default)] - pub plugins: PluginsConfig, + /// The removed `[plugins]` section (WebAssembly format plugins). The top + /// level accepts unknown keys, so without this a leftover section would + /// be ignored and the repositories of its formats would answer 503. + #[serde(default, rename = "plugins")] + removed_plugins: Option, /// Docker / OCI registry (`/v2`): size limits, registry tokens, upload /// and blob-link expiry. #[serde(default)] @@ -1503,63 +1503,6 @@ fn default_nuget_max_nuspec_compression_ratio() -> u64 { forge_format_nuget::NupkgLimits::default().max_nuspec_compression_ratio } -/// `plugins.*` — WebAssembly format plugins. -/// -/// The format plugin built into the server (NuGet) is always installed. -/// Further ones are WebAssembly components, each listed explicitly with the -/// SHA-256 of its component: a directory under `dir` that is not listed is -/// never loaded, and a listed one whose component does not match its pin, or -/// whose id is taken by a built-in format, fails startup. Every instance must -/// therefore load the same plugins (the same configuration and files), since -/// a repository of a format an instance lacks is unavailable there. -/// -/// ```toml -/// [plugins] -/// dir = "/etc/forge/plugins" # holds /plugin.toml + its component -/// -/// [plugins.formats] -/// example = "sha256:3f1c…" # format id → pinned component digest -/// ``` -/// -/// As environment variables: `FORGE_PLUGINS__DIR`, -/// `FORGE_PLUGINS__FORMATS__EXAMPLE`. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PluginsConfig { - /// Directory with one subdirectory per plugin, named by its format id. - #[serde(default)] - pub dir: Option, - /// The plugins to load: format id → SHA-256 of its component (lowercase - /// hex, optionally prefixed `sha256:`). - #[serde(default)] - pub formats: std::collections::BTreeMap, -} - -impl PluginsConfig { - fn validate(&self) -> Result<(), ConfigError> { - if self.formats.is_empty() { - return Ok(()); - } - if self.dir.is_none() { - return Err(ConfigError::Invalid( - "plugins.formats lists plugins but plugins.dir is not set".into(), - )); - } - for (id, sha256) in &self.formats { - forge_plugin::manifest::validate_id(id) - .map_err(|e| ConfigError::Invalid(format!("plugins.formats.{id}: {e}")))?; - let hex = sha256.trim(); - let hex = hex.strip_prefix("sha256:").unwrap_or(hex); - if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(ConfigError::Invalid(format!( - "plugins.formats.{id} must be the component's SHA-256 (64 hex digits)" - ))); - } - } - Ok(()) - } -} - /// `web.*` — static SPA serving. /// /// When `enabled`, the server serves the built TanStack Start SPA from @@ -2265,10 +2208,16 @@ impl Config { self.validate_crypto()?; self.proxy.validate()?; self.nuget.validate()?; - self.plugins.validate()?; self.docker.validate()?; self.validate_workers()?; self.validate_insecure_defaults()?; + if self.removed_plugins.is_some() { + return Err(ConfigError::Invalid( + "plugins was removed: Forge loads no WebAssembly format plugins, only its \ + built-in formats. Remove the [plugins] section (FORGE_PLUGINS__*)" + .into(), + )); + } if self.server.removed_max_upload_bytes.is_some() { return Err(ConfigError::Invalid( "server.max_upload_bytes was removed: set each repository's upload limit \ @@ -4163,9 +4112,17 @@ mod tests { ); } - // `[plugins.max_upload_bytes]` is gone: limits are per repository. - let per_format = format!("{MINIMAL}\n[plugins.max_upload_bytes]\nnuget = 4096\n"); - assert!(Config::from_figment(fig_from_toml(&per_format)).is_err()); + // `[plugins]` is gone with WebAssembly plugins. + for plugins in [ + "[plugins.max_upload_bytes]\nnuget = 4096\n", + "[plugins]\ndir = \"/etc/forge/plugins\"\n", + "[plugins.formats]\nexample = \"sha256:00\"\n", + ] { + let err = Config::from_figment(fig_from_toml(&format!("{MINIMAL}\n{plugins}"))) + .unwrap_err() + .to_string(); + assert!(err.contains("plugins was removed"), "{err}"); + } } #[test] diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 314160af..e958bed8 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -121,7 +121,6 @@ async fn main() -> anyhow::Result<()> { .context("ensure auth realms")?; let bind_addr = config.server.bind_addr().context("parse server.bind")?; - let plugins = forge_server::plugins::load(&config.plugins).context("load format plugins")?; // Registry (`/v2`) tokens are signed with one key shared by every // instance: docker.token_secret, or the install's generated key. let registry_keys = @@ -130,10 +129,7 @@ async fn main() -> anyhow::Result<()> { .context("load the registry token key")?; let state = AppState::new(pool, blob_store, cache, config) .with_envelope(envelope) - .with_registry_keys(registry_keys) - .with_plugins(plugins) - .map_err(anyhow::Error::msg) - .context("install format plugins")?; + .with_registry_keys(registry_keys); let layer = principal_layer(&state)?; let serve_options = forge_server::serve::ServeOptions::from_config(&state.config); diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index aef2e61d..89314ea1 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -1,13 +1,12 @@ -//! Format plugins: the built-in ones, loading the WebAssembly ones at -//! startup, and serving the repositories of their formats under each -//! format's mount, `{mount}/{repo}/...` (`/repository` unless its -//! `plugin.toml` names another; see `docs/format-plugins.md`). +//! Format plugins: the built-in ones, and serving the repositories of their +//! formats under each format's mount, `{mount}/{repo}/...` (`/repository` +//! unless its `plugin.toml` names another; see `docs/format-plugins.md`). //! //! The dispatcher does what [`crate::dispatch`] does for the NuGet feed — //! resolve the repository, authorize the request's action (GET/HEAD read, //! PUT/POST write, DELETE delete), refuse an offline repository — and then //! lets the format's plugin decide what the request is -//! ([`FormatPlugin::handle`]), whether it is built in or a component. +//! ([`FormatPlugin::handle`]). //! Everything that touches data stays here: //! //! * **Queries** the plugin asks for run through the same read ops as every @@ -30,7 +29,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use crate::feed::{byte_range, ByteRange, PackageBytes}; -use anyhow::Context; use axum::body::Body; use axum::extract::{Request, State}; use axum::http::{header, HeaderValue, Method, StatusCode}; @@ -49,11 +47,10 @@ use forge_plugin::api::{ self as protocol, Action, Answer, ChangeRequest, FormatPlugin, Loaded, PackageInfo, PackageRef, PublishRequest, Query, Rendered, Upload, }; -use forge_plugin::{HostLimits, Plugin}; +use forge_plugin::Plugin; use futures_util::{StreamExt, TryStreamExt}; use tokio_util::io::StreamReader; -use crate::config::PluginsConfig; use crate::dispatch::{ action_for_method, authz_denied, offline_response, offline_serves_nothing, principal_allows_protocol_mutations, HostedWriteAdapter, @@ -108,39 +105,6 @@ pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec .expect("plugins/nuget/plugin.toml is valid")] } -/// Load the WebAssembly plugins `config` lists, each from `/`, -/// checked against its pinned digest. Any failure fails startup: a -/// configured format that is silently missing would leave its repositories -/// unavailable. -pub fn load(config: &PluginsConfig) -> anyhow::Result>> { - let Some(dir) = &config.dir else { - return Ok(Vec::new()); - }; - let limits = HostLimits::default(); - config - .formats - .iter() - .map(|(id, sha256)| { - let path = dir.join(id); - let plugin = Plugin::load(&path, sha256, &limits) - .with_context(|| format!("load format plugin {id:?} from {}", path.display()))?; - anyhow::ensure!( - plugin.id() == id, - "the plugin in {} declares format id {:?}, not {id:?}", - path.display(), - plugin.id() - ); - tracing::info!( - format = id, - version = %plugin.manifest().version, - sha256 = plugin.sha256().unwrap_or_default(), - "format plugin loaded" - ); - Ok(plugin) - }) - .collect() -} - /// Axum handler mounted at `/repository/{repo}` (and everything under it). pub async fn dispatch(State(state): State, req: Request) -> Response { let mut response = dispatch_request(state, req).await; @@ -308,8 +272,7 @@ struct Ctx<'a> { impl Ctx<'_> { /// Run a protocol call of the plugin off the async workers: rendering a - /// large index takes a while, and a component's call may run for as long - /// as its time limit. A plugin that panics answers 500. + /// large listing takes a while. A plugin that panics answers 500. async fn call( &self, f: impl FnOnce(&dyn FormatPlugin) -> R + Send + 'static, diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index 68d8c6da..8f7ed1fa 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -54,8 +54,8 @@ pub struct AppState { /// Authorization decision service: every read/write/admin gate routes /// through this. pub authz: Authz, - /// Package-format registry, keyed by `Format::id()`: the built-in format - /// plugin (NuGet) and any loaded WebAssembly plugins. + /// Package-format registry, keyed by `Format::id()`: the format plugins + /// (NuGet). pub formats: Arc, /// Parsed server configuration. pub config: Arc, @@ -139,10 +139,8 @@ impl Instance { } /// Format registry: maps a format id (the `repository.format` column) to its -/// [`Format`] and its plugin. The built-in format plugin -/// ([`crate::plugins::builtin`]: NuGet), Docker and raw are always -/// there; WebAssembly plugins are added at startup -/// ([`FormatRegistry::with_plugin`]). +/// [`Format`] and its plugin: the built-in format plugin +/// ([`crate::plugins::builtin`]: NuGet), Docker and raw. /// /// Docker ([`DOCKER_FORMAT`]) and raw ([`RAW_FORMAT`]) are installed but have /// no [`Format`]: their repositories hold images (manifests and blobs) or @@ -152,8 +150,7 @@ impl Instance { /// them. pub struct FormatRegistry { formats: BTreeMap>, - /// Every format's plugin (built in or WebAssembly), by format id: its - /// client protocol. + /// Every format's plugin, by format id: its client protocol. plugins: BTreeMap>, } @@ -179,8 +176,7 @@ impl FormatRegistry { pub fn with_format(mut self, format: Arc) -> Result { let id = format.id().to_string(); if self.is_installed(&id) { - let built_in = - Self::is_native(&id) || self.plugins.get(&id).is_some_and(|p| p.is_builtin()); + let built_in = Self::is_native(&id) || self.plugins.contains_key(&id); return Err(if built_in { format!("format {id:?} is built into this server") } else { @@ -192,7 +188,7 @@ impl FormatRegistry { } /// Add a format plugin: its [`Format`] and its client protocol. Its id - /// must be free: a WebAssembly plugin never replaces a built-in format. + /// must be free: a plugin never replaces another format. pub fn with_plugin(mut self, plugin: Arc) -> Result { let format = Arc::new(forge_plugin::PluginFormat::new(&plugin)); self = self.with_format(format)?; @@ -210,7 +206,7 @@ impl FormatRegistry { self.plugins.get(id).cloned() } - /// The plugin formats (built in and WebAssembly), by format id. + /// The plugin formats, by format id. pub fn plugins(&self) -> impl Iterator> { self.plugins.values() } @@ -456,9 +452,9 @@ impl AppState { self } - /// Install the loaded WebAssembly plugins (see [`crate::plugins::load`]) - /// next to the built-in formats. A plugin whose format id is taken - /// (a built-in plugin, or another plugin) is an error. + /// Install further format plugins next to the built-in formats (the + /// tests' own). A plugin whose format id is taken (a built-in plugin, or + /// another plugin) is an error. pub fn with_plugins(mut self, plugins: Vec>) -> Result { let mut registry = FormatRegistry::new(self.config.nuget.limits()); for plugin in plugins { diff --git a/crates/server/tests/admin.rs b/crates/server/tests/admin.rs index ceaedefa..797bc8d5 100644 --- a/crates/server/tests/admin.rs +++ b/crates/server/tests/admin.rs @@ -3284,7 +3284,7 @@ async fn blob_cache_statistics_name_their_instance() { format!("admin{}@example.com", common::unique()), "adminpw".into(), )), - plugins_block: Some(format!( + extra_toml: Some(format!( "[blob_cache]\npath = {:?}\n", dir.path().join("cache").display().to_string() )), diff --git a/crates/server/tests/common/mod.rs b/crates/server/tests/common/mod.rs index 8e19d163..5df2a2bb 100644 --- a/crates/server/tests/common/mod.rs +++ b/crates/server/tests/common/mod.rs @@ -205,7 +205,7 @@ fn config_toml(db_url: &str, s3_endpoint: &str, opts: &ServerOpts) -> String { let web_assets_dir = opts.web_assets_dir.as_deref(); let oidc_block = opts.oidc_block.as_deref(); let crypto_block = opts.crypto_block.as_deref(); - let plugins_block = opts.plugins_block.as_deref(); + let extra_toml = opts.extra_toml.as_deref(); let s3_credentials = opts .s3_credentials .clone() @@ -259,12 +259,12 @@ poll_interval_seconds = 1 {web} {oidc} {crypto} -{plugins} +{extra} "#, bucket = S3_BUCKET, oidc = oidc_block.unwrap_or(""), crypto = crypto_block.unwrap_or(""), - plugins = plugins_block.unwrap_or(""), + extra = extra_toml.unwrap_or(""), ) } @@ -314,9 +314,9 @@ pub struct ServerOpts { /// Extra raw TOML lines for the `[auth]` section (e.g. /// `realm_cache_ttl_seconds`). pub auth_extra: Option, - /// When set, this raw TOML is appended last (the `[plugins]` section, or - /// any other), and the format plugins it lists are loaded. - pub plugins_block: Option, + /// When set, this raw TOML is appended last (a section of its own, such + /// as `[docker]`). + pub extra_toml: Option, /// Format plugins installed next to the built-in ones, as if built in. pub extra_plugins: Vec>, /// When set, this raw TOML replaces the static keys at the end of the @@ -337,7 +337,7 @@ impl Default for ServerOpts { public_base_url: None, server_extra: None, auth_extra: None, - plugins_block: None, + extra_toml: None, extra_plugins: Vec::new(), s3_credentials: None, } @@ -398,8 +398,6 @@ pub async fn start_with(pool: PgPool, opts: ServerOpts) -> TestServer { .expect("seed auth realms"); let cache = Cache::l1_only(config.cache.l1_max_capacity); - let mut plugins = forge_server::plugins::load(&config.plugins).expect("load format plugins"); - plugins.extend(opts.extra_plugins.iter().cloned()); let registry_keys = forge_server::registry::load_token_keys(&config, &pool, envelope.as_deref()) .await @@ -407,7 +405,7 @@ pub async fn start_with(pool: PgPool, opts: ServerOpts) -> TestServer { let state = AppState::new(pool.clone(), store, cache, config) .with_envelope(envelope) .with_registry_keys(registry_keys) - .with_plugins(plugins) + .with_plugins(opts.extra_plugins.clone()) .expect("install format plugins"); let layer = principal_layer(&state).expect("principal layer"); diff --git a/crates/server/tests/plugin_contract.rs b/crates/server/tests/plugin_contract.rs index c5de24bf..25c25e06 100644 --- a/crates/server/tests/plugin_contract.rs +++ b/crates/server/tests/plugin_contract.rs @@ -198,7 +198,7 @@ async fn start_probe(extra_toml: Option) -> TestServer { ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), extra_plugins: vec![Plugin::builtin(PROBE_TOML, Probe).unwrap()], - plugins_block: extra_toml, + extra_toml, ..Default::default() }, ) @@ -349,6 +349,37 @@ async fn a_format_is_served_under_its_own_mount_and_kinds() { assert_eq!(nuget["mount"], "/v3"); } +/// A repository of a format the server does not have (its plugin removed +/// since) answers 503 under a plugin's mount, and so does an administrator's +/// delete of one of its versions: it is never read as another format. +#[tokio::test] +async fn a_repository_whose_format_is_missing_is_unavailable() { + let srv = start_probe(None).await; + let name = format!("orphan{}", unique()); + let mut row = forge_db::repository_dao::NewRepository::hosted(name.clone()); + row.format = "gone".into(); + forge_db::repository_dao::insert_repository(&srv.pool, row) + .await + .expect("insert repository"); + let (u, p) = srv.admin_basic(); + for (method, url) in [ + (reqwest::Method::GET, format!("/probe/{name}/nothing")), + ( + reqwest::Method::DELETE, + format!("/admin/repositories/{name}/packages/demo/1.0.0"), + ), + ] { + let resp = srv + .client() + .request(method, format!("{}{url}", srv.base_url)) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE, "{url}"); + } +} + #[tokio::test] async fn uploads_take_the_first_part_and_assets_their_names() { let srv = start_probe(None).await; diff --git a/crates/server/tests/plugins.rs b/crates/server/tests/plugins.rs deleted file mode 100644 index 7cddcb13..00000000 --- a/crates/server/tests/plugins.rs +++ /dev/null @@ -1,1222 +0,0 @@ -//! Format plugins end to end, through the example plugin `plugins/helm` -//! (Helm chart repositories; Forge does not build it in): installed as if -//! built in, its repositories created through the admin API and used through -//! `/repository/{repo}` the way Helm clients use a chart repository — and the -//! same crate built as a WebAssembly component, loaded at startup. - -mod common; - -use std::io::Write; - -use common::{start_with, unique, ServerOpts, TestServer}; -use reqwest::StatusCode; -use serde_json::{json, Value}; -use sha2::Digest; - -/// The format id the Helm component is installed under (the Helm plugin -/// installed as if built in holds `helm`). -const WASM_HELM: &str = "helm-wasm"; - -/// The `[plugins]` block loading the Helm plugin built as a component, -/// served under a mount of its own (`/charts`). -fn wasm_helm_plugins_block() -> String { - let dir = forge_plugin::testing::example_plugin_with("helm", WASM_HELM, "mount = \"/charts\""); - let sha = forge_plugin::testing::component_sha256(&dir); - let parent = dir.parent().expect("plugin directory parent"); - format!("[plugins]\ndir = {parent:?}\n\n[plugins.formats]\n{WASM_HELM} = \"sha256:{sha}\"\n") -} - -/// The Helm plugin, to install as if built in. -fn helm_plugin() -> std::sync::Arc { - forge_plugin::Plugin::builtin(forge_format_helm::PLUGIN_TOML, forge_format_helm::Helm) - .expect("plugins/helm/plugin.toml is valid") -} - -/// A server with the Helm plugin installed as if built in. -async fn start_helm(mut opts: ServerOpts) -> TestServer { - opts.extra_plugins.push(helm_plugin()); - start_with( - common::pool().await, - ServerOpts { - bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - ..opts - }, - ) - .await -} - -/// A gzipped chart archive declaring `name` at `version`. -fn chart(name: &str, version: &str) -> Vec { - chart_with_values(name, version, "replicas: 1\n") -} - -/// [`chart`] with `values` as its `values.yaml`. -fn chart_with_values(name: &str, version: &str, values: &str) -> Vec { - chart_archive( - name, - &format!("apiVersion: v2\nname: {name}\nversion: {version}\ndescription: {name} chart\n"), - values, - ) -} - -/// A gzipped chart archive of the chart `name` with `chart_yaml` as its -/// Chart.yaml. -fn chart_archive(name: &str, chart_yaml: &str, values: &str) -> Vec { - tgz(&[ - (&format!("{name}/values.yaml"), values), - (&format!("{name}/Chart.yaml"), chart_yaml), - ]) -} - -/// A gzipped ustar archive of `(path, contents)` entries. -fn tgz(entries: &[(&str, &str)]) -> Vec { - let mut tar = Vec::new(); - for (path, data) in entries { - let mut header = [0u8; 512]; - header[..path.len()].copy_from_slice(path.as_bytes()); - header[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes()); - header[156] = b'0'; - header[257..263].copy_from_slice(b"ustar\0"); - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - tar.extend_from_slice(&header); - tar.extend_from_slice(data.as_bytes()); - tar.resize(tar.len().div_ceil(512) * 512, 0); - } - tar.extend_from_slice(&[0u8; 1024]); - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(&tar).unwrap(); - gz.finish().unwrap() -} - -async fn create_repo(srv: &TestServer, body: Value) -> reqwest::Response { - let (u, p) = srv.admin_basic(); - srv.client() - .post(format!("{}/admin/repositories", srv.base_url)) - .basic_auth(&u, Some(&p)) - .json(&body) - .send() - .await - .unwrap() -} - -/// Create a hosted Helm repository and return its name. -async fn helm_repo(srv: &TestServer, anonymous_read: bool) -> String { - hosted_repo(srv, "helm", anonymous_read).await -} - -/// Create a hosted repository of `format` and return its name. -async fn hosted_repo(srv: &TestServer, format: &str, anonymous_read: bool) -> String { - let name = format!("charts{}", unique()); - let resp = create_repo( - srv, - json!({ "name": name, "kind": "hosted", "format": format, "anonymous_read": anonymous_read }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - name -} - -/// The admin's username and an API token: what `helm repo add --username -/// --password` would be given. -async fn credentials(srv: &TestServer) -> (String, String) { - (srv.admin_user.clone().unwrap(), srv.admin_token().await) -} - -async fn push( - srv: &TestServer, - repo: &str, - creds: &(String, String), - body: Vec, -) -> reqwest::Response { - push_at(srv, "/repository", repo, creds, body).await -} - -/// [`push`] to a repository served under `mount`. -async fn push_at( - srv: &TestServer, - mount: &str, - repo: &str, - creds: &(String, String), - body: Vec, -) -> reqwest::Response { - srv.client() - .post(format!("{}{mount}/{repo}/api/charts", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .body(body) - .send() - .await - .unwrap() -} - -async fn get_index( - srv: &TestServer, - repo: &str, - creds: Option<&(String, String)>, -) -> reqwest::Response { - get_index_at(srv, "/repository", repo, creds).await -} - -/// [`get_index`] of a repository served under `mount`. -async fn get_index_at( - srv: &TestServer, - mount: &str, - repo: &str, - creds: Option<&(String, String)>, -) -> reqwest::Response { - let req = srv - .client() - .get(format!("{}{mount}/{repo}/index.yaml", srv.base_url)); - let req = match creds { - Some((u, p)) => req.basic_auth(u, Some(p)), - None => req, - }; - req.send().await.unwrap() -} - -#[tokio::test] -async fn a_helm_repository_round_trip() { - let srv = start_helm(ServerOpts::default()).await; - let repo = helm_repo(&srv, false).await; - let creds = credentials(&srv).await; - - // Publish: the raw body, and the multipart field `helm cm-push` sends. - let first = chart("Demo", "v1.0"); - let resp = push(&srv, &repo, &creds, first.clone()).await; - assert_eq!(resp.status(), StatusCode::CREATED); - assert_eq!( - resp.json::().await.unwrap(), - json!({ "saved": true }) - ); - let form = reqwest::multipart::Form::new() - .text("note", "skipped before the chart") - .part( - "chart", - reqwest::multipart::Part::bytes(chart("Demo", "1.1.0")).file_name("Demo-1.1.0.tgz"), - ); - let resp = srv - .client() - .post(format!("{}/repository/{repo}/api/charts", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .multipart(form) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - // The same version again (names match case-insensitively), and - // something that is not a chart. - let resp = push(&srv, &repo, &creds, chart("demo", "1.0.0")).await; - assert_eq!(resp.status(), StatusCode::CONFLICT); - let resp = push(&srv, &repo, &creds, b"not a chart".to_vec()).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp.text().await.unwrap().contains("not a chart archive")); - // A chart that Helm would install as another one: Helm loads the last - // Chart.yaml of an archive. - let twin = tgz(&[ - ( - "demo/Chart.yaml", - "apiVersion: v2\nname: demo\nversion: 3.0.0\n", - ), - ( - "evil/Chart.yaml", - "apiVersion: v2\nname: evil\nversion: 6.6.6\n", - ), - ]); - let resp = push(&srv, &repo, &creds, twin).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp - .text() - .await - .unwrap() - .contains("more than one top-level directory")); - - // The index lists both versions, newest first, as published. - let resp = get_index(&srv, &repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.headers()["x-content-type-options"], "nosniff"); - let index: Value = resp.json().await.unwrap(); - let entries = index["entries"]["Demo"].as_array().expect("Demo entries"); - let versions: Vec<&str> = entries - .iter() - .map(|e| e["version"].as_str().unwrap()) - .collect(); - assert_eq!(versions, ["1.1.0", "v1.0"]); - assert_eq!(entries[1]["description"], "Demo chart"); - assert_eq!( - entries[1]["digest"], - hex::encode(sha2::Sha256::digest(&first)), - "the digest is the stored package's SHA-256" - ); - - // The URL the index points at serves the chart, streamed by the host. - let url = entries[1]["urls"][0].as_str().unwrap(); - let path = url - .strip_prefix("http://test.local") - .expect("absolute URL on the public base"); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), first.as_slice()); - let resp = srv - .client() - .head(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.headers()["content-length"], first.len().to_string()); - - // The ChartMuseum listing of one chart. - let resp = srv - .client() - .get(format!( - "{}/repository/{repo}/api/charts/demo", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.json::() - .await - .unwrap() - .as_array() - .unwrap() - .len(), - 2 - ); - - // Delete a version: gone from the index and from downloads. - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/demo/1.0.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = get_index(&srv, &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["Demo"].as_array().unwrap().len(), 1); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/demo/1.0.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // The package detail of the web UI reads the plugin's summary. - let session = srv - .login( - &srv.admin_user.clone().unwrap(), - &srv.admin_pass.clone().unwrap(), - ) - .await; - let detail: Value = session - .apply( - srv.client() - .get(format!("{}/browse/packages/{repo}/demo", srv.base_url)), - ) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(detail["original_id"], "Demo"); - assert_eq!(detail["versions"].as_array().unwrap().len(), 1); -} - -/// #318: a holder of the built-in `writer` role uploads a new chart to a -/// hosted Helm repository and reads it back, but cannot delete a version. -#[tokio::test] -async fn a_writer_uploads_charts_but_deletes_none() { - let srv = start_helm(ServerOpts::default()).await; - let repo = helm_repo(&srv, false).await; - let (au, ap) = srv.admin_basic(); - let username = format!("charts-writer{}@example.com", unique()); - let user_id = srv - .create_user(&username, "pw12345678-charts", "reader") - .await; - let resp = srv - .client() - .put(format!("{}/admin/users/{user_id}/roles", srv.base_url)) - .basic_auth(&au, Some(&ap)) - .json(&json!({ "role": "writer" })) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NO_CONTENT); - let token: Value = srv - .client() - .post(format!("{}/admin/users/{user_id}/tokens", srv.base_url)) - .basic_auth(&au, Some(&ap)) - .json(&json!({ "name": "ci" })) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - let creds = (username, token["token"].as_str().unwrap().to_string()); - - let resp = push(&srv, &repo, &creds, chart("Fresh", "0.1.0")).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let index: Value = get_index(&srv, &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["Fresh"].as_array().unwrap().len(), 1); - let resp = srv - .client() - .delete(format!( - "{}/repository/{repo}/api/charts/fresh/0.1.0", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::FORBIDDEN); -} - -#[tokio::test] -async fn plugin_repositories_are_authorized_like_feeds() { - let srv = start_with( - common::fresh_pool().await, - ServerOpts { - bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - extra_plugins: vec![helm_plugin()], - ..Default::default() - }, - ) - .await; - srv.set_auth_settings(serde_json::json!({"anonymous_mode": "all"})) - .await; - let private = helm_repo(&srv, false).await; - let public = helm_repo(&srv, true).await; - let creds = credentials(&srv).await; - let resp = push(&srv, &public, &creds, chart("web", "1.0.0")).await; - assert_eq!(resp.status(), StatusCode::CREATED); - - // Reading: anonymous only where anonymous reads are allowed. - let resp = get_index(&srv, &private, None).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - assert!(resp.headers().contains_key("www-authenticate")); - let resp = get_index(&srv, &public, None).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert!(index["entries"]["web"].is_array()); - // An unknown repository answers like one the caller may not read. - let resp = index_of_unknown(&srv).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - - // Writing: never anonymously, and never with a password (a browser - // credential), only with an API token. - let resp = srv - .client() - .post(format!("{}/repository/{public}/api/charts", srv.base_url)) - .body(chart("web", "2.0.0")) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - let (user, password) = srv.admin_password_basic(); - let resp = push(&srv, &public, &(user, password), chart("web", "2.0.0")).await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - assert!(resp.text().await.unwrap().contains("API token")); -} - -async fn index_of_unknown(srv: &TestServer) -> reqwest::Response { - get_index(srv, &format!("missing{}", unique()), None).await -} - -#[tokio::test] -async fn formats_keep_to_their_own_mounts_and_kinds() { - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let charts = helm_repo(&srv, false).await; - let nuget = format!("nuget{}", unique()); - let resp = create_repo(&srv, json!({ "name": nuget, "kind": "hosted" })).await; - assert_eq!(resp.status(), StatusCode::CREATED); - - // A Helm repository is not a NuGet feed, and the other way around. - let resp = srv - .client() - .get(format!("{}/v3/{charts}/index.json", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - let resp = get_index(&srv, &nuget, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // The admin API lists what can be created. - let (u, p) = srv.admin_basic(); - let formats: Value = srv - .client() - .get(format!("{}/admin/formats", srv.base_url)) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(formats[0]["id"], "nuget"); - assert_eq!(formats[1]["id"], "docker"); - assert_eq!(formats[2]["id"], "helm"); - assert_eq!(formats[2]["builtin"], true); - assert_eq!(formats[2]["kinds"], json!(["hosted", "proxy", "group"])); - assert_eq!(formats[2]["mount"], "/repository"); - assert!(formats[2].get("sha256").is_none()); - - // A proxy needs its upstream; unknown formats are refused. - let resp = create_repo( - &srv, - json!({ "name": format!("p{}", unique()), "kind": "proxy", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - let resp = create_repo( - &srv, - json!({ "name": format!("n{}", unique()), "kind": "hosted", "format": "npm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp.text().await.unwrap().contains("helm")); - - // A group serves one format: a NuGet member is refused, a Helm one is - // served through the group. - let group = format!("group{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": group, "kind": "group", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let add = |member: String, priority: i32| { - srv.client() - .post(format!( - "{}/admin/repositories/{group}/members", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .json(&json!({ "member": member, "priority": priority })) - .send() - }; - assert_eq!( - add(nuget.clone(), 1).await.unwrap().status(), - StatusCode::BAD_REQUEST - ); - assert_eq!( - add(charts.clone(), 1).await.unwrap().status(), - StatusCode::CREATED - ); - let resp = push(&srv, &charts, &creds, chart("api", "3.2.1")).await; - assert_eq!(resp.status(), StatusCode::CREATED); - let index: Value = get_index(&srv, &group, Some(&creds)) - .await - .json() - .await - .unwrap(); - let url = index["entries"]["api"][0]["urls"][0] - .as_str() - .unwrap() - .to_string(); - assert!( - url.contains(&format!("/repository/{group}/charts/api/3.2.1/")), - "{url}" - ); - let resp = srv - .client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - // A group publishes nothing. - let resp = push(&srv, &group, &creds, chart("api", "3.2.2")).await; - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); -} - -#[tokio::test] -async fn a_repository_has_an_upload_limit_of_its_own() { - let srv = start_helm(ServerOpts::default()).await; - let name = format!("charts{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": name, "kind": "hosted", "format": "helm", "max_upload_bytes": 1048576 }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let creds = credentials(&srv).await; - let resp = push(&srv, &name, &creds, chart("small", "1.0.0")).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let resp = push(&srv, &name, &creds, vec![0u8; 2 * 1024 * 1024]).await; - assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE); - assert_eq!( - resp.text().await.unwrap(), - "upload exceeds limit of 1048576 bytes" - ); -} - -#[tokio::test] -async fn a_repository_whose_plugin_is_missing_is_unavailable() { - // A server without the plugin, over a repository created with it, under - // a mount another plugin serves. - let srv = start_helm(ServerOpts::default()).await; - let name = format!("orphan{}", unique()); - let mut row = forge_db::repository_dao::NewRepository::hosted(name.clone()); - row.format = WASM_HELM.into(); - forge_db::repository_dao::insert_repository(&srv.pool, row) - .await - .expect("insert repository"); - let (u, p) = srv.admin_basic(); - let resp = get_index(&srv, &name, Some(&(u.clone(), p.clone()))).await; - assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); - let resp = srv - .client() - .delete(format!( - "{}/admin/repositories/{name}/packages/demo/1.0.0", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); -} - -/// The Helm crate built as a WebAssembly component serves its repositories -/// exactly as the built-in one does: one contract, two ways to run it. -#[tokio::test] -async fn a_webassembly_plugin_serves_like_a_builtin_one() { - let srv = start_helm(ServerOpts { - plugins_block: Some(wasm_helm_plugins_block()), - ..Default::default() - }) - .await; - let creds = credentials(&srv).await; - let (u, p) = srv.admin_basic(); - let formats: Value = srv - .client() - .get(format!("{}/admin/formats", srv.base_url)) - .basic_auth(&u, Some(&p)) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - let wasm = formats - .as_array() - .unwrap() - .iter() - .find(|f| f["id"] == WASM_HELM) - .expect("the component is listed"); - assert_eq!(wasm["builtin"], false); - assert_eq!(wasm["sha256"].as_str().map(str::len), Some(64)); - - assert_eq!(wasm["mount"], "/charts"); - - let repo = hosted_repo(&srv, WASM_HELM, false).await; - let package = chart("Demo", "v1.0"); - let resp = push_at(&srv, "/charts", &repo, &creds, package.clone()).await; - assert_eq!(resp.status(), StatusCode::CREATED); - let resp = push_at(&srv, "/charts", &repo, &creds, package.clone()).await; - assert_eq!(resp.status(), StatusCode::CONFLICT); - let resp = push_at(&srv, "/charts", &repo, &creds, b"not a chart".to_vec()).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - // Served under its own mount only. - let resp = get_index(&srv, &repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let index: Value = get_index_at(&srv, "/charts", &repo, Some(&creds)) - .await - .json() - .await - .unwrap(); - let entry = &index["entries"]["Demo"][0]; - assert_eq!(entry["version"], "v1.0"); - assert_eq!(entry["digest"], hex::encode(sha2::Sha256::digest(&package))); - let url = entry["urls"][0].as_str().unwrap(); - assert!(url.contains(&format!("/charts/{repo}/charts/")), "{url}"); - let resp = srv - .client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), package.as_slice()); -} - -/// A WebAssembly plugin never replaces a built-in format. -#[test] -fn a_component_cannot_take_a_builtin_format_id() { - let dir = forge_plugin::testing::example_plugin_as("helm", "nuget"); - let plugin = forge_plugin::Plugin::load( - &dir, - &forge_plugin::testing::component_sha256(&dir), - &forge_plugin::HostLimits::default(), - ) - .unwrap(); - let err = forge_server::state::FormatRegistry::new(Default::default()) - .with_plugin(plugin) - .err() - .expect("nuget is taken"); - assert!(err.contains("built into this server"), "{err}"); -} - -/// A chart repository upstream of a Helm proxy, serving `index.yaml` and -/// chart archives under `/stable`, and counting chart downloads. -struct ChartUpstream { - base: String, - charts: std::sync::Arc>>>, - downloads: std::sync::Arc, -} - -impl ChartUpstream { - async fn start(index: String) -> Self { - use axum::extract::{Path, State}; - use axum::response::IntoResponse; - use std::sync::atomic::Ordering; - - type Shared = ( - String, - std::sync::Arc>>>, - std::sync::Arc, - ); - let charts = std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); - let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let app = axum::Router::new() - .route( - "/stable/index.yaml", - axum::routing::get(|State((index, _, _)): State| async move { index }), - ) - .route( - "/stable/charts/{file}", - axum::routing::get( - |State((_, charts, downloads)): State, Path(file): Path| async move { - match charts.lock().unwrap().get(&file).cloned() { - Some(bytes) => { - downloads.fetch_add(1, Ordering::SeqCst); - bytes.into_response() - } - None => StatusCode::NOT_FOUND.into_response(), - } - }, - ), - ) - .with_state((index, charts.clone(), downloads.clone())); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base = format!("http://{}/stable", listener.local_addr().unwrap()); - tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - ChartUpstream { - base, - charts, - downloads, - } - } - - fn publish(&self, file: &str, bytes: Vec) { - self.charts.lock().unwrap().insert(file.into(), bytes); - } - - fn downloads(&self) -> usize { - self.downloads.load(std::sync::atomic::Ordering::SeqCst) - } -} - -/// Check `index` against the types Helm gives an index's fields (its -/// `repo.ChartVersion` and `chart.Metadata`). Helm reads a JSON index with -/// Go's `encoding/json` and refuses the whole index when one value has -/// another type; fields it does not know it ignores. -fn assert_helm_types(index: &Value) { - const STRINGS: &[&str] = &[ - "name", - "home", - "version", - "description", - "icon", - "apiVersion", - "condition", - "tags", - "appVersion", - "kubeVersion", - "type", - "created", - "digest", - ]; - let strings = |v: &Value| v.as_array().is_some_and(|a| a.iter().all(Value::is_string)); - let fields = |v: &Value, strs: &[&str], lists: &[&str], bools: &[&str], any: &[&str]| { - v.as_object().is_some_and(|o| { - o.iter().all(|(k, v)| { - (strs.contains(&k.as_str()) && v.is_string()) - || (lists.contains(&k.as_str()) && strings(v)) - || (bools.contains(&k.as_str()) && v.is_boolean()) - || (any.contains(&k.as_str()) && v.is_array()) - }) - }) - }; - assert!(index["apiVersion"].is_string()); - for (chart, versions) in index["entries"].as_object().expect("entries") { - for version in versions.as_array().expect("versions") { - for (key, value) in version.as_object().expect("a chart version") { - let ok = match key.as_str() { - key if STRINGS.contains(&key) => value.is_string(), - "sources" | "keywords" | "urls" => strings(value), - "deprecated" | "removed" => value.is_boolean(), - "annotations" => value - .as_object() - .is_some_and(|m| m.values().all(Value::is_string)), - "maintainers" => value.as_array().is_some_and(|a| { - a.iter() - .all(|m| fields(m, &["name", "email", "url"], &[], &[], &[])) - }), - "dependencies" => value.as_array().is_some_and(|a| { - a.iter().all(|d| { - fields( - d, - &["name", "version", "repository", "condition", "alias"], - &["tags"], - &["enabled"], - &["import-values"], - ) - }) - }), - _ => true, - }; - assert!(ok, "{chart}: Helm cannot read {key} = {value}"); - } - } - } -} - -/// A Chart.yaml (or an upstream's YAML index) may hold a number or a -/// boolean where Helm wants a string, as Helm's YAML reader accepts: the -/// index lists it as a string, so it does not break the index (#206). -#[tokio::test] -async fn an_index_lists_scalars_as_the_strings_helm_reads() { - const QUIRKS: &str = concat!( - "appVersion: 2\n", - "kubeVersion: 1.30\n", - "icon: 7\n", - "keywords: [web, 8, true]\n", - "annotations:\n", - " artifacthub.io/prerelease: false\n", - " revision: 3\n", - " nested: {a: b}\n", - "maintainers:\n", - " - name: 42\n", - " email: ops@example.com\n", - " extra: [x]\n", - "dependencies:\n", - " - name: redis\n", - " version: 18\n", - " repository: https://charts.example\n", - " enabled: true\n", - " tags: [cache, 1]\n", - " import-values: [data]\n", - "deprecated: false\n", - "home: {not: a string}\n", - "sources: just-one\n", - ); - let upstream_index = format!( - "apiVersion: v1\nentries:\n quirky:\n - apiVersion: v2\n name: quirky\n \ - version: 1.0.0\n urls: [charts/quirky-1.0.0.tgz]\n{}", - QUIRKS - .lines() - .map(|l| format!(" {l}\n")) - .collect::() - ); - let upstream = ChartUpstream::start(upstream_index).await; - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let hosted = helm_repo(&srv, false).await; - let archive = chart_archive( - "quirks", - &format!("apiVersion: v2\nname: quirks\nversion: 0.1.0\n{QUIRKS}"), - "replicas: 1\n", - ); - let resp = push(&srv, &hosted, &creds, archive).await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let group = format!("group{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": group, "kind": "group", "format": "helm" }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let (u, p) = srv.admin_basic(); - for (member, priority) in [(&hosted, 10), (&proxy, 1)] { - let resp = srv - .client() - .post(format!( - "{}/admin/repositories/{group}/members", - srv.base_url - )) - .basic_auth(&u, Some(&p)) - .json(&json!({ "member": member, "priority": priority })) - .send() - .await - .unwrap(); - assert!(resp.status().is_success(), "{}", resp.text().await.unwrap()); - } - - for (repo, chart) in [ - (&hosted, "quirks"), - (&proxy, "quirky"), - (&group, "quirks"), - (&group, "quirky"), - ] { - let resp = get_index(&srv, repo, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert_helm_types(&index); - let entry = &index["entries"][chart][0]; - assert_eq!(entry["appVersion"], "2", "{repo}: {entry}"); - assert_eq!(entry["kubeVersion"], "1.30"); - assert_eq!(entry["keywords"], json!(["web", "8", "true"])); - assert_eq!(entry["annotations"]["artifacthub.io/prerelease"], "false"); - assert_eq!(entry["annotations"]["revision"], "3"); - assert!(entry["annotations"].get("nested").is_none()); - assert_eq!( - entry["maintainers"][0], - json!({"name": "42", "email": "ops@example.com"}) - ); - assert_eq!(entry["dependencies"][0]["version"], "18"); - assert_eq!(entry["dependencies"][0]["tags"], json!(["cache", "1"])); - assert_eq!(entry["dependencies"][0]["enabled"], true); - assert_eq!(entry["deprecated"], false); - assert!(entry.get("home").is_none() && entry.get("sources").is_none()); - } -} - -/// A Helm proxy lists its upstream's charts in its index, serves each from -/// the upstream once and from its cache after, and publishes nothing. -#[tokio::test] -async fn a_helm_proxy_serves_its_upstreams_charts() { - let index = "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n version: 1.0.0\n description: demo chart\n urls: [charts/demo-1.0.0.tgz]\n other:\n - apiVersion: v2\n name: other\n version: 2.0.0\n urls: [charts/other-2.0.0.tgz]\n twin:\n - apiVersion: v2\n name: twin\n version: 1.0.0\n urls: [charts/twin-1.0.0.tgz]\n".to_string(); - let upstream = ChartUpstream::start(index).await; - let demo = chart("demo", "1.0.0"); - upstream.publish("demo-1.0.0.tgz", demo.clone()); - // Helm would install it as `evil` 6.6.6, its last Chart.yaml. - upstream.publish( - "twin-1.0.0.tgz", - tgz(&[ - ( - "twin/Chart.yaml", - "apiVersion: v2\nname: twin\nversion: 1.0.0\n", - ), - ( - "twin/Chart.yaml", - "apiVersion: v2\nname: evil\nversion: 6.6.6\n", - ), - ]), - ); - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - // The index lists the upstream's charts, nothing downloaded yet. - let resp = get_index(&srv, &proxy, Some(&creds)).await; - assert_eq!(resp.status(), StatusCode::OK); - let index: Value = resp.json().await.unwrap(); - assert_eq!(index["entries"]["other"][0]["version"], "2.0.0"); - let url = index["entries"]["demo"][0]["urls"][0] - .as_str() - .unwrap() - .to_string(); - assert!( - url.ends_with(&format!( - "/repository/{proxy}/charts/demo/1.0.0/demo-1.0.0.tgz" - )), - "{url}" - ); - assert_eq!(index["entries"]["demo"][0]["description"], "demo chart"); - assert_eq!(upstream.downloads(), 0); - - // The chart comes from the upstream once, then from the cache. - let get = |url: String| { - srv.client() - .get(url.replace("http://test.local", &srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - }; - for _ in 0..2 { - let resp = get(url.clone()).await.unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), demo.as_slice()); - } - assert_eq!(upstream.downloads(), 1); - - // A version the upstream does not have is not found. - let resp = get(format!( - "{}/repository/{proxy}/charts/other/2.0.0/other-2.0.0.tgz", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - // A chart that is not what Helm would install is not cached. - let resp = get(format!( - "{}/repository/{proxy}/charts/twin/1.0.0/twin-1.0.0.tgz", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert_eq!(upstream.downloads(), 2); - - // The ChartMuseum view of one chart, from the upstream's index. - let resp = get(format!( - "{}/repository/{proxy}/api/charts/demo", - srv.base_url - )) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let versions: Value = resp.json().await.unwrap(); - assert_eq!(versions[0]["version"], "1.0.0"); - - // A proxy publishes nothing. - let resp = push(&srv, &proxy, &creds, chart("demo", "1.0.1")).await; - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); -} - -/// `len` lowercase letters and digits that PostgreSQL cannot compress, so -/// that an index entry holding them is as long as they are. -fn incompressible(len: usize) -> String { - const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789"; - let mut block = sha2::Sha256::digest(b"incompressible"); - let mut out = String::with_capacity(len); - while out.len() < len { - out.extend( - block - .iter() - .map(|b| char::from(ALPHABET[usize::from(*b) % ALPHABET.len()])) - .take(len - out.len()), - ); - block = sha2::Sha256::digest(&block[..]); - } - out -} - -/// #231: a chart version longer than 128 characters (an OCI tag's limit) -/// is refused on push with 400, not stored until its index entry fails the -/// insert (500); a URL naming one answers 404, in a hosted repository and in -/// a proxy, which neither records it nor asks its upstream; and a proxy -/// leaves an upstream's such version out of its listings. -#[tokio::test] -async fn chart_versions_longer_than_an_oci_tag_are_refused() { - let huge = format!("1.0.0-{}", incompressible(4096)); - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let get = |path: String| { - srv.client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - }; - let long_rows = |repo: String| { - let pool = srv.pool.clone(); - async move { - sqlx::query_scalar::<_, i64>( - "SELECT count(*) FROM proxy_cached_version v JOIN repository r \ - ON r.id = v.repository_id WHERE r.name = $1 AND length(v.version) > 128", - ) - .bind(repo) - .fetch_one(&pool) - .await - .unwrap() - } - }; - - let repo = helm_repo(&srv, false).await; - let resp = push(&srv, &repo, &creds, chart("demo", &huge)).await; - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); - assert!(resp - .text() - .await - .unwrap() - .contains("longer than 128 characters")); - let longest = format!("1.0.0-{}", "a".repeat(122)); - for (version, status) in [ - (format!("{longest}a"), StatusCode::BAD_REQUEST), - (longest.clone(), StatusCode::CREATED), - ] { - let resp = push(&srv, &repo, &creds, chart("demo", &version)).await; - assert_eq!(resp.status(), status, "{}", version.len()); - } - let resp = get(format!("/repository/{repo}/charts/demo/{longest}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let resp = get(format!("/repository/{repo}/charts/demo/{huge}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let index = format!( - "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n \ - version: 1.0.0\n urls: [charts/demo-1.0.0.tgz]\n - apiVersion: v2\n \ - name: demo\n version: {huge}\n urls: [charts/demo-huge.tgz]\n" - ); - let upstream = ChartUpstream::start(index).await; - upstream.publish("demo-huge.tgz", chart("demo", &huge)); - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - let resp = get(format!("/repository/{proxy}/charts/demo/{huge}/demo.tgz")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - assert_eq!(upstream.downloads(), 0); - assert_eq!(long_rows(proxy.clone()).await, 0); - - // The upstream's longer version is left out of the proxy's listings. - let resp = get(format!("/repository/{proxy}/api/charts/demo")) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let versions: Value = resp.json().await.unwrap(); - assert_eq!(versions.as_array().unwrap().len(), 1); - assert_eq!(versions[0]["version"], "1.0.0"); - let index: Value = get_index(&srv, &proxy, Some(&creds)) - .await - .json() - .await - .unwrap(); - assert_eq!(index["entries"]["demo"].as_array().unwrap().len(), 1); - assert_eq!(long_rows(proxy).await, 0); -} - -/// A Helm index may list a chart on another host (a project's GitHub -/// releases): the proxy downloads it from there, a host its upstream's -/// index names (#233). -#[tokio::test] -async fn a_helm_proxy_downloads_a_chart_its_upstreams_index_lists_on_another_host() { - let releases = ChartUpstream::start(String::new()).await; - let demo = chart("demo", "1.0.0"); - releases.publish("demo-1.0.0.tgz", demo.clone()); - let index = format!( - "apiVersion: v1\nentries:\n demo:\n - apiVersion: v2\n name: demo\n \ - version: 1.0.0\n urls:\n - {}/charts/demo-1.0.0.tgz\n", - releases.base - ); - let upstream = ChartUpstream::start(index).await; - - let srv = start_helm(ServerOpts::default()).await; - let creds = credentials(&srv).await; - let proxy = format!("proxy{}", unique()); - let resp = create_repo( - &srv, - json!({ "name": proxy, "kind": "proxy", "format": "helm", "remote_url": upstream.base }), - ) - .await; - assert_eq!( - resp.status(), - StatusCode::CREATED, - "{}", - resp.text().await.unwrap() - ); - - let resp = srv - .client() - .get(format!( - "{}/repository/{proxy}/charts/demo/1.0.0/demo-1.0.0.tgz", - srv.base_url - )) - .basic_auth(&creds.0, Some(&creds.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.bytes().await.unwrap().as_ref(), demo.as_slice()); - assert_eq!((upstream.downloads(), releases.downloads()), (0, 1)); -} diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 206f5b15..221eb419 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -817,7 +817,7 @@ async fn chunked_uploads_cache_the_blob_not_its_chunks() { common::pool().await, common::ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - plugins_block: Some(format!( + extra_toml: Some(format!( "[blob_cache]\npath = {:?}\nmax_bytes = 1048576\nmin_free_bytes = 0\n", dir.path().join("cache").display().to_string() )), @@ -2560,7 +2560,7 @@ async fn admin_validates_docker_repositories() { .iter() .find(|f| f["id"] == "docker") .expect("docker is listed"); - assert_eq!(docker["builtin"], true); + assert!(docker.get("builtin").is_none(), "{docker}"); assert_eq!(docker["kinds"], json!(["hosted", "proxy", "group"])); assert_eq!(docker["mount"], "/v2"); @@ -2600,7 +2600,7 @@ async fn confirming_a_blob_records_the_use_of_its_link() { common::pool().await, common::ServerOpts { bootstrap_admin: Some((format!("admin{}@example.com", unique()), "adminpw".into())), - plugins_block: Some("[docker]\nunreferenced_blob_ttl_seconds = 3600\n".into()), + extra_toml: Some("[docker]\nunreferenced_blob_ttl_seconds = 3600\n".into()), ..Default::default() }, ) diff --git a/docker/licenses/about.toml b/docker/licenses/about.toml index 5c9a6528..0a4bf7ff 100644 --- a/docker/licenses/about.toml +++ b/docker/licenses/about.toml @@ -12,10 +12,6 @@ # check that a new one is before adding it. accepted = [ "Apache-2.0", - # Apache-2.0 plus an additional permission (the LLVM exception, waiving - # attribution for compiled-in code): Wasmtime and Cranelift, the runtime - # of format plugins. - "Apache-2.0 WITH LLVM-exception", "MIT", "ISC", "BSD-3-Clause", diff --git a/docs/changes/2.0.0/228.fixed.md b/docs/changes/2.0.0/228.fixed.md deleted file mode 100644 index 0dea4138..00000000 --- a/docs/changes/2.0.0/228.fixed.md +++ /dev/null @@ -1,4 +0,0 @@ -Format plugins: the host stores no manifest summary over 64 KiB from a -WebAssembly format plugin. Every listing hands each version's summary to -the plugin, so a plugin that summarized without a bound let a publisher make -every listing take gigabytes. diff --git a/docs/changes/2.0.0/229.security.md b/docs/changes/2.0.0/229.security.md deleted file mode 100644 index b6875d95..00000000 --- a/docs/changes/2.0.0/229.security.md +++ /dev/null @@ -1,10 +0,0 @@ -WebAssembly format plugins: the memory ceiling (`memory_mib`) did not bound -the host memory a component could take. It applied to each of up to 16 -linear memories on its own, left tables unbounded (a `table.grow` to -hundreds of millions of elements), and did not cover what a call returns: a -list whose elements all point at one large string was copied into the host -once per element. A buggy or malicious component could so make Forge run -out of memory on its first call. Now an instance's memories and tables -count against its ceiling together, and the strings and lists one call -returns, and the random bytes it asks WASI for, are bounded by it too; a -component that goes past it fails that call like any other trap. diff --git a/docs/changes/2.0.0/234.fixed.md b/docs/changes/2.0.0/234.fixed.md deleted file mode 100644 index 9e4c6abf..00000000 --- a/docs/changes/2.0.0/234.fixed.md +++ /dev/null @@ -1,5 +0,0 @@ -Format plugins: a plugin could take the mount `/images`, the web UI's image -pages, so a refresh, bookmark or shared link of an image page reached the -plugin instead of the web UI. `/images` and `/invite` are now reserved like -the other pages, and a test keeps the reserved mounts in step with the web -UI's routes. diff --git a/docs/deployment.md b/docs/deployment.md index d7a259b3..9d5bb49a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -889,11 +889,6 @@ notices within about 15 s, drops its in-memory cache and resubscribes. name ([format-plugins.md](./format-plugins.md#security-model)); to keep internal services out of reach even of an upstream that names them, block them on the outbound proxy or with network policy. -- **Format plugins.** Every instance must load the same WebAssembly plugins - (the same `[plugins]` section and component files): a repository whose - format is missing on an instance answers 503 there. The built-in plugin - (NuGet) comes with the image. See - [format-plugins.md](./format-plugins.md#loading-webassembly-plugins). ### Redis is a trust boundary diff --git a/docs/format-plugins.md b/docs/format-plugins.md index fd0e7c6b..278e8421 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -9,21 +9,10 @@ stays Forge's own. That covers storage, metadata, authentication, RBAC and package selectors, hosted, proxy and group repositories, retention, and the streaming of package bytes. -A plugin runs one of two ways, and the server cannot tell them apart: - -- **Built in.** The crate is linked into the server and called directly, like - any other Rust code. Built-in plugins need no configuration. Forge ships - one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). -- **As a WebAssembly component.** The same kind of crate, built for - `wasm32-wasip2`, loaded at startup from a digest the operator pinned and - run in a sandbox. This is how formats that are not part of Forge are added - without rebuilding it. - -[`plugins/helm`](../plugins/helm), classic Helm chart repositories -(`index.yaml`, ChartMuseum uploads), is the **example plugin**: the plugin -tests run it both ways, and it is the crate to start a new plugin from, but -Forge does not build it in. Helm charts are served by the registry -([docker.md](docker.md#helm-charts)). +Every plugin is built in: a crate under `plugins//` linked into the +server and called directly, like any other Rust code. Plugins need no +configuration. Forge ships one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). +Helm charts are served by the registry ([docker.md](docker.md#helm-charts)). > **Status: preview.** The plugin contract is `forge:format@0.3.0`. It may > change before 1.x of the contract; a plugin declares the contract version it @@ -31,29 +20,22 @@ Forge does not build it in. Helm charts are served by the registry ## Contents -- [One contract, two ways to run it](#one-contract-two-ways-to-run-it) +- [Layout](#layout) - [How it fits together](#how-it-fits-together) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) -- [Loading WebAssembly plugins](#loading-webassembly-plugins) - [Security model](#security-model) - [Writing a plugin](#writing-a-plugin) - [Limitations and roadmap](#limitations-and-roadmap) -- [Design notes: compared with artifact-keeper](#design-notes-compared-with-artifact-keeper) -## One contract, two ways to run it +## Layout -Every plugin, built in or not, has the same layout: +A plugin is a workspace crate with a `plugin.toml`: ``` -plugins/helm/ -├── Cargo.toml # a workspace crate; feature `component` builds it as a component -├── plugin.toml # id, name, version, contract version; component file and limits -└── src/lib.rs # impl FormatPlugin for Helm - plugins/nuget/ ├── Cargo.toml -├── plugin.toml # mount "/v3", kinds hosted/proxy/group +├── plugin.toml # id, name, version, contract version; mount "/v3", kinds hosted/proxy/group ├── src/lib.rs # impl FormatPlugin for Nuget ├── src/protocol.rs # the V3 protocol: handle and render ├── src/upstream.rs # proxies: reading another V3 feed (upstream) @@ -64,24 +46,17 @@ The pieces: | Crate | What it holds | | --- | --- | -| [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, the same contract as WIT ([`wit/format.wit`](../crates/plugin-api/wit/format.wit)), and, behind its `component` feature, the glue that exports a plugin as a component (`export_component!`). Depends on nothing but `std`. | +| [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, and, behind its `testing` feature, helpers that run a plugin's protocol against canned answers. Depends on nothing but `std`. | | [`plugins/`](../plugins) | One crate per plugin, implementing `FormatPlugin`. | -| [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin, built in or WebAssembly), the Wasmtime runtime whose `WasmPlugin` implements `FormatPlugin` by calling into a component, and `PluginFormat`, which adapts any plugin to Forge's `Format` seam. | -| [`forge-server`](../crates/server/src/plugins.rs) | The built-in plugins (`plugins::builtin`), loading the configured components, and the `{mount}/{repo}/...` dispatcher. | - -The trait mirrors the WIT world one to one: a component exports exactly the -trait's methods, and the host turns a loaded component back into a -`FormatPlugin`. So the dispatcher, the hosted write path, groups and -retention run the same code for both. The test -[`the_component_answers_like_the_builtin_plugin`](../crates/plugin/tests/component.rs) -runs the example plugin both ways and checks that every answer matches. +| [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin: its `plugin.toml` and its implementation) and `PluginFormat`, which adapts it to Forge's `Format` seam. | +| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`) and the `{mount}/{repo}/...` dispatcher. | ## How it fits together | | Forge (the host) | The plugin | | --- | --- | --- | | Identity | Calls the format for every id and version it sees | `canonical_id`, `validate_id`, `normalize_version`, `compare_versions` | -| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file (a component through a lent reader), returns id, version, manifest, dependencies | +| Publishing | Streams the upload to a staging file, enforces the format's upload limit, authorizes against the package selector, stores the blob content-addressed, writes the version in one transaction, records trusted-publishing provenance | `parse_package`: reads the staged file, returns id, version, manifest, dependencies | | Reads | Runs the queries the plugin asks for through the repository read path (hosted, proxy or group). Package selectors, group membership, prefer-hosted and reserved prefixes all apply | `handle` decides what a request is; `render` turns the loaded data into the response body | | Downloads | Streams the blob from S3, counts the download, answers `HEAD` without touching S3, and a single `Range` when the local blob cache is on | Names the version (`download`) | | Unlist, relist, delete | Hosted write path, with the repository's permissions and selectors; a package outside the caller's selector answers 404, like an unknown one | Names the version and the success response | @@ -116,8 +91,7 @@ in this order: hosted, and the caller must present an API token or a workload identity. -The contract is [`crates/plugin-api/src/lib.rs`](../crates/plugin-api/src/lib.rs), -and as WIT [`crates/plugin-api/wit/format.wit`](../crates/plugin-api/wit/format.wit). +The contract is [`crates/plugin-api/src/lib.rs`](../crates/plugin-api/src/lib.rs). Its queries are targeted: | Query | Answer | @@ -144,11 +118,7 @@ if present: `description`, and `dependency_groups` (the same shape as NuGet's). Every listing hands each version's summary to the plugin, so a summary -should stay small. The host stores none larger than 64 KiB from a -WebAssembly plugin, and logs a warning instead; a built-in plugin bounds its -own. The example plugin's summary, which holds a chart version's index -entry, takes at most 16 KiB: past that, its description is cut to 1 KiB, -then its largest fields are left out until it fits. +should stay small. ## Proxy repositories @@ -204,10 +174,9 @@ cached. ## Using a plugin `GET /admin/formats` lists every format the server can create repositories -of: the built-in plugin (`builtin: true`; NuGet), Docker, raw files, and the -loaded WebAssembly plugins (with the `sha256` of their component), each with -the repository kinds it supports and where its repositories are served. The -web UI's *New repository* form offers them. Plugins serve **hosted** and +of: the plugins (NuGet), Docker and raw files, each with the repository +kinds it supports and where its repositories are served. The web UI's *New +repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream side (NuGet does). Docker has proxies of its own. @@ -224,7 +193,11 @@ curl -u admin@example.com:$ADMIN_TOKEN -H 'content-type: application/json' \ ``` As with every format, the password is an **API token**. A password typed at -the web login is not a publishing credential. +the web login is not a publishing credential. The admin API refuses to +create a repository of a format the server does not have, and to put +repositories of different formats in one group. A repository whose format +the server no longer has stays in the database: it answers 503, is never +read as another format, and retention skips it (a failed run is recorded). The web UI's *Browse* screen filters repositories by format (`GET /browse/repos?format=`), and a repository's screen shows its @@ -240,70 +213,9 @@ a limit may be at most what a push can stage, `server.max_staging_bytes` less `nuget.max_nuspec_bytes` (`GET /admin/formats` reports it as `max_upload_bytes_limit`, with each format's `default_max_upload_bytes`). -## Loading WebAssembly plugins - -A WebAssembly plugin is a directory holding a `plugin.toml` and the component -it names: - -``` -/etc/forge/plugins/ -└── example/ - ├── plugin.toml - └── forge_format_example.wasm -``` - -The server loads only the plugins its configuration lists, each pinned to -the SHA-256 of its component: - -```toml -[plugins] -dir = "/etc/forge/plugins" - -[plugins.formats] -example = "sha256:c2053ffcbcc8cd38061813f5c42436464b7917c86fa3096eb849e653a0f1bf0a" -``` - -or, as environment variables: - -```bash -FORGE_PLUGINS__DIR=/etc/forge/plugins -FORGE_PLUGINS__FORMATS__EXAMPLE=sha256:c2053ffc… -``` - -- **Pinning is mandatory.** A component whose digest differs from its pin, - a `plugin.toml` whose `id` is not its directory name, a contract version - this server does not implement, or a component that does not instantiate - all fail startup. A directory under `dir` that is not listed is never - read. -- **Built-in formats come first.** A component cannot take the id of Docker, - raw or a built-in plugin (`nuget`); configuring one fails startup. -- **Every instance loads the same plugins.** A repository whose format is - not loaded on an instance answers 503 there; it is never read as another - format. The admin API refuses to create a repository of a format that is - not loaded, and to put repositories of different formats in one group. -- **Build them** with `cargo xtask plugins`. It builds every plugin in - `plugins/` as a component, installs it under - `target/plugins/installed//`, and prints the `[plugins]` section with - the digests. Built-in plugins are commented out there: their component - loads only under another id. A plugin whose `plugin.toml` names no - `component` is built in only, and skipped (NuGet, see - [NuGet as a plugin](#nuget-as-a-plugin)). In a container, bake the directory into an - image derived from Forge's, or mount it from a volume (a ConfigMap's 1 MiB - limit leaves little room for components). -- **Limits.** A plugin's `plugin.toml` may ask for a memory ceiling per - instance (`memory_mib`, default 64, at most 512; it covers the instance's - linear memories and tables together, and what one call returns to the - host), fuel per call (`fuel`, - default 5·10⁹, at most 5·10¹⁰) and a wall-clock budget per call - (`timeout_ms`, default 5,000, at most 30,000). Asking for more than the - maximum fails startup. Built-in plugins ignore `[limits]`. -- **Removing a plugin** leaves its repositories in the database. They answer - 503 until the plugin is back, and retention skips them (a failed run is - recorded). - ## Security model -The host enforces these for every plugin, built in or not: +The host enforces these for every plugin: - **No data beyond the caller's.** The plugin reads only through host queries, and those run with the caller's package selector and group @@ -319,9 +231,8 @@ The host enforces these for every plugin, built in or not: answers 400, a request naming one answers like one naming an unknown package (404), and a proxy leaves such versions of an upstream listing out. Ids and versions are keys of database indexes, whose entries - PostgreSQL limits to about 2,700 bytes. The plugins allow less: NuGet 100 - characters for an id and 64 for a version, the example plugin 250 for a - chart name and 128 for a version. + PostgreSQL limits to about 2,700 bytes. NuGet allows less: 100 characters + for an id and 64 for a version. - **Upstream requests go where the upstream points.** A proxy's plugin reaches the network only through the GETs the host sends for it, and the host sends them, and downloads packages, only from the origin of the @@ -338,45 +249,13 @@ The host enforces these for every plugin, built in or not: `Content-Disposition: attachment`). A plugin cannot set cookies, CORS headers, or redirect a client elsewhere. -A built-in plugin is part of the server: it is reviewed, tested and released -with it, and runs in-process like the rest of Forge's code. A WebAssembly -plugin is code from elsewhere, so it is also sandboxed: - -- **Integrity.** Only components pinned in the server's configuration are - loaded, by digest. There is no upload or install API: changing what runs - takes a configuration change and a restart, like any other code - deployment. -- **No ambient authority.** The component gets an empty WASI context: no - files, no sockets, no environment, no arguments; only clocks and - randomness. Its stdout and stderr go nowhere. It imports exactly one - host interface, the reader for the package being parsed. What it can make - the host request for a proxy is bounded by the rule above. -- **Resource limits.** Each call runs with a fresh fuel budget and an epoch - deadline (10 ms ticks), in an instance with a memory ceiling. The ceiling - bounds the host memory an instance can take: all its linear memories and - tables count against it together (each table element as a pointer), the - strings and lists one call returns are copied into the host only up to - it (a string that several list elements point at is copied once per - element, and counted so), and so are the random bytes it asks WASI for. A - trap (out of fuel, past the deadline, out of memory, a guest panic, a - result past the ceiling) fails that call with a 500 or 400 and discards - the instance. The sandbox tests include a gzip bomb, an instance whose - memories together exceed the ceiling, a `table.grow` past it, and a - result whose list elements all point at one string. -- **Isolation from the server's threads.** Guest code runs on dedicated - threads, never on the async runtime's workers. The quick rule calls - (`canonical-id`, `compare-versions`, …) and the slow calls (parsing a - package, rendering a response) have separate pools, so a long render - cannot hold up the rule calls every request makes. - -Plugins are still code you run: a plugin with a bug can serve wrong -metadata for its own repositories. Review a plugin like any other -dependency before you pin it. +A plugin is part of the server: it is reviewed, tested and released with +it, and runs in-process like the rest of Forge's code. The documents an upstream serves decide which hosts a proxy's plugin can reach. A public feed whose package metadata names an internal URL (a project URL or a chart's home page pointing at `http://10.0.0.5:9200/`) makes -that host reachable for a proxy of that feed, and a plugin can read what it +that host reachable for a proxy of that feed, and the plugin reads what it answers back into what the proxy serves. The rule bounds a plugin to the hosts its upstream points at, public or internal; where internal services must stay out of Forge's reach altogether, confine its egress with network @@ -384,9 +263,9 @@ policy (an outbound proxy, a Kubernetes `NetworkPolicy`). ## Writing a plugin -Start from the example plugin, [`plugins/helm`](../plugins/helm): a crate -in `plugins//` with a `plugin.toml`, a member of the workspace, -depending on `forge-plugin-api` and implementing its trait: +A plugin is a crate in `plugins//` with a `plugin.toml`, a member of the +workspace, depending on `forge-plugin-api` and implementing its trait +([`plugins/nuget`](../plugins/nuget) is a complete one): ```rust use forge_plugin_api::{Action, FormatPlugin, Loaded, Rendered, Repository, Request /* … */}; @@ -397,9 +276,6 @@ pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); #[derive(Default)] pub struct MyFormat; -#[cfg(feature = "component")] -forge_plugin_api::export_component!(MyFormat); - impl FormatPlugin for MyFormat { // identity: canonical_id, validate_id, normalize_version, compare_versions // packages: parse_package, manifest_summary @@ -408,12 +284,6 @@ impl FormatPlugin for MyFormat { } ``` -```toml -# Cargo.toml -[features] -component = ["forge-plugin-api/component"] -``` - ```toml # plugin.toml [plugin] @@ -421,7 +291,6 @@ id = "example" # the format id repositories are created with name = "Example packages" version = "0.1.0" api = "0.3.0" # the forge:format contract -component = "forge_format_example.wasm" # the component build's file mount = "/repository" # optional: served at {mount}/{repo}/… kinds = ["hosted", "group"] # optional: the repository kinds it serves # ("proxy" too when it implements `upstream`) @@ -431,9 +300,6 @@ package = "package" package_type = "application/octet-stream" manifest = "manifest" manifest_type = "application/octet-stream" - -[limits] # optional, as a component (see above) -timeout_ms = 2000 ``` A mount is `/` and one lowercase word; the server's own paths (`/admin`, @@ -441,24 +307,16 @@ A mount is `/` and one lowercase word; the server's own paths (`/admin`, may share a mount. List `proxy` among the kinds only when the plugin implements `upstream`. -Then either: - -- **build it in**: add the crate to `forge-server`'s dependencies and to - `plugins::builtin()` in - [`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs); or -- **ship it as a component**: `cargo xtask plugins` builds it (the crate as a - `cdylib` for `wasm32-wasip2` with the `component` feature) and prints the - configuration that loads it. - -A few rules hold either way: +Then build it in: add the crate to `forge-server`'s dependencies and to +`plugins::builtin()` in +[`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs). A few +rules hold: - **Keep methods pure.** The host may call any method from any thread, in - any order, any number of times; a component's instances are replaced at - will. + any order, any number of times. - **Read packages as a stream.** `parse_package` gets a `Read + Seek` over - the staged package (in a component, a reader the host lends with `size` and - `read-at`). Read only what you need, and bound everything you decompress: - a gzip bomb should fail fast, and a component runs under a fuel budget. + the staged package. Read only what you need, and bound everything you + decompress: a gzip bomb should fail fast. - **Ask for data, don't assume it.** `handle` returns `Action::Load(queries)`; `render` gets each query with its answer and may load more. Build absolute URLs from the repository's `base_url`. Groups and kinds are already @@ -468,17 +326,10 @@ A few rules hold either way: (`testing::serve`), as [`plugins/nuget/tests/protocol.rs`](../plugins/nuget/tests/protocol.rs) does; `testing::converse` runs `upstream` against canned upstream documents ([`plugins/nuget/tests/upstream.rs`](../plugins/nuget/tests/upstream.rs)). -- **Test both ways.** The crate's unit tests run with the workspace. - [`crates/plugin/tests/component.rs`](../crates/plugin/tests/component.rs) - loads the component and compares it with the crate built in, and - [`crates/server/tests/plugins.rs`](../crates/server/tests/plugins.rs) runs - a server with both; copy them for a new plugin. -- **Check it like CI does:** `cargo xtask plugins-check` lints the component - build and builds it. - -A component can also come from another language, as long as it implements -the `forge:format/format-plugin` world. The host provides WASI 0.2 imports -with no capabilities. +- **Test it in a server.** The crate's unit tests run with the workspace. + [`crates/server/tests/plugin_contract.rs`](../crates/server/tests/plugin_contract.rs) + runs a server with a small plugin of its own installed; copy it for a new + plugin. ## Limitations and roadmap @@ -508,8 +359,8 @@ to lift them: format with another rule (PEP 440's `1.0a1`) needs a contract function for it. - **Listings are bounded.** A response may load at most 2,048 answers, so - a listing of every package of a repository (the example plugin's - `index.yaml`) covers at most about 2,000 packages. + a listing of every package of a repository covers at most about 2,000 + packages. - **An upstream document is read whole.** A proxy reads each upstream document it needs (cached for the metadata TTL, and at most `proxy.max_metadata_bytes`, 32 MiB by default); the contract has no ranged @@ -530,11 +381,6 @@ index's resources, the flat container, the registration (bounded to the newest pages a request needs), catalog entries for the package delay of unlisted versions, and the search service. -One part is missing: **a component build.** The crate reads packages with -`zip`, whose `typed-path` dependency needs a nightly compiler for -`wasm32-wasip2`, so NuGet runs built in only (its `plugin.toml` names no -`component`). - ### Why Docker is not a plugin The contract models a repository as package versions, each one file with an @@ -559,34 +405,3 @@ several files in one form, always answer ranges and conditional requests, and their proxies follow redirects and re-validate with `If-None-Match`. They live in the server (`crates/server/src/raw`, served under `/raw`) and are registered like Docker, with selectors on paths. - -## Design notes: compared with artifact-keeper - -[artifact-keeper](https://github.com/artifact-keeper/artifact-keeper) also -runs format plugins as WebAssembly components under Wasmtime. Its native -formats are Rust handlers in `backend/src/formats/.rs` behind one -`FormatHandler` trait, and `formats/wasm.rs` wraps a loaded plugin in the -same trait, so services do not care which kind they hold. Forge follows that -shape: `FormatPlugin` is the one trait, the built-in plugins are its native -implementations, and `WasmPlugin` is the wrapper. Its WIT contract is the -`artifact-keeper:format@1.0.0` package (`backend/src/wit/format-plugin.wit`). -We studied the version of October 2026 and kept the idea while changing the -contract: - -| | artifact-keeper | Forge | -| --- | --- | --- | -| Native formats | Handlers in the backend crate; the WASM contract is a separate, narrower interface | Built-in plugins are crates implementing the same contract a component exports; any of them also builds as a component | -| Package bytes into the plugin | Whole artifact as `list` (uploads are read into memory) | A lent reader over the staged file; the plugin reads what it needs | -| Data for a request | Every artifact row of the repository, on every request | Targeted queries, bounded per request | -| Repository kinds | Hosted in practice (remote and virtual repositories see only their own rows) | Hosted, group, and proxy: the plugin reads the upstream through the host | -| Authorization of data | Repository visibility | Repository grants plus package selectors, group visibility and the dependency-confusion guards | -| Uploads | A separate generic upload endpoint; the plugin validates and parses | The plugin routes the publish; the host streams it through the hosted write path | -| Response headers | Passed through as the plugin sets them | Allowlist plus the feed's hardening headers | -| Execution limits | Fuel and memory | Fuel, memory and an epoch deadline; guest code off the async workers | -| Installing plugins | Admin API from git or a ZIP, Ed25519-signed when required | Operator configuration, pinned by SHA-256; no install API | -| Instances | A new store and instance per call | Instances reused per plugin, a fresh budget per call, recycled after a trap or 10,000 calls | - -The example plugin shows the difference in practice: a 64 MiB component serves -a repository index of any size the query bounds allow, and a chart is never -copied into the sandbox, which streams it through the lent reader and keeps -only its `Chart.yaml`. diff --git a/forge.toml.example b/forge.toml.example index e38f2a8e..d214c648 100644 --- a/forge.toml.example +++ b/forge.toml.example @@ -159,18 +159,6 @@ max_total_uncompressed_bytes = 17179869184 # (default: 16 GiB) total declared s max_nuspec_bytes = 1048576 # (default: 1 MiB) also caps an upstream .nuspec a proxy verifies max_nuspec_compression_ratio = 200 # (default: 200) -# Optional: WebAssembly format plugins — further package formats loaded at -# startup (docs/format-plugins.md). The built-in one (NuGet) needs nothing here. -# Only the plugins listed under [plugins.formats] are loaded, each from -# // and pinned to its component's SHA-256; a mismatch, or an id a -# built-in format holds, fails startup. `cargo xtask plugins` builds the -# plugins of plugins/ as components and prints this section with their digests. -# [plugins] -# dir = "/etc/forge/plugins" -# -# [plugins.formats] -# example = "sha256:<64 hex digits>" - [docker] # The Docker / OCI registry under /v2 (see docs/docker.md). # max_blob_bytes = 1073741824 # (default: 1 GiB, or what an explicit server.max_staging_bytes holds) largest layer a proxy downloads; staged on disk (pushes follow the repository's upload limit) diff --git a/plugins/README.md b/plugins/README.md index ebef6cf4..482620bf 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,42 +1,24 @@ # Format plugins Package formats for Forge, one crate per directory. Each implements -`FormatPlugin` from [`forge-plugin-api`](../crates/plugin-api) and runs -either way: linked into the server (built in), or as a WebAssembly component -loaded at startup. See [docs/format-plugins.md](../docs/format-plugins.md) for -the design, the contract and how to operate them. +`FormatPlugin` from [`forge-plugin-api`](../crates/plugin-api) and is linked +into the server. See [docs/format-plugins.md](../docs/format-plugins.md) for +the design, the contract and how to write one. -| Plugin | Format id | Runs | What it serves | -| --- | --- | --- | --- | -| [`nuget`](nuget) | `nuget` | built in only | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | -| [`helm`](helm) | `helm` | the example plugin: not built in; the plugin tests run it built in and as a component | Classic Helm chart repositories: `index.yaml`, chart downloads, ChartMuseum-compatible push, delete and listing; hosted, proxy and group repositories | +| Plugin | Format id | What it serves | +| --- | --- | --- | +| [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | A plugin directory holds: -- `Cargo.toml`: a member of the workspace, with a `component` feature - (`component = ["forge-plugin-api/component"]`) unless it is built in only; +- `Cargo.toml`: a member of the workspace; - `plugin.toml`: the format id (also the directory's name), display name, - version, the `forge:format` contract version it targets (`api`), + version, the `forge:format` contract version it targets (`api`), and optionally its mount, repository kinds (`proxy` when it implements - `upstream`) and `[assets]` names, and, for the component build, the - component's file name and optional `[limits]`; -- `src/`: `impl FormatPlugin`, `pub const PLUGIN_TOML: &str = - forge_plugin_api::plugin_toml!();`, and - `#[cfg(feature = "component")] forge_plugin_api::export_component!(…);`. + `upstream`) and `[assets]` names; +- `src/`: `impl FormatPlugin`, and `pub const PLUGIN_TOML: &str = + forge_plugin_api::plugin_toml!();`. -The workspace builds, lints and tests the plugins natively like every other -crate. As components (needs the `wasm32-wasip2` target): - -```bash -rustup target add wasm32-wasip2 # once -cargo xtask plugins # build + install under target/plugins/installed/, print the [plugins] config -cargo xtask plugins-check # clippy of the component build, and the build, as CI runs them -``` - -Both skip a plugin whose `plugin.toml` names no `component`. NuGet is one: -its `.nupkg` reader (`zip`, through `typed-path`) needs a nightly compiler for -`wasm32-wasip2`. - -The tests that load a component build it through -`forge_plugin::testing::example_plugin`, so `cargo test --workspace` needs the -`wasm32-wasip2` target as well. +The server installs it in `plugins::builtin()` +([`crates/server/src/plugins.rs`](../crates/server/src/plugins.rs)). The +workspace builds, lints and tests the plugins like every other crate. diff --git a/plugins/helm/Cargo.toml b/plugins/helm/Cargo.toml deleted file mode 100644 index 6c3570c1..00000000 --- a/plugins/helm/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "forge-format-helm" -# The plugin's own version: keep it equal to `version` in plugin.toml. -version = "0.1.0" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish.workspace = true -description = "Helm chart repositories for Forge, as a format plugin" - -# Built into the server; `--features component` builds it as a WebAssembly -# component instead (`cargo xtask plugins`). -[features] -component = ["forge-plugin-api/component"] - -[dependencies] -forge-plugin-api = { path = "../../crates/plugin-api" } -# .tgz charts: gzip with a pure-Rust backend (no C in a wasm build). -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } -# Chart.yaml, and the index.yaml it serves. -yaml-rust2 = { version = "0.13", default-features = false } -serde_json = { workspace = true } - -[dev-dependencies] -forge-plugin-api = { path = "../../crates/plugin-api", features = ["testing"] } diff --git a/plugins/helm/plugin.toml b/plugins/helm/plugin.toml deleted file mode 100644 index e389e057..00000000 --- a/plugins/helm/plugin.toml +++ /dev/null @@ -1,15 +0,0 @@ -# The example Forge format plugin: classic Helm chart repositories (see -# plugins/README.md). Forge does not build it in. -[plugin] -id = "helm" -name = "Helm charts" -version = "0.1.0" -api = "0.3.0" -component = "forge_format_helm.wasm" -description = "Helm chart repositories: `helm repo add` / `helm pull`, ChartMuseum-compatible push and delete; proxies of other chart repositories." -# Proxies mirror another chart repository through the plugin's upstream side. -kinds = ["hosted", "proxy", "group"] - -[limits] -memory_mib = 64 -timeout_ms = 5000 diff --git a/plugins/helm/src/chart.rs b/plugins/helm/src/chart.rs deleted file mode 100644 index 3b0a150c..00000000 --- a/plugins/helm/src/chart.rs +++ /dev/null @@ -1,614 +0,0 @@ -//! Reading a chart archive: `/Chart.yaml` inside a gzipped tarball, -//! found the way Helm finds it. -//! -//! A chart's name and version decide where Forge stores it and whether a -//! publisher may push it, so Forge takes them from the Chart.yaml Helm loads -//! when it installs the chart, or refuses the archive. Helm -//! (`loader.LoadArchiveFiles`, `LoadFiles`) reads the whole archive, strips -//! the first component of every name ([`chart_path`]), and reads every file -//! it then calls `Chart.yaml`, then every `requirements.yaml`, into the -//! chart's metadata: the last one wins. So Forge reads the whole archive -//! too ([`tar`] reads it as Go does), and refuses one with more than one -//! top-level directory, more than one Chart.yaml or requirements.yaml, or a -//! requirements.yaml that declares more than dependencies. - -use std::io::{self, Read}; - -use serde_json::{Map, Value}; -use yaml_rust2::Yaml; - -use crate::semver::Version; -use crate::tar; - -/// The largest Chart.yaml (or requirements.yaml) read. -const MAX_CHART_YAML: u64 = 1024 * 1024; - -/// How much of a chart is decompressed. The whole archive is read, so that -/// no Chart.yaml goes unseen; a chart that unpacks to more, or a gzip bomb, -/// is refused. (Helm itself loads at most 100 MiB of files.) -const MAX_UNPACKED: u64 = 256 * 1024 * 1024; - -/// The longest chart name. -const MAX_NAME: usize = 250; - -/// The UTF-8 byte order mark, which Helm strips from every file it loads. -const BOM: &[u8] = b"\xEF\xBB\xBF"; - -/// What a chart declares. -pub struct Chart { - /// `name`, as written. - pub name: String, - /// `version`, as written. - pub version: String, - /// The normalized version. - pub normalized: Version, - /// The raw Chart.yaml. - pub chart_yaml: Vec, - /// Chart.yaml as JSON. - pub metadata: Map, -} - -/// Whether `name` is a chart name: letters, digits, `.`, `_` and `-`, -/// starting with a letter or digit. -pub fn valid_name(name: &str) -> bool { - !name.is_empty() - && name.len() <= MAX_NAME - && name.starts_with(|c: char| c.is_ascii_alphanumeric()) - && name - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) - && !name.contains("..") -} - -/// Read the chart in a `.tgz`. -pub fn read(tgz: impl Read) -> Result { - read_unpacking(tgz, MAX_UNPACKED) -} - -/// [`read`], decompressing at most `max_unpacked` bytes. -fn read_unpacking(tgz: impl Read, max_unpacked: u64) -> Result { - // Go's gzip reader, which Helm uses, reads concatenated gzip members as - // one stream. - let unpacked = Bounded { - inner: flate2::read::MultiGzDecoder::new(tgz), - left: max_unpacked, - max: max_unpacked, - }; - let mut archive = tar::Archive::new(unpacked); - let lossy = |name: &[u8]| String::from_utf8_lossy(name).into_owned(); - let mut top_dir: Option> = None; - let (mut chart_yaml, mut requirements) = (None, None); - while let Some(entry) = archive - .next_entry() - .map_err(|e| format!("not a chart archive: {e}"))? - { - // Helm skips directories and global PAX headers. - if entry.is_dir() || entry.typeflag == b'g' { - continue; - } - let (top, path) = chart_path(&entry.name)?; - let dir = top_dir.get_or_insert_with(|| top.to_vec()); - if dir.as_slice() != top { - return Err(format!( - "the archive holds more than one top-level directory: {:?} and {:?}", - lossy(dir), - lossy(top) - )); - } - // Keep Chart.yaml and requirements.yaml; skip the rest. - let found = match path.as_slice() { - b"Chart.yaml" => &mut chart_yaml, - b"requirements.yaml" => &mut requirements, - _ => continue, - }; - if found.is_some() { - return Err(format!( - "the archive holds more than one /{}", - lossy(&path) - )); - } - let data = archive - .read_data(MAX_CHART_YAML) - .map_err(|e| format!("{}: {e}", lossy(&entry.name)))?; - *found = Some(strip_bom(data)); - } - let chart_yaml = - chart_yaml.ok_or_else(|| "the archive has no /Chart.yaml".to_string())?; - if let Some(requirements) = requirements { - check_requirements(&requirements)?; - } - parse_chart_yaml(chart_yaml) -} - -/// What Helm makes of an entry's name (`loader.LoadArchiveFiles`): the -/// top-level directory it strips, and the file's path in the chart, -/// cleaned. An error for a name Helm refuses. -fn chart_path(name: &[u8]) -> Result<(&[u8], Vec), String> { - // A name with a backslash is split at backslashes (a chart packaged on - // Windows), and its `/` are then part of a name. - let separator = if name.contains(&b'\\') { b'\\' } else { b'/' }; - let (top, rest) = match name.iter().position(|&b| b == separator) { - Some(i) => (&name[..i], &name[i + 1..]), - None => (name, &name[name.len()..]), - }; - let rest: Vec = rest - .iter() - .map(|&b| if b == separator { b'/' } else { b }) - .collect(); - let refused = |why: &str| format!("{why}: {:?}", String::from_utf8_lossy(name)); - if rest.starts_with(b"/") { - return Err(refused("the archive holds an absolute path")); - } - let path = clean(&rest); - if path == b"." { - return Err(refused("the archive holds a file outside a directory")); - } - if path.starts_with(b"..") { - return Err(refused("the archive holds a path to a parent directory")); - } - if let [drive, b':', b'/', ..] = path.as_slice() { - if drive.is_ascii_alphabetic() { - return Err(refused("the archive holds a path with a drive letter")); - } - } - if top == b"Chart.yaml" { - return Err(refused("the archive holds a directory named Chart.yaml")); - } - Ok((top, path)) -} - -/// Go's `path.Clean` of a relative path: no empty or `.` elements, a `..` -/// removing the element before it, and `.` for nothing at all. -fn clean(path: &[u8]) -> Vec { - let mut kept: Vec<&[u8]> = Vec::new(); - for element in path.split(|&b| b == b'/') { - match element { - b"" | b"." => {} - b".." if kept.last().is_some_and(|last| *last != b"..") => { - kept.pop(); - } - element => kept.push(element), - } - } - if kept.is_empty() { - b".".to_vec() - } else { - kept.join(&b'/') - } -} - -/// `data` without a leading byte order mark. -fn strip_bom(mut data: Vec) -> Vec { - if data.starts_with(BOM) { - data.drain(..BOM.len()); - } - data -} - -/// Refuse a `requirements.yaml` that declares more than dependencies. Helm -/// reads it (Helm 2's list of dependencies) into the chart's metadata after -/// Chart.yaml, so a `name` or a `version` in it would be the chart's. -fn check_requirements(yaml: &[u8]) -> Result<(), String> { - let text = std::str::from_utf8(yaml).map_err(|_| "requirements.yaml is not UTF-8")?; - let docs = crate::yaml::load(text).map_err(|e| format!("requirements.yaml: {e}"))?; - match docs.first() { - None | Some(Yaml::Null) => Ok(()), - Some(Yaml::Hash(map)) if map.keys().all(|k| k.as_str() == Some("dependencies")) => Ok(()), - _ => Err("requirements.yaml declares more than dependencies".into()), - } -} - -/// A reader of at most `max` bytes, which fails, rather than ending early, -/// when its source holds more. -struct Bounded { - /// The source. - inner: R, - /// The bytes it may still read. - left: u64, - /// The bytes it may read in all. - max: u64, -} - -impl Read for Bounded { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() { - return Ok(0); - } - if self.left == 0 { - return match self.inner.read(&mut [0u8])? { - 0 => Ok(0), - _ => Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("it unpacks to more than {} bytes", self.max), - )), - }; - } - let len = buf - .len() - .min(usize::try_from(self.left).unwrap_or(usize::MAX)); - let n = self.inner.read(&mut buf[..len])?; - self.left -= n as u64; - Ok(n) - } -} - -/// Parse a Chart.yaml. -pub fn parse_chart_yaml(chart_yaml: Vec) -> Result { - let text = std::str::from_utf8(&chart_yaml).map_err(|_| "Chart.yaml is not UTF-8")?; - let docs = crate::yaml::load(text).map_err(|e| format!("Chart.yaml: {e}"))?; - let Some(Value::Object(metadata)) = docs.first().map(to_json) else { - return Err("Chart.yaml is not a mapping".into()); - }; - let field = |key: &str| -> Result { - match metadata.get(key) { - Some(Value::String(s)) if !s.trim().is_empty() => Ok(s.trim().to_string()), - Some(Value::Number(n)) => Ok(n.to_string()), - _ => Err(format!("Chart.yaml has no {key}")), - } - }; - let name = field("name")?; - if !valid_name(&name) { - return Err(format!("invalid chart name {name:?}")); - } - let version = field("version")?; - let normalized = Version::parse(&version)?; - field("apiVersion")?; - Ok(Chart { - name, - version, - normalized, - chart_yaml, - metadata, - }) -} - -/// A YAML value as JSON. Scalars YAML would read as numbers keep their text -/// when JSON cannot hold them exactly (`version: 1.10` stays "1.10"). -pub fn to_json(yaml: &Yaml) -> Value { - match yaml { - Yaml::String(s) => Value::String(s.clone()), - Yaml::Integer(i) => Value::from(*i), - Yaml::Real(r) => Value::String(r.clone()), - Yaml::Boolean(b) => Value::Bool(*b), - Yaml::Array(items) => Value::Array(items.iter().map(to_json).collect()), - Yaml::Hash(map) => Value::Object( - map.iter() - .map(|(k, v)| { - let key = match k { - Yaml::String(s) | Yaml::Real(s) => s.clone(), - Yaml::Integer(i) => i.to_string(), - Yaml::Boolean(b) => b.to_string(), - _ => String::new(), - }; - (key, to_json(v)) - }) - .collect(), - ), - Yaml::Null | Yaml::BadValue | Yaml::Alias(_) => Value::Null, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tar::tests::{archive, entry, file, header, long, pax, seal, END}; - use std::io::Write; - - /// `tar`, gzipped. - fn gzip(tar: &[u8]) -> Vec { - let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); - gz.write_all(tar).unwrap(); - gz.finish().unwrap() - } - - /// A gzipped archive of `(path, contents)` regular files. - pub(crate) fn tgz(entries: &[(&str, &[u8])]) -> Vec { - gzip(&archive(entries)) - } - - const GOOD: &[u8] = b"apiVersion: v2\nname: good\nversion: 1.0.0\n"; - const EVIL: &[u8] = b"apiVersion: v2\nname: evil\nversion: 6.6.6\n"; - - /// The name and version [`read`] finds in the archive `tar`, gzipped. - fn identity(tar: &[u8]) -> Result { - read(&gzip(tar)[..]).map(|chart| format!("{} {}", chart.name, chart.version)) - } - - /// The error [`read`] refuses the archive `tar` with. - fn refusal(tar: &[u8]) -> String { - identity(tar).unwrap_err() - } - - #[test] - fn reads_the_top_level_chart_yaml() { - let archive = tgz(&[ - ("demo/charts/sub/Chart.yaml", b"apiVersion: v2\nname: sub\nversion: 9.9.9\n"), - ( - "demo/Chart.yaml", - b"apiVersion: v2\nname: Demo\nversion: v1.10\nappVersion: 2.0\ndescription: A demo\n", - ), - ]); - let chart = read(&archive[..]).unwrap(); - assert_eq!(chart.name, "Demo"); - assert_eq!(chart.version, "v1.10"); - assert_eq!(chart.normalized.to_string(), "1.10.0"); - assert_eq!(chart.metadata["appVersion"], Value::String("2.0".into())); - assert_eq!( - chart.metadata["description"], - Value::String("A demo".into()) - ); - } - - #[test] - fn refuses_what_is_not_a_chart() { - assert!(read(&b"not gzip"[..]).is_err()); - assert!(read(&tgz(&[("demo/values.yaml", b"a: 1\n")])[..]).is_err()); - let no_version = tgz(&[("demo/Chart.yaml", b"apiVersion: v2\nname: demo\n")]); - assert!(read(&no_version[..]).is_err()); - let bad_name = tgz(&[( - "demo/Chart.yaml", - b"apiVersion: v2\nname: ../x\nversion: 1.0.0\n", - )]); - assert!(read(&bad_name[..]).is_err()); - } - - #[test] - fn refuses_versions_longer_than_an_oci_tag() { - let chart_yaml = |version: &str| { - format!("apiVersion: v2\nname: demo\nversion: {version}\n").into_bytes() - }; - let longest = format!( - "1.0.0-{}", - "a".repeat(crate::semver::MAX_VERSION_LENGTH - 6) - ); - let chart = parse_chart_yaml(chart_yaml(&longest)).unwrap(); - assert_eq!(chart.version, longest); - assert_eq!(chart.normalized.to_string(), longest); - for longer in [format!("{longest}a"), format!("1.0.0-{}", "x".repeat(4096))] { - let err = parse_chart_yaml(chart_yaml(&longer)).err().unwrap(); - assert!(err.contains("longer than 128 characters"), "{err}"); - } - } - - #[test] - fn chart_names() { - assert!(valid_name("nginx-ingress")); - assert!(valid_name("my_chart.v2")); - for bad in ["", "-x", ".x", "a/b", "a..b", "a b", "a%2f"] { - assert!(!valid_name(bad), "{bad:?}"); - } - } - - #[test] - fn reads_charts_as_tools_pack_them() { - // `helm package`: Chart.yaml first, a long name in a PAX header, no - // directories. - let long_name = format!("demo/templates/{}.yaml", "x".repeat(120)); - let tar = [ - &file("demo/Chart.yaml", GOOD)[..], - &file("demo/values.yaml", b"replicas: 1\n"), - &pax(b'x', &[("path", &long_name)]), - &file(&long_name[..99], b"kind: ConfigMap\n"), - &file("demo/charts/sub/Chart.yaml", EVIL), - &END, - ] - .concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // `git archive` (a global header first) and GNU tar (directories, - // GNU long names). - let tar = [ - &pax(b'g', &[("comment", "8b4c2f0")])[..], - &header(b"demo/", b'5', 0), - &header(b"demo/templates/", b'5', 0), - &long(b'L', &long_name), - &file(&long_name[..99], b""), - &file("demo/Chart.yaml", GOOD), - &END, - ] - .concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // Packed inside the chart's directory: `./Chart.yaml`. - let tar = archive(&[("./Chart.yaml", GOOD), ("./values.yaml", b"")]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // On Windows: backslashes. - let tar = archive(&[("demo\\templates\\a.yaml", b""), ("demo\\Chart.yaml", GOOD)]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - // In two gzip members. - let first = gzip(&file("demo/values.yaml", b"")); - let second = gzip(&[&file("demo/Chart.yaml", GOOD)[..], &END].concat()); - let chart = read(&[first, second].concat()[..]).unwrap(); - assert_eq!(chart.name, "good"); - } - - #[test] - fn finds_the_chart_yaml_helm_finds() { - // Helm strips the first component of a name and cleans the rest; - // a name with a backslash is split at backslashes only. - for name in [ - "demo/./Chart.yaml", - "demo/templates/../Chart.yaml", - "demo/Chart.yaml/", - "demo/x\\Chart.yaml", - ] { - let tar = archive(&[(name, GOOD)]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0", "{name:?}"); - } - // `./demo/Chart.yaml` is the file `demo/Chart.yaml` of the - // directory `.` to Helm, not its Chart.yaml. - let tar = archive(&[("./demo/Chart.yaml", GOOD)]); - assert!(refusal(&tar).contains("no /Chart.yaml")); - // A directory by its mode is skipped like any directory, and a - // symlink has no data. - let mut dir = header(b"demo/Chart.yaml", b'0', EVIL.len() as u64); - dir[100..108].copy_from_slice(b"0040755\0"); - seal(&mut dir); - let tar = [&entry(dir, EVIL)[..], &file("demo/Chart.yaml", GOOD), &END].concat(); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - let tar = [&header(b"demo/Chart.yaml", b'2', 0)[..], &END].concat(); - assert!(refusal(&tar).contains("not a mapping")); - // Helm strips a byte order mark. - let tar = archive(&[("demo/Chart.yaml", &[BOM, GOOD].concat())]); - assert_eq!(read(&gzip(&tar)[..]).unwrap().chart_yaml, GOOD); - } - - #[test] - fn refuses_more_than_one_top_level_directory() { - // The archive of the issue: Forge stored `good` 1.0.0, Helm installs - // `evil` 6.6.6. - let tar = archive(&[("a/Chart.yaml", GOOD), ("b/Chart.yaml", EVIL)]); - assert!(refusal(&tar).contains("more than one top-level directory")); - let tar = archive(&[("demo/Chart.yaml", GOOD), ("other/templates/a.yaml", b"")]); - assert!(refusal(&tar).contains("more than one top-level directory")); - let tar = archive(&[("demo/Chart.yaml", GOOD), ("./demo/values.yaml", b"")]); - assert!(refusal(&tar).contains("more than one top-level directory")); - } - - #[test] - fn refuses_more_than_one_chart_yaml() { - // Helm loads every one, the last one winning. - for second in [ - "demo/Chart.yaml", - "demo/./Chart.yaml", - "demo/x/../Chart.yaml", - "demo/Chart.yaml/", - "demo\\Chart.yaml", - ] { - let tar = archive(&[("demo/Chart.yaml", GOOD), (second, EVIL)]); - let err = refusal(&tar); - assert!( - err.contains("more than one /Chart.yaml"), - "{second:?}: {err}" - ); - } - } - - #[test] - fn sees_every_entry_helm_sees() { - let good = file("demo/Chart.yaml", GOOD); - let evil = file("demo/Chart.yaml", EVIL); - let refused = |tar: &[u8]| { - let err = refusal(tar); - assert!(err.contains("more than one /Chart.yaml"), "{err}"); - }; - // A PAX size of 0: what the header calls data is an entry. - refused( - &[ - &good[..], - &pax(b'x', &[("size", "0")]), - &entry(header(b"demo/values.yaml", b'0', evil.len() as u64), &evil), - &END, - ] - .concat(), - ); - // A symlink with a size has no data either. - refused( - &[ - &good[..], - &header(b"demo/link", b'2', evil.len() as u64), - &evil, - &END, - ] - .concat(), - ); - // A long link name does not take the long name before it. - refused( - &[ - &good[..], - &long(b'L', "demo/Chart.yaml"), - &long(b'K', "target"), - &file("demo/a", EVIL), - &END, - ] - .concat(), - ); - // A GNU header's times are not a prefix. - let mut gnu = header(b"demo/Chart.yaml", b'0', EVIL.len() as u64); - gnu[257..265].copy_from_slice(b"ustar \0"); - gnu[345..347].copy_from_slice(b"7\0"); - seal(&mut gnu); - refused(&[&good[..], &entry(gnu, EVIL), &END].concat()); - // A second gzip member. - let tgz = [gzip(&good), gzip(&[&evil[..], &END].concat())].concat(); - let err = read(&tgz[..]).err().unwrap(); - assert!(err.contains("more than one /Chart.yaml"), "{err}"); - // Not the end of the archive: a zero block, then more. - let tar = [&good[..], &[0; 512], &evil, &END].concat(); - assert!(refusal(&tar).contains("not a chart archive")); - } - - #[test] - fn refuses_names_helm_refuses() { - for name in [ - "demo//Chart.yaml", - "demo/../../etc/passwd", - "demo/..", - "Chart.yaml", - "README.md", - "Chart.yaml/values.yaml", - "demo/c:/x", - "demo\\c:\\x", - ] { - let tar = archive(&[(name, GOOD), ("demo/Chart.yaml", GOOD)]); - assert!(refusal(&tar).contains("the archive holds a"), "{name:?}"); - } - } - - #[test] - fn refuses_a_requirements_yaml_that_renames_the_chart() { - let dependencies = b"dependencies:\n - name: redis\n version: 17.0.0\n"; - for requirements in [&dependencies[..], b"", b"# none\n"] { - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", requirements), - ]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - } - // Helm reads it into the chart's metadata after Chart.yaml. - for requirements in [ - &b"name: evil\nversion: 6.6.6\n"[..], - b"dependencies: []\nVersion: 6.6.6\n", - b"- name: evil\n", - ] { - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", requirements), - ]); - assert!(refusal(&tar).contains("more than dependencies")); - } - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/requirements.yaml", dependencies), - ("demo/requirements.yaml", dependencies), - ]); - assert!(refusal(&tar).contains("more than one /requirements.yaml")); - // A subchart's is the subchart's. - let tar = archive(&[ - ("demo/Chart.yaml", GOOD), - ("demo/charts/sub/requirements.yaml", b"name: evil\n"), - ]); - assert_eq!(identity(&tar).unwrap(), "good 1.0.0"); - } - - #[test] - fn refuses_an_archive_it_cannot_read_to_the_end() { - let tar = archive(&[("demo/Chart.yaml", GOOD), ("demo/big", &[7; 4096])]); - let len = tar.len() as u64; - assert!(read_unpacking(&gzip(&tar)[..], len).is_ok()); - let err = read_unpacking(&gzip(&tar)[..], len - 1).err().unwrap(); - assert!(err.contains("unpacks to more than"), "{err}"); - } - - #[test] - fn cleans_paths_like_go() { - for (path, clean_path) in [ - ("", "."), - ("a//b/./c/", "a/b/c"), - ("a/../..", ".."), - ("../a/../b", "../b"), - ("a/b/../../c", "c"), - (".", "."), - ] { - assert_eq!(clean(path.as_bytes()), clean_path.as_bytes(), "{path:?}"); - } - } -} diff --git a/plugins/helm/src/lib.rs b/plugins/helm/src/lib.rs deleted file mode 100644 index fefdda69..00000000 --- a/plugins/helm/src/lib.rs +++ /dev/null @@ -1,997 +0,0 @@ -//! Helm chart repositories for Forge. -//! -//! Served under a Forge repository's base URL: -//! -//! * `GET /index.yaml` — the repository index (`helm repo add`, `helm repo -//! update`); every chart and version the caller may read; -//! * `GET /charts/{name}/{version}/{name}-{version}.tgz` — a chart (the URL -//! the index points at); -//! * `POST /api/charts` — upload a chart, as the request body -//! (`curl --data-binary @chart.tgz`) or as the multipart field `chart` -//! (`helm cm-push`); -//! * `DELETE /api/charts/{name}/{version}` — delete a version; -//! * `GET /api/charts`, `/api/charts/{name}`, `/api/charts/{name}/{version}` -//! — the ChartMuseum JSON listing. -//! -//! Chart names are matched case-insensitively (the canonical id is the -//! lowercased name); versions are SemVer, normalized (`v1.2` → `1.2.0`), and -//! at most 128 characters ([`semver::MAX_VERSION_LENGTH`]). -//! -//! A proxy repository mirrors another chart repository ([`upstream`]): its -//! index lists the upstream's charts, each downloaded and cached on first -//! use. -//! -//! This is the example plugin (`docs/format-plugins.md`): Forge does not -//! build it in, and serves Helm charts from its OCI registry instead. The -//! plugin tests install it as if built in ([`Helm`], [`PLUGIN_TOML`]); with -//! the `component` feature the same crate builds as a WebAssembly component -//! of the `forge:format` world (`cargo xtask plugins`). - -mod chart; -mod semver; -mod tar; -mod upstream; -mod yaml; - -use std::cmp::{Ordering, Reverse}; -use std::collections::BTreeMap; - -use forge_plugin_api::{ - Action, Answer, ChangeRequest, Dependency, DependencyGroup, FetchResult, FormatError, - FormatPlugin, Loaded, PackageInfo, PackageRef, PackageSource, ParsedPackage, PublishRequest, - Query, Remote, Rendered, Repository, RepositoryKind, Request, Response, SearchHit, SearchQuery, - Upload, UpstreamOp, UpstreamStep, -}; -use serde_json::{json, Map, Value}; - -use semver::Version; - -/// This plugin's `plugin.toml`: its format id, name and version. -pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); - -/// Searches page through the repository this many charts at a time. -const PAGE: u32 = 1000; - -/// The Chart.yaml keys an index entry carries (Helm's chart metadata). -const METADATA_KEYS: &[&str] = &[ - "apiVersion", - "name", - "version", - "kubeVersion", - "description", - "type", - "keywords", - "home", - "sources", - "dependencies", - "maintainers", - "icon", - "appVersion", - "deprecated", - "annotations", - "condition", - "tags", -]; - -/// The most a chart version's manifest summary takes, as JSON. Every -/// `index.yaml` request renders every version from its summary, so this -/// bounds what a version adds to the index: 300 versions make under 5 MB, -/// where a Chart.yaml of 1 MiB each made 300 MB. A typical chart's summary -/// takes a few hundred bytes to a few KiB. -const MAX_SUMMARY: usize = 16 * 1024; - -/// The longest description, in bytes, that a summary over [`MAX_SUMMARY`] -/// keeps. Helm asks for a single sentence, and `helm search` shows its first -/// 50 characters. -const MAX_DESCRIPTION: usize = 1024; - -/// The Chart.yaml keys a summary over [`MAX_SUMMARY`] gives up last: what -/// Helm needs to list, choose and install a version, each a short string or -/// a boolean. -const ESSENTIAL_KEYS: &[&str] = &[ - "apiVersion", - "name", - "version", - "appVersion", - "kubeVersion", - "type", - "deprecated", -]; - -/// The Helm chart format. -#[derive(Debug, Default, Clone, Copy)] -pub struct Helm; - -#[cfg(feature = "component")] -forge_plugin_api::export_component!(Helm); - -impl FormatPlugin for Helm { - fn canonical_id(&self, id: &str) -> String { - id.to_ascii_lowercase() - } - - fn validate_id(&self, id: &str) -> Result<(), FormatError> { - if chart::valid_name(id) { - Ok(()) - } else { - Err(FormatError::InvalidId(format!("invalid chart name {id:?}"))) - } - } - - fn normalize_version(&self, version: &str) -> Result { - Version::parse(version) - .map(|v| v.to_string()) - .map_err(FormatError::InvalidVersion) - } - - fn compare_versions(&self, a: &str, b: &str) -> Ordering { - match (Version::parse(a), Version::parse(b)) { - (Ok(a), Ok(b)) => a.cmp(&b), - _ => a.cmp(b), - } - } - - fn parse_package(&self, package: &mut dyn PackageSource) -> Result { - let chart = chart::read(package).map_err(FormatError::InvalidPackage)?; - let dependencies = dependencies(&chart.metadata); - Ok(ParsedPackage { - id: chart.name.to_ascii_lowercase(), - original_id: chart.name, - version: chart.normalized.to_string(), - version_original: chart.version, - listed: true, - manifest: chart.chart_yaml, - dependencies: if dependencies.is_empty() { - Vec::new() - } else { - vec![DependencyGroup { - target: None, - dependencies, - }] - }, - }) - } - - fn manifest_summary(&self, manifest: &[u8]) -> Option { - let chart = chart::parse_chart_yaml(manifest.to_vec()).ok()?; - Some(summary(&chart.metadata)) - } - - fn handle(&self, repo: &Repository, req: &Request) -> Action { - let read = matches!(req.method.as_str(), "GET" | "HEAD"); - let segments: Vec<&str> = req.path.trim_start_matches('/').split('/').collect(); - match (req.method.as_str(), segments.as_slice()) { - (_, ["index.yaml"]) if read => Action::Load(vec![search_page(repo, 0)]), - (_, ["charts", name, version, file]) if read => match package_ref(name, version) { - Some(target) if file.ends_with(".tgz") => Action::Download(target), - _ => Action::Respond(not_found()), - }, - (_, ["api", "charts"]) if read => Action::Load(vec![search_page(repo, 0)]), - (_, ["api", "charts", name]) if read => match decode(name) { - Some(name) if chart::valid_name(&name) => { - Action::Load(vec![Query::Packages(name.to_ascii_lowercase())]) - } - _ => Action::Respond(not_found()), - }, - (_, ["api", "charts", name, version]) if read => match package_ref(name, version) { - Some(target) => Action::Load(vec![Query::Package(target)]), - None => Action::Respond(not_found()), - }, - ("POST", ["api", "charts"]) => { - let multipart = req - .header("content-type") - .is_some_and(|v| v.to_ascii_lowercase().starts_with("multipart/form-data")); - Action::Publish(PublishRequest { - upload: if multipart { - Upload::MultipartField("chart".into()) - } else { - Upload::Body - }, - done: json_response(201, &json!({ "saved": true })), - }) - } - ("DELETE", ["api", "charts", name, version]) => match package_ref(name, version) { - Some(target) => Action::Delete(ChangeRequest { - target, - done: json_response(200, &json!({ "deleted": true })), - }), - None => Action::Respond(not_found()), - }, - _ => Action::Respond(not_found()), - } - } - - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let segments: Vec<&str> = req.path.trim_start_matches('/').split('/').collect(); - match segments.as_slice() { - ["index.yaml"] | ["api", "charts"] => render_listing(repo, &segments, data), - ["api", "charts", _] => match data.first().map(|l| &l.answer) { - Some(Answer::Packages(versions)) if !versions.is_empty() => { - let mut versions: Vec<&PackageInfo> = versions.iter().collect(); - newest_first(&mut versions); - let entries: Vec = - versions.into_iter().map(|p| entry(repo, p)).collect(); - Rendered::Respond(json_response(200, &Value::Array(entries))) - } - _ => Rendered::Respond(not_found()), - }, - ["api", "charts", _, _] => match data.first().map(|l| &l.answer) { - Some(Answer::Package(Some(p))) => { - Rendered::Respond(json_response(200, &entry(repo, p))) - } - _ => Rendered::Respond(not_found()), - }, - _ => Rendered::Respond(not_found()), - } - } - - fn upstream( - &self, - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], - ) -> Result { - upstream::step(remote, op, fetched) - } -} - -/// The manifest summary of a chart's metadata (its Chart.yaml, or an index -/// entry): the chart metadata an index entry carries, and the description -/// and dependency groups Forge's web UI shows, within [`MAX_SUMMARY`]. -fn summary(metadata: &Map) -> String { - let deps: Vec = dependencies(metadata) - .into_iter() - .map(|d| json!({ "id": d.id, "range": d.range })) - .collect(); - let mut summary = Map::new(); - summary.insert("chart".into(), Value::Object(helm_metadata(metadata))); - summary.insert( - "description".into(), - metadata.get("description").cloned().unwrap_or(Value::Null), - ); - summary.insert( - "dependency_groups".into(), - if deps.is_empty() { - json!([]) - } else { - json!([{ "target_framework": null, "dependencies": deps }]) - }, - ); - bounded(&mut summary) -} - -/// `summary` (a manifest summary's `chart`, with its `description` and -/// `dependency_groups` when it has them) cut down to [`MAX_SUMMARY`], and -/// its JSON. -/// -/// A summary within the budget stays as it is. One over it has its -/// description cut to [`MAX_DESCRIPTION`] bytes and an ellipsis, then loses -/// whole fields, the largest first and the [`ESSENTIAL_KEYS`] last, until it -/// fits. So what is left of a field is all of it, never part of a list or a -/// map, and the same summary always comes out the same. -fn bounded(summary: &mut Map) -> String { - let text = serde_json::to_string(summary).unwrap_or_default(); - if text.len() <= MAX_SUMMARY { - return text; - } - // The description, in both of its places. - let mut description = None; - if let Some(Value::Object(chart)) = summary.get_mut("chart") { - if let Some(Value::String(long)) = chart.get_mut("description") { - let end = long.floor_char_boundary(MAX_DESCRIPTION); - if end < long.len() { - long.truncate(end); - long.push('…'); - } - } - description = chart.get("description").cloned(); - } - if summary.contains_key("description") { - summary.insert("description".into(), description.unwrap_or(Value::Null)); - } - // Then whole fields, the chart's and the summary's own, ordered by - // (essential, largest first, the summary's before the chart's, key). - let mut fields: Vec<(bool, Reverse, bool, String)> = Vec::new(); - for (key, value) in summary.iter() { - match value { - Value::Object(chart) if key == "chart" => { - fields.extend(chart.iter().map(|(key, value)| { - let essential = ESSENTIAL_KEYS.contains(&key.as_str()); - (essential, Reverse(field_len(key, value)), true, key.clone()) - })); - } - _ => fields.push((false, Reverse(field_len(key, value)), false, key.clone())), - } - } - fields.sort(); - let mut size = serde_json::to_string(summary).map_or(0, |text| text.len()); - for (_, Reverse(len), in_chart, key) in fields { - if size <= MAX_SUMMARY { - break; - } - let object = if in_chart { - match summary.get_mut("chart") { - Some(Value::Object(chart)) => chart, - _ => continue, - } - } else { - &mut *summary - }; - // The field, and the comma between it and its neighbour. - size = size.saturating_sub(len + usize::from(object.len() > 1)); - object.remove(&key); - } - serde_json::to_string(summary).unwrap_or_default() -} - -/// The length of `"key":value` in JSON. -fn field_len(key: &str, value: &Value) -> usize { - Value::from(key).to_string().len() + 1 + value.to_string().len() -} - -/// The [`METADATA_KEYS`] of `metadata`, each as the type Helm's -/// `chart.Metadata` declares for it. -/// -/// Helm reads an index that is valid JSON with Go's `encoding/json`, straight -/// into those types, and refuses the whole index when one value has another -/// type. A Chart.yaml (or a YAML index of an upstream) may hold -/// `appVersion: 2` or an annotation `false`: Helm's YAML reader turns such a -/// scalar into the string its field wants, so the chart is valid, but its -/// JSON number or boolean would break every chart of the index. So scalars -/// become strings where Helm wants strings, and a value that has no such -/// form (a mapping where a string belongs) is left out. -fn helm_metadata(metadata: &Map) -> Map { - metadata - .iter() - .filter_map(|(key, value)| { - let value = match key.as_str() { - "keywords" | "sources" => string_list(value), - "deprecated" => value.is_boolean().then(|| value.clone()), - "annotations" => string_map(value), - "maintainers" => objects(value, |maintainer| { - pick(maintainer, &["name", "email", "url"], &[], &[]) - }), - "dependencies" => objects(value, |dependency| { - let mut picked = pick( - dependency, - &["name", "version", "repository", "condition", "alias"], - &["tags"], - &["enabled"], - ); - // `[]interface{}`: any list. - if let Some(values @ Value::Array(_)) = dependency.get("import-values") { - picked.insert("import-values".into(), values.clone()); - } - picked - }), - key if METADATA_KEYS.contains(&key) => scalar_string(value), - _ => None, - }?; - Some((key.clone(), value)) - }) - .collect() -} - -/// A scalar as the string Helm's YAML reader makes of it. -fn scalar_string(value: &Value) -> Option { - match value { - Value::String(_) => Some(value.clone()), - Value::Number(n) => Some(Value::String(n.to_string())), - Value::Bool(b) => Some(Value::String(b.to_string())), - _ => None, - } -} - -/// A list of strings (`[]string`): its scalars as strings. -fn string_list(value: &Value) -> Option { - let items = value.as_array()?; - Some(Value::Array( - items.iter().filter_map(scalar_string).collect(), - )) -} - -/// A string map (`map[string]string`): its scalar values as strings. -fn string_map(value: &Value) -> Option { - let map = value.as_object()?; - Some(Value::Object( - map.iter() - .filter_map(|(k, v)| Some((k.clone(), scalar_string(v)?))) - .collect(), - )) -} - -/// A list of objects (`[]*Maintainer`, `[]*Dependency`), each made by `one`. -fn objects( - value: &Value, - one: impl Fn(&Map) -> Map, -) -> Option { - let items = value.as_array()?; - Some(Value::Array( - items - .iter() - .filter_map(|item| Some(Value::Object(one(item.as_object()?)))) - .collect(), - )) -} - -/// The `strings`, `lists` (of strings) and `bools` of `object`, as such. -fn pick( - object: &Map, - strings: &[&str], - lists: &[&str], - bools: &[&str], -) -> Map { - let mut picked = Map::new(); - for (key, value) in object { - let value = if strings.contains(&key.as_str()) { - scalar_string(value) - } else if lists.contains(&key.as_str()) { - string_list(value) - } else if bools.contains(&key.as_str()) { - value.is_boolean().then(|| value.clone()) - } else { - None - }; - if let Some(value) = value { - picked.insert(key.clone(), value); - } - } - picked -} - -/// Chart.yaml `dependencies` as Forge dependencies. -fn dependencies(metadata: &Map) -> Vec { - let Some(Value::Array(deps)) = metadata.get("dependencies") else { - return Vec::new(); - }; - deps.iter() - .filter_map(|d| { - let name = d.get("name")?.as_str()?; - Some(Dependency { - id: name.to_string(), - range: d.get("version").and_then(Value::as_str).map(str::to_string), - }) - }) - .collect() -} - -fn json_response(status: u16, value: &Value) -> Response { - Response::new(status, "application/json", value.to_string()) -} - -fn not_found() -> Response { - json_response(404, &json!({ "error": "not found" })) -} - -/// Percent-decode one path segment. -fn decode(segment: &str) -> Option { - let bytes = segment.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'%' { - let hex = segment.get(i + 1..i + 3)?; - out.push(u8::from_str_radix(hex, 16).ok()?); - i += 3; - } else { - out.push(bytes[i]); - i += 1; - } - } - String::from_utf8(out).ok() -} - -/// `(canonical id, normalized version)` of a name and version from a URL. -fn package_ref(name: &str, version: &str) -> Option { - let name = decode(name)?; - let version = Version::parse(&decode(version)?).ok()?; - chart::valid_name(&name).then(|| PackageRef { - id: name.to_ascii_lowercase(), - version: version.to_string(), - }) -} - -/// A page of every chart of `repo`: a proxy's (and a group's proxies') -/// upstream charts included, not only the ones it has cached. -fn search_page(repo: &Repository, skip: u32) -> Query { - Query::Search(SearchQuery { - text: String::new(), - skip, - take: PAGE, - prerelease: true, - semver2: true, - upstream: repo.kind != RepositoryKind::Hosted, - }) -} - -/// The index (`/index.yaml`) or the ChartMuseum listing (`/api/charts`): -/// page through the search, then load every chart's versions, then render. -fn render_listing(repo: &Repository, segments: &[&str], data: &[Loaded]) -> Rendered { - let mut total = 0u32; - let mut pages = 0u32; - let mut ids: Vec = Vec::new(); - let mut hits: Vec<&SearchHit> = Vec::new(); - let mut charts: Vec> = Vec::new(); - for loaded in data { - match (&loaded.query, &loaded.answer) { - (Query::Search(_), Answer::Search(result)) => { - total = total.max(result.total); - pages += 1; - ids.extend( - result - .hits - .iter() - .map(|h| h.original_id.to_ascii_lowercase()), - ); - hits.extend(result.hits.iter()); - } - // Unlisted versions stay downloadable but are not offered. - (Query::Packages(_), Answer::Packages(versions)) => { - let mut listed: Vec<&PackageInfo> = versions.iter().filter(|p| p.listed).collect(); - if !listed.is_empty() { - newest_first(&mut listed); - charts.push(listed); - } - } - _ => {} - } - } - // Search pages are all loaded in the second round, charts in the next. - let loaded_charts = data.iter().any(|l| matches!(l.query, Query::Packages(_))); - if !loaded_charts { - if pages * PAGE < total && pages == 1 { - let more = (1..total.div_ceil(PAGE)) - .map(|p| search_page(repo, p * PAGE)) - .collect(); - return Rendered::Load(more); - } - // A proxy's charts are its upstream's: the search lists each with - // its versions and the newest one's metadata, so the index comes - // from it rather than from reading the upstream once per chart. - if repo.kind == RepositoryKind::Proxy { - let mut entries = Map::new(); - for hit in hits { - entries.insert( - hit.original_id.clone(), - Value::Array(hit_entries(repo, hit)), - ); - } - return render_index(segments, entries, None); - } - if !ids.is_empty() { - ids.sort(); - ids.dedup(); - return Rendered::Load(ids.into_iter().map(Query::Packages).collect()); - } - } - - // A chart is listed under the name its newest version is published as - // (names match case-insensitively). - let mut by_name: BTreeMap> = BTreeMap::new(); - for versions in charts { - by_name - .entry(versions[0].original_id.clone()) - .or_default() - .extend(versions); - } - let mut entries = Map::new(); - let mut latest: Option<&str> = None; - for (name, versions) in by_name { - for p in &versions { - if let Some(published) = p.published.as_deref() { - if latest.is_none_or(|l| published > l) { - latest = Some(published); - } - } - } - let list: Vec = versions.into_iter().map(|p| entry(repo, p)).collect(); - entries.insert(name, Value::Array(list)); - } - render_index(segments, entries, latest) -} - -/// The index (`/index.yaml`) or the ChartMuseum listing (`/api/charts`) of -/// `entries`, generated as of `latest`. -fn render_index(segments: &[&str], entries: Map, latest: Option<&str>) -> Rendered { - if segments == ["api", "charts"] { - return Rendered::Respond(json_response(200, &Value::Object(entries))); - } - let mut index = Map::new(); - index.insert("apiVersion".into(), json!("v1")); - index.insert("entries".into(), Value::Object(entries)); - if let Some(latest) = latest { - index.insert("generated".into(), json!(latest)); - } - // JSON is YAML; Helm reads either. - Rendered::Respond( - Response::new(200, "application/x-yaml", Value::Object(index).to_string()) - .with_header("cache-control", "no-cache"), - ) -} - -/// The index entries of a chart a proxy's search found, newest first. The -/// newest version carries the chart's metadata; the others, which the -/// search does not describe, their name, version and URL. -fn hit_entries(repo: &Repository, hit: &SearchHit) -> Vec { - let chart = stored_chart(hit.summary.as_deref()); - let mut versions: Vec<_> = hit.versions.iter().collect(); - versions.sort_by( - |a, b| match (Version::parse(&a.version), Version::parse(&b.version)) { - (Ok(a), Ok(b)) => b.cmp(&a), - _ => b.version.cmp(&a.version), - }, - ); - versions - .into_iter() - .map(|v| { - let mut entry = if v.version == hit.latest_version { - chart.clone() - } else { - chart - .get("apiVersion") - .map(|api| Map::from_iter([("apiVersion".to_string(), api.clone())])) - .unwrap_or_default() - }; - entry.insert("name".into(), json!(hit.original_id)); - entry.insert("version".into(), json!(v.version_original)); - entry.insert( - "urls".into(), - json!([chart_url(repo, &hit.original_id, &v.version)]), - ); - Value::Object(entry) - }) - .collect() -} - -/// Where a chart version is downloaded from this repository. -fn chart_url(repo: &Repository, name: &str, version: &str) -> String { - format!( - "{}/charts/{}/{version}/{name}-{version}.tgz", - repo.base_url, - name.to_ascii_lowercase(), - ) -} - -fn newest_first(versions: &mut [&PackageInfo]) { - versions.sort_by( - |a, b| match (Version::parse(&a.version), Version::parse(&b.version)) { - (Ok(a), Ok(b)) => b.cmp(&a), - _ => b.version.cmp(&a.version), - }, - ); -} - -/// The chart metadata of a stored manifest summary, typed for Helm again (a -/// summary stored before may not be). A summary this plugin makes fits in -/// [`MAX_SUMMARY`]; one over it, stored before summaries were bounded, has -/// its chart cut down the same way ([`bounded`]). -fn stored_chart(summary: Option<&str>) -> Map { - let Some(text) = summary else { - return Map::new(); - }; - let chart = serde_json::from_str::(text) - .ok() - .and_then(|s| match s.get("chart") { - Some(Value::Object(map)) => Some(helm_metadata(map)), - _ => None, - }) - .unwrap_or_default(); - if text.len() <= MAX_SUMMARY { - return chart; - } - let mut summary = Map::from_iter([("chart".to_string(), Value::Object(chart))]); - bounded(&mut summary); - match summary.remove("chart") { - Some(Value::Object(chart)) => chart, - _ => Map::new(), - } -} - -/// One chart version as the index and the ChartMuseum API list it. -fn entry(repo: &Repository, p: &PackageInfo) -> Value { - let mut chart = stored_chart(p.summary.as_deref()); - chart.insert("name".into(), json!(p.original_id)); - chart.insert("version".into(), json!(p.version_original)); - chart.insert( - "urls".into(), - json!([chart_url(repo, &p.original_id, &p.version)]), - ); - if let Some(published) = &p.published { - chart.insert("created".into(), json!(published)); - } - if let Some(sha256) = &p.sha256 { - chart.insert("digest".into(), json!(sha256)); - } - Value::Object(chart) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn index_metadata_has_the_types_helm_reads() { - let chart = chart::parse_chart_yaml( - b"apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: 2\ndeprecated: true\n\ - annotations: {a: false, b: 1.5, c: [x]}\nkeywords: [k, 3]\nhome: [x]\nextra: 1\n" - .to_vec(), - ) - .unwrap(); - let metadata = helm_metadata(&chart.metadata); - assert_eq!( - Value::Object(metadata), - json!({ - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2", - "deprecated": true, - "annotations": {"a": "false", "b": "1.5"}, - "keywords": ["k", "3"], - }) - ); - } - - fn summary_of(chart_yaml: &str) -> Value { - let chart = chart::parse_chart_yaml(chart_yaml.as_bytes().to_vec()).unwrap(); - let text = summary(&chart.metadata); - assert!(text.len() <= MAX_SUMMARY, "{} bytes", text.len()); - serde_json::from_str(&text).unwrap() - } - - fn package_info(summary: &str) -> PackageInfo { - PackageInfo { - original_id: "demo".into(), - version: "1.0.0".into(), - version_original: "1.0.0".into(), - listed: true, - published: None, - sha256: None, - sha512: None, - size: None, - summary: Some(summary.into()), - } - } - - fn hosted() -> Repository { - forge_plugin_api::testing::repository(RepositoryKind::Hosted, "/repository") - } - - #[test] - fn a_typical_charts_summary_is_all_of_its_metadata() { - let summary = summary_of( - "apiVersion: v2\nname: web\nversion: 1.2.0\nappVersion: \"1.27\"\n\ - kubeVersion: \">=1.25.0-0\"\ndescription: A web server\ntype: application\n\ - keywords: [web, http]\nhome: https://example.com/web\n\ - sources: [https://github.com/example/web]\nicon: https://example.com/web.png\n\ - maintainers:\n - name: Ada\n email: ada@example.com\n\ - dependencies:\n - name: redis\n version: \">=17\"\n \ - repository: https://charts.example.com\n condition: redis.enabled\n\ - annotations:\n artifacthub.io/license: Apache-2.0\n", - ); - let chart = json!({ - "apiVersion": "v2", - "name": "web", - "version": "1.2.0", - "appVersion": "1.27", - "kubeVersion": ">=1.25.0-0", - "description": "A web server", - "type": "application", - "keywords": ["web", "http"], - "home": "https://example.com/web", - "sources": ["https://github.com/example/web"], - "icon": "https://example.com/web.png", - "maintainers": [{"name": "Ada", "email": "ada@example.com"}], - "dependencies": [{ - "name": "redis", - "version": ">=17", - "repository": "https://charts.example.com", - "condition": "redis.enabled", - }], - "annotations": {"artifacthub.io/license": "Apache-2.0"}, - }); - assert_eq!( - summary, - json!({ - "chart": chart, - "description": "A web server", - "dependency_groups": [{ - "target_framework": null, - "dependencies": [{"id": "redis", "range": ">=17"}], - }], - }) - ); - // The index entry carries all of it (and the stored version's name - // and version). - let entry = entry(&hosted(), &package_info(&summary.to_string())); - for (key, value) in chart.as_object().unwrap() { - if key != "name" && key != "version" { - assert_eq!(&entry[key], value, "{key}"); - } - } - } - - #[test] - fn a_huge_description_is_cut_short() { - // 990 kB of a three-byte character: the cut falls inside one. - let description = "€".repeat(330_000); - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: \"2.0\"\n\ - keywords: [a, b]\ndescription: {description}\n" - )); - let cut = format!("{}…", "€".repeat(MAX_DESCRIPTION / 3)); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2.0", - "keywords": ["a", "b"], - "description": cut, - }, - "description": cut, - "dependency_groups": [], - }) - ); - let entry = entry(&hosted(), &package_info(&summary.to_string())); - assert!(entry.to_string().len() <= MAX_SUMMARY); - assert_eq!(entry["description"], json!(cut)); - } - - #[test] - fn the_largest_fields_go_first_and_the_essential_ones_last() { - let big = "x".repeat(20_000); - // Long annotations go, and nothing else. - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\ndescription: A demo\n\ - keywords: [a]\nannotations:\n changes: {big}\n license: MIT\n" - )); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "description": "A demo", - "keywords": ["a"], - }, - "description": "A demo", - "dependency_groups": [], - }) - ); - // A long essential field goes only after every other one. - let summary = summary_of(&format!( - "apiVersion: v2\nname: demo\nversion: 1.0.0\nappVersion: {big}\n\ - description: A demo\nkeywords: [a]\ntype: application\n" - )); - assert_eq!( - summary, - json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "type": "application", - }, - }) - ); - } - - #[test] - fn an_oversized_stored_summary_renders_a_bounded_entry() { - // As stored before summaries were bounded, and not typed for Helm. - let big = "x".repeat(1 << 20); - let stored = json!({ - "chart": { - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": 2, - "description": big, - "keywords": ["a"], - "annotations": {"changes": big}, - }, - "description": big, - "dependency_groups": [], - }) - .to_string(); - let expected = json!({ - "apiVersion": "v2", - "name": "demo", - "version": "1.0.0", - "appVersion": "2", - "description": format!("{}…", &big[..MAX_DESCRIPTION]), - "keywords": ["a"], - }); - let check = |mut entry: Value| { - assert!(entry.to_string().len() <= MAX_SUMMARY); - let entry = entry.as_object_mut().unwrap(); - for key in ["urls", "created", "digest"] { - entry.remove(key); - } - assert_eq!(Value::Object(entry.clone()), expected); - }; - check(entry(&hosted(), &package_info(&stored))); - let hit = SearchHit { - original_id: "demo".into(), - latest_version: "1.0.0".into(), - versions: vec![forge_plugin_api::VersionInfo { - original_id: "demo".into(), - version: "1.0.0".into(), - version_original: "1.0.0".into(), - listed: true, - }], - summary: Some(stored), - total_downloads: None, - }; - let proxy = forge_plugin_api::testing::repository(RepositoryKind::Proxy, "/repository"); - let entries = hit_entries(&proxy, &hit); - check(entries.into_iter().next().unwrap()); - } - - #[test] - fn versions_longer_than_an_oci_tag_are_refused() { - let longest = format!("1.0.0-{}", "a".repeat(semver::MAX_VERSION_LENGTH - 6)); - let longer = format!("{longest}a"); - assert_eq!(Helm.normalize_version(&longest).unwrap(), longest); - assert!(matches!( - Helm.normalize_version(&longer), - Err(FormatError::InvalidVersion(_)) - )); - assert_eq!( - package_ref("Demo", &longest), - Some(PackageRef { - id: "demo".into(), - version: longest.clone(), - }) - ); - assert_eq!(package_ref("demo", &longer), None); - - // A URL naming a longer version is not found: nothing is loaded, - // downloaded or deleted. - let repo = Repository { - name: "charts".into(), - kind: RepositoryKind::Proxy, - base_url: "https://forge.example.com/repository/charts".into(), - }; - let download = Helm.handle( - &repo, - &Request::new("GET", &format!("/charts/demo/{longest}/demo.tgz")), - ); - assert!(matches!(download, Action::Download(_)), "{download:?}"); - let huge = format!("1.0.0-{}", "x".repeat(4096)); - for (method, path) in [ - ("GET", format!("/charts/demo/{longer}/demo.tgz")), - ("GET", format!("/charts/demo/{huge}/demo.tgz")), - ("GET", format!("/api/charts/demo/{huge}")), - ("DELETE", format!("/api/charts/demo/{huge}")), - ] { - let action = Helm.handle(&repo, &Request::new(method, &path)); - assert!( - matches!(&action, Action::Respond(r) if r.status == 404), - "{method} {path}: {action:?}" - ); - } - } - - #[test] - fn plugin_toml_names_this_crate() { - let value = |key: &str| { - PLUGIN_TOML - .lines() - .filter_map(|line| line.split_once('=')) - .find(|(k, _)| k.trim() == key) - .map(|(_, v)| v.trim().trim_matches('"').to_string()) - }; - assert_eq!(value("id").as_deref(), Some("helm")); - assert_eq!(value("version").as_deref(), Some(env!("CARGO_PKG_VERSION"))); - assert_eq!(value("api").as_deref(), Some(forge_plugin_api::API)); - assert_eq!( - value("component"), - Some(format!("{}.wasm", env!("CARGO_CRATE_NAME"))) - ); - } -} diff --git a/plugins/helm/src/semver.rs b/plugins/helm/src/semver.rs deleted file mode 100644 index 4faf1c3e..00000000 --- a/plugins/helm/src/semver.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! SemVer 2.0.0, as Helm reads chart versions: a leading `v` and missing -//! minor/patch components are accepted and normalized away. - -use std::cmp::Ordering; - -/// The longest chart version, as written and normalized. Helm pushes a chart -/// to an OCI registry under its version as the tag (with `+` written `_`), -/// and a tag is at most 128 characters (the OCI distribution spec's -/// `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}`), so no chart a Helm user can push to -/// an OCI registry has a longer version. Without a limit a multi-kilobyte -/// prerelease passed parsing and failed the `(repository_id, name, version)` -/// index insert in PostgreSQL, so a push answered 500 instead of 400. A -/// version is ASCII, so this counts bytes. -pub const MAX_VERSION_LENGTH: usize = 128; - -/// A parsed version. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Version { - major: u64, - minor: u64, - patch: u64, - pre: Vec, - build: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum Ident { - Num(u64), - Alpha(String), -} - -impl Version { - /// Parse `raw`, or say why it is not a version. A version longer than - /// [`MAX_VERSION_LENGTH`], as written or normalized, is not one. - pub fn parse(raw: &str) -> Result { - let written = raw.trim(); - // Before parsing, so an oversized version is not copied around first. - if written.len() > MAX_VERSION_LENGTH { - return Err(too_long(written)); - } - let s = written.strip_prefix(['v', 'V']).unwrap_or(written); - let (rest, build) = match s.split_once('+') { - Some((rest, build)) => (rest, Some(build)), - None => (s, None), - }; - let (core, pre) = match rest.split_once('-') { - Some((core, pre)) => (core, Some(pre)), - None => (rest, None), - }; - let mut parts = core.split('.'); - let mut number = |required: bool| -> Result { - match parts.next() { - Some(p) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => p - .parse() - .map_err(|_| format!("version {raw:?}: {p} is too large")), - Some(p) => Err(format!("version {raw:?}: {p:?} is not a number")), - None if required => Err(format!("version {raw:?} has no major version")), - None => Ok(0), - } - }; - let major = number(true)?; - let minor = number(false)?; - let patch = number(false)?; - if parts.next().is_some() { - return Err(format!("version {raw:?} has more than three numbers")); - } - let identifiers = |text: &str| -> Result<(), String> { - let ok = text.split('.').all(|i| { - !i.is_empty() && i.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') - }); - if ok { - Ok(()) - } else { - Err(format!("version {raw:?}: invalid identifiers {text:?}")) - } - }; - let pre = match pre { - Some(pre) => { - identifiers(pre)?; - pre.split('.') - .map(|i| { - // A number with a leading zero is not a numeric - // identifier; keep it as text rather than refuse it. - match i.parse::() { - Ok(n) if !(i.len() > 1 && i.starts_with('0')) => Ident::Num(n), - _ => Ident::Alpha(i.to_string()), - } - }) - .collect() - } - None => Vec::new(), - }; - let build = match build { - Some(build) => { - identifiers(build)?; - build.to_string() - } - None => String::new(), - }; - let version = Version { - major, - minor, - patch, - pre, - build, - }; - // Normalizing can lengthen a version (`1-x` → `1.0.0-x`). - let normalized = version.to_string(); - if normalized.len() > MAX_VERSION_LENGTH { - return Err(format!( - "version {written:?} normalizes to {normalized:?}, longer than \ - {MAX_VERSION_LENGTH} characters" - )); - } - Ok(version) - } -} - -/// Why `written` is not a version: it is longer than [`MAX_VERSION_LENGTH`]. -/// The reason names only its start. -fn too_long(written: &str) -> String { - let start: String = written.chars().take(MAX_VERSION_LENGTH).collect(); - format!( - "version {:?} is longer than {MAX_VERSION_LENGTH} characters", - format!("{start}…") - ) -} - -impl std::fmt::Display for Version { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}.{}.{}", self.major, self.minor, self.patch)?; - for (i, ident) in self.pre.iter().enumerate() { - f.write_str(if i == 0 { "-" } else { "." })?; - match ident { - Ident::Num(n) => write!(f, "{n}")?, - Ident::Alpha(s) => f.write_str(s)?, - } - } - if !self.build.is_empty() { - write!(f, "+{}", self.build)?; - } - Ok(()) - } -} - -impl Ord for Version { - /// SemVer precedence; build metadata, which precedence ignores, breaks - /// ties so that distinct versions never compare equal. - fn cmp(&self, other: &Self) -> Ordering { - (self.major, self.minor, self.patch) - .cmp(&(other.major, other.minor, other.patch)) - .then_with(|| match (self.pre.is_empty(), other.pre.is_empty()) { - (true, true) => Ordering::Equal, - (true, false) => Ordering::Greater, - (false, true) => Ordering::Less, - (false, false) => { - for (a, b) in self.pre.iter().zip(&other.pre) { - let order = match (a, b) { - (Ident::Num(a), Ident::Num(b)) => a.cmp(b), - (Ident::Num(_), Ident::Alpha(_)) => Ordering::Less, - (Ident::Alpha(_), Ident::Num(_)) => Ordering::Greater, - (Ident::Alpha(a), Ident::Alpha(b)) => a.cmp(b), - }; - if order != Ordering::Equal { - return order; - } - } - self.pre.len().cmp(&other.pre.len()) - } - }) - .then_with(|| self.build.cmp(&other.build)) - } -} - -impl PartialOrd for Version { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn v(s: &str) -> Version { - Version::parse(s).unwrap() - } - - #[test] - fn normalizes() { - assert_eq!(v("v1.2.3").to_string(), "1.2.3"); - assert_eq!(v("1.2").to_string(), "1.2.0"); - assert_eq!(v("1").to_string(), "1.0.0"); - assert_eq!(v("01.002.3").to_string(), "1.2.3"); - assert_eq!(v("1.0.0-rc.01+build.5").to_string(), "1.0.0-rc.01+build.5"); - for bad in [ - "", - "x", - "1.2.3.4", - "1..2", - "1.0.0-", - "1.0.0+", - "1.0.0-a..b", - "1.0.0-a_b", - ] { - assert!(Version::parse(bad).is_err(), "{bad:?}"); - } - } - - #[test] - fn refuses_versions_longer_than_an_oci_tag() { - let longest = format!("1.0.0-{}", "a".repeat(MAX_VERSION_LENGTH - 6)); - assert_eq!(longest.len(), MAX_VERSION_LENGTH); - assert_eq!(v(&longest).to_string(), longest); - // Surrounding whitespace is not part of the version. - assert_eq!(v(&format!(" {longest}\n")).to_string(), longest); - let err = Version::parse(&format!("{longest}a")).unwrap_err(); - assert!(err.contains("longer than 128 characters"), "{err}"); - // A multi-kilobyte prerelease, named only by its start. - let err = Version::parse(&format!("1.0.0-{}", "x".repeat(4096))).unwrap_err(); - assert!(err.len() < 200, "{err}"); - // Normalized, `1-…` gains `.0.0`: the written form fits, not the - // normalized one. - let short = format!("1-{}", "a".repeat(MAX_VERSION_LENGTH - 2)); - assert_eq!(short.len(), MAX_VERSION_LENGTH); - let err = Version::parse(&short).unwrap_err(); - assert!(err.contains("normalizes to"), "{err}"); - assert!(Version::parse(&format!("1-{}", "a".repeat(MAX_VERSION_LENGTH - 6))).is_ok()); - } - - #[test] - fn orders_by_precedence() { - let ordered = [ - "1.0.0-alpha", - "1.0.0-alpha.1", - "1.0.0-alpha.beta", - "1.0.0-beta", - "1.0.0-beta.2", - "1.0.0-beta.11", - "1.0.0-rc.1", - "1.0.0", - "1.0.0+build", - "1.9.0", - "1.10.0", - "2.0.0", - ]; - for pair in ordered.windows(2) { - assert!(v(pair[0]) < v(pair[1]), "{} < {}", pair[0], pair[1]); - } - // Forge's rule: a normalized version is a prerelease when it has a `-`. - assert!(v("1.0.0-beta").to_string().contains('-')); - assert!(!v("v1.0+build").to_string().contains('-')); - } -} diff --git a/plugins/helm/src/tar.rs b/plugins/helm/src/tar.rs deleted file mode 100644 index a60df243..00000000 --- a/plugins/helm/src/tar.rs +++ /dev/null @@ -1,914 +0,0 @@ -//! A tar reader that sees a chart archive the way Helm does. -//! -//! Helm reads a chart with Go's `archive/tar`, so an entry's name, its type -//! and where its data ends are whatever that reader makes of the headers. -//! Forge must make the same of them, or the two disagree about which file -//! is the chart's `Chart.yaml`. [`Archive`] follows Go's `Reader.Next` for -//! every archive it accepts: ustar, PAX, GNU and STAR headers and their -//! checksums, PAX `path` and `size` records (of the last extended header -//! before an entry), GNU long names and long link names, global PAX headers -//! as entries of their own, entry types without data, and Go's end of -//! archive. It refuses what it does not model: sparse files, entry types -//! charts do not use, and a few archives Go reads (a stream that ends inside -//! padding, an extension header over [`MAX_HEADER_DATA`], a negative size). - -use std::io::{self, Read}; - -/// The size of a header, and the unit data is padded to. -const BLOCK: usize = 512; - -/// The longest PAX header or GNU long name read. Go reads up to 1 MiB; the -/// names in a chart are far shorter. -const MAX_HEADER_DATA: u64 = 64 * 1024; - -/// An entry of an archive, as Go's `tar.Reader.Next` returns it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Entry { - /// Its name (`Header.Name`): the header's own, joined to a ustar prefix, - /// or taken from a PAX `path` record or a GNU long name. Bytes, as in a - /// Go string. - pub name: Vec, - /// Its type flag. An old-style regular entry (`\0`) reads as `0`, or as - /// a directory (`5`) when its name ends with `/`, as in Go. - pub typeflag: u8, - /// Its mode field (`Header.Mode`). - pub mode: i64, - /// The length of its data: 0 for the types that have none (links, - /// devices, directories, FIFOs), whatever the header says. - pub size: u64, -} - -impl Entry { - /// Whether Go's `Header.FileInfo().IsDir()` holds: a directory by its - /// type, or by the file type bits of its mode, whatever its type. - pub fn is_dir(&self) -> bool { - // Go converts the mode to a 32-bit `fs.FileMode`. - self.typeflag == b'5' || (self.mode as u32) & !0o7777 == 0o040000 - } -} - -/// The entries of a tar stream, one at a time. -pub struct Archive { - reader: R, - /// The current entry's data not read yet. - unread: u64, - /// The padding after the current entry's data. - padding: u64, - /// Whether the end of the archive was reached. - ended: bool, -} - -impl Archive { - /// Read the archive in `reader`. - pub fn new(reader: R) -> Self { - Self { - reader, - unread: 0, - padding: 0, - ended: false, - } - } - - /// The next entry, after skipping what is left of the current one; - /// `None` at the end of the archive. Extended headers and GNU long names - /// apply to the entry after them and are not entries; a global PAX - /// header (`g`) is one, as in Go, and ends what came before it. - pub fn next_entry(&mut self) -> io::Result> { - if self.ended { - return Ok(None); - } - skip(&mut self.reader, self.unread + self.padding)?; - self.unread = 0; - self.padding = 0; - // What extension headers say about the entry after them. Go keeps - // the last extended header, and the last long name. - let mut pax: Option = None; - let mut long_name: Vec = Vec::new(); - loop { - let Some(header) = self.read_header()? else { - self.ended = true; - return Ok(None); - }; - match header.typeflag { - b'x' | b'g' => { - let records = Pax::parse(&self.special_data(&header)?)?; - if header.typeflag == b'g' { - let path = records.path.filter(|p| !p.is_empty()); - return Ok(Some(Entry { - name: path.unwrap_or(header.name), - typeflag: b'g', - mode: 0, - size: 0, - })); - } - pax = Some(records); - } - b'L' | b'K' => { - let data = self.special_data(&header)?; - // A long link name only names a link's target. - if header.typeflag == b'L' { - long_name = c_string(&data).to_vec(); - } - } - b'S' => return Err(invalid("the archive holds a GNU sparse file")), - typeflag => { - let mut name = header.name; - let mut size = header.size; - if let Some(pax) = pax { - pax.check()?; - if let Some(path) = pax.path.filter(|p| !p.is_empty()) { - name = path; - } - if let Some(value) = pax.size.filter(|s| !s.is_empty()) { - size = decimal(&value).ok_or_else(|| invalid("bad PAX size record"))?; - } - } - // A GNU long name wins over a PAX path, in either order. - if !long_name.is_empty() { - name = long_name; - } - let typeflag = match typeflag { - 0 if name.ends_with(b"/") => b'5', - 0 => b'0', - other => other, - }; - if !matches!(typeflag, b'0'..=b'7') { - return Err(invalid(&format!( - "the archive holds an entry of type {:?}", - char::from(typeflag) - ))); - } - let size = u64::try_from(size).map_err(|_| invalid("negative tar size"))?; - // Links, devices, directories and FIFOs have no data, - // whatever their size says. - let size = if matches!(typeflag, b'1'..=b'6') { - 0 - } else { - size - }; - self.unread = size; - self.padding = padding(size); - return Ok(Some(Entry { - name, - typeflag, - mode: header.mode, - size, - })); - } - } - } - } - - /// The current entry's data. An entry larger than `max` bytes is an - /// error. - pub fn read_data(&mut self, max: u64) -> io::Result> { - if self.unread > max { - return Err(invalid(&format!("larger than {max} bytes"))); - } - let data = read_exact(&mut self.reader, self.unread)?; - self.unread = 0; - Ok(data) - } - - /// The next header; `None` at the end of the archive: the end of the - /// stream before a header or after one zero block, or two zero blocks. - /// A zero block followed by anything else is an error, as in Go. - fn read_header(&mut self) -> io::Result> { - let mut block = [0u8; BLOCK]; - if !read_block(&mut self.reader, &mut block)? { - return Ok(None); - } - if block.iter().all(|&b| b == 0) { - if !read_block(&mut self.reader, &mut block)? || block.iter().all(|&b| b == 0) { - return Ok(None); - } - return Err(invalid("a zero block in the middle of the archive")); - } - let header = Header::parse(&block)?; - if header.size < 0 { - return Err(invalid("negative tar size")); - } - Ok(Some(header)) - } - - /// The data of an extension header, and its padding skipped. - fn special_data(&mut self, header: &Header) -> io::Result> { - let size = header.size as u64; - if size > MAX_HEADER_DATA { - return Err(invalid("oversized tar extension header")); - } - let data = read_exact(&mut self.reader, size)?; - skip(&mut self.reader, padding(size))?; - Ok(data) - } -} - -/// The fields of a header block that Go's reader uses. -struct Header { - /// The name, joined to the prefix of the header's format. - name: Vec, - /// The type flag, as written. - typeflag: u8, - /// The size field, which an extended header may replace. - size: i64, - /// The mode field. - mode: i64, -} - -/// The header formats Go tells apart by their magic. -#[derive(PartialEq)] -enum Format { - /// No magic: no prefix, no user or device fields. - V7, - /// `ustar\0`, which PAX archives use too: a 155-byte prefix. - Ustar, - /// `ustar \0`: times where ustar has its prefix. - Gnu, - /// `ustar\0` and `tar\0` at the end: a 131-byte prefix, then times. - Star, -} - -impl Header { - /// Parse a header block as Go's `Reader.readHeader` does. - fn parse(block: &[u8; BLOCK]) -> io::Result { - let format = format(block).ok_or_else(|| invalid("bad tar header checksum"))?; - let mut fields = Numbers::default(); - let mode = fields.numeric(&block[100..108]); - fields.numeric(&block[108..116]); // uid - fields.numeric(&block[116..124]); // gid - let size = fields.numeric(&block[124..136]); - fields.numeric(&block[136..148]); // mtime - if format != Format::V7 { - fields.numeric(&block[329..337]); // devmajor - fields.numeric(&block[337..345]); // devminor - } - let prefix = match format { - Format::V7 => &[], - Format::Ustar => c_string(&block[345..500]), - Format::Star => { - fields.numeric(&block[476..488]); // atime - fields.numeric(&block[488..500]); // ctime - c_string(&block[345..476]) - } - // A GNU header has times where ustar has its prefix. Go reads - // them as a prefix after all when they are no times (old Go - // versions wrote GNU headers with a prefix) and it is ASCII. - Format::Gnu => { - let mut times = Numbers::default(); - if block[345] != 0 { - times.numeric(&block[345..357]); - } - if block[357] != 0 { - times.numeric(&block[357..369]); - } - let prefix = c_string(&block[345..500]); - if times.failed && prefix.is_ascii() { - prefix - } else { - &[] - } - } - }; - if fields.failed { - return Err(invalid("bad tar header field")); - } - let name = c_string(&block[0..100]); - Ok(Self { - name: if prefix.is_empty() { - name.to_vec() - } else { - [prefix, b"/", name].concat() - }, - typeflag: block[156], - size, - mode, - }) - } -} - -/// The format of a header block; `None` when its checksum is wrong. Go -/// accepts the unsigned sum of the bytes and the signed one (Sun tar). -fn format(block: &[u8; BLOCK]) -> Option { - let mut fields = Numbers::default(); - let stored = fields.octal(&block[148..156]); - let (mut unsigned, mut signed) = (0i64, 0i64); - for (i, &b) in block.iter().enumerate() { - // The checksum field counts as spaces. - let b = if (148..156).contains(&i) { b' ' } else { b }; - unsigned += i64::from(b); - signed += i64::from(b as i8); - } - if fields.failed || (stored != unsigned && stored != signed) { - return None; - } - let (magic, version, trailer) = (&block[257..263], &block[263..265], &block[508..512]); - Some(if magic == b"ustar\0" && trailer == b"tar\0" { - Format::Star - } else if magic == b"ustar\0" { - Format::Ustar - } else if magic == b"ustar " && version == b" \0" { - Format::Gnu - } else { - Format::V7 - }) -} - -/// Go's reader of numeric header fields; `failed` is set by any field it -/// cannot read, like Go's `parser.err`. -#[derive(Default)] -struct Numbers { - failed: bool, -} - -impl Numbers { - /// An octal field, or GNU base-256 (two's complement) when its first - /// bit is set (Go's `parseNumeric`). - fn numeric(&mut self, field: &[u8]) -> i64 { - if field[0] & 0x80 == 0 { - return self.octal(field); - } - let inverse = if field[0] & 0x40 != 0 { 0xff } else { 0 }; - let mut n: u64 = 0; - for (i, &b) in field.iter().enumerate() { - let mut b = b ^ inverse; - if i == 0 { - b &= 0x7f; - } - if n >> 56 > 0 { - self.failed = true; - return 0; - } - n = n << 8 | u64::from(b); - } - if n >> 63 > 0 { - self.failed = true; - return 0; - } - if inverse == 0xff { - !(n as i64) - } else { - n as i64 - } - } - - /// An octal field: NULs and spaces trimmed from both ends, then the - /// digits up to the first NUL (Go's `parseOctal`). - fn octal(&mut self, field: &[u8]) -> i64 { - let padding = |b: &u8| *b == b' ' || *b == 0; - let start = field.iter().position(|b| !padding(b)); - let end = field.iter().rposition(|b| !padding(b)); - let (Some(start), Some(end)) = (start, end) else { - return 0; - }; - let mut n: u64 = 0; - for &b in c_string(&field[start..=end]) { - let next = match b { - b'0'..=b'7' => n - .checked_mul(8) - .and_then(|n| n.checked_add(u64::from(b - b'0'))), - _ => None, - }; - let Some(next) = next else { - self.failed = true; - return 0; - }; - n = next; - } - n as i64 - } -} - -/// The records of a PAX extended header that bear on an entry. A key given -/// twice keeps its last value, as in Go. -#[derive(Default)] -struct Pax { - /// `path`: the entry's name. - path: Option>, - /// `size`: the length of the entry's data. - size: Option>, - /// `uid`. - uid: Option>, - /// `gid`. - gid: Option>, - /// `atime`, `mtime` and `ctime`. - times: [Option>; 3], -} - -impl Pax { - /// Parse the records of a PAX header (`" =\n"`, - /// Go's `parsePAX`). GNU sparse records, which describe a sparse file's - /// layout and may rename it, are refused. - fn parse(mut data: &[u8]) -> io::Result { - let bad = || invalid("bad PAX record"); - let mut pax = Self::default(); - while !data.is_empty() { - // The length, of the whole record, ends at the first space. - let space = data.iter().position(|&b| b == b' ').ok_or_else(bad)?; - let length = decimal(&data[..space]).ok_or_else(bad)?; - if length < 5 || length > data.len() as i64 { - return Err(bad()); - } - let length = length as usize; - if length <= space + 1 || data[length - 1] != b'\n' { - return Err(bad()); - } - let record = &data[space + 1..length - 1]; - data = &data[length..]; - let equals = record.iter().position(|&b| b == b'=').ok_or_else(bad)?; - let (key, value) = (&record[..equals], record[equals + 1..].to_vec()); - let field = match key { - b"" => return Err(bad()), - b"path" | b"linkpath" | b"uname" | b"gname" if value.contains(&0) => { - return Err(bad()) - } - _ if key.contains(&0) => return Err(bad()), - _ if key.starts_with(b"GNU.sparse.") => { - return Err(invalid("the archive holds a GNU sparse file")) - } - b"path" => &mut pax.path, - b"size" => &mut pax.size, - b"uid" => &mut pax.uid, - b"gid" => &mut pax.gid, - b"atime" => &mut pax.times[0], - b"mtime" => &mut pax.times[1], - b"ctime" => &mut pax.times[2], - _ => continue, - }; - *field = Some(value); - } - Ok(pax) - } - - /// Refuse the records Go cannot apply to an entry (`mergePAX`). Empty - /// values are ignored, as in Go. - fn check(&self) -> io::Result<()> { - let bad = || invalid("bad PAX record"); - for id in [&self.uid, &self.gid].into_iter().flatten() { - if !id.is_empty() { - decimal(id).ok_or_else(bad)?; - } - } - // Seconds, and an optional fraction. - for time in self.times.iter().flatten().filter(|t| !t.is_empty()) { - let (seconds, fraction) = match time.iter().position(|&b| b == b'.') { - Some(dot) => (&time[..dot], &time[dot + 1..]), - None => (&time[..], &[][..]), - }; - decimal(seconds).ok_or_else(bad)?; - if !fraction.iter().all(u8::is_ascii_digit) { - return Err(bad()); - } - } - Ok(()) - } -} - -/// A decimal integer with an optional sign, as Go's `strconv.ParseInt(s, -/// 10, 64)` reads it. -fn decimal(text: &[u8]) -> Option { - let (negative, digits) = match text.split_first() { - Some((b'-', rest)) => (true, rest), - Some((b'+', rest)) => (false, rest), - _ => (false, text), - }; - if digits.is_empty() { - return None; - } - let mut n: i64 = 0; - for &b in digits { - if !b.is_ascii_digit() { - return None; - } - let digit = i64::from(b - b'0'); - n = n.checked_mul(10)?; - n = if negative { - n.checked_sub(digit)? - } else { - n.checked_add(digit)? - }; - } - Some(n) -} - -/// An error for an archive Forge does not read. -fn invalid(msg: &str) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, msg.to_string()) -} - -/// The padding after `size` bytes of data, to the end of their last block. -fn padding(size: u64) -> u64 { - size.next_multiple_of(BLOCK as u64) - size -} - -/// Fill `block`; `false` at a clean end of the stream. -fn read_block(r: &mut impl Read, block: &mut [u8; BLOCK]) -> io::Result { - let mut filled = 0; - while filled < BLOCK { - match r.read(&mut block[filled..])? { - 0 if filled == 0 => return Ok(false), - 0 => return Err(invalid("truncated tar header")), - n => filled += n, - } - } - Ok(true) -} - -/// The next `len` bytes; an error when the stream ends first. -fn read_exact(r: &mut impl Read, len: u64) -> io::Result> { - let mut data = Vec::new(); - r.take(len).read_to_end(&mut data)?; - if data.len() as u64 != len { - return Err(invalid("truncated tar entry")); - } - Ok(data) -} - -/// Skip `len` bytes; an error when the stream ends first. -fn skip(r: &mut impl Read, len: u64) -> io::Result<()> { - let skipped = io::copy(&mut r.take(len), &mut io::sink())?; - if skipped != len { - return Err(invalid("truncated tar entry")); - } - Ok(()) -} - -/// The bytes up to the first NUL (Go's `parseString`). -fn c_string(bytes: &[u8]) -> &[u8] { - let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); - &bytes[..end] -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - - /// A ustar header (mode 0644) with a valid checksum. - pub(crate) fn header(name: &[u8], typeflag: u8, size: u64) -> [u8; BLOCK] { - let mut header = [0u8; BLOCK]; - header[..name.len()].copy_from_slice(name); - header[100..108].copy_from_slice(b"0000644\0"); - header[124..136].copy_from_slice(format!("{size:011o}\0").as_bytes()); - header[156] = typeflag; - header[257..263].copy_from_slice(b"ustar\0"); - header[263..265].copy_from_slice(b"00"); - seal(&mut header); - header - } - - /// Write the checksum of `header`, after changing it. - pub(crate) fn seal(header: &mut [u8; BLOCK]) { - header[148..156].fill(b' '); - let sum: u32 = header.iter().map(|&b| u32::from(b)).sum(); - header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - } - - /// `header` followed by `data`, padded to whole blocks. - pub(crate) fn entry(header: [u8; BLOCK], data: &[u8]) -> Vec { - let mut out = header.to_vec(); - out.extend_from_slice(data); - out.resize(out.len().next_multiple_of(BLOCK), 0); - out - } - - /// A regular file. - pub(crate) fn file(path: &str, data: &[u8]) -> Vec { - entry(header(path.as_bytes(), b'0', data.len() as u64), data) - } - - /// A GNU long name (`L`) or long link name (`K`). - pub(crate) fn long(typeflag: u8, name: &str) -> Vec { - let data = [name.as_bytes(), b"\0"].concat(); - entry(header(b"././@LongLink", typeflag, data.len() as u64), &data) - } - - /// A PAX header of type `typeflag` (`x` or `g`) holding `records`. - pub(crate) fn pax(typeflag: u8, records: &[(&str, &str)]) -> Vec { - let mut data = Vec::new(); - for (key, value) in records { - let body = format!(" {key}={value}\n"); - // The length counts its own digits. - let mut length = body.len(); - while length != length.to_string().len() + body.len() { - length = length.to_string().len() + body.len(); - } - data.extend_from_slice(format!("{length}{body}").as_bytes()); - } - entry( - header(b"PaxHeaders.0/x", typeflag, data.len() as u64), - &data, - ) - } - - /// The end of an archive: two zero blocks. - pub(crate) const END: [u8; 2 * BLOCK] = [0; 2 * BLOCK]; - - /// A minimal ustar archive of `(path, contents)` entries. - pub(crate) fn archive(entries: &[(&str, &[u8])]) -> Vec { - let mut out: Vec = entries - .iter() - .flat_map(|(path, data)| file(path, data)) - .collect(); - out.extend_from_slice(&END); - out - } - - /// Every entry of `tar`: its name, type and data. - fn entries(tar: &[u8]) -> io::Result)>> { - let mut archive = Archive::new(tar); - let mut out = Vec::new(); - while let Some(entry) = archive.next_entry()? { - let data = archive.read_data(u64::MAX)?; - assert_eq!(data.len() as u64, entry.size); - let name = String::from_utf8_lossy(&entry.name).into_owned(); - out.push((name, entry.typeflag, data)); - } - Ok(out) - } - - /// The names of the entries of `tar`. - fn names(tar: &[u8]) -> Vec { - entries(tar) - .unwrap() - .into_iter() - .map(|(n, _, _)| n) - .collect() - } - - #[test] - fn reads_entries_and_their_data() { - let tar = archive(&[ - ("demo/values.yaml", b"replicas: 1\n"), - ("demo/Chart.yaml", b"name: demo\n"), - ]); - assert_eq!( - entries(&tar).unwrap(), - [ - ("demo/values.yaml".into(), b'0', b"replicas: 1\n".to_vec()), - ("demo/Chart.yaml".into(), b'0', b"name: demo\n".to_vec()), - ] - ); - // Data not read is skipped. - let mut archive = Archive::new(&tar[..]); - archive.next_entry().unwrap(); - assert!(archive.read_data(4).is_err()); - let second = archive.next_entry().unwrap().unwrap(); - assert_eq!(second.name, b"demo/Chart.yaml"); - assert_eq!(archive.next_entry().unwrap(), None); - assert_eq!(archive.next_entry().unwrap(), None); - // Truncated: in the data, in the padding, in a header. - assert!(entries(&tar[..520]).is_err()); - assert!(entries(&tar[..700]).is_err()); - assert!(entries(&tar[..1030]).is_err()); - } - - #[test] - fn ends_like_go() { - let one = file("demo/a", b"a"); - // The stream ends before a header, or after one zero block. - assert_eq!(names(&one), ["demo/a"]); - assert_eq!(names(&[&one[..], &[0; BLOCK]].concat()), ["demo/a"]); - // Two zero blocks end the archive; what follows is not read. - let after = [&one[..], &END, &file("demo/b", b"b")].concat(); - assert_eq!(names(&after), ["demo/a"]); - // A zero block followed by a header is an error, not the end. - let hidden = [&one[..], &[0; BLOCK], &file("demo/b", b"b")].concat(); - assert!(entries(&hidden).is_err()); - } - - #[test] - fn checks_header_checksums() { - let mut bad = header(b"demo/a", b'0', 0); - bad[0] = b'x'; - assert!(entries(&[&bad[..], &END].concat()).is_err()); - // Go accepts the signed sum too (Sun tar). - let mut signed = header("demo/é".as_bytes(), b'0', 0); - let sum: i64 = signed - .iter() - .enumerate() - .map(|(i, &b)| if (148..156).contains(&i) { b' ' } else { b }) - .map(|b| i64::from(b as i8)) - .sum(); - signed[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); - assert_eq!(names(&[&signed[..], &END].concat()), ["demo/é"]); - } - - #[test] - fn reads_numeric_fields_like_go() { - // Leading NULs and spaces are padding, not the end of the number. - let mut padded = header(b"demo/a", b'0', 0); - padded[124..136].copy_from_slice(b"\0 0000000003"); - seal(&mut padded); - let tar = [&entry(padded, b"abc")[..], &END].concat(); - assert_eq!(entries(&tar).unwrap()[0].2, b"abc"); - // GNU base-256. - let mut binary = header(b"demo/a", b'0', 0); - binary[124..136].copy_from_slice(&[0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3]); - seal(&mut binary); - let tar = [&entry(binary, b"abc")[..], &END].concat(); - assert_eq!(entries(&tar).unwrap()[0].2, b"abc"); - // What Go cannot read: a sign, a digit that is not octal, a mode - // in hexadecimal. - for (field, value) in [ - (124..136, &b"+0000000003\0"[..]), - (124..136, b"0000000009\0\0"), - (100..108, b"0x644\0\0\0"), - ] { - let mut bad = header(b"demo/a", b'0', 3); - bad[field].copy_from_slice(value); - seal(&mut bad); - assert!(entries(&[&entry(bad, b"abc")[..], &END].concat()).is_err()); - } - } - - #[test] - fn joins_a_ustar_prefix_but_not_gnu_times() { - let mut ustar = header(b"Chart.yaml", b'0', 0); - ustar[345..349].copy_from_slice(b"demo"); - seal(&mut ustar); - assert_eq!(names(&[&ustar[..], &END].concat()), ["demo/Chart.yaml"]); - - // A GNU header has times where ustar has its prefix. - let gnu = |times: &[u8]| { - let mut gnu = header(b"demo/Chart.yaml", b'0', 0); - gnu[257..265].copy_from_slice(b"ustar \0"); - gnu[345..345 + times.len()].copy_from_slice(times); - seal(&mut gnu); - names(&[&gnu[..], &END].concat()) - }; - assert_eq!(gnu(b"14567012345\0"), ["demo/Chart.yaml"]); - // Junk that reads as octal times stays times. - assert_eq!(gnu(b"7\0junk"), ["demo/Chart.yaml"]); - // Junk that does not is a prefix to Go (old Go versions wrote one), - // when it is ASCII. - assert_eq!(gnu(b"evil"), ["evil/demo/Chart.yaml"]); - assert_eq!(gnu("évil".as_bytes()), ["demo/Chart.yaml"]); - - // STAR: a shorter prefix, then times. - let mut star = header(b"Chart.yaml", b'0', 0); - star[345..349].copy_from_slice(b"demo"); - star[508..512].copy_from_slice(b"tar\0"); - seal(&mut star); - assert_eq!(names(&[&star[..], &END].concat()), ["demo/Chart.yaml"]); - - // V7, without a magic: no prefix. - let mut v7 = header(b"demo/Chart.yaml", b'0', 0); - v7[257..265].fill(0); - v7[345..349].copy_from_slice(b"evil"); - seal(&mut v7); - assert_eq!(names(&[&v7[..], &END].concat()), ["demo/Chart.yaml"]); - } - - #[test] - fn applies_pax_records_like_go() { - // A PAX path renames the next entry only. - let tar = [ - &pax(b'x', &[("mtime", "1.5"), ("path", "demo/values.yaml")])[..], - &file("x", b"a: 1\n"), - &file("demo/b", b""), - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/values.yaml", "demo/b"]); - // A key given twice keeps its last value; an empty one keeps the - // header's name. - let twice = [ - &pax(b'x', &[("path", "demo/a"), ("path", "demo/b")])[..], - &file("x", b""), - &pax(b'x', &[("path", "")]), - &file("demo/c", b""), - &END, - ] - .concat(); - assert_eq!(names(&twice), ["demo/b", "demo/c"]); - // Only the last extended header before an entry counts. - let last = [ - &pax(b'x', &[("path", "demo/a")])[..], - &pax(b'x', &[("mtime", "1")]), - &file("demo/b", b""), - &END, - ] - .concat(); - assert_eq!(names(&last), ["demo/b"]); - - // A PAX size moves the end of the entry's data: what its header - // calls data is the next entry. - let chart = file("demo/Chart.yaml", b"name: demo\n"); - let sized = [ - &pax(b'x', &[("size", "0")])[..], - &entry(header(b"demo/a", b'0', chart.len() as u64), &chart), - &END, - ] - .concat(); - assert_eq!(names(&sized), ["demo/a", "demo/Chart.yaml"]); - - // Records Go cannot apply, and sparse files. - for records in [ - &[("size", "x")][..], - &[("uid", "1x")], - &[("mtime", "1.x")], - &[("GNU.sparse.major", "1"), ("GNU.sparse.minor", "0")], - &[("GNU.sparse.name", "demo/Chart.yaml")], - ] { - let tar = [&pax(b'x', records)[..], &file("demo/a", b""), &END].concat(); - assert!(entries(&tar).is_err(), "{records:?}"); - } - // A record longer than the header. - let malformed = [ - &entry(header(b"x", b'x', 11), b"13 path=ab\n")[..], - &file("demo/a", b""), - &END, - ] - .concat(); - assert!(entries(&malformed).is_err()); - } - - #[test] - fn applies_gnu_long_names_like_go() { - // A long name renames the next entry; a long link name does not, - // and does not take the long name before it. - let tar = [ - &long(b'L', "demo/values.yaml")[..], - &file("x", b""), - &long(b'L', "demo/Chart.yaml"), - &long(b'K', "demo/target"), - &file("demo/a", b""), - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/values.yaml", "demo/Chart.yaml"]); - // A long name wins over a PAX path, before it or after it. - for (first, second) in [ - (long(b'L', "demo/long"), pax(b'x', &[("path", "demo/pax")])), - (pax(b'x', &[("path", "demo/pax")]), long(b'L', "demo/long")), - ] { - let tar = [&first[..], &second, &file("x", b""), &END].concat(); - assert_eq!(names(&tar), ["demo/long"]); - } - } - - #[test] - fn a_global_header_is_an_entry_of_its_own() { - // It ends what came before it: the long name applies to no entry. - let tar = [ - &long(b'L', "demo/Chart.yaml")[..], - &pax(b'g', &[("comment", "abc")]), - &file("demo/a", b""), - &END, - ] - .concat(); - let entries = entries(&tar).unwrap(); - assert_eq!(entries[0].1, b'g'); - assert_eq!(entries[1].0, "demo/a"); - } - - #[test] - fn types_without_data_have_none() { - // A symlink's size does not make the next header its data. - let chart = file("demo/Chart.yaml", b"name: demo\n"); - for typeflag in *b"123456" { - let tar = [ - &header(b"demo/link", typeflag, chart.len() as u64)[..], - &chart, - &END, - ] - .concat(); - assert_eq!(names(&tar), ["demo/link", "demo/Chart.yaml"]); - } - // An old-style regular entry named with a `/` is a directory. - let tar = [ - &header(b"demo/", 0, chart.len() as u64)[..], - &entry(header(b"demo/a", 0, 1), b"a"), - &chart, - &END, - ] - .concat(); - let entries = entries(&tar).unwrap(); - assert_eq!((entries[0].1, entries[1].1), (b'5', b'0')); - assert_eq!(entries[2].0, "demo/Chart.yaml"); - } - - #[test] - fn a_directory_by_its_mode() { - let mut dir = header(b"demo/Chart.yaml", b'0', 0); - dir[100..108].copy_from_slice(b"0040755\0"); - seal(&mut dir); - let tar = [&dir[..], &END].concat(); - let mut archive = Archive::new(&tar[..]); - assert!(archive.next_entry().unwrap().unwrap().is_dir()); - let tar = [&header(b"demo/a", b'0', 0)[..], &END].concat(); - let mut archive = Archive::new(&tar[..]); - assert!(!archive.next_entry().unwrap().unwrap().is_dir()); - } - - #[test] - fn refuses_what_it_does_not_model() { - for typeflag in *b"SDMVA" { - let tar = [&header(b"demo/a", typeflag, 0)[..], &END].concat(); - assert!(entries(&tar).is_err(), "{}", char::from(typeflag)); - } - let mut negative = header(b"demo/a", b'0', 0); - negative[124..136].copy_from_slice(&[0xff; 12]); - seal(&mut negative); - assert!(entries(&[&negative[..], &END].concat()).is_err()); - } -} diff --git a/plugins/helm/src/upstream.rs b/plugins/helm/src/upstream.rs deleted file mode 100644 index 8d6be726..00000000 --- a/plugins/helm/src/upstream.rs +++ /dev/null @@ -1,709 +0,0 @@ -//! The upstream side of Helm proxies: another chart repository, read through -//! the host ([`FormatPlugin::upstream`](forge_plugin_api::FormatPlugin::upstream)). -//! -//! A chart repository is one document, its `index.yaml`: every chart, every -//! version, and where each version's archive is. Every operation reads it -//! (the host caches it for the repository's metadata TTL). The upstream's -//! index ([`Remote::index`]) is just its base URL. -//! -//! Parsing a large index takes long (half a second and some 200 MB for a -//! 10 MB one), and a proxy's chart listing runs several operations per -//! chart, so the built-in plugin keeps the parsed index while the upstream -//! serves the same document, and parses one index at a time -//! ([`parsed_index`]). - -use std::collections::BTreeMap; -use std::sync::Arc; - -use forge_plugin_api::{ - Fetch, FetchResult, Fetched, FormatError, Freshness, PackageMetadata, PackageRef, - PackageVersions, Remote, SearchQuery, UpstreamAnswer, UpstreamHit, UpstreamOp, UpstreamSearch, - UpstreamStep, VersionMetadata, -}; -use serde_json::{json, Map, Value}; - -use crate::chart::{self, to_json}; -use crate::semver::Version; - -/// The default page of an upstream search. -const DEFAULT_TAKE: usize = 20; - -/// One chart version an upstream index lists. Compact, since parsed -/// indexes are kept ([`parsed_index`]): the entry itself stays JSON text, -/// read back only for the versions an operation describes. -struct Entry { - /// The chart name, as written. - name: String, - version: Version, - /// Its `created`. - created: Option, - /// The first of its `urls`. - url: Option, - /// The index entry (Chart.yaml's metadata, `urls`, `created`, `digest`), - /// as JSON. - metadata: Box, -} - -impl Entry { - /// The index entry ([`Entry::metadata`]). - fn metadata(&self) -> Map { - serde_json::from_str(&self.metadata).unwrap_or_default() - } -} - -/// An upstream index: each chart's versions by canonical name, ascending. -type Index = BTreeMap>; - -/// One step of the conversation for `op` ([`crate::Helm`]'s -/// `FormatPlugin::upstream`). -pub(crate) fn step( - remote: &Remote, - op: &UpstreamOp, - fetched: &[FetchResult], -) -> Result { - let base = match op { - UpstreamOp::Index => remote.url.trim_end_matches('/').to_string(), - _ => match remote - .index - .as_deref() - .and_then(|i| serde_json::from_str::(i).ok()) - .and_then(|i| i.get("base")?.as_str().map(str::to_string)) - { - Some(base) => base, - None => return Ok(failed("the upstream's index is not resolved")), - }, - }; - // Nothing to read: a chart repository has no manifest without its - // chart, so the host takes the manifest from the package. - if let UpstreamOp::Manifest(_) = op { - return Ok(done(UpstreamAnswer::Manifest(None))); - } - let url = format!("{base}/index.yaml"); - let index = match fetched - .iter() - .find(|f| f.fetch.url == url) - .map(|f| &f.outcome) - { - None => { - return Ok(UpstreamStep::Fetch(vec![Fetch { - url, - freshness: Freshness::Metadata, - }])) - } - Some(Fetched::Found(body)) => match parsed_index(&url, body) { - Ok(index) => index, - Err(e) => return Ok(failed(format!("upstream index {url}: {e}"))), - }, - Some(Fetched::NotFound) => return Ok(failed(format!("upstream index {url} returned 404"))), - Some(Fetched::Failed(reason)) => return Ok(failed(reason.clone())), - }; - Ok(match op { - UpstreamOp::Index => done(UpstreamAnswer::Index(json!({ "base": base }).to_string())), - UpstreamOp::Versions(id) => { - done(UpstreamAnswer::Versions(index.get(id).map(|versions| { - versions.iter().map(|e| e.version.to_string()).collect() - }))) - } - UpstreamOp::Package(target) => { - done(UpstreamAnswer::Package(package_url(&base, &index, target))) - } - UpstreamOp::Manifest(_) => unreachable!("answered above"), - UpstreamOp::Metadata(wanted) => metadata(&index, wanted, false), - UpstreamOp::Dating(wanted) => metadata(&index, wanted, true), - UpstreamOp::Search(query) => done(UpstreamAnswer::Search(Some(search(&index, query)))), - }) -} - -fn done(answer: UpstreamAnswer) -> UpstreamStep { - UpstreamStep::Done(answer) -} - -fn failed(reason: impl Into) -> UpstreamStep { - done(UpstreamAnswer::Failed(reason.into())) -} - -/// The parsed `index.yaml` at `url` whose bytes are `body`. -/// -/// The built-in plugin keeps what it parsed, one index per URL, as many as -/// fit in [`parsed::BUDGET`] (one budget for every Helm proxy of the -/// process), and reuses one while the upstream serves the same bytes (the -/// host hands every operation the document it caches for the metadata TTL): -/// comparing the bytes takes a millisecond where parsing takes half a second. -/// Since the result depends on the bytes alone, it is the same for every -/// repository that fetches them. Parses run one at a time: concurrent -/// operations on a document nobody has parsed yet wait for the first parse -/// instead of each running their own, and memory holds one parse at most. -/// -/// A group asks its members in turn for every chart, so the indexes of all -/// its upstreams are in use at once. Making room by evicting the least -/// recently used would evict the very index the group asks for next, and -/// every chart would parse every member's index again. So only an index -/// nobody used for [`parsed::IDLE`] makes room for a new one; while every -/// kept index is in use, a new one that does not fit is served without -/// being kept. Past the budget, a group keeps the indexes that fit and -/// parses only the others for each chart. -/// -/// A WebAssembly build parses for every operation: its instances are -/// recycled and their memory is bounded, so it keeps nothing. -fn parsed_index(url: &str, body: &[u8]) -> Result, String> { - #[cfg(not(target_family = "wasm"))] - { - parsed::get_or_parse(url, body) - } - #[cfg(target_family = "wasm")] - { - let _ = url; - parse_index(body).map(Arc::new) - } -} - -#[cfg(not(target_family = "wasm"))] -mod parsed { - use std::sync::{Arc, Mutex, PoisonError}; - use std::time::{Duration, Instant}; - - use super::{parse_index, Index}; - - /// What the kept indexes may hold together ([`retained`]), about twice - /// the size of their documents, so about 60 MB of `index.yaml`: the - /// indexes of a few dozen typical chart repositories, or of a handful of - /// the largest public ones. A new index that does not fit takes the - /// place of kept ones that went unused for [`IDLE`], or is not kept. One - /// that alone is larger is kept while nothing else is. - pub(super) const BUDGET: usize = 128 * 1024 * 1024; - - /// How long a kept index must go unused before a new one may take its - /// place. A group's listing uses each of its members' indexes for every - /// chart, so the ones it keeps stay in place until it is done. - pub(super) const IDLE: Duration = Duration::from_secs(60); - - /// A parsed index and the bytes it was parsed from. - pub(super) struct Parsed { - url: String, - body: Arc<[u8]>, - index: Arc, - /// What it holds ([`retained`]). - bytes: usize, - /// When it was last used. - used: Instant, - } - - impl Parsed { - pub(super) fn new(url: &str, body: &[u8], index: Arc, now: Instant) -> Self { - Self { - url: url.to_string(), - bytes: retained(body, &index), - body: body.into(), - index, - used: now, - } - } - } - - pub(super) struct State { - parsed: Vec, - } - - impl State { - pub(super) const fn new() -> Self { - Self { parsed: Vec::new() } - } - - /// The bytes and the index kept for `url`. - pub(super) fn find(&self, url: &str) -> Option<(Arc<[u8]>, Arc)> { - let p = self.parsed.iter().find(|p| p.url == url)?; - Some((Arc::clone(&p.body), Arc::clone(&p.index))) - } - - /// Marks `index`, kept for `url`, as used at `now`. - pub(super) fn touch(&mut self, url: &str, index: &Arc, now: Instant) { - if let Some(p) = self - .parsed - .iter_mut() - .find(|p| p.url == url && Arc::ptr_eq(&p.index, index)) - { - p.used = now; - } - } - - /// Keeps `parsed` as the index of its URL, if it fits in `budget` - /// once the others unused for [`IDLE`] since it was parsed gave way, - /// the least recently used first, or nothing else is kept. Returns - /// the indexes it does not keep, `parsed` among them when it does - /// not fit. - pub(super) fn keep(&mut self, parsed: Parsed, budget: usize) -> Vec { - let now = parsed.used; - let mut removed = Vec::new(); - if let Some(i) = self.parsed.iter().position(|p| p.url == parsed.url) { - removed.push(self.parsed.swap_remove(i)); - } - let mut total: usize = self.parsed.iter().map(|p| p.bytes).sum(); - while total + parsed.bytes > budget { - let Some(oldest) = (0..self.parsed.len()) - .filter(|&i| now.saturating_duration_since(self.parsed[i].used) >= IDLE) - .min_by_key(|&i| self.parsed[i].used) - else { - break; - }; - let oldest = self.parsed.swap_remove(oldest); - total -= oldest.bytes; - removed.push(oldest); - } - if total + parsed.bytes > budget && !self.parsed.is_empty() { - // The kept ones are in use: keeping this one would push out - // what the next operation asks for. - removed.push(parsed); - } else { - self.parsed.push(parsed); - } - removed - } - } - - static STATE: Mutex = Mutex::new(State::new()); - - /// Held while an index is parsed. - static PARSING: Mutex<()> = Mutex::new(()); - - pub(super) fn get_or_parse(url: &str, body: &[u8]) -> Result, String> { - get_or_parse_in(&STATE, url, body, Instant::now, BUDGET) - } - - /// [`get_or_parse`] with the indexes kept in `state`, the time from - /// `clock`, and `budget`. - pub(super) fn get_or_parse_in( - state: &Mutex, - url: &str, - body: &[u8], - clock: impl Fn() -> Instant, - budget: usize, - ) -> Result, String> { - if let Some(index) = lookup(state, url, body, clock()) { - return Ok(index); - } - let _parsing = PARSING.lock().unwrap_or_else(PoisonError::into_inner); - // Parsed while this one waited. - if let Some(index) = lookup(state, url, body, clock()) { - return Ok(index); - } - let index = Arc::new(parse_index(body)?); - // Sized, copied and dropped outside the lock: a large index takes a - // moment. - let parsed = Parsed::new(url, body, Arc::clone(&index), clock()); - let removed = state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .keep(parsed, budget); - drop(removed); - Ok(index) - } - - /// The index kept in `state` for `url`, when it was parsed from `body`. - fn lookup(state: &Mutex, url: &str, body: &[u8], now: Instant) -> Option> { - let (kept, index) = state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .find(url)?; - // Compared outside the lock: a large index takes a moment. - if *kept != *body { - return None; - } - state - .lock() - .unwrap_or_else(PoisonError::into_inner) - .touch(url, &index, now); - Some(index) - } - - /// About what `index`, parsed from `body`, holds while kept: the - /// document and the entries. - pub(super) fn retained(body: &[u8], index: &Index) -> usize { - let entries: usize = index - .iter() - .map(|(name, versions)| { - name.len() - + versions - .iter() - .map(|e| { - std::mem::size_of_val(e) - + e.name.len() - + e.created.as_ref().map_or(0, String::len) - + e.url.as_ref().map_or(0, String::len) - + e.metadata.len() - }) - .sum::() - }) - .sum(); - body.len() + entries - } -} - -/// Read an `index.yaml` (or a JSON one: JSON is YAML). Entries that are not -/// a valid chart version (name, version) are left out. -fn parse_index(body: &[u8]) -> Result { - let text = std::str::from_utf8(body).map_err(|_| "not UTF-8".to_string())?; - let docs = crate::yaml::load(text).map_err(|e| format!("not YAML: {e}"))?; - let doc = docs.first().map(to_json).unwrap_or(Value::Null); - let Some(entries) = doc.get("entries").and_then(Value::as_object) else { - // An empty repository may write `entries: {}` or leave it out, but - // a document without `apiVersion` is no chart repository index. - return match doc.get("apiVersion") { - Some(_) => Ok(Index::new()), - None => Err("not a chart repository index (no apiVersion)".into()), - }; - }; - let mut index = Index::new(); - for versions in entries.values() { - let Some(versions) = versions.as_array() else { - continue; - }; - for entry in versions { - let Some(metadata) = entry.as_object() else { - continue; - }; - let (Some(name), Some(version)) = ( - metadata.get("name").and_then(Value::as_str), - metadata.get("version").and_then(|v| match v { - Value::String(s) => Some(s.clone()), - Value::Number(n) => Some(n.to_string()), - _ => None, - }), - ) else { - continue; - }; - let (true, Ok(version)) = (chart::valid_name(name), Version::parse(&version)) else { - continue; - }; - index - .entry(name.to_ascii_lowercase()) - .or_default() - .push(Entry { - name: name.to_string(), - version, - created: metadata - .get("created") - .and_then(Value::as_str) - .map(str::to_string), - url: metadata - .get("urls") - .and_then(Value::as_array) - .and_then(|urls| urls.first()) - .and_then(Value::as_str) - .map(str::to_string), - metadata: serde_json::to_string(metadata).unwrap_or_default().into(), - }); - } - } - for versions in index.values_mut() { - versions.sort_by(|a, b| a.version.cmp(&b.version)); - versions.dedup_by(|a, b| a.version == b.version); - } - Ok(index) -} - -/// Where a version's archive is: the first of its `urls`, resolved against -/// the repository (`charts/x-1.0.0.tgz`, `/x-1.0.0.tgz`, or absolute). A -/// version the index does not list gets the conventional -/// `{base}/{name}-{version}.tgz`, which an upstream without it answers with -/// a 404. -fn package_url(base: &str, index: &Index, target: &PackageRef) -> String { - let listed = index.get(&target.id).and_then(|versions| { - versions - .iter() - .find(|e| e.version.to_string() == target.version) - }); - let url = listed.and_then(|e| e.url.clone()); - match url { - Some(url) => resolve(base, &url), - None => format!("{base}/{}-{}.tgz", target.id, target.version), - } -} - -/// `url` resolved against the repository's base URL. -fn resolve(base: &str, url: &str) -> String { - if url.starts_with("https://") || url.starts_with("http://") { - return url.to_string(); - } - if let Some(path) = url.strip_prefix('/') { - // Relative to the host: keep the base's scheme and authority. - let authority_end = base - .find("://") - .map(|i| i + 3) - .and_then(|start| base[start..].find('/').map(|end| start + end)) - .unwrap_or(base.len()); - return format!("{}/{path}", &base[..authority_end]); - } - format!("{base}/{}", url.trim_start_matches("./")) -} - -/// The index's metadata of `wanted.versions`: every chart version is -/// listed, published and created when its entry says (`created`), and -/// summarized like its Chart.yaml. -fn metadata(index: &Index, wanted: &PackageVersions, dating: bool) -> UpstreamStep { - let Some(versions) = index.get(&wanted.id) else { - return done(UpstreamAnswer::Metadata(PackageMetadata::default())); - }; - done(UpstreamAnswer::Metadata(PackageMetadata { - // The name the newest version is published under. - original_id: versions.last().map(|e| e.name.clone()), - versions: versions - .iter() - .filter(|e| wanted.versions.contains(&e.version.to_string())) - .map(|e| VersionMetadata { - version: e.version.to_string(), - original_id: Some(e.name.clone()), - listed: true, - published: e.created.clone(), - created: if dating { e.created.clone() } else { None }, - summary: Some(crate::summary(&e.metadata())), - }) - .collect(), - })) -} - -/// A page of the charts whose name matches `query`: each term must match, -/// `id:x` and `packageid:x` exactly, any other as a part of the name. -fn search(index: &Index, query: &SearchQuery) -> UpstreamSearch { - let terms: Vec = query - .text - .split_whitespace() - .map(str::to_ascii_lowercase) - .collect(); - let matches = |name: &str| { - terms.iter().all(|term| { - match term - .strip_prefix("packageid:") - .or_else(|| term.strip_prefix("id:")) - { - Some(exact) => name == exact, - None => name.contains(term.as_str()), - } - }) - }; - let matched: Vec<&Vec> = index - .iter() - .filter(|(name, _)| matches(name)) - .map(|(_, versions)| versions) - .filter(|versions| !versions.is_empty()) - .collect(); - let take = match query.take { - 0 => DEFAULT_TAKE, - n => usize::try_from(n).unwrap_or(usize::MAX), - }; - let hits = matched - .iter() - .skip(usize::try_from(query.skip).unwrap_or(usize::MAX)) - .take(take) - .map(|versions| { - let newest = versions.last().expect("non-empty"); - UpstreamHit { - id: newest.name.clone(), - versions: versions.iter().map(|e| e.version.to_string()).collect(), - summary: Some(crate::summary(&newest.metadata())), - total_downloads: None, - } - }) - .collect(); - UpstreamSearch { - total: matched.len() as u64, - hits, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn urls_resolve_against_the_repository() { - let base = "https://charts.example/stable"; - assert_eq!( - resolve(base, "charts/x-1.0.0.tgz"), - "https://charts.example/stable/charts/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "./x-1.0.0.tgz"), - "https://charts.example/stable/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "/dl/x-1.0.0.tgz"), - "https://charts.example/dl/x-1.0.0.tgz" - ); - assert_eq!( - resolve(base, "https://cdn.example/x-1.0.0.tgz"), - "https://cdn.example/x-1.0.0.tgz" - ); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_parsed_index_is_kept_while_the_upstream_serves_the_same_document() { - let url = "https://kept.example/stable/index.yaml"; - let body = "apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - // Concurrent operations on a new document share one parse. - let first: Vec> = std::thread::scope(|s| { - let parses: Vec<_> = (0..8) - .map(|_| s.spawn(|| parsed_index(url, body.as_bytes()).unwrap())) - .collect(); - parses.into_iter().map(|p| p.join().unwrap()).collect() - }); - assert!(first.iter().all(|p| Arc::ptr_eq(p, &first[0]))); - assert!(Arc::ptr_eq( - &parsed_index(url, body.as_bytes()).unwrap(), - &first[0] - )); - - // A changed document is parsed again. - let changed = format!("{body} - name: demo\n version: 2.0.0\n"); - let again = parsed_index(url, changed.as_bytes()).unwrap(); - assert!(!Arc::ptr_eq(&again, &first[0])); - assert_eq!(again["demo"].len(), 2); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_group_of_many_upstreams_parses_each_index_once() { - // A group asks its members in turn for every chart: each member's - // index is parsed for the first chart and kept for the others. - let body = "apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let urls: Vec = (0..12) - .map(|n| format!("https://member{n}.example/index.yaml")) - .collect(); - let first: Vec> = urls - .iter() - .map(|url| parsed_index(url, body.as_bytes()).unwrap()) - .collect(); - for _chart in 0..3 { - for (url, first) in urls.iter().zip(&first) { - assert!(Arc::ptr_eq( - &parsed_index(url, body.as_bytes()).unwrap(), - first - )); - } - } - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn kept_indexes_fit_in_the_budget() { - use std::time::{Duration, Instant}; - - let body = b"apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let index = Arc::new(parse_index(body).unwrap()); - let one = parsed::retained(body, &index); - assert!(one > body.len()); - let kept = |state: &parsed::State| -> Vec<&str> { - ["a", "b", "c", "d", "e"] - .into_iter() - .filter(|url| state.find(url).is_some()) - .collect() - }; - let start = Instant::now(); - let at = |secs| start + Duration::from_secs(secs); - let parsed = |url, secs| parsed::Parsed::new(url, body, Arc::clone(&index), at(secs)); - - let mut state = parsed::State::new(); - for (secs, url) in [(0, "a"), (1, "b"), (2, "c")] { - state.keep(parsed(url, secs), 3 * one); - } - assert_eq!(kept(&state), ["a", "b", "c"]); - // While the kept ones are in use, one that does not fit is not kept. - state.touch("a", &index, at(30)); - state.keep(parsed("d", 31), 3 * one); - assert_eq!(kept(&state), ["a", "b", "c"]); - // One unused for IDLE gives way, the least recently used first. - let later = 2 + parsed::IDLE.as_secs(); - state.keep(parsed("d", later), 3 * one); - assert_eq!(kept(&state), ["a", "c", "d"]); - // A document parsed again replaces the index of its URL. - state.keep(parsed("c", later), 3 * one); - assert_eq!(kept(&state), ["a", "c", "d"]); - - // An index larger than the budget is kept while nothing else is. - let mut state = parsed::State::new(); - state.keep(parsed("e", 0), one - 1); - assert_eq!(kept(&state), ["e"]); - } - - #[cfg(not(target_family = "wasm"))] - #[test] - fn a_group_past_the_budget_parses_only_the_indexes_that_do_not_fit() { - use std::cell::Cell; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - let body = b"apiVersion: v1\nentries:\n demo:\n - name: demo\n version: 1.0.0\n"; - let one = parsed::retained(body, &parse_index(body).unwrap()); - let state = Mutex::new(parsed::State::new()); - let start = Instant::now(); - let elapsed = Cell::new(Duration::ZERO); - let clock = || start + elapsed.get(); - let members: Vec = (0..6) - .map(|n| format!("https://member{n}.example/index.yaml")) - .collect(); - let mut last: Vec>> = vec![None; members.len()]; - // Asks the members from `first` on for `charts` charts in turn, - // 10 ms apart, as a group's listing does; how many indexes that - // parsed. - let mut list = |first: usize, charts: usize| { - let mut parses = 0; - for _ in 0..charts { - for (m, url) in members.iter().enumerate().skip(first) { - elapsed.set(elapsed.get() + Duration::from_millis(10)); - let index = parsed::get_or_parse_in(&state, url, body, clock, 4 * one).unwrap(); - if !last[m].as_ref().is_some_and(|l| Arc::ptr_eq(l, &index)) { - parses += 1; - } - last[m] = Some(index); - } - } - parses - }; - - // Four of the six indexes fit: those are parsed once, the other two - // for every chart. - assert_eq!(list(0, 50), 4 + 2 * 50); - // Once members 0 and 1 go unused, 4 and 5 take their places. - elapsed.set(elapsed.get() + parsed::IDLE); - assert_eq!(list(2, 50), 2); - } - - #[test] - fn an_index_keeps_valid_chart_versions() { - let index = parse_index( - b"apiVersion: v1\nentries:\n Demo:\n - name: Demo\n version: v1.2\n - name: Demo\n version: 1.0.0\n - name: Demo\n version: not-a-version\n ../evil:\n - name: ../evil\n version: 1.0.0\n", - ) - .unwrap(); - assert_eq!(index.keys().collect::>(), ["demo"]); - let versions: Vec = index["demo"] - .iter() - .map(|e| e.version.to_string()) - .collect(); - assert_eq!(versions, ["1.0.0", "1.2.0"]); - assert!(parse_index(b"apiVersion: v1\n").unwrap().is_empty()); - assert!(parse_index(b"portal").is_err()); - } - - #[test] - fn an_index_leaves_out_versions_longer_than_an_oci_tag() { - let longest = format!( - "1.0.0-{}", - "a".repeat(crate::semver::MAX_VERSION_LENGTH - 6) - ); - let mut body = String::from("apiVersion: v1\nentries:\n demo:\n"); - for version in [ - "1.0.0".to_string(), - longest.clone(), - format!("{longest}a"), - format!("2.0.0-{}", "x".repeat(4096)), - ] { - body.push_str(&format!(" - name: demo\n version: {version}\n")); - } - let index = parse_index(body.as_bytes()).unwrap(); - let versions: Vec = index["demo"] - .iter() - .map(|e| e.version.to_string()) - .collect(); - assert_eq!(versions, [longest, "1.0.0".to_string()]); - } -} diff --git a/plugins/helm/src/yaml.rs b/plugins/helm/src/yaml.rs deleted file mode 100644 index bc23a7dd..00000000 --- a/plugins/helm/src/yaml.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Loading the YAML a chart repository serves and stores (`Chart.yaml`, -//! `index.yaml`) within bounds. -//! -//! `yaml-rust2` builds the document tree by cloning an anchored node for -//! every alias that names it, so a few hundred bytes of nested anchors -//! expand to gigabytes. Charts and chart indexes have no use for that, but -//! both are written by others (a pusher, an upstream), so a document is -//! first walked as parser events, which costs no more than its size, and -//! refused when its aliases would expand it by more than -//! [`MAX_ALIAS_NODES`] nodes or when it nests deeper than [`MAX_DEPTH`]. - -use std::collections::HashMap; -use yaml_rust2::parser::{Event, EventReceiver, Parser}; -use yaml_rust2::{Yaml, YamlLoader}; - -/// The most nodes that aliases may add to a document, all aliases -/// together. Anchors in a chart or an index are rare and small. -pub const MAX_ALIAS_NODES: u64 = 100_000; - -/// The deepest nesting of sequences and mappings a document may have. A -/// chart index nests about six levels deep. -pub const MAX_DEPTH: usize = 64; - -/// Load every document of `text`, refusing one that aliases would expand -/// past [`MAX_ALIAS_NODES`] or that nests past [`MAX_DEPTH`]. -pub fn load(text: &str) -> Result, String> { - let mut budget = Budget::default(); - Parser::new_from_str(text) - .load(&mut budget, true) - .map_err(|e| e.to_string())?; - if budget.deepest > MAX_DEPTH { - return Err(format!("nested deeper than {MAX_DEPTH} levels")); - } - if budget.alias_nodes > MAX_ALIAS_NODES { - return Err(format!( - "its aliases expand to more than {MAX_ALIAS_NODES} nodes" - )); - } - YamlLoader::load_from_str(text).map_err(|e| e.to_string()) -} - -/// Counts what loading a document would build, without building it. -#[derive(Default)] -struct Budget { - /// Nodes built so far, aliases expanded. - nodes: u64, - /// Nodes the aliases added. - alias_nodes: u64, - /// Expanded size of each anchored node, by anchor id. - anchors: HashMap, - /// Open sequences and mappings: their anchor id and `nodes` before them. - open: Vec<(usize, u64)>, - deepest: usize, -} - -impl EventReceiver for Budget { - fn on_event(&mut self, event: Event) { - match event { - Event::Scalar(_, _, anchor, _) => { - self.nodes = self.nodes.saturating_add(1); - if anchor > 0 { - self.anchors.insert(anchor, 1); - } - } - Event::SequenceStart(anchor, _) | Event::MappingStart(anchor, _) => { - self.open.push((anchor, self.nodes)); - self.deepest = self.deepest.max(self.open.len()); - self.nodes = self.nodes.saturating_add(1); - } - Event::SequenceEnd | Event::MappingEnd => { - if let Some((anchor, before)) = self.open.pop() { - if anchor > 0 { - self.anchors.insert(anchor, self.nodes - before); - } - } - } - Event::Alias(anchor) => { - let size = self.anchors.get(&anchor).copied().unwrap_or(1); - self.nodes = self.nodes.saturating_add(size); - self.alias_nodes = self.alias_nodes.saturating_add(size); - } - Event::Nothing - | Event::StreamStart - | Event::StreamEnd - | Event::DocumentStart - | Event::DocumentEnd => {} - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `levels` anchors, each a sequence of ten aliases of the previous one. - fn laughs(levels: usize) -> String { - let mut doc = - String::from("name: demo\nversion: 1.0.0\na0: &a0 [x, x, x, x, x, x, x, x, x, x]\n"); - for i in 1..=levels { - let prev = format!("*a{}", i - 1); - doc.push_str(&format!("a{i}: &a{i} [{}]\n", vec![prev; 10].join(", "))); - } - doc - } - - #[test] - fn plain_documents_and_small_anchors_load() { - let docs = load("name: demo\nversion: 1.0.0\nkeywords: [a, b]\n").unwrap(); - assert_eq!(docs[0]["name"].as_str(), Some("demo")); - // A shared block reused a few times is fine. - let docs = load(&laughs(2)).unwrap(); - assert_eq!(docs[0]["a2"][9][9][9].as_str(), Some("x")); - } - - #[test] - fn alias_bombs_are_refused_before_they_expand() { - // 6 levels expand to over a million nodes (gigabytes once built); - // the check costs what the 500 bytes cost. - let err = load(&laughs(6)).unwrap_err(); - assert!(err.contains("aliases"), "{err}"); - let err = load(&laughs(30)).unwrap_err(); - assert!(err.contains("aliases"), "{err}"); - } - - #[test] - fn deep_nesting_is_refused() { - let deep = format!("{}{}", "[".repeat(200), "]".repeat(200)); - assert!(load(&deep).unwrap_err().contains("deeper")); - let mut block = String::new(); - for depth in 0..100 { - block.push_str(&format!("{}k{depth}:\n", " ".repeat(depth))); - } - assert!(load(&block).unwrap_err().contains("deeper")); - assert!(load(&format!("{}{}", "[".repeat(10), "]".repeat(10))).is_ok()); - } -} diff --git a/plugins/helm/tests/upstream.rs b/plugins/helm/tests/upstream.rs deleted file mode 100644 index e99d9bf7..00000000 --- a/plugins/helm/tests/upstream.rs +++ /dev/null @@ -1,255 +0,0 @@ -//! Helm proxies without a server: the upstream side run through -//! `forge_plugin_api::testing::converse` against a canned `index.yaml`, and a -//! proxy's index rendered through `testing::serve` from the search the host -//! answers with the upstream's charts. - -use std::collections::BTreeMap; - -use forge_format_helm::Helm; -use forge_plugin_api::testing::{converse, repository, serve, Conversed}; -use forge_plugin_api::{ - Answer, Fetch, Fetched, Freshness, PackageRef, PackageVersions, Query, Remote, RepositoryKind, - Request, SearchHit, SearchQuery, SearchResult, UpstreamAnswer, UpstreamOp, VersionInfo, -}; -use serde_json::{json, Value}; - -const BASE: &str = "https://charts.example/stable"; - -const INDEX: &str = r#"apiVersion: v1 -entries: - Nginx: - - apiVersion: v2 - name: Nginx - version: 2.1.0 - appVersion: "1.25" - description: A web server - created: "2026-05-01T10:00:00.123456789Z" - digest: abc - urls: - - charts/Nginx-2.1.0.tgz - - apiVersion: v2 - name: Nginx - version: v1.0 - description: An old web server - created: "2024-01-01T00:00:00Z" - urls: - - https://cdn.example/Nginx-1.0.0.tgz - redis: - - apiVersion: v2 - name: redis - version: 7.0.0 - urls: [redis-7.0.0.tgz] -generated: "2026-05-01T10:00:00Z" -"#; - -fn resolved() -> Remote { - Remote { - url: format!("{BASE}/"), - index: Some(json!({ "base": BASE }).to_string()), - } -} - -fn run(remote: &Remote, op: UpstreamOp, docs: &BTreeMap) -> Conversed { - converse(&Helm, remote, &op, |fetch: &Fetch| { - docs.get(&fetch.url).cloned().unwrap_or(Fetched::NotFound) - }) - .expect("the conversation ends") -} - -fn upstream() -> BTreeMap { - BTreeMap::from([( - format!("{BASE}/index.yaml"), - Fetched::Found(INDEX.as_bytes().to_vec()), - )]) -} - -fn package(id: &str, version: &str) -> PackageRef { - PackageRef { - id: id.into(), - version: version.into(), - } -} - -#[test] -fn the_index_is_the_repository_base() { - let unresolved = Remote { - url: format!("{BASE}/"), - index: None, - }; - let done = run(&unresolved, UpstreamOp::Index, &upstream()); - assert_eq!( - done.fetches, - [Fetch { - url: format!("{BASE}/index.yaml"), - freshness: Freshness::Metadata, - }] - ); - let UpstreamAnswer::Index(index) = done.answer else { - panic!("{:?}", done.answer) - }; - assert_eq!( - serde_json::from_str::(&index).unwrap(), - json!({ "base": BASE }) - ); - - // Not a chart repository: a failure, never an empty one. - let portal = BTreeMap::from([( - format!("{BASE}/index.yaml"), - Fetched::Found(b"sign in".to_vec()), - )]); - assert!(matches!( - run(&unresolved, UpstreamOp::Index, &portal).answer, - UpstreamAnswer::Failed(_) - )); - assert!(matches!( - run(&unresolved, UpstreamOp::Index, &BTreeMap::new()).answer, - UpstreamAnswer::Failed(m) if m.contains("404") - )); -} - -#[test] -fn versions_and_packages_come_from_the_index() { - let remote = resolved(); - assert_eq!( - run(&remote, UpstreamOp::Versions("nginx".into()), &upstream()).answer, - UpstreamAnswer::Versions(Some(vec!["1.0.0".into(), "2.1.0".into()])) - ); - assert_eq!( - run(&remote, UpstreamOp::Versions("mysql".into()), &upstream()).answer, - UpstreamAnswer::Versions(None) - ); - // Relative and absolute URLs; a version the index does not list gets - // the conventional location. - for (target, url) in [ - ( - package("nginx", "2.1.0"), - format!("{BASE}/charts/Nginx-2.1.0.tgz"), - ), - ( - package("nginx", "1.0.0"), - "https://cdn.example/Nginx-1.0.0.tgz".to_string(), - ), - (package("redis", "7.0.0"), format!("{BASE}/redis-7.0.0.tgz")), - (package("redis", "6.0.0"), format!("{BASE}/redis-6.0.0.tgz")), - ] { - assert_eq!( - run(&remote, UpstreamOp::Package(target), &upstream()).answer, - UpstreamAnswer::Package(url) - ); - } - // A chart repository has no manifest without its chart. - let manifest = run( - &remote, - UpstreamOp::Manifest(package("nginx", "2.1.0")), - &upstream(), - ); - assert!(manifest.fetches.is_empty()); - assert_eq!(manifest.answer, UpstreamAnswer::Manifest(None)); -} - -#[test] -fn metadata_and_dating_read_the_entries() { - let wanted = PackageVersions { - id: "nginx".into(), - versions: vec!["1.0.0".into(), "2.1.0".into(), "9.9.9".into()], - }; - let UpstreamAnswer::Metadata(metadata) = - run(&resolved(), UpstreamOp::Dating(wanted.clone()), &upstream()).answer - else { - panic!("metadata") - }; - assert_eq!(metadata.original_id.as_deref(), Some("Nginx")); - assert_eq!(metadata.versions.len(), 2); - let newest = &metadata.versions[1]; - assert_eq!(newest.version, "2.1.0"); - assert!(newest.listed); - assert_eq!( - newest.created.as_deref(), - Some("2026-05-01T10:00:00.123456789Z") - ); - let summary: Value = serde_json::from_str(newest.summary.as_deref().unwrap()).unwrap(); - assert_eq!(summary["description"], "A web server"); - assert_eq!(summary["chart"]["appVersion"], "1.25"); - - let UpstreamAnswer::Metadata(plain) = - run(&resolved(), UpstreamOp::Metadata(wanted), &upstream()).answer - else { - panic!("metadata") - }; - assert!(plain.versions.iter().all(|v| v.created.is_none())); -} - -#[test] -fn search_matches_chart_names() { - let query = |text: &str, skip: u32, take: u32| SearchQuery { - text: text.into(), - skip, - take, - prerelease: true, - semver2: true, - upstream: true, - }; - let search = |q: SearchQuery| match run(&resolved(), UpstreamOp::Search(q), &upstream()).answer - { - UpstreamAnswer::Search(Some(page)) => page, - other => panic!("{other:?}"), - }; - let all = search(query("", 0, 0)); - assert_eq!(all.total, 2); - assert_eq!(all.hits[0].id, "Nginx"); - assert_eq!(all.hits[0].versions, ["1.0.0", "2.1.0"]); - assert_eq!(all.hits[1].id, "redis"); - assert_eq!(search(query("ngi", 0, 0)).total, 1); - assert_eq!(search(query("id:ngi", 0, 0)).total, 0); - assert_eq!(search(query("packageid:NGINX", 0, 0)).hits[0].id, "Nginx"); - let second = search(query("", 1, 1)); - assert_eq!((second.total, second.hits.len()), (2, 1)); - assert_eq!(second.hits[0].id, "redis"); -} - -#[test] -fn a_proxys_index_lists_what_its_upstream_search_found() { - let repo = repository(RepositoryKind::Proxy, "/repository"); - let summary = json!({ "chart": { - "apiVersion": "v2", "name": "Nginx", "version": "2.1.0", "appVersion": "1.25" - }, "description": "A web server", "dependency_groups": [] }) - .to_string(); - let version = |v: &str| VersionInfo { - original_id: "Nginx".into(), - version: v.into(), - version_original: v.into(), - listed: true, - }; - let served = serve(&Helm, &repo, &Request::new("GET", "/index.yaml"), |query| { - match query { - // The upstream's charts too, not only the cached ones. - Query::Search(q) if q.upstream => Answer::Search(SearchResult { - total: 1, - hits: vec![SearchHit { - original_id: "Nginx".into(), - latest_version: "2.1.0".into(), - versions: vec![version("1.0.0"), version("2.1.0")], - summary: Some(summary.clone()), - total_downloads: None, - }], - }), - other => panic!("unexpected {other:?}"), - } - }); - // One search, no per-chart reads of the upstream. - assert_eq!(served.queries.len(), 1); - let index: Value = serde_json::from_str(&served.text()).unwrap(); - let entries = index["entries"]["Nginx"].as_array().unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0]["version"], "2.1.0"); - assert_eq!(entries[0]["appVersion"], "1.25"); - assert_eq!( - entries[0]["urls"][0], - "https://forge.example.com/repository/repo/charts/nginx/2.1.0/Nginx-2.1.0.tgz" - ); - // An older version: name, version and URL (the search describes the - // newest only). - assert_eq!(entries[1]["version"], "1.0.0"); - assert_eq!(entries[1]["apiVersion"], "v2"); - assert!(entries[1].get("appVersion").is_none()); -} diff --git a/plugins/nuget/Cargo.toml b/plugins/nuget/Cargo.toml index 4a9c272c..5a3ae9a8 100644 --- a/plugins/nuget/Cargo.toml +++ b/plugins/nuget/Cargo.toml @@ -8,10 +8,6 @@ license.workspace = true publish.workspace = true description = "NuGet V3 feeds for Forge, as a format plugin" -# Built into the server only, for now: the `zip` reader's `typed-path` -# dependency needs nightly Rust on wasm32-wasip2, so there is no `component` -# feature (and no `component` in plugin.toml) yet. - [dependencies] forge-plugin-api = { path = "../../crates/plugin-api" } diff --git a/plugins/nuget/src/lib.rs b/plugins/nuget/src/lib.rs index c5dd472d..0b6c3a1d 100644 --- a/plugins/nuget/src/lib.rs +++ b/plugins/nuget/src/lib.rs @@ -17,9 +17,7 @@ //! Ids are matched case-insensitively (`ToLowerInvariant`); versions are //! NuGet versions, normalized (`1.0` → `1.0.0`). //! -//! Forge links this plugin in ([`Nuget`], [`PLUGIN_TOML`]). It is built in -//! only for now: the `zip` reader it parses packages with does not build for -//! `wasm32-wasip2` on stable Rust, so it has no component build yet. +//! Forge links this plugin in ([`Nuget`], [`PLUGIN_TOML`]). pub mod model; pub mod nuspec; @@ -115,8 +113,6 @@ mod tests { assert_eq!(value("version").as_deref(), Some(env!("CARGO_PKG_VERSION"))); assert_eq!(value("api").as_deref(), Some(forge_plugin_api::API)); assert_eq!(value("mount").as_deref(), Some("/v3")); - // Built in only (see the crate docs). - assert_eq!(value("component"), None); } #[test] diff --git a/web/src/api/repositories.ts b/web/src/api/repositories.ts index 57af6f08..4d4c1cd7 100644 --- a/web/src/api/repositories.ts +++ b/web/src/api/repositories.ts @@ -183,16 +183,13 @@ export interface UpdateRepo { /** * An installed package format (`FormatView` of `GET /admin/formats`): the - * built-in format plugin (NuGet), Docker, raw and every loaded WebAssembly - * plugin. + * format plugins (NuGet), Docker and raw. */ export interface FormatView { /** The format id (`RepoView.format`). */ id: string /** Display name. */ name: string - /** Built into the server rather than a WebAssembly plugin. */ - builtin: boolean /** The repository kinds it supports (proxy only with a host-side upstream, e.g. NuGet). */ kinds: RepoKind[] /** Where its repositories are served: `{mount}/{repository}`. */ @@ -201,8 +198,6 @@ export interface FormatView { version?: string /** Plugin description. */ description?: string - /** SHA-256 of a WebAssembly plugin's component. */ - sha256?: string /** The upload limit of its hosted repositories without one of their own. */ default_max_upload_bytes?: number /** The largest upload limit a repository may be given; `null`: unbounded. */ @@ -220,7 +215,7 @@ export interface InvalidateProxyCache { /** What a proxy cache invalidation removed (`Invalidated`). */ export interface ProxyCacheInvalidated { - /** Remembered upstream "not found" answers removed (NuGet, plugins). */ + /** Remembered upstream "not found" answers removed (NuGet). */ not_found_cleared: number /** Cached tags due for re-validation against the upstream (docker). */ tags_expired: number diff --git a/web/src/components/PackageConnect.tsx b/web/src/components/PackageConnect.tsx index 16c67105..9dd15787 100644 --- a/web/src/components/PackageConnect.tsx +++ b/web/src/components/PackageConnect.tsx @@ -13,7 +13,7 @@ interface Snippet { * The client commands for a repository of `format` at `url` (its * `/browse/repos` address), named `repo` in the client's configuration. * Push commands are left out unless `canPush`. A format without commands of - * its own (a WebAssembly plugin's) gets the repository URL. + * its own gets the repository URL. */ export function connectSnippets( format: string, diff --git a/web/src/components/RepoFormatBadge.tsx b/web/src/components/RepoFormatBadge.tsx index b9970a95..fe7b9ce6 100644 --- a/web/src/components/RepoFormatBadge.tsx +++ b/web/src/components/RepoFormatBadge.tsx @@ -4,7 +4,7 @@ export { formatLabel } /** * The repository format (`nuget` | `docker` | `raw`) with its display - * name and a generic glyph. Other formats (WebAssembly plugins) show their + * name and a generic glyph. Other formats (other plugins) show their * raw id, like `RepoKindBadge`. */ export function RepoFormatBadge({ format }: { format: string }) { diff --git a/web/src/components/RepositoryForm.test.tsx b/web/src/components/RepositoryForm.test.tsx index 8ce41414..f0afded7 100644 --- a/web/src/components/RepositoryForm.test.tsx +++ b/web/src/components/RepositoryForm.test.tsx @@ -33,7 +33,6 @@ import { const example: FormatView = { id: 'example', name: 'Example packages', - builtin: false, kinds: ['hosted', 'group'], mount: '/repository', version: '0.1.0', diff --git a/web/src/components/RepositoryForm.tsx b/web/src/components/RepositoryForm.tsx index ac91609e..caa45be7 100644 --- a/web/src/components/RepositoryForm.tsx +++ b/web/src/components/RepositoryForm.tsx @@ -133,7 +133,6 @@ function originOf(url: string | null | undefined): string | null { export const NUGET_FORMAT: FormatView = { id: 'nuget', name: 'NuGet', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3', } @@ -145,7 +144,6 @@ export const NUGET_FORMAT: FormatView = { export const DOCKER_FORMAT: FormatView = { id: 'docker', name: 'Docker / OCI', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2', } @@ -158,7 +156,6 @@ export const DOCKER_FORMAT: FormatView = { export const RAW_FORMAT: FormatView = { id: 'raw', name: 'Raw files', - builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/raw', } @@ -412,7 +409,6 @@ export function RepositoryForm({ [NUGET_FORMAT, DOCKER_FORMAT, RAW_FORMAT].find((f) => f.id === values.format) ?? { id: values.format, name: values.format, - builtin: false, kinds: ['hosted', 'group'], mount: '/repository', } @@ -485,7 +481,7 @@ export function RepositoryForm({ > {available.map((f) => ( ))} @@ -493,7 +489,7 @@ export function RepositoryForm({ { mockApi({ repositories: [], formats: [ - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, - { id: 'acme', name: 'Acme packages', builtin: false, kinds: ['hosted', 'group'], mount: '/repository' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'acme', name: 'Acme packages', kinds: ['hosted', 'group'], mount: '/repository' }, ], onCreate: (body) => { posted = body @@ -242,8 +242,7 @@ describe('admin repositories', () => { await renderAt('/manage/repositories/new') await user.type(await screen.findByLabelText('Name'), 'pkgs') - // A WebAssembly plugin is named with its id. - await screen.findByRole('option', { name: 'Acme packages (plugin acme)' }) + await screen.findByRole('option', { name: 'Acme packages' }) await user.selectOptions(screen.getByLabelText('Format'), 'acme') await user.click(screen.getByRole('button', { name: 'Create repository' })) @@ -275,8 +274,8 @@ describe('admin repositories', () => { mockApi({ repositories: [], formats: [ - { id: 'docker', name: 'Docker / OCI', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'docker', name: 'Docker / OCI', kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ], onCreate: (body) => { posted = body as Record @@ -315,8 +314,8 @@ describe('admin repositories', () => { mockApi({ repositories: [], formats: [ - { id: 'docker', name: 'Docker / OCI', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'docker', name: 'Docker / OCI', kinds: ['hosted', 'proxy', 'group'], mount: '/v2' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ], onCreate: () => new Response('another docker repository already answers on this host', { diff --git a/web/src/test/fakeRbacApi.ts b/web/src/test/fakeRbacApi.ts index cb758717..b55061cf 100644 --- a/web/src/test/fakeRbacApi.ts +++ b/web/src/test/fakeRbacApi.ts @@ -203,7 +203,7 @@ export class FakeRbacApi { if (path === '/admin/me') return json(s.me) if (path === '/admin/formats') { return json([ - { id: 'nuget', name: 'NuGet', builtin: true, kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, + { id: 'nuget', name: 'NuGet', kinds: ['hosted', 'proxy', 'group'], mount: '/v3' }, ]) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 60977781..acb14c61 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -76,9 +76,6 @@ fn main() -> ExitCode { "prepare" => prepare(), "prepare-check" => prepare_check(), "clean" => clean(), - // Format plugins (WebAssembly components in plugins/) - "plugins" => plugins(), - "plugins-check" => plugins_check(), // Docker image (build it; run it against the dev infra) "docker-build" => docker_build(), "docker-run" => docker_run(), @@ -148,13 +145,6 @@ Rust build & quality: prepare-check Migrate, then verify the committed .sqlx/ cache is in sync (CI's drift guard). clean Remove build artifacts (cargo clean). -Format plugins as WebAssembly components (plugins//; the workspace builds -them natively, these need the wasm32-wasip2 target: `rustup target add wasm32-wasip2`): - plugins Build every plugin as a component and install it under - target/plugins/installed//, printing the [plugins] - configuration that loads them (with digests). - plugins-check clippy -D warnings of every plugin's component build, and the build. - Docker image (single container: web SPA + Rust server): docker-build Build the application image from ./Dockerfile (multi-stage: pnpm builds the SPA, cargo builds forge-server release). @@ -734,158 +724,6 @@ fn clean() -> Result<(), String> { Cmd::new("cargo").arg("clean").run() } -// ── Format plugins ─────────────────────────────────────────────────────────── - -/// Format plugins: one workspace crate per directory (`plugins//`). -const PLUGINS: &str = "plugins"; - -/// Where the components are built and installed (apart from `target/`, so a -/// wasm build never invalidates the host's). -const PLUGINS_TARGET: &str = "target/plugins"; - -/// The plugin directories with a component build (`plugins//` whose -/// `plugin.toml` names a `component`), sorted. A plugin without one is built -/// in only. -fn plugin_dirs() -> Result, String> { - let mut dirs: Vec<_> = std::fs::read_dir(PLUGINS) - .map_err(|e| format!("read {PLUGINS}/: {e}"))? - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|dir| dir.join("plugin.toml").is_file() && plugin_component(dir).is_ok()) - .collect(); - dirs.sort(); - Ok(dirs) -} - -/// The `component = "…"` of a plugin.toml (a plain line scan: xtask has no -/// TOML parser, and the key is a single string). -fn plugin_component(dir: &std::path::Path) -> Result { - let manifest = dir.join("plugin.toml"); - let text = std::fs::read_to_string(&manifest) - .map_err(|e| format!("read {}: {e}", manifest.display()))?; - text.lines() - .filter_map(|line| line.trim().strip_prefix("component")) - .filter_map(|rest| rest.trim().strip_prefix('=')) - .map(|value| value.trim().trim_matches('"').to_string()) - .next() - .ok_or_else(|| format!("{} has no component", manifest.display())) -} - -/// Whether the server links the plugin in `dir` (a path dependency of -/// forge-server, not only of its tests): its id is then taken by the -/// built-in plugin. -fn plugin_is_builtin(dir: &std::path::Path) -> bool { - let server = std::fs::read_to_string("crates/server/Cargo.toml").unwrap_or_default(); - let dependencies = server - .split("[dev-dependencies]") - .next() - .unwrap_or_default(); - let path = format!("\"../../{}\"", dir.display()).replace('\\', "/"); - dependencies.contains(&path) -} - -/// The SHA-256 of `file` (`sha256sum`, or `shasum -a 256` on macOS). -fn sha256_of(file: &std::path::Path) -> Option { - let tools: [(&str, &[&str]); 2] = [("sha256sum", &[]), ("shasum", &["-a", "256"])]; - tools.iter().find_map(|(tool, args)| { - let out = std::process::Command::new(tool) - .args(*args) - .arg(file) - .output() - .ok()?; - let text = String::from_utf8(out.stdout).ok()?; - out.status - .success() - .then(|| text.split_whitespace().next().map(str::to_string))? - }) -} - -/// `cargo ` of the plugin crate in `dir` for wasm32-wasip2 with its -/// `component` feature, into [`PLUGINS_TARGET`]. -fn plugin_cargo(sub: &str, dir: &std::path::Path) -> Cmd { - Cmd::new("cargo") - .args([sub, "--locked", "--target", "wasm32-wasip2"]) - .args(["--features", "component", "--manifest-path"]) - .arg(dir.join("Cargo.toml")) - .args(["--target-dir", PLUGINS_TARGET]) -} - -/// Build the component of the plugin in `dir` (the crate as a `cdylib`, with -/// the `plugin` profile); returns its path. -fn build_component(dir: &std::path::Path) -> Result { - plugin_cargo("rustc", dir) - .args(["--lib", "--crate-type", "cdylib", "--profile", "plugin"]) - .run()?; - Ok(std::path::Path::new(PLUGINS_TARGET) - .join("wasm32-wasip2/plugin") - .join(plugin_component(dir)?)) -} - -/// Build every plugin as a component and install it as an operator would -/// (plugin.toml plus component under `target/plugins/installed//`). -fn plugins() -> Result<(), String> { - let mut pins = Vec::new(); - for dir in plugin_dirs()? { - let id = dir - .file_name() - .and_then(|n| n.to_str()) - .ok_or("plugin directory name is not UTF-8")? - .to_string(); - let component = plugin_component(&dir)?; - let built = build_component(&dir)?; - let installed = std::path::Path::new(PLUGINS_TARGET) - .join("installed") - .join(&id); - std::fs::create_dir_all(&installed) - .map_err(|e| format!("create {}: {e}", installed.display()))?; - std::fs::copy(dir.join("plugin.toml"), installed.join("plugin.toml")) - .map_err(|e| format!("copy plugin.toml: {e}"))?; - std::fs::copy(&built, installed.join(&component)) - .map_err(|e| format!("copy {}: {e}", built.display()))?; - let pin = sha256_of(&installed.join(&component)).map_or_else( - || "".to_string(), - |sha| format!("sha256:{sha}"), - ); - pins.push((id, pin, plugin_is_builtin(&dir))); - } - let dir = std::env::current_dir() - .map_err(|e| e.to_string())? - .join(PLUGINS_TARGET) - .join("installed"); - println!( - "\nInstalled into {}. To load them, add to forge.toml:\n", - dir.display() - ); - println!( - "[plugins]\ndir = {:?}\n\n[plugins.formats]", - dir.display().to_string() - ); - for (id, pin, builtin) in pins { - if builtin { - println!( - "# {id} is built into the server: its component loads only under another id\n\ - # (rename its directory, `id` in its plugin.toml and the key here).\n\ - # {id} = \"{pin}\"" - ); - } else { - println!("{id} = \"{pin}\""); - } - } - Ok(()) -} - -/// What CI checks of every plugin beyond the workspace's own fmt, clippy and -/// tests: that it lints clean and builds as a WebAssembly component. -fn plugins_check() -> Result<(), String> { - for dir in plugin_dirs()? { - plugin_cargo("clippy", &dir) - .args(["--", "-D", "warnings"]) - .run()?; - build_component(&dir)?; - } - Ok(()) -} - // ── Docker image (single-container web SPA + Rust server) ──────────────────── /// Build the single application image from the root Dockerfile (multi-stage: