diff --git a/.sqlx/query-4af28572d09bd9cd95aab5d51ef0d72449ff02c2a15f01bbb4aaed626d8a8420.json b/.sqlx/query-4af28572d09bd9cd95aab5d51ef0d72449ff02c2a15f01bbb4aaed626d8a8420.json deleted file mode 100644 index 496227dc..00000000 --- a/.sqlx/query-4af28572d09bd9cd95aab5d51ef0d72449ff02c2a15f01bbb4aaed626d8a8420.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE oci_manifest\n SET summary = summary || jsonb_build_object('platform', $4::text)\n WHERE repository_id = $1 AND image = $2\n AND summary->>'config_digest' = $3\n AND NOT summary ? 'platform'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4af28572d09bd9cd95aab5d51ef0d72449ff02c2a15f01bbb4aaed626d8a8420" -} diff --git a/.sqlx/query-8955e4c8c98c7e4c42f2db4754aa56d81e3cd7072972cb7c5092531f8c96a416.json b/.sqlx/query-8955e4c8c98c7e4c42f2db4754aa56d81e3cd7072972cb7c5092531f8c96a416.json deleted file mode 100644 index 3704e601..00000000 --- a/.sqlx/query-8955e4c8c98c7e4c42f2db4754aa56d81e3cd7072972cb7c5092531f8c96a416.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS (\n SELECT 1 FROM oci_manifest\n WHERE repository_id = $1 AND image = $2\n AND summary->>'config_digest' = $3\n AND NOT summary ? 'platform'\n ) AS \"exists!\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists!", - "type_info": "Bool", - "origin": "Expression" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "8955e4c8c98c7e4c42f2db4754aa56d81e3cd7072972cb7c5092531f8c96a416" -} diff --git a/.sqlx/query-2523846362d2e9c13f843174d4c4f7077f782b2f4f02113d54d90532e726fd26.json b/.sqlx/query-a4623a9b33cff1324f6273292aab80f74f4404fd06326af81e89750c53d111c5.json similarity index 95% rename from .sqlx/query-2523846362d2e9c13f843174d4c4f7077f782b2f4f02113d54d90532e726fd26.json rename to .sqlx/query-a4623a9b33cff1324f6273292aab80f74f4404fd06326af81e89750c53d111c5.json index dc7d5644..3ccf7ea0 100644 --- a/.sqlx/query-2523846362d2e9c13f843174d4c4f7077f782b2f4f02113d54d90532e726fd26.json +++ b/.sqlx/query-a4623a9b33cff1324f6273292aab80f74f4404fd06326af81e89750c53d111c5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n id, name, format, kind, anonymous_read, online, config,\n remote_url, package_delay_seconds, upstream_username, upstream_password,\n upstream_password_enc, metadata_ttl_seconds, not_found_ttl_seconds,\n allow_redeploy, created_at, updated_at\n FROM repository\n WHERE format = 'docker' AND config ? 'host' AND lower(config->>'host') = lower($1)\n ", + "query": "\n SELECT\n id, name, format, kind, anonymous_read, online, config,\n remote_url, package_delay_seconds, upstream_username, upstream_password,\n upstream_password_enc, metadata_ttl_seconds, not_found_ttl_seconds,\n allow_redeploy, created_at, updated_at\n FROM repository\n WHERE format = $2 AND config ? 'host' AND lower(config->>'host') = lower($1)\n ", "describe": { "columns": [ { @@ -193,6 +193,7 @@ ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -216,5 +217,5 @@ false ] }, - "hash": "2523846362d2e9c13f843174d4c4f7077f782b2f4f02113d54d90532e726fd26" + "hash": "a4623a9b33cff1324f6273292aab80f74f4404fd06326af81e89750c53d111c5" } diff --git a/AGENTS.md b/AGENTS.md index 148a62f6..4c735a7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,9 +58,9 @@ after its format id, linked into the server like `crates/formats/nuget` under `crates/` (no `crates/format-`) or as a module of the server: it implements `FormatPlugin` with the layout [crates/formats/README.md](crates/formats/README.md) describes. Docker / OCI -is in `crates/formats/docker` but still native: the server's registry serves -it under `/v2`, not through `FormatPlugin`. Until it implements -`FormatPlugin`, it is not the model for a new format. +(`crates/formats/docker`) is one too, an images format: it speaks the +registry protocol, and the server's image store (`crates/server/src/images`) +keeps the images. ## Changelog: a fragment per change, never CHANGELOG.md diff --git a/Cargo.lock b/Cargo.lock index 562fab73..1fbacae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,34 +1736,16 @@ dependencies = [ [[package]] name = "forge-format-docker" -version = "1.1.0" +version = "1.0.0" dependencies = [ - "async-trait", - "axum", - "base64 0.23.1", - "bytes", - "chrono", "flate2", - "forge-cache", - "forge-core", - "forge-crypto", - "forge-db", - "forge-storage", - "futures-util", + "forge-plugin-api", "hex", - "jsonwebtoken", - "rand 0.10.3", - "reqwest", "serde", "serde_json", "sha2 0.11.1", - "sqlx", "thiserror", - "tokio", - "tokio-util", - "tracing", "url", - "uuid", ] [[package]] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index cbf54fd5..6c84d5c4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -250,12 +250,12 @@ groups. See [docs/raw.md](docs/raw.md). ### Repositories of other formats (format plugins) -Every package format is a format plugin, a crate in +Every format is a format plugin, a crate in [`crates/formats/`](./crates/formats) linked into the server (see -[docs/format-plugins.md](./docs/format-plugins.md)): NuGet, npm and raw. A -new format goes in `crates/formats/` too, never in a `crates/format-` -crate or the server. Docker / OCI is in `crates/formats/docker` but still -native, served by the server's registry under `/v2`. A repository of a plugin +[docs/format-plugins.md](./docs/format-plugins.md)): Docker / OCI, NuGet, npm +and raw. A new format goes in `crates/formats/` too, never in a +`crates/format-` crate or the server. Docker / OCI is an images format, +served under `/v2` over the server's image store. A repository of a plugin format is created with `"format": ""` (hosted, proxy or group; a plugin format has proxies when it implements the upstream side) and served under `/repository//` unless the plugin names another mount (NuGet's is diff --git a/README.md b/README.md index 92fcb31e..864aebe9 100644 --- a/README.md +++ b/README.md @@ -57,17 +57,17 @@ A Cargo workspace of small crates plus the web SPA: | `core` | `forge-core` | Domain logic: repo dispatch, auth chain, authz/RBAC, sessions, OIDC, retention. | | `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | | `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | -| `formats/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `npm`, `raw`. | -| `formats/docker` | `forge-format-docker` | Docker / OCI registry, still native: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | -| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, npm at `/npm`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. | +| `formats/` | `forge-format-*` | Format plugins, built into the server: `docker` (the distribution-spec `/v2` API, manifests, upstream registries), `nuget`, `npm`, `raw`. | +| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, npm at `/npm`, raw files at `/raw`, the Docker / OCI registry at `/v2`), the file store of files formats, the image store and registry tokens of images formats, `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | -**Every format is a plugin in `crates/formats/`.** A package format is a -crate in `crates/formats//` implementing `FormatPlugin`, linked into the -server. Docker / OCI is in `crates/formats/docker` but still native: the -server's registry serves it under `/v2`, not through `FormatPlugin`. +**Every format is a plugin in `crates/formats/`.** A format is a crate in +`crates/formats//` implementing `FormatPlugin`, linked into the server. +Docker / OCI is one too, an images format: the plugin speaks the protocol, +and the server's image store (`crates/server/src/images`) keeps the +images. **Single-origin serving.** `forge-server` mounts each format plugin's feed (NuGet at `/v3`, npm at `/npm`, others at `/repository`), the Docker / OCI registry (`/v2`), diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index c60765a7..e4422a31 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -53,6 +53,13 @@ pub trait Format: Send + Sync { false } + /// Whether the format's repositories hold images (Docker / OCI): their + /// contents are the image store's (`oci_*`), served by the registry, and + /// none of the package seams below apply to them. + fn stores_images(&self) -> bool { + false + } + /// Parse an uploaded package into format-neutral [`ParsedUpload`] data. fn parse_upload(&self, file: &mut dyn ReadSeek) -> Result; diff --git a/crates/core/src/repo/proxy.rs b/crates/core/src/repo/proxy.rs index 5e0539e0..211d9076 100644 --- a/crates/core/src/repo/proxy.rs +++ b/crates/core/src/repo/proxy.rs @@ -673,7 +673,13 @@ impl ProxyRepo { } else { (self.refuse(repo, &fetch.url), Vec::new(), Trouble::None) }; - (api::FetchResult { fetch, outcome }, named, down) + let result = api::FetchResult { + fetch, + outcome, + status: None, + headers: Vec::new(), + }; + (result, named, down) } }) .buffered(FETCH_CONCURRENCY) diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index 3475b138..69329ebb 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -228,7 +228,12 @@ impl RetentionWorker { run_id: uuid::Uuid, ) -> Result { let kind = RepoKind::from(repo.kind.as_str()); - if repo.format == forge_db::model::DOCKER_FORMAT { + // An images format's repositories (Docker) hold images. + if self + .formats + .resolve(&repo.format) + .is_some_and(|format| format.stores_images()) + { return self.process_images(repo, kind, run_id).await; } // A files format's repositories (raw) hold files under paths. diff --git a/crates/db/src/model.rs b/crates/db/src/model.rs index 31a8470a..28755c41 100644 --- a/crates/db/src/model.rs +++ b/crates/db/src/model.rs @@ -202,9 +202,6 @@ impl Repository { /// `repository.format` of NuGet repositories (the column default). pub const NUGET_FORMAT: &str = "nuget"; -/// `repository.format` of Docker / OCI registry repositories. -pub const DOCKER_FORMAT: &str = "docker"; - /// `repository.config` key of [`Repository::docker_host`]. pub const DOCKER_HOST_KEY: &str = "host"; @@ -221,11 +218,6 @@ pub const WRITABLE_MEMBER_ID_KEY: &str = "writable_member_id"; pub const MAX_UPLOAD_BYTES_KEY: &str = "max_upload_bytes"; impl Repository { - /// Whether this is a Docker / OCI registry repository. - pub fn is_docker(&self) -> bool { - self.format == DOCKER_FORMAT - } - /// Docker option (`config.host`): the host name this repository answers /// on by itself (host-based routing), so its images keep short names. pub fn docker_host(&self) -> Option<&str> { diff --git a/crates/db/src/oci_dao.rs b/crates/db/src/oci_dao.rs index e32628c3..0eedb5ac 100644 --- a/crates/db/src/oci_dao.rs +++ b/crates/db/src/oci_dao.rs @@ -21,11 +21,12 @@ const TOUCH_INTERVAL_SECS: f64 = 3600.0; // ── Repositories ──────────────────────────────────────────────────────────── -/// The docker repository that serves the host name `host` (host-based -/// routing, `config.host`), case-insensitively. +/// The repository of the images format `format` that serves the host name +/// `host` (host-based routing, `config.host`), case-insensitively. pub async fn find_repository_by_host( pool: &PgPool, host: &str, + format: &str, ) -> sqlx::Result> { sqlx::query_as!( Repository, @@ -36,9 +37,10 @@ pub async fn find_repository_by_host( upstream_password_enc, metadata_ttl_seconds, not_found_ttl_seconds, allow_redeploy, created_at, updated_at FROM repository - WHERE format = 'docker' AND config ? 'host' AND lower(config->>'host') = lower($1) + WHERE format = $2 AND config ? 'host' AND lower(config->>'host') = lower($1) "#, host, + format, ) .fetch_optional(pool) .await @@ -838,57 +840,59 @@ pub async fn set_manifest_summary( Ok(()) } -/// Record the platform of the image config `config_digest` on the image's -/// manifests that name it and have none yet (a proxy caches a manifest -/// before its config). -pub async fn set_config_platform( +/// The media type of the config `config_digest`, as a manifest of the image +/// that names it as its config says (its summary's `config_digest` and +/// `config_media_type`), if one does. +pub async fn config_media_type( pool: &PgPool, repository_id: i64, image: &str, config_digest: &str, - platform: &str, -) -> sqlx::Result<()> { - sqlx::query!( +) -> sqlx::Result> { + sqlx::query_scalar( r#" - UPDATE oci_manifest - SET summary = summary || jsonb_build_object('platform', $4::text) + SELECT summary->>'config_media_type' FROM oci_manifest WHERE repository_id = $1 AND image = $2 AND summary->>'config_digest' = $3 - AND NOT summary ? 'platform' + AND summary ? 'config_media_type' + LIMIT 1 "#, - repository_id, - image, - config_digest, - platform, ) - .execute(pool) - .await?; - Ok(()) + .bind(repository_id) + .bind(image) + .bind(config_digest) + .fetch_optional(pool) + .await } -/// Whether a manifest of the image names `config_digest` as its config and -/// has no platform recorded yet. -pub async fn config_needs_platform( +/// Merge `part`, what the config `config_digest` holds, into the summaries +/// of the image's manifests that name it as their config and hold none of +/// `keys` yet (a proxy caches a manifest before its config). +pub async fn complete_config_summary( pool: &PgPool, repository_id: i64, image: &str, config_digest: &str, -) -> sqlx::Result { - sqlx::query_scalar!( + part: &JsonValue, + keys: &[String], +) -> sqlx::Result<()> { + sqlx::query( r#" - SELECT EXISTS ( - SELECT 1 FROM oci_manifest - WHERE repository_id = $1 AND image = $2 - AND summary->>'config_digest' = $3 - AND NOT summary ? 'platform' - ) AS "exists!" + UPDATE oci_manifest + SET summary = summary || $4 + WHERE repository_id = $1 AND image = $2 + AND summary->>'config_digest' = $3 + AND NOT summary ?| $5 "#, - repository_id, - image, - config_digest, ) - .fetch_one(pool) - .await + .bind(repository_id) + .bind(image) + .bind(config_digest) + .bind(part) + .bind(keys) + .execute(pool) + .await?; + Ok(()) } /// Which of `digests` are manifests of the image. diff --git a/crates/formats/README.md b/crates/formats/README.md index 09104ab1..a5130828 100644 --- a/crates/formats/README.md +++ b/crates/formats/README.md @@ -7,14 +7,11 @@ the design, the contract and how to write one. | Plugin | Format id | What it serves | | --- | --- | --- | +| [`docker`](docker) | `docker` | Docker / OCI registries at `/v2`: push and pull images, indexes and artifacts (Helm charts, signatures, SBOMs); hosted, proxy and group repositories ([docs/docker.md](../../docs/docker.md)) | | [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | | [`npm`](npm) | `npm` | npm registries at `/npm` for npm, pnpm, Yarn and Bun: publish, install, dist-tags, deprecations, unpublishing, search, audits and logins; hosted, proxy and group repositories ([docs/npm.md](../../docs/npm.md)) | | [`raw`](raw) | `raw` | Raw files at `/raw`: files under paths, uploads with `PUT` or a form, ranges, checksum sidecars, listings; hosted, proxy and group repositories ([docs/raw.md](../../docs/raw.md)) | -[`docker`](docker) holds Docker / OCI (`forge-format-docker`), which is still -native: the server's registry serves it under `/v2`, not through -`FormatPlugin` ([docs/docker.md](../../docs/docker.md)). - A plugin directory holds: - `Cargo.toml`: a member of the workspace; its `version` is the plugin's; @@ -22,9 +19,10 @@ A plugin directory holds: Descriptor` that its `descriptor()` returns: the format id (also the directory's name), display name, version (`env!("CARGO_PKG_VERSION")`), description, mount, repository kinds (`Proxy` when it implements - `upstream`), its `storage` (`Packages`, or `Files` for a files format such - as raw), its asset names, and how a proxy sends a token - (`upstream_token`: `Bearer` for npm). + `upstream`), its `storage` (`Packages`, `Files` for a files format such + as raw, or `Images` for an images format such as Docker), its asset + names, and how a proxy sends a token (`upstream_token`: `Bearer` for + npm). The server installs it in `plugins::builtin()` ([`crates/server/src/plugins.rs`](../server/src/plugins.rs)), whose tests diff --git a/crates/formats/docker/Cargo.toml b/crates/formats/docker/Cargo.toml index 16970a83..d9b79844 100644 --- a/crates/formats/docker/Cargo.toml +++ b/crates/formats/docker/Cargo.toml @@ -1,41 +1,25 @@ [package] name = "forge-format-docker" -version.workspace = true +# The plugin's own version, its descriptor's. +version = "1.0.0" edition.workspace = true rust-version.workspace = true license.workspace = true publish.workspace = true +description = "Docker / OCI registries for Forge, as a format plugin" [dependencies] -forge-core = { path = "../../core" } -forge-db = { path = "../../db" } -forge-storage = { path = "../../storage" } -forge-cache = { path = "../../cache" } -forge-crypto = { path = "../../crypto" } +forge-plugin-api = { path = "../../plugin-api" } -axum = { workspace = true } -tokio = { workspace = true, features = ["rt", "sync"] } -tokio-util = { workspace = true } -futures-util = { workspace = true } -bytes = "1" -async-trait = { workspace = true } -sqlx = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -chrono = { workspace = true } -uuid = { workspace = true } thiserror = { workspace = true } -tracing = { workspace = true } -sha2 = { workspace = true } -hex = { workspace = true } -base64 = { workspace = true } +# Upstream URLs and the pages a tag list links to. url = { workspace = true } -reqwest = { workspace = true } -# Registry Bearer tokens (HS256), the same crate the OIDC verifier uses. -jsonwebtoken = { workspace = true } -rand = { workspace = true } # Helm chart archives (.tgz), read for their README and values (`chart`). flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } [dev-dependencies] -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +forge-plugin-api = { path = "../../plugin-api", features = ["testing"] } +sha2 = { workspace = true } +hex = { workspace = true } diff --git a/crates/formats/docker/src/chart.rs b/crates/formats/docker/src/chart.rs index cc9321bd..ed8eb8a7 100644 --- a/crates/formats/docker/src/chart.rs +++ b/crates/formats/docker/src/chart.rs @@ -22,12 +22,11 @@ pub mod media { pub const PROVENANCE: &str = "application/vnd.cncf.helm.chart.provenance.v1.prov"; } -/// Largest config blob read for its metadata. -pub const MAX_CONFIG_BYTES: i64 = 1024 * 1024; - -/// Largest chart archive read for its README and values, as stored -/// (compressed). Charts are a few KiB to a few hundred KiB. -pub const MAX_ARCHIVE_BYTES: i64 = 16 * 1024 * 1024; +/// Whether a layer of a chart is its packaged archive: Helm's media type, +/// or the plain one of the experimental OCI support of Helm before 3.8. +pub fn is_archive(media_type: &str) -> bool { + media_type == media::CONTENT || media_type == "application/tar+gzip" +} /// Most bytes of an archive decompressed while looking for its README and /// values: Helm's own limit on a whole chart (`MaxDecompressedChartSize`). @@ -169,7 +168,7 @@ pub struct ChartFiles { /// `/charts//README.md` a subchart's. At most /// [`MAX_UNPACKED_BYTES`] are decompressed; an archive that is not gzip or /// tar, or ends early, yields what was found before. -pub fn read_files(archive: &[u8]) -> ChartFiles { +pub fn read_files(archive: impl Read) -> ChartFiles { let mut found = ChartFiles::default(); let decoder = flate2::read::MultiGzDecoder::new(archive); let mut tar = TarReader { @@ -491,7 +490,7 @@ mod tests { ("demo/values.yaml", b"replicas: 1\n"), ("demo/templates/values.yaml", b"not: this\n"), ]); - let files = read_files(&archive); + let files = read_files(&archive[..]); let readme = files.readme.unwrap(); assert_eq!(readme.name, "Readme.MD"); assert_eq!(readme.text, "# Demo\n"); @@ -523,7 +522,7 @@ mod tests { let pax = format!("{length}{line}"); entry(&mut tar, "PaxHeaders/values", pax.as_bytes(), b'x'); entry(&mut tar, "values.yaml-short", b"x: 1\n", b'0'); - let files = read_files(&gzip(tar)); + let files = read_files(&gzip(tar)[..]); let readme = files.readme.unwrap(); assert!(readme.truncated); assert_eq!(readme.text.len(), MAX_FILE_TEXT); @@ -532,14 +531,14 @@ mod tests { #[test] fn what_is_not_a_chart_yields_nothing() { - assert_eq!(read_files(b"not gzip"), ChartFiles::default()); + assert_eq!(read_files(&b"not gzip"[..]), ChartFiles::default()); assert_eq!( - read_files(&gzip(b"not a tar header".to_vec())), + read_files(&gzip(b"not a tar header".to_vec())[..]), ChartFiles::default() ); // Paths outside the chart, or the chart's directory alone. let archive = tgz(&[("demo/../README.md", b"out"), ("README.md", b"top")]); - assert_eq!(read_files(&archive), ChartFiles::default()); + assert_eq!(read_files(&archive[..]), ChartFiles::default()); // A truncated archive keeps what came before the cut. let mut noise = Vec::with_capacity(64 * 1024); let mut x: u32 = 1; diff --git a/crates/formats/docker/src/digest.rs b/crates/formats/docker/src/digest.rs index 9bdeab64..93edaf76 100644 --- a/crates/formats/docker/src/digest.rs +++ b/crates/formats/docker/src/digest.rs @@ -1,16 +1,13 @@ //! OCI content digests (`:`). //! -//! The CAS is keyed by SHA-256, and every staged file is hashed with SHA-256 -//! and SHA-512 at once, so both registered algorithms of the image spec are -//! supported: a `sha512:` digest names the same CAS blob as its `sha256:` -//! twin, it is only verified against the other hash. Any other algorithm is -//! refused as unsupported. +//! Both registered algorithms of the image spec are supported, `sha256` and +//! `sha512`: the host hashes everything it stores with both, and verifies +//! content against either. Any other algorithm is refused as unsupported. +//! A digest crosses the plugin contract in the canonical form [`Digest`] +//! keeps. use std::fmt; -use base64::Engine; -use sha2::{Digest as _, Sha256, Sha512}; - /// A digest algorithm Forge can verify. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Algorithm { @@ -78,24 +75,6 @@ impl Digest { Ok(Digest(s.to_string())) } - /// The SHA-256 digest of `bytes`. - pub fn sha256_of(bytes: &[u8]) -> Self { - Digest(format!("sha256:{}", hex::encode(Sha256::digest(bytes)))) - } - - /// The digest of `bytes` with `algorithm`. - pub fn of(algorithm: Algorithm, bytes: &[u8]) -> Self { - match algorithm { - Algorithm::Sha256 => Self::sha256_of(bytes), - Algorithm::Sha512 => Digest(format!("sha512:{}", hex::encode(Sha512::digest(bytes)))), - } - } - - /// A `sha256:` digest from the CAS's lowercase hex key. - pub fn from_sha256_hex(hex: &str) -> Result { - Self::parse(&format!("sha256:{hex}")) - } - pub fn algorithm(&self) -> Algorithm { if self.0.starts_with("sha512:") { Algorithm::Sha512 @@ -113,16 +92,9 @@ impl Digest { &self.0 } - /// Whether this digest names content with the given CAS hashes (the - /// lowercase hex SHA-256 and the standard-base64 SHA-512 the CAS - /// computes while staging). - pub fn matches(&self, sha256_hex: &str, sha512_base64: &str) -> bool { - match self.algorithm() { - Algorithm::Sha256 => self.hex() == sha256_hex, - Algorithm::Sha512 => base64::engine::general_purpose::STANDARD - .decode(sha512_base64) - .is_ok_and(|raw| hex::encode(raw) == self.hex()), - } + /// The canonical text, as the plugin contract carries digests. + pub fn into_string(self) -> String { + self.0 } } @@ -160,6 +132,8 @@ fn valid_encoded_grammar(s: &str) -> bool { #[cfg(test)] mod tests { + use sha2::{Digest as _, Sha512}; + use super::*; const EMPTY_SHA256: &str = @@ -170,11 +144,13 @@ mod tests { let d = Digest::parse(EMPTY_SHA256).unwrap(); assert_eq!(d.algorithm(), Algorithm::Sha256); assert_eq!(d.hex().len(), 64); - assert_eq!(Digest::sha256_of(b""), d); + assert_eq!(d.as_str(), EMPTY_SHA256); - let d512 = Digest::of(Algorithm::Sha512, b""); - assert_eq!(Digest::parse(d512.as_str()).unwrap(), d512); + let empty512 = format!("sha512:{}", hex::encode(Sha512::digest(b""))); + let d512 = Digest::parse(&empty512).unwrap(); assert_eq!(d512.algorithm(), Algorithm::Sha512); + assert_eq!(d512.hex().len(), 128); + assert_eq!(d512.into_string(), empty512); } #[test] @@ -200,14 +176,4 @@ mod tests { Err(DigestError::Unsupported(_)) )); } - - #[test] - fn matches_cas_hashes() { - let sha256 = hex::encode(Sha256::digest(b"hello")); - let sha512 = base64::engine::general_purpose::STANDARD.encode(Sha512::digest(b"hello")); - assert!(Digest::sha256_of(b"hello").matches(&sha256, &sha512)); - assert!(Digest::of(Algorithm::Sha512, b"hello").matches(&sha256, &sha512)); - assert!(!Digest::sha256_of(b"world").matches(&sha256, &sha512)); - assert!(!Digest::of(Algorithm::Sha512, b"world").matches(&sha256, &sha512)); - } } diff --git a/crates/formats/docker/src/error.rs b/crates/formats/docker/src/error.rs index ca631ffd..cb0b4934 100644 --- a/crates/formats/docker/src/error.rs +++ b/crates/formats/docker/src/error.rs @@ -1,7 +1,10 @@ -//! Distribution-spec error responses: `{"errors":[{"code","message","detail"}]}`. +//! Distribution-spec error responses: +//! `{"errors":[{"code","message","detail"}]}`, for the errors of the +//! protocol and for what the host refused or failed ([`ImageError`]). -use axum::http::{header, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; +use forge_plugin_api::images::{ImageError, Permission}; +use forge_plugin_api::Response; +use serde_json::{json, Value}; /// The error codes of the distribution spec (plus Docker's `UNAVAILABLE`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -47,19 +50,18 @@ impl Code { } /// An error answered to a registry client. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct OciError { - pub status: StatusCode, + pub status: u16, pub code: Code, pub message: String, - pub detail: Option, - /// Extra response headers (a `WWW-Authenticate` challenge, `Range` of an - /// upload, `Retry-After`). - pub headers: Vec<(header::HeaderName, HeaderValue)>, + pub detail: Option, + /// Extra response headers (`Range` of an upload, `Retry-After`). + pub headers: Vec<(String, String)>, } impl OciError { - pub fn new(status: StatusCode, code: Code, message: impl Into) -> Self { + pub fn new(status: u16, code: Code, message: impl Into) -> Self { OciError { status, code, @@ -69,238 +71,248 @@ impl OciError { } } - pub fn with_detail(mut self, detail: serde_json::Value) -> Self { + pub fn with_detail(mut self, detail: Value) -> Self { self.detail = Some(detail); self } - pub fn with_header(mut self, name: header::HeaderName, value: HeaderValue) -> Self { - self.headers.push((name, value)); + pub fn with_header(mut self, name: &str, value: &str) -> Self { + self.headers.push((name.to_string(), value.to_string())); self } pub fn blob_unknown(digest: &str) -> Self { - Self::new( - StatusCode::NOT_FOUND, - Code::BlobUnknown, - "blob unknown to registry", - ) - .with_detail(serde_json::json!({ "digest": digest })) + Self::new(404, Code::BlobUnknown, "blob unknown to registry") + .with_detail(json!({ "digest": digest })) } pub fn manifest_unknown(reference: &str) -> Self { - Self::new( - StatusCode::NOT_FOUND, - Code::ManifestUnknown, - "manifest unknown", - ) - .with_detail(serde_json::json!({ "reference": reference })) + Self::new(404, Code::ManifestUnknown, "manifest unknown") + .with_detail(json!({ "reference": reference })) } pub fn name_unknown(name: &str) -> Self { Self::new( - StatusCode::NOT_FOUND, + 404, Code::NameUnknown, "repository name not known to registry", ) - .with_detail(serde_json::json!({ "name": name })) + .with_detail(json!({ "name": name })) } pub fn name_invalid(name: &str) -> Self { - Self::new( - StatusCode::BAD_REQUEST, - Code::NameInvalid, - "invalid repository name", - ) - .with_detail(serde_json::json!({ "name": name })) + Self::new(400, Code::NameInvalid, "invalid repository name") + .with_detail(json!({ "name": name })) } pub fn upload_unknown() -> Self { Self::new( - StatusCode::NOT_FOUND, + 404, Code::BlobUploadUnknown, "blob upload unknown to registry", ) } pub fn digest_invalid(message: impl Into) -> Self { - Self::new(StatusCode::BAD_REQUEST, Code::DigestInvalid, message) + Self::new(400, Code::DigestInvalid, message) } pub fn manifest_invalid(message: impl Into) -> Self { - Self::new(StatusCode::BAD_REQUEST, Code::ManifestInvalid, message) + Self::new(400, Code::ManifestInvalid, message) } pub fn denied(message: impl Into) -> Self { - Self::new(StatusCode::FORBIDDEN, Code::Denied, message) + Self::new(403, Code::Denied, message) } pub fn unsupported(message: impl Into) -> Self { - Self::new(StatusCode::METHOD_NOT_ALLOWED, Code::Unsupported, message) + Self::new(405, Code::Unsupported, message) } pub fn too_large(limit: u64) -> Self { Self::new( - StatusCode::PAYLOAD_TOO_LARGE, + 413, Code::SizeInvalid, format!("content exceeds the limit of {limit} bytes"), ) } - /// The service cannot answer now (staging full, upstream down, offline - /// repository): 503 with `Retry-After`. - pub fn unavailable(message: impl Into) -> Self { - Self::new(StatusCode::SERVICE_UNAVAILABLE, Code::Unavailable, message) - .with_header(header::RETRY_AFTER, HeaderValue::from_static("30")) + /// An upload chunk that does not fit: 416 with the `Range` received so + /// far. + pub fn chunk(message: &str, size: u64) -> Self { + Self::new(416, Code::BlobUploadInvalid, message).with_header("range", &upload_range(size)) } - /// An internal failure; the details go to the log, not the client. - pub fn internal(context: &str, error: impl std::fmt::Display) -> Self { - tracing::error!(error = %error, "registry: {context}"); - Self::new( - StatusCode::INTERNAL_SERVER_ERROR, - Code::Unavailable, - "internal error", - ) + /// The service cannot answer now: 503 with `Retry-After`. + pub fn unavailable(message: impl Into) -> Self { + Self::new(503, Code::Unavailable, message).with_header("retry-after", "30") } -} -impl From for OciError { - fn from(e: sqlx::Error) -> Self { - OciError::internal("database error", e) - } -} - -impl OciError { - /// A failed read from the object store (through the local blob cache): - /// 503 with `Retry-After` while S3 cannot be reached, a server error - /// otherwise. - pub(crate) fn storage_read(context: &str, error: forge_storage::S3Error) -> Self { - if error.is_unavailable() { - tracing::warn!(%error, "registry: {context}: the object store is unavailable"); - OciError::object_store_unavailable() - } else { - OciError::internal(context, error) + /// The JSON response. + pub fn into_response(self) -> Response { + let mut error = json!({ + "code": self.code.as_str(), + "message": self.message, + }); + if let Some(detail) = self.detail { + error["detail"] = detail; } - } - - fn object_store_unavailable() -> Self { - OciError::unavailable("the object store is unavailable; retry later") + let body = json!({ "errors": [error] }).to_string(); + let mut response = Response::new(self.status, "application/json", body); + response.headers.extend(self.headers); + response } } -/// A request body that could not be read: the client aborted it, or it -/// stalled past the idle timeout. Travels inside the I/O error staging the -/// body fails with. -#[derive(Debug)] -pub(crate) struct RequestBodyError(pub axum::Error); +/// `Range` of an upload: the inclusive end of the bytes received. +pub fn upload_range(size: u64) -> String { + format!("0-{}", size.saturating_sub(1)) +} -impl std::fmt::Display for RequestBodyError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "reading the request body: {}", self.0) +/// The token-scope actions a permission needs. +pub fn scope_actions(permission: Permission) -> &'static str { + match permission { + Permission::Read => "pull", + Permission::Write => "pull,push", + Permission::Delete => "delete", } } -impl std::error::Error for RequestBodyError {} - -impl From for OciError { - fn from(e: forge_storage::CasError) -> Self { - match e { - forge_storage::CasError::TooLarge { limit } => OciError::too_large(limit), - forge_storage::CasError::StagingFull { .. } => { - OciError::unavailable("temporary storage is full; retry later") - } - e if e.is_unavailable() => { - tracing::warn!(error = %e, "registry: the object store is unavailable"); - OciError::object_store_unavailable() +/// What the host refused or failed, as the client sees it. +pub fn of(error: &ImageError) -> OciError { + match error { + ImageError::NotAnEndpoint => { + OciError::new(404, Code::NameUnknown, "not a registry endpoint") + } + ImageError::NameInvalid { name, hint } => { + let invalid = OciError::name_invalid(name); + match hint { + Some(hint) => invalid.with_detail(json!({ "name": name, "hint": hint })), + None => invalid, } - forge_storage::CasError::Io(e) - if e.get_ref() - .is_some_and(|inner| inner.is::()) => - { - tracing::debug!(error = %e, "registry: the upload body could not be read"); - OciError::new( - StatusCode::BAD_REQUEST, - Code::BlobUploadInvalid, - "the request body could not be read", - ) + } + ImageError::NameUnknown { name } => OciError::name_unknown(name), + ImageError::AuthenticationRequired { scope } => { + let required = OciError::new(401, Code::Unauthorized, "authentication required"); + match scope { + Some((name, permission)) => required + .with_detail(json!({ "name": name, "action": scope_actions(*permission) })), + None => required, } - other => OciError::internal("storage error", other), } - } -} - -impl IntoResponse for OciError { - fn into_response(self) -> Response { - let mut error = serde_json::json!({ - "code": self.code.as_str(), - "message": self.message, - }); - if let Some(detail) = self.detail { - error["detail"] = detail; + ImageError::InvalidToken => { + OciError::new(401, Code::Unauthorized, "invalid or expired registry token") + } + ImageError::InvalidCredentials => { + OciError::new(401, Code::Unauthorized, "invalid credentials") + } + ImageError::Denied { name, permission } => { + OciError::denied("requested access to the resource is denied") + .with_detail(json!({ "name": name, "action": scope_actions(*permission) })) } - let body = serde_json::json!({ "errors": [error] }).to_string(); - let mut response = ( - self.status, - [( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - )], - body, + ImageError::Unsupported { status, message } => { + OciError::new(*status, Code::Unsupported, message.clone()) + } + ImageError::Offline => OciError::unavailable("repository is offline"), + ImageError::BlobUnknown { digest } => OciError::blob_unknown(digest), + ImageError::ManifestUnknown { reference } => OciError::manifest_unknown(reference), + ImageError::UploadUnknown => OciError::upload_unknown(), + ImageError::DigestMismatch => { + OciError::digest_invalid("the uploaded content does not match the digest") + } + ImageError::ChunkOutOfOrder { size } => { + OciError::chunk("the chunk does not continue the upload", *size) + } + ImageError::ChunkLength { size } => { + OciError::chunk("the chunk length does not match its Content-Range", *size) + } + ImageError::TooLarge { limit } => OciError::too_large(*limit), + ImageError::BodyUnreadable => OciError::new( + 400, + Code::BlobUploadInvalid, + "the request body could not be read", + ), + ImageError::ManifestInvalid(message) => OciError::manifest_invalid(message.clone()), + ImageError::ManifestBlobUnknown { digest, manifest } => OciError::new( + 400, + Code::ManifestBlobUnknown, + if *manifest { + "the index references a manifest unknown to this repository" + } else { + "the manifest references a blob unknown to this repository" + }, + ) + .with_detail(json!({ "digest": digest })), + ImageError::TagImmutable { tag } => OciError::denied(format!( + "tag {tag:?} already exists and tags are immutable in this repository" + )) + .with_detail(json!({ "tag": tag })), + ImageError::BlobReferenced { digest } => OciError::new( + 409, + Code::Denied, + "the blob is referenced by a manifest of this image; delete the manifest first", ) - .into_response(); - for (name, value) in self.headers { - response.headers_mut().insert(name, value); + .with_detail(json!({ "digest": digest })), + ImageError::NoLongerAllowed => { + OciError::denied("the workload identity may no longer change this repository") } - response + ImageError::RateLimited => OciError::new( + 429, + Code::TooManyRequests, + "the upstream registry rate-limits this proxy; retry later", + ), + ImageError::Unavailable(message) => OciError::unavailable(message.clone()), + ImageError::Internal => OciError::new(500, Code::Unavailable, "internal error"), } } -/// Result alias of registry handlers. -pub type OciResult = Result; - #[cfg(test)] mod tests { - use forge_storage::{CasError, S3Error}; - use super::*; - fn down() -> S3Error { - S3Error::Unavailable("connection refused".into()) + fn body(response: &Response) -> Value { + serde_json::from_slice(&response.body).unwrap() + } + + fn header<'a>(response: &'a Response, name: &str) -> Option<&'a str> { + response + .headers + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.as_str()) } #[test] - fn an_unavailable_object_store_answers_503_with_retry_after() { - // Also when a stream read from S3 failed while it was staged (the - // finish of a chunked upload). - for e in [ - CasError::S3(down()), - CasError::Io(std::io::Error::other(down())), - ] { - let e = OciError::from(e); - assert_eq!(e.status, StatusCode::SERVICE_UNAVAILABLE); - assert_eq!(e.code, Code::Unavailable); - assert!(e - .headers - .iter() - .any(|(name, _)| name == header::RETRY_AFTER)); - } - let e = OciError::storage_read("blob read", down()); - assert_eq!(e.status, StatusCode::SERVICE_UNAVAILABLE); - let e = OciError::storage_read("blob read", S3Error::S3("access denied".into())); - assert_eq!(e.status, StatusCode::INTERNAL_SERVER_ERROR); + fn errors_carry_their_code_status_and_detail() { + let response = of(&ImageError::Denied { + name: "docker/app".into(), + permission: Permission::Write, + }) + .into_response(); + assert_eq!(response.status, 403); + let denied = body(&response); + assert_eq!(denied["errors"][0]["code"], "DENIED"); + assert_eq!(denied["errors"][0]["detail"]["action"], "pull,push"); + + let response = of(&ImageError::ChunkOutOfOrder { size: 100 }).into_response(); + assert_eq!(response.status, 416); + assert_eq!(header(&response, "range"), Some("0-99")); + + let response = + of(&ImageError::Unavailable("the object store is down".into())).into_response(); + assert_eq!(response.status, 503); + assert_eq!(header(&response, "retry-after"), Some("30")); + assert_eq!(body(&response)["errors"][0]["code"], "UNAVAILABLE"); + + let response = of(&ImageError::Internal).into_response(); + assert_eq!(response.status, 500); + assert_eq!(body(&response)["errors"][0]["message"], "internal error"); } #[test] - fn an_unreadable_request_body_is_the_clients_error() { - let aborted = std::io::Error::other(RequestBodyError(axum::Error::new( - "connection reset by peer", - ))); - let e = OciError::from(CasError::Io(aborted)); - assert_eq!(e.status, StatusCode::BAD_REQUEST); - assert_eq!(e.code, Code::BlobUploadInvalid); - // Staging's own I/O errors stay server errors. - let e = OciError::from(CasError::Io(std::io::Error::other("disk full"))); - assert_eq!(e.status, StatusCode::INTERNAL_SERVER_ERROR); + fn upload_ranges() { + assert_eq!(upload_range(0), "0-0"); + assert_eq!(upload_range(1), "0-0"); + assert_eq!(upload_range(100), "0-99"); } } diff --git a/crates/formats/docker/src/lib.rs b/crates/formats/docker/src/lib.rs index 5e35fff2..1f856c9d 100644 --- a/crates/formats/docker/src/lib.rs +++ b/crates/formats/docker/src/lib.rs @@ -1,40 +1,286 @@ -//! Docker / OCI registry format: the distribution-spec v1.1 API under -//! `/v2`, with hosted, proxy and group repositories over Forge's CAS. +//! Docker / OCI registries for Forge, as a format plugin: the +//! distribution-spec v1.1 API under `/v2`, over the host's image store. //! -//! * [`digest`], [`reference`], [`manifest`] — the protocol's grammar and -//! manifest parsing (Docker v2 schema 2 manifests and manifest lists, OCI -//! image manifests and indexes, artifacts with `artifactType`/`subject`); -//! * [`token`] — the registry Bearer tokens of the Docker token protocol; -//! * [`chart`] — Helm charts stored as artifacts: their metadata, README and +//! * [`path`], [`digest`], [`reference`]: the protocol's paths and the +//! grammar of its names, references and digests; +//! * [`manifest`]: Docker v2 schema 2 manifests and manifest lists, OCI +//! image manifests and indexes, artifacts with `artifactType` and +//! `subject`, and what Forge shows of them (their summaries); +//! * [`chart`]: Helm charts stored as artifacts: their metadata, README and //! values, for the web UI; -//! * [`Registry`] — the service answering every `/v2` request; -//! * [`import`] — copying images from another registry (a Nexus hosted -//! repository) into a hosted repository. +//! * [`error`]: the distribution-spec error answers; +//! * [`upstream`]: the URLs of an upstream registry, and its tag lists and +//! referrers. //! -//! Repositories are addressed by path (`//`) or, -//! with `config.host`, by host name (`/`). +//! Forge links this plugin in ([`Docker`], [`DESCRIPTOR`]). The host keeps +//! everything else: the image store (blobs and manifests by digest, tags, +//! upload sessions, groups, proxies' caching, garbage collection), the +//! token service at `/v2/token`, resolving a `` (by path, +//! `/`, or by a repository's own host) and authorizing +//! a request. pub mod chart; pub mod digest; pub mod error; -pub mod gc; -pub mod import; pub mod manifest; pub mod path; +mod protocol; pub mod reference; -pub mod token; - -mod auth; -mod group; -mod proxy; -mod pull; -mod push; -mod service; - -pub use proxy::proxied_image; -pub use service::{ - registry_origin, PublishRecorder, Registry, RegistryConfig, RegistryParts, WorkloadCheck, +pub mod upstream; + +use forge_plugin_api::images::{ + ArtifactFiles, BlobInfo, ConfigBlob, ImageError, ImagePath, ManifestBlob, ParsedManifest, + Permission, TextFile, +}; +use forge_plugin_api::{ + Action, Assets, Descriptor, FetchResult, FormatError, FormatPlugin, Loaded, PackageSource, + Remote, Repository, RepositoryKind, Request, Response, Storage, UpstreamOp, UpstreamStep, + UpstreamToken, +}; + +use crate::manifest::{media, ManifestError, EMPTY_JSON, EMPTY_JSON_DIGEST}; +use crate::path::Endpoint; + +/// What Docker / OCI says about itself: its format id, name, version and +/// mount. +pub const DESCRIPTOR: Descriptor = Descriptor { + id: "docker", + name: "Docker / OCI", + version: env!("CARGO_PKG_VERSION"), + description: "Docker / OCI registries: push and pull images, indexes and artifacts (Helm \ + charts, signatures, SBOMs) with docker, podman, containerd, crane, helm and \ + cosign; proxies of Docker Hub and other registries.", + // Clients always talk to `/v2/` at the root of a host; the host keeps + // `/v2/`, `/v2/token` and `/v2/_catalog`. + mount: "/v2", + kinds: &[ + RepositoryKind::Hosted, + RepositoryKind::Proxy, + RepositoryKind::Group, + ], + // Blobs and manifests by digest, tags and upload sessions, in the + // host's image store. + storage: Storage::Images, + assets: Assets::DEFAULT, + // The host's image store answers an upstream's Bearer challenge itself. + upstream_token: UpstreamToken::None, }; -/// `repository.format` of registry repositories. -pub const FORMAT: &str = forge_db::model::DOCKER_FORMAT; +/// The Docker / OCI format. +#[derive(Debug, Clone, Copy, Default)] +pub struct Docker; + +impl FormatPlugin for Docker { + fn descriptor(&self) -> &'static Descriptor { + &DESCRIPTOR + } + + /// An image name: the spec's repository-name grammar + /// ([`reference::valid_name`]). + fn validate_id(&self, id: &str) -> Result<(), FormatError> { + if reference::valid_name(id) { + Ok(()) + } else { + Err(FormatError::InvalidId(format!( + "{id:?} is not an image name" + ))) + } + } + + /// An image-name prefix a group reserves: letters, digits, `.`, `_`, + /// `-` and `/` (a namespace such as `acme/`), lowercased. + fn reserved_prefix(&self, prefix: &str) -> Result { + let valid = !prefix.is_empty() + && prefix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-' | b'/')); + if valid { + Ok(prefix.to_ascii_lowercase()) + } else { + Err(FormatError::InvalidId(format!( + "{prefix:?} is not an image name prefix (letters, digits, '.', '_', '-', '/')" + ))) + } + } + + fn handle(&self, _repo: &Repository, req: &Request) -> Action { + protocol::handle(req) + } + + fn render(&self, _repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + protocol::render(req, data) + } + + fn upstream( + &self, + remote: &Remote, + op: &UpstreamOp, + fetched: &[FetchResult], + ) -> Result { + upstream::step(remote, op, fetched) + } + + fn image_path(&self, method: &str, path: &str) -> Result { + let (endpoint, rest) = path::split(path).ok_or(ImageError::NotAnEndpoint)?; + let name = endpoint.name().unwrap_or_default().to_string(); + if !reference::valid_name(&name) { + return Err(ImageError::NameInvalid { name, hint: None }); + } + Ok(ImagePath { + name, + endpoint: rest, + permission: permission(&endpoint, method), + }) + } + + fn image_error(&self, error: &ImageError) -> Response { + error::of(error).into_response() + } + + fn parse_manifest( + &self, + manifest: &[u8], + content_type: Option<&str>, + ) -> Result { + let parsed = manifest::parse(manifest, content_type).map_err(|e| { + FormatError::MalformedManifest(match e { + ManifestError::UnsupportedMediaType(m) => { + format!("unsupported manifest media type {m:?}") + } + other => other.to_string(), + }) + })?; + Ok(contract(&parsed)) + } + + fn config_summary(&self, media_type: &str, config: &[u8]) -> Option { + manifest::config_summary(media_type, config).map(|part| part.to_string()) + } + + fn artifact_files(&self, blob: &mut dyn PackageSource) -> ArtifactFiles { + let files = chart::read_files(blob); + let text = |file: chart::ChartFile| TextFile { + name: file.name, + text: file.text, + truncated: file.truncated, + }; + ArtifactFiles { + readme: files.readme.map(text), + values: files.values.map(text), + } + } +} + +/// What a request of `method` to `endpoint` needs. +fn permission(endpoint: &Endpoint, method: &str) -> Permission { + match (endpoint, method) { + (Endpoint::UploadStart { .. } | Endpoint::Upload { .. }, _) => Permission::Write, + (_, "PUT" | "POST" | "PATCH") => Permission::Write, + (_, "DELETE") => Permission::Delete, + _ => Permission::Read, + } +} + +/// A parsed manifest as the plugin contract carries it. +fn contract(parsed: &manifest::ParsedManifest) -> ParsedManifest { + ParsedManifest { + media_type: parsed.media_type.clone(), + blobs: parsed + .blob_references() + .into_iter() + .map(|d| ManifestBlob { + digest: d.digest.to_string(), + size: d.size.max(0) as u64, + // Layers that live elsewhere by definition. + required: !media::is_non_distributable(&d.media_type), + // Clients commonly reference the empty JSON as an + // artifact's config without uploading it. + known: (d.digest.as_str() == EMPTY_JSON_DIGEST).then(|| EMPTY_JSON.to_vec()), + }) + .collect(), + manifests: parsed + .manifest_references() + .into_iter() + .map(|d| d.digest.to_string()) + .collect(), + subject: parsed.subject.as_ref().map(|s| s.digest.to_string()), + artifact_type: parsed.artifact_type.clone(), + config: parsed.summary_config().map(|c| ConfigBlob { + digest: c.digest.to_string(), + media_type: c.media_type.clone(), + size: c.size.max(0) as u64, + }), + files: parsed.chart_archive().map(|a| BlobInfo { + digest: a.digest.to_string(), + size: a.size.max(0) as u64, + }), + summary: parsed.summary().to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths_split_into_names_endpoints_and_permissions() { + let split = |method: &str, path: &str| Docker.image_path(method, path); + let manifest = split("PUT", "/docker/team/app/manifests/1").unwrap(); + assert_eq!(manifest.name, "docker/team/app"); + assert_eq!(manifest.endpoint, "/manifests/1"); + assert_eq!(manifest.permission, Permission::Write); + assert_eq!( + split("GET", "/a/b/manifests/x").unwrap().permission, + Permission::Read + ); + assert_eq!( + split("HEAD", "/a/b/blobs/sha256:00").unwrap().permission, + Permission::Read + ); + assert_eq!( + split("DELETE", "/a/b/manifests/x").unwrap().permission, + Permission::Delete + ); + // Every request of an upload session writes. + for method in ["GET", "DELETE"] { + assert_eq!( + split(method, "/a/b/blobs/uploads/x").unwrap().permission, + Permission::Write + ); + } + assert_eq!(split("GET", "/x"), Err(ImageError::NotAnEndpoint)); + assert!(matches!( + split("GET", "/A/b/manifests/x"), + Err(ImageError::NameInvalid { .. }) + )); + } + + #[test] + fn manifests_name_what_a_push_needs() { + let body = serde_json::json!({ + "schemaVersion": 2, + "mediaType": media::OCI_MANIFEST, + "config": {"mediaType": media::OCI_EMPTY, "digest": EMPTY_JSON_DIGEST, "size": 2}, + "layers": [ + {"mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "digest": format!("sha256:{}", "11".repeat(32)), "size": 5, + "urls": ["https://example.com/layer"]} + ], + "subject": {"mediaType": media::OCI_MANIFEST, + "digest": format!("sha256:{}", "22".repeat(32)), "size": 9} + }); + let parsed = Docker + .parse_manifest(body.to_string().as_bytes(), None) + .unwrap(); + assert_eq!(parsed.media_type, media::OCI_MANIFEST); + assert_eq!(parsed.blobs.len(), 2); + assert_eq!(parsed.blobs[0].known.as_deref(), Some(EMPTY_JSON)); + assert!(!parsed.blobs[1].required); + assert_eq!(parsed.subject, Some(format!("sha256:{}", "22".repeat(32)))); + assert!(parsed.config.is_none(), "the empty JSON says nothing"); + let refused = Docker.parse_manifest(b"{}", Some("application/vnd.example")); + assert!( + matches!(refused, Err(FormatError::MalformedManifest(m)) if m.contains("unsupported")) + ); + } +} diff --git a/crates/formats/docker/src/manifest.rs b/crates/formats/docker/src/manifest.rs index 7aefb6be..4420e6e9 100644 --- a/crates/formats/docker/src/manifest.rs +++ b/crates/formats/docker/src/manifest.rs @@ -9,7 +9,9 @@ //! what a manifest keeps alive in the CAS and what a proxy fetches. use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use crate::chart; use crate::digest::Digest; /// The media types Forge stores as manifests. @@ -63,7 +65,14 @@ pub const EMPTY_JSON_DIGEST: &str = /// Largest manifest Forge accepts or proxies. The spec asks registries to /// accept at least 4 MiB. -pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_MANIFEST_BYTES: usize = forge_plugin_api::images::MAX_MANIFEST_BYTES; + +/// Config blobs larger than this are not read for the summary. +pub const MAX_CONFIG_SUMMARY_BYTES: i64 = 1024 * 1024; + +/// Most bytes of a manifest's annotations kept in its summary (for the +/// referrers API), so one manifest cannot bloat every listing. +const MAX_SUMMARY_ANNOTATIONS: usize = 16 * 1024; /// Whether the manifest is a single image/artifact or a list of manifests. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -155,6 +164,86 @@ impl ParsedManifest { pub fn layers_size(&self) -> i64 { self.layers.iter().map(|l| l.size).sum() } + + /// Whether it is a Helm chart (`helm push`): its config is the chart's + /// metadata. + pub fn is_chart(&self) -> bool { + self.kind == ManifestKind::Image + && (self + .config + .as_ref() + .is_some_and(|c| c.media_type == chart::media::CONFIG) + || self.artifact_type.as_deref() == Some(chart::media::CONFIG)) + } + + /// The config whose content completes the summary: an image's (its + /// platform) or a chart's (its metadata), when it is small enough to + /// read. + pub fn summary_config(&self) -> Option<&Descriptor> { + self.config.as_ref().filter(|c| { + (is_image_config(&c.media_type) || c.media_type == chart::media::CONFIG) + && c.size <= MAX_CONFIG_SUMMARY_BYTES + }) + } + + /// A chart's packaged archive, whose README and values the web UI shows. + pub fn chart_archive(&self) -> Option<&Descriptor> { + if !self.is_chart() { + return None; + } + self.layers + .iter() + .find(|l| chart::is_archive(&l.media_type)) + } + + /// Display data of the manifest itself: an index's platforms; an image's + /// layer count and total, and its config's media type and digest; its + /// annotations, for the referrers API. Its config's part comes from + /// [`config_summary`] once the config is at hand. + pub fn summary(&self) -> Value { + let mut summary = Map::new(); + match self.kind { + ManifestKind::Index => { + let platforms: Vec = self + .manifests + .iter() + .filter_map(|m| m.platform.as_ref()) + .filter(|p| !p.architecture.is_empty() && p.architecture != "unknown") + .map(|p| p.label()) + .collect(); + summary.insert("platforms".into(), json!(platforms)); + } + ManifestKind::Image => { + summary.insert("layers_size".into(), json!(self.layers_size())); + summary.insert("layers".into(), json!(self.layers.len())); + if let Some(config) = &self.config { + summary.insert("config_media_type".into(), json!(config.media_type)); + summary.insert("config_digest".into(), json!(config.digest.as_str())); + } + } + } + if let Some(annotations) = &self.annotations { + let value = Value::Object(annotations.clone()); + if value.to_string().len() <= MAX_SUMMARY_ANNOTATIONS { + summary.insert("annotations".into(), value); + } + } + Value::Object(summary) + } +} + +/// The part of a summary a config holds: an image config's platform, a +/// chart's metadata. `None` when it holds nothing worth showing. +pub fn config_summary(media_type: &str, config: &[u8]) -> Option { + if is_image_config(media_type) { + let platform = config_platform(config)?; + return Some(json!({ "platform": platform.label() })); + } + if media_type == chart::media::CONFIG { + let metadata = chart::metadata(config)?; + return Some(json!({ "chart": metadata })); + } + None } /// Why a manifest was refused. @@ -478,7 +567,9 @@ mod tests { #[test] fn empty_json_digest_is_right() { - assert_eq!(Digest::sha256_of(EMPTY_JSON).as_str(), EMPTY_JSON_DIGEST); + use sha2::Digest as _; + let digest = format!("sha256:{}", hex::encode(sha2::Sha256::digest(EMPTY_JSON))); + assert_eq!(digest, EMPTY_JSON_DIGEST); } #[test] @@ -487,4 +578,66 @@ mod tests { assert_eq!(p.label(), "linux/amd64"); assert!(config_platform(b"{}").is_none()); } + + #[test] + fn summaries_name_platforms_layers_and_the_config() { + let index = serde_json::json!({ + "schemaVersion": 2, + "mediaType": media::OCI_INDEX, + "manifests": [ + {"mediaType": media::OCI_MANIFEST, "digest": d(4), "size": 7, + "platform": {"architecture": "arm64", "os": "linux", "variant": "v8"}}, + {"mediaType": media::OCI_MANIFEST, "digest": d(5), "size": 7, + "platform": {"architecture": "unknown", "os": "unknown"}} + ] + }); + let summary = parse(index.to_string().as_bytes(), None).unwrap().summary(); + assert_eq!(summary["platforms"], serde_json::json!(["linux/arm64/v8"])); + + let image = serde_json::json!({ + "schemaVersion": 2, + "mediaType": media::OCI_MANIFEST, + "config": {"mediaType": media::OCI_IMAGE_CONFIG, "digest": d(1), "size": 10}, + "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": d(2), "size": 100}], + "annotations": {"org.opencontainers.image.title": "app"} + }); + let parsed = parse(image.to_string().as_bytes(), None).unwrap(); + let summary = parsed.summary(); + assert_eq!(summary["layers_size"], 100); + assert_eq!(summary["config_digest"], d(1)); + assert_eq!( + summary["annotations"]["org.opencontainers.image.title"], + "app" + ); + assert_eq!(parsed.summary_config().unwrap().digest.as_str(), d(1)); + assert!(!parsed.is_chart()); + assert_eq!( + config_summary( + media::OCI_IMAGE_CONFIG, + br#"{"architecture":"amd64","os":"linux"}"# + ), + Some(serde_json::json!({"platform": "linux/amd64"})) + ); + assert_eq!(config_summary("application/vnd.example", b"{}"), None); + } + + #[test] + fn a_chart_names_its_metadata_and_archive() { + let body = serde_json::json!({ + "schemaVersion": 2, + "mediaType": media::OCI_MANIFEST, + "config": {"mediaType": chart::media::CONFIG, "digest": d(1), "size": 100}, + "layers": [{"mediaType": chart::media::CONTENT, "digest": d(2), "size": 1000}] + }); + let parsed = parse(body.to_string().as_bytes(), None).unwrap(); + assert!(parsed.is_chart()); + assert_eq!(parsed.chart_archive().unwrap().digest.as_str(), d(2)); + assert_eq!(parsed.summary_config().unwrap().digest.as_str(), d(1)); + let part = config_summary( + chart::media::CONFIG, + br#"{"name":"demo","version":"1.0.0"}"#, + ) + .unwrap(); + assert_eq!(part["chart"]["name"], "demo"); + } } diff --git a/crates/formats/docker/src/path.rs b/crates/formats/docker/src/path.rs index e0d50b82..e7d98428 100644 --- a/crates/formats/docker/src/path.rs +++ b/crates/formats/docker/src/path.rs @@ -3,6 +3,11 @@ //! A repository name may itself contain `/` (and words like `blobs` or //! `manifests` as components), but no reference, digest or upload id does, //! so endpoints are recognized from the **end** of the path. +//! +//! The host routes `/v2` and answers `/v2/`, `/v2/token` and `/v2/_catalog` +//! itself; it hands the format the path below `/v2`, which [`split`] splits +//! into the name and the rest, and then the rest, which [`of_name`] parses +//! again with the name the host resolved. /// A `/v2/` endpoint, with the raw (not yet validated) name and argument. #[derive(Debug, Clone, PartialEq, Eq)] @@ -42,6 +47,21 @@ impl Endpoint { } } +/// A path below `/v2` (`/team/app/manifests/1`) as the endpoint of a name, +/// with the part after the name (`/manifests/1`). `None` for anything that +/// is not the endpoint of a name. +pub fn split(path: &str) -> Option<(Endpoint, String)> { + let endpoint = parse(&format!("/v2{path}"))?; + let rest = path.strip_prefix('/')?.strip_prefix(endpoint.name()?)?; + let rest = rest.to_string(); + Some((endpoint, rest)) +} + +/// The endpoint `rest` (what [`split`] left after the name) of `name`. +pub fn of_name(name: &str, rest: &str) -> Option { + parse(&format!("/v2/{name}{rest}")).filter(|endpoint| endpoint.name() == Some(name)) +} + /// Parse a request path. `None` for anything that is not a `/v2` endpoint. pub fn parse(path: &str) -> Option { let rest = path.strip_prefix("/v2")?; @@ -180,4 +200,29 @@ mod tests { assert_eq!(parse("/v2/x"), None); assert_eq!(parse("/v2/x/y"), None); } + + #[test] + fn splits_the_name_off_and_back_on() { + let (endpoint, rest) = split("/docker/team/app/manifests/1.0").unwrap(); + assert_eq!( + endpoint, + Endpoint::Manifest { + name: "docker/team/app".into(), + reference: "1.0".into() + } + ); + assert_eq!(rest, "/manifests/1.0"); + assert_eq!(of_name("docker/team/app", &rest), Some(endpoint)); + let (_, rest) = split("/a/blobs/uploads/").unwrap(); + assert_eq!(rest, "/blobs/uploads/"); + assert_eq!( + of_name("other", "/blobs/uploads/"), + Some(Endpoint::UploadStart { + name: "other".into() + }) + ); + assert_eq!(split("/token"), None); + assert_eq!(split("/x"), None); + assert_eq!(split(""), None); + } } diff --git a/crates/formats/docker/src/protocol.rs b/crates/formats/docker/src/protocol.rs new file mode 100644 index 00000000..6cccc2b0 --- /dev/null +++ b/crates/formats/docker/src/protocol.rs @@ -0,0 +1,489 @@ +//! The distribution-spec v1.1 API of an image: what each request is +//! ([`handle`]), and the answer once the image store answered +//! ([`render`]). +//! +//! The host has resolved the name, authorized the request and picked where +//! a change lands; the request's path is the endpoint after the name. + +use forge_plugin_api::images::{ + ChunkRange, ImageAnswer, ImageError, ImageName, ImageQuery, ManifestInfo, Mount, Reference, + Started, UploadState, +}; +use forge_plugin_api::{Action, Answer, Loaded, Query, Request, Response}; +use serde_json::{json, Value}; + +use crate::digest::{Digest, DigestError}; +use crate::error::{self, Code, OciError}; +use crate::manifest::media; +use crate::path::{self, Endpoint}; +use crate::reference::{self, ReferenceError}; + +/// Tags returned by one `tags/list` page when the client sets no `n`, and +/// the most one page returns. +const DEFAULT_TAG_PAGE: u32 = 10_000; + +/// What a request is. +pub fn handle(req: &Request) -> Action { + let Some((image, endpoint)) = endpoint(req) else { + return respond(error::of(&ImageError::NotAnEndpoint)); + }; + match request(image, &endpoint, req) { + Ok(action) => action, + Err(e) => respond(e), + } +} + +/// The answer once the store answered the request's query. +pub fn render(req: &Request, data: &[Loaded]) -> Action { + let (Some((image, _)), Some(last)) = (endpoint(req), data.last()) else { + return respond(error::of(&ImageError::Internal)); + }; + let (Query::Image(query), Answer::Image(answer)) = (&last.query, &last.answer) else { + return respond(error::of(&ImageError::Internal)); + }; + match answered(image, req, query, answer) { + Ok(action) => action, + Err(e) => respond(e), + } +} + +/// The image of the request and its endpoint. +fn endpoint(req: &Request) -> Option<(&ImageName, Endpoint)> { + let image = req.image.as_ref()?; + let endpoint = path::of_name(&image.name, &req.path)?; + Some((image, endpoint)) +} + +fn respond(error: OciError) -> Action { + Action::Respond(error.into_response()) +} + +fn load(query: ImageQuery) -> Result { + Ok(Action::Load(vec![Query::Image(query)])) +} + +fn request(image: &ImageName, endpoint: &Endpoint, req: &Request) -> Result { + let name = image.image.clone(); + match (endpoint, req.method.as_str()) { + (Endpoint::Manifest { reference, .. }, "GET" | "HEAD") => { + let parsed = parse_reference(reference).map_err(|e| { + // A malformed reference names nothing that can exist. + if e.code == Code::ManifestInvalid { + OciError::manifest_unknown(reference) + } else { + e + } + })?; + load(ImageQuery::Manifest { + image: name, + reference: parsed, + }) + } + (Endpoint::Manifest { reference, .. }, "PUT") => { + let reference = parse_reference(reference)?; + let mut tags: Vec = Vec::new(); + if let Reference::Tag(tag) = &reference { + tags.push(tag.clone()); + } + for (key, value) in req.query_pairs() { + if key == "tag" { + if !reference::valid_tag(&value) { + return Err(OciError::manifest_invalid(format!("invalid tag {value:?}"))); + } + if !tags.contains(&value) { + tags.push(value); + } + } + } + load(ImageQuery::PutManifest { + image: name, + reference, + tags, + }) + } + (Endpoint::Manifest { reference, .. }, "DELETE") => load(ImageQuery::DeleteManifest { + image: name, + reference: parse_reference(reference)?, + }), + (Endpoint::Blob { digest, .. }, "GET" | "HEAD") => load(ImageQuery::Blob { + image: name, + digest: parse_digest(digest)?.into_string(), + }), + (Endpoint::Blob { digest, .. }, "DELETE") => load(ImageQuery::DeleteBlob { + image: name, + digest: parse_digest(digest)?.into_string(), + }), + (Endpoint::UploadStart { .. }, "POST") => { + let mount = match req.query_param("mount") { + Some(digest) => Some(Mount { + digest: parse_digest(&digest)?.into_string(), + from: req + .query_param("from") + .filter(|from| reference::valid_name(from)), + }), + None => None, + }; + // A monolithic upload: checked now, finished once the session + // stands. + if let Some(digest) = req.query_param("digest") { + parse_digest(&digest)?; + } + load(ImageQuery::StartUpload { image: name, mount }) + } + (Endpoint::Upload { id, .. }, "GET") => load(ImageQuery::UploadStatus(id.clone())), + (Endpoint::Upload { id, .. }, "PATCH") => load(ImageQuery::AppendUpload { + id: id.clone(), + range: content_range(req)?, + }), + (Endpoint::Upload { id, .. }, "PUT") => { + let digest = req.query_param("digest").ok_or_else(|| { + OciError::digest_invalid("the digest query parameter is required") + })?; + load(ImageQuery::FinishUpload { + id: id.clone(), + digest: parse_digest(&digest)?.into_string(), + range: content_range(req)?, + }) + } + (Endpoint::Upload { id, .. }, "DELETE") => load(ImageQuery::CancelUpload(id.clone())), + (Endpoint::TagsList { .. }, "GET") => { + let mut n: Option = None; + for (key, value) in req.query_pairs() { + if key == "n" { + // A page larger than the most one returns is that many. + let parsed: u64 = value.parse().map_err(|_| { + OciError::new(400, Code::Unsupported, format!("invalid n {value:?}")) + })?; + n = Some(parsed.min(u64::from(DEFAULT_TAG_PAGE)) as u32); + } + } + let page = n.unwrap_or(DEFAULT_TAG_PAGE); + if page == 0 { + return Ok(Action::Respond(tag_list(&image.name, &[], None))); + } + load(ImageQuery::Tags { + image: name, + last: req.query_param("last"), + n: page, + }) + } + (Endpoint::Referrers { digest, .. }, "GET") => load(ImageQuery::Referrers { + image: name, + digest: parse_digest(digest)?.into_string(), + artifact_type: req.query_param("artifactType"), + }), + _ => Err(OciError::unsupported("method not allowed on this endpoint")), + } +} + +fn answered( + image: &ImageName, + req: &Request, + query: &ImageQuery, + answer: &ImageAnswer, +) -> Result { + let name = &image.name; + let respond = |response: Response| Ok(Action::Respond(response)); + match (query, answer) { + (ImageQuery::Manifest { reference, .. }, ImageAnswer::Manifest(found)) => { + match failed(found)? { + Some(_) => Ok(Action::ServeManifest { + image: image.image.clone(), + reference: reference.clone(), + }), + None => Err(OciError::manifest_unknown(reference.as_str())), + } + } + (ImageQuery::Blob { digest, .. }, ImageAnswer::Blob(found)) => match failed(found)? { + Some(_) => Ok(Action::ServeBlob { + image: image.image.clone(), + digest: digest.clone(), + }), + None => Err(OciError::blob_unknown(digest)), + }, + (ImageQuery::Tags { n, .. }, ImageAnswer::Tags(found)) => match failed(found)? { + Some(page) => { + let next = page.more.then_some(*n); + respond(tag_list(name, &page.tags, next)) + } + None => Err(OciError::name_unknown(name)), + }, + (ImageQuery::Referrers { artifact_type, .. }, ImageAnswer::Referrers(found)) => { + respond(referrers(failed(found)?, artifact_type.is_some())) + } + (ImageQuery::StartUpload { .. }, ImageAnswer::Started(started)) => { + match failed(started)? { + Started::Mounted(blob) => respond(blob_created(name, &blob.digest)), + Started::Session(state) => match req.query_param("digest") { + // A monolithic upload: the body is the whole blob. + Some(digest) => load(ImageQuery::FinishUpload { + id: state.id.clone(), + digest: parse_digest(&digest)?.into_string(), + range: None, + }), + None => respond(upload_accepted(202, name, state)), + }, + } + } + (ImageQuery::UploadStatus(_), ImageAnswer::Upload(state)) => { + respond(upload_accepted(204, name, failed(state)?)) + } + (ImageQuery::AppendUpload { .. }, ImageAnswer::Upload(state)) => { + respond(upload_accepted(202, name, failed(state)?)) + } + (ImageQuery::FinishUpload { digest, .. }, ImageAnswer::Blob(stored)) => { + failed(stored)?; + respond(blob_created(name, digest)) + } + (ImageQuery::CancelUpload(_), ImageAnswer::Done(done)) => { + failed(done)?; + respond(Response::empty(204)) + } + ( + ImageQuery::PutManifest { + reference, tags, .. + }, + ImageAnswer::Stored(stored), + ) => { + let stored = match stored { + Ok(stored) => stored, + Err(ImageError::DigestMismatch) => { + return Err(OciError::digest_invalid( + "the manifest does not match the digest it was pushed as", + )) + } + Err(ImageError::BodyUnreadable) => { + return Err(OciError::manifest_invalid( + "the request body could not be read", + )) + } + Err(e) => return Err(error::of(e)), + }; + let mut response = Response::empty(201) + .with_header( + "location", + &format!("/v2/{name}/manifests/{}", stored.digest), + ) + .with_header("docker-content-digest", &stored.digest); + if let Some(subject) = &stored.subject { + response = response.with_header("oci-subject", subject); + } + // A push by digest with `?tag=` parameters names every tag it set. + if matches!(reference, Reference::Digest(_)) { + for tag in tags { + response = response.with_header("oci-tag", tag); + } + } + respond(response) + } + ( + ImageQuery::DeleteManifest { .. } | ImageQuery::DeleteBlob { .. }, + ImageAnswer::Done(done), + ) => { + failed(done)?; + respond(Response::empty(202)) + } + _ => Err(error::of(&ImageError::Internal)), + } +} + +/// An answer's value, or its error as the client sees it. +fn failed(answer: &Result) -> Result<&T, OciError> { + answer.as_ref().map_err(error::of) +} + +pub(crate) fn parse_digest(raw: &str) -> Result { + Digest::parse(raw).map_err(|e| match e { + DigestError::Invalid(_) => OciError::digest_invalid(format!("invalid digest {raw:?}")), + DigestError::Unsupported(_) => OciError::digest_invalid(format!( + "unsupported digest algorithm in {raw:?}: use sha256 or sha512" + )), + }) +} + +fn parse_reference(raw: &str) -> Result { + match reference::Reference::parse(raw) { + Ok(reference::Reference::Tag(tag)) => Ok(Reference::Tag(tag)), + Ok(reference::Reference::Digest(digest)) => Ok(Reference::Digest(digest.into_string())), + Err(ReferenceError::InvalidTag) => { + Err(OciError::manifest_invalid(format!("invalid tag {raw:?}")) + .with_detail(json!({ "reference": raw }))) + } + Err(ReferenceError::Digest(_)) => Err(parse_digest(raw) + .err() + .unwrap_or_else(|| OciError::digest_invalid(format!("invalid digest {raw:?}")))), + } +} + +/// `Content-Range: -` of a chunk (also accepting a `bytes` +/// prefix some clients add). +fn content_range(req: &Request) -> Result, OciError> { + let Some(value) = req.header("content-range") else { + return Ok(None); + }; + let invalid = || OciError::new(416, Code::BlobUploadInvalid, "invalid Content-Range"); + let text = value.trim(); + let text = text + .strip_prefix("bytes=") + .or_else(|| text.strip_prefix("bytes ")) + .unwrap_or(text); + let text = text.split('/').next().unwrap_or(text); + let (start, end) = text.split_once('-').ok_or_else(invalid)?; + let start: u64 = start.trim().parse().map_err(|_| invalid())?; + let end: u64 = end.trim().parse().map_err(|_| invalid())?; + if end < start { + return Err(invalid()); + } + Ok(Some(ChunkRange { start, end })) +} + +/// The answer of an upload session in progress. +fn upload_accepted(status: u16, name: &str, upload: &UploadState) -> Response { + Response::empty(status) + .with_header( + "location", + &format!("/v2/{name}/blobs/uploads/{}", upload.id), + ) + .with_header("range", &error::upload_range(upload.size)) + .with_header("docker-upload-uuid", &upload.id) +} + +/// The 201 answer of a stored blob. +fn blob_created(name: &str, digest: &str) -> Response { + Response::empty(201) + .with_header("location", &format!("/v2/{name}/blobs/{digest}")) + .with_header("docker-content-digest", digest) +} + +/// A `tags/list` page, linking to the next one when there is one (`n`: the +/// page size). +fn tag_list(name: &str, tags: &[String], next: Option) -> Response { + let body = json!({ "name": name, "tags": tags }).to_string(); + let response = Response::new(200, "application/json", body); + match (next, tags.last()) { + (Some(n), Some(last)) => { + let query: String = url::form_urlencoded::Serializer::new(String::new()) + .append_pair("n", &n.to_string()) + .append_pair("last", last) + .finish(); + response.with_header( + "link", + &format!("; rel=\"next\""), + ) + } + _ => response, + } +} + +/// The referrers of a manifest, as an OCI index. +fn referrers(found: &[ManifestInfo], filtered: bool) -> Response { + let manifests: Vec = found.iter().map(descriptor).collect(); + let body = json!({ + "schemaVersion": 2, + "mediaType": media::OCI_INDEX, + "manifests": manifests, + }); + let response = Response::new(200, media::OCI_INDEX, body.to_string()); + if filtered { + response.with_header("oci-filters-applied", "artifactType") + } else { + response + } +} + +/// A referrer's descriptor: its annotations are kept in its summary. +fn descriptor(manifest: &ManifestInfo) -> Value { + let mut descriptor = json!({ + "mediaType": manifest.media_type, + "digest": manifest.digest, + "size": manifest.size, + }); + if let Some(artifact_type) = &manifest.artifact_type { + descriptor["artifactType"] = json!(artifact_type); + } + let annotations = manifest + .summary + .as_deref() + .and_then(|summary| serde_json::from_str::(summary).ok()) + .and_then(|summary| summary.get("annotations").cloned()) + .filter(|a| a.as_object().is_some_and(|o| !o.is_empty())); + if let Some(annotations) = annotations { + descriptor["annotations"] = annotations; + } + descriptor +} + +#[cfg(test)] +mod tests { + use super::*; + + fn req(method: &str, target: &str) -> Request { + let (path, query) = target.split_once('?').unwrap_or((target, "")); + let mut req = Request::new(method, path).with_image("team/app", "docker/team/app"); + req.query = query.to_string(); + req + } + + fn sha(n: u8) -> String { + format!("sha256:{}", format!("{n:02x}").repeat(32)) + } + + #[test] + fn content_ranges() { + let with = |value: &str| { + content_range(&req("PATCH", "/blobs/uploads/x").with_header("content-range", value)) + }; + assert_eq!( + content_range(&req("PATCH", "/blobs/uploads/x")).unwrap(), + None + ); + assert_eq!( + with("0-99").unwrap(), + Some(ChunkRange { start: 0, end: 99 }) + ); + assert_eq!( + with("bytes 100-199/*").unwrap(), + Some(ChunkRange { + start: 100, + end: 199 + }) + ); + assert_eq!(with("9-1").unwrap_err().status, 416); + } + + #[test] + fn a_get_of_a_malformed_reference_names_an_unknown_manifest() { + let Action::Respond(response) = handle(&req("GET", "/manifests/-bad")) else { + panic!("expected a response"); + }; + assert_eq!(response.status, 404); + let Action::Load(queries) = handle(&req("GET", &format!("/manifests/{}", sha(1)))) else { + panic!("expected a load"); + }; + assert_eq!( + queries, + [Query::Image(ImageQuery::Manifest { + image: "team/app".into(), + reference: Reference::Digest(sha(1)), + })] + ); + } + + #[test] + fn tag_pages_link_to_the_next() { + let response = tag_list("docker/team/app", &["a".into(), "b c".into()], Some(2)); + let link = response + .headers + .iter() + .find(|(k, _)| k == "link") + .map(|(_, v)| v.as_str()); + assert_eq!( + link, + Some("; rel=\"next\"") + ); + assert!(!tag_list("x", &[], Some(2)) + .headers + .iter() + .any(|(k, _)| k == "link")); + } +} diff --git a/crates/formats/docker/src/push.rs b/crates/formats/docker/src/push.rs deleted file mode 100644 index bf563676..00000000 --- a/crates/formats/docker/src/push.rs +++ /dev/null @@ -1,1060 +0,0 @@ -//! Pushes and deletes: blob uploads (monolithic, chunked, cross-repository -//! mounts), manifests and tags. -//! -//! Upload chunks are CAS blobs held by `oci_upload_part` rows, so a session -//! survives on any instance. When an upload consists of one chunk (what -//! docker, podman and containerd send) that chunk already is the blob and is -//! linked as is; only multi-chunk uploads are assembled into a new blob. - -use std::collections::HashSet; -use std::io::Cursor; - -use axum::body::Body; -use axum::http::{header, HeaderMap, HeaderValue, Request, StatusCode}; -use axum::response::{IntoResponse, Response}; -use bytes::Bytes; -use forge_core::auth::Principal; -use forge_core::authz::Level; -use forge_core::repo::{PackageMutation, PublishAuthorizationContext, WriteError}; -use forge_db::model::{RepoKind, Repository}; -use forge_db::oci_dao::{self, LinkOutcome, NewManifest, TagWrite, UploadRow}; -use forge_storage::BlobRef; -use futures_util::{StreamExt, TryStreamExt}; -use tokio_util::io::StreamReader; -use uuid::Uuid; - -use crate::digest::Digest; -use crate::error::{Code, OciError, OciResult, RequestBodyError}; -use crate::manifest::{ - self, config_platform, is_image_config, media, ManifestError, ManifestKind, ParsedManifest, - EMPTY_JSON, EMPTY_JSON_DIGEST, MAX_MANIFEST_BYTES, -}; -use crate::pull::{header_value, parse_digest, parse_reference}; -use crate::reference::{self, Reference}; -use crate::service::{location, Decision, Registry, RequestCtx, Target}; - -/// Config blobs larger than this are not read for the platform summary. -const MAX_CONFIG_SUMMARY_BYTES: i64 = 1024 * 1024; - -/// A manifest [`Registry::commit_manifest`] stored. -pub(crate) struct CommittedManifest { - pub id: i64, - /// Whether the image did not hold the manifest before. - pub created: bool, - /// What became of each tag. - pub tags: Vec<(String, TagWrite)>, -} - -/// Refuse to commit a link whose blob turned out to be gone or being -/// deleted: dropping the transaction rolls the link row back, so no link -/// ever exists without its reference. -pub(crate) fn require_linked(outcome: LinkOutcome) -> OciResult<()> { - match outcome { - LinkOutcome::BlobUnavailable => Err(OciError::unavailable( - "the blob is being removed from storage; retry the upload", - )), - LinkOutcome::Inserted | LinkOutcome::AlreadyLinked => Ok(()), - } -} - -/// A request body as an `AsyncRead`. A body that cannot be read fails with a -/// [`RequestBodyError`] inside the I/O error, which answers 400. -fn body_reader(body: Body) -> impl tokio::io::AsyncRead + Unpin + Send { - StreamReader::new( - body.into_data_stream() - .map_err(|e| std::io::Error::other(RequestBodyError(e))), - ) -} - -/// Read a whole (small) request body, refusing more than `limit` bytes. -async fn read_body(body: Body, limit: usize) -> OciResult { - let mut stream = body.into_data_stream(); - let mut out = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| { - OciError::new( - StatusCode::BAD_REQUEST, - Code::ManifestInvalid, - format!("reading the body: {e}"), - ) - })?; - if out.len() + chunk.len() > limit { - return Err(OciError::too_large(limit as u64)); - } - out.extend_from_slice(&chunk); - } - Ok(Bytes::from(out)) -} - -/// An upload session id from the URL. -fn upload_id(raw: &str) -> OciResult { - Uuid::parse_str(raw).map_err(|_| OciError::upload_unknown()) -} - -/// `Range` of an upload: the inclusive end of the bytes received. -fn upload_range(size: i64) -> HeaderValue { - header_value(&format!("0-{}", (size - 1).max(0))) -} - -/// The 202 answer of an upload session in progress. -fn upload_accepted(status: StatusCode, name: &str, upload: &UploadRow) -> Response { - let mut response = status.into_response(); - let headers = response.headers_mut(); - headers.insert( - header::LOCATION, - header_value(&location(name, &format!("blobs/uploads/{}", upload.id))), - ); - headers.insert(header::RANGE, upload_range(upload.size)); - headers.insert("docker-upload-uuid", header_value(&upload.id.to_string())); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0)); - response -} - -/// The 201 answer of a stored blob. -fn blob_created(name: &str, digest: &str) -> Response { - let mut response = StatusCode::CREATED.into_response(); - let headers = response.headers_mut(); - headers.insert( - header::LOCATION, - header_value(&location(name, &format!("blobs/{digest}"))), - ); - headers.insert("docker-content-digest", header_value(digest)); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0)); - response -} - -/// `Content-Range: -` of a chunk (also accepting a `bytes` -/// prefix some clients add). -fn content_range(headers: &HeaderMap) -> OciResult> { - let Some(value) = headers.get(header::CONTENT_RANGE) else { - return Ok(None); - }; - let invalid = || { - OciError::new( - StatusCode::RANGE_NOT_SATISFIABLE, - Code::BlobUploadInvalid, - "invalid Content-Range", - ) - }; - let text = value.to_str().map_err(|_| invalid())?.trim(); - let text = text - .strip_prefix("bytes=") - .or_else(|| text.strip_prefix("bytes ")) - .unwrap_or(text); - let text = text.split('/').next().unwrap_or(text); - let (start, end) = text.split_once('-').ok_or_else(invalid)?; - let start: i64 = start.trim().parse().map_err(|_| invalid())?; - let end: i64 = end.trim().parse().map_err(|_| invalid())?; - if start < 0 || end < start { - return Err(invalid()); - } - Ok(Some((start, end))) -} - -impl Registry { - fn grace(&self) -> chrono::Duration { - self.inner.config.delete_grace - } - - /// The largest blob a push to `repo` may carry: its own upload limit, - /// else the registry's default. - pub(crate) fn blob_limit(&self, repo: &Repository) -> u64 { - repo.max_upload_bytes() - .unwrap_or(self.inner.config.default_max_upload_bytes) - } - - // ── Uploads ───────────────────────────────────────────────────────────── - - pub(crate) async fn start_upload( - &self, - ctx: &RequestCtx, - req: Request, - ) -> OciResult { - let t = ctx.serving(); - let name = &ctx.target.name; - let query = req.uri().query().unwrap_or_default().to_string(); - let params: Vec<(String, String)> = url::form_urlencoded::parse(query.as_bytes()) - .into_owned() - .collect(); - let param = |k: &str| { - params - .iter() - .find(|(key, _)| key == k) - .map(|(_, v)| v.clone()) - }; - - if let Some(mount) = param("mount") { - let digest = parse_digest(&mount)?; - if self.mount(ctx, &digest, param("from").as_deref()).await? { - return Ok(blob_created(name, digest.as_str())); - } - // Not mountable: fall back to a regular upload session. - } - if let Some(digest) = param("digest") { - // Monolithic upload: the body is the whole blob. - let digest = parse_digest(&digest)?; - let staged = self - .inner - .cas - .stage(body_reader(req.into_body()), self.blob_limit(&t.repo)) - .await?; - if !digest.matches(&staged.sha256, &staged.sha512_base64) { - return Err(OciError::digest_invalid( - "the uploaded content does not match the digest", - )); - } - let stored = self.inner.cas.commit(staged).await?; - let mut tx = self.inner.pool.begin().await?; - oci_dao::insert_link_ingested( - &mut tx, - t.repo.id, - &t.image, - digest.as_str(), - &stored.sha256, - stored.size as i64, - stored.ingest_token, - ) - .await - .map_err(OciError::from) - .and_then(require_linked)?; - tx.commit().await?; - return Ok(blob_created(name, digest.as_str())); - } - let id = Uuid::new_v4(); - oci_dao::create_upload(&self.inner.pool, id, t.repo.id, &t.image).await?; - let upload = UploadRow { - id, - repository_id: t.repo.id, - image: t.image.clone(), - size: 0, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - }; - Ok(upload_accepted(StatusCode::ACCEPTED, name, &upload)) - } - - /// Link `digest` into the target image without uploading it: from the - /// image `from` the caller may read, or (without `from`) when the image - /// already has it. `false` when it cannot be mounted. - async fn mount( - &self, - ctx: &RequestCtx, - digest: &Digest, - from: Option<&str>, - ) -> OciResult { - let t = ctx.serving(); - let pool = &self.inner.pool; - if oci_dao::find_link(pool, t.repo.id, &t.image, digest.as_str()) - .await? - .is_some() - { - // The client is about to name it in a manifest: record the use - // now, or the sweeper may release it first. - oci_dao::touch_link( - pool, - t.repo.id, - &t.image, - digest.as_str(), - chrono::Duration::zero(), - ) - .await?; - return Ok(true); - } - let Some(from) = from.filter(|f| reference::valid_name(f)) else { - return Ok(false); - }; - let Ok(Some(source)) = self.resolve(&ctx.site, from).await else { - return Ok(false); - }; - // What the source would serve: an offline hosted repository serves - // nothing (an offline proxy still serves its cache). - let serves = match source.repo.kind() { - RepoKind::Hosted => source.repo.online, - RepoKind::Proxy => true, - RepoKind::Group => false, - }; - if !serves || self.decide(&ctx.caller, &source, Level::Read).await != Decision::Allow { - return Ok(false); - } - let Some(link) = - oci_dao::find_link(pool, source.repo.id, &source.image, digest.as_str()).await? - else { - return Ok(false); - }; - let mut tx = pool.begin().await?; - match oci_dao::insert_link_existing( - &mut tx, - t.repo.id, - &t.image, - digest.as_str(), - &link.blob_sha256, - link.size, - ) - .await? - { - LinkOutcome::BlobUnavailable => { - tx.rollback().await?; - Ok(false) - } - LinkOutcome::Inserted | LinkOutcome::AlreadyLinked => { - tx.commit().await?; - Ok(true) - } - } - } - - async fn upload_of(&self, ctx: &RequestCtx, id: &str) -> OciResult { - let t = ctx.serving(); - oci_dao::get_upload(&self.inner.pool, upload_id(id)?, t.repo.id, &t.image) - .await? - .ok_or_else(OciError::upload_unknown) - } - - pub(crate) async fn upload_status(&self, ctx: &RequestCtx, id: &str) -> OciResult { - let upload = self.upload_of(ctx, id).await?; - Ok(upload_accepted( - StatusCode::NO_CONTENT, - &ctx.target.name, - &upload, - )) - } - - pub(crate) async fn cancel_upload(&self, ctx: &RequestCtx, id: &str) -> OciResult { - let upload = self.upload_of(ctx, id).await?; - let mut tx = self.inner.pool.begin().await?; - let deleted = oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; - tx.commit().await?; - if !deleted { - return Err(OciError::upload_unknown()); - } - Ok(StatusCode::NO_CONTENT.into_response()) - } - - /// Store one chunk of an upload. `expected_start` is where the chunk - /// must begin (the session's size when the request started); the whole - /// blob may be at most `max` bytes. - async fn append_chunk( - &self, - upload: &UploadRow, - range: Option<(i64, i64)>, - body: Body, - max: u64, - ) -> OciResult { - let out_of_order = |size: i64| { - OciError::new( - StatusCode::RANGE_NOT_SATISFIABLE, - Code::BlobUploadInvalid, - "the chunk does not continue the upload", - ) - .with_header(header::RANGE, upload_range(size)) - }; - if let Some((start, _)) = range { - if start != upload.size { - return Err(out_of_order(upload.size)); - } - } - let remaining = max.saturating_sub(upload.size.max(0) as u64); - let staged = self - .inner - .cas - .stage(body_reader(body), remaining) - .await - .map_err(|e| match e { - forge_storage::CasError::TooLarge { .. } => OciError::too_large(max), - other => other.into(), - })?; - if let Some((start, end)) = range { - if staged.size as i64 != end - start + 1 { - return Err(OciError::new( - StatusCode::RANGE_NOT_SATISFIABLE, - Code::BlobUploadInvalid, - "the chunk length does not match its Content-Range", - ) - .with_header(header::RANGE, upload_range(upload.size))); - } - } - if staged.size == 0 { - return Ok(upload.clone()); - } - // The first chunk may be the whole blob (a monolithic upload, or a - // layer streamed in one PATCH, as docker, podman and containerd - // send it), so it joins the local blob cache like any pushed blob. - // Later chunks are only read back once, to assemble the blob: - // caching them would evict content that is downloaded. - let stored = if upload.size == 0 { - self.inner.cas.commit(staged).await? - } else { - self.inner.cas.commit_uncached(staged).await? - }; - let mut tx = self.inner.pool.begin().await?; - let Some(locked) = - oci_dao::lock_upload(&mut tx, upload.id, upload.repository_id, &upload.image).await? - else { - return Err(OciError::upload_unknown()); - }; - if locked.size != upload.size { - // Another chunk landed meanwhile. - return Err(out_of_order(locked.size)); - } - oci_dao::add_upload_part( - &mut tx, - &locked, - stored.size as i64, - &stored.sha256, - &stored.sha512_base64, - stored.ingest_token, - ) - .await?; - tx.commit().await?; - Ok(UploadRow { - size: locked.size + stored.size as i64, - updated_at: chrono::Utc::now(), - ..locked - }) - } - - pub(crate) async fn patch_upload( - &self, - ctx: &RequestCtx, - id: &str, - req: Request, - ) -> OciResult { - let upload = self.upload_of(ctx, id).await?; - let range = content_range(req.headers())?; - let max = self.blob_limit(&ctx.serving().repo); - let upload = self - .append_chunk(&upload, range, req.into_body(), max) - .await?; - Ok(upload_accepted( - StatusCode::ACCEPTED, - &ctx.target.name, - &upload, - )) - } - - pub(crate) async fn finish_upload( - &self, - ctx: &RequestCtx, - id: &str, - req: Request, - ) -> OciResult { - let t = ctx.serving(); - let upload = self.upload_of(ctx, id).await?; - let digest = url::form_urlencoded::parse(req.uri().query().unwrap_or_default().as_bytes()) - .find(|(k, _)| k == "digest") - .map(|(_, v)| v.into_owned()) - .ok_or_else(|| OciError::digest_invalid("the digest query parameter is required"))?; - let digest = parse_digest(&digest)?; - let range = content_range(req.headers())?; - let max = self.blob_limit(&t.repo); - let upload = self - .append_chunk(&upload, range, req.into_body(), max) - .await?; - let pool = &self.inner.pool; - let parts = oci_dao::upload_parts(pool, upload.id).await?; - - let mismatch = - || OciError::digest_invalid("the uploaded content does not match the digest"); - // The blob: the single chunk itself, or the chunks assembled. - let mut tx; - match parts.as_slice() { - [part] => { - if !digest.matches(&part.blob_sha256, &part.sha512_base64) { - self.drop_upload(upload.id).await; - return Err(mismatch()); - } - tx = pool.begin().await?; - if oci_dao::lock_upload(&mut tx, upload.id, t.repo.id, &t.image) - .await? - .is_none_or(|u| u.size != upload.size) - { - return Err(OciError::upload_unknown()); - } - // Linked before the chunk's reference is released, so the - // blob never drops to zero references. - if oci_dao::insert_link_existing( - &mut tx, - t.repo.id, - &t.image, - digest.as_str(), - &part.blob_sha256, - part.size, - ) - .await? - == LinkOutcome::BlobUnavailable - { - tx.rollback().await?; - return Err(OciError::internal( - "upload finish", - "the uploaded chunk vanished from the CAS", - )); - } - oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; - } - parts => { - let store = self.inner.store.clone(); - let keys: Vec = parts.iter().map(|p| p.s3_key.clone()).collect(); - let chunks = futures_util::stream::iter(keys) - .then(move |key| { - let store = store.clone(); - async move { - store - .get(&key) - .await - .map(|s| tokio_util::io::ReaderStream::new(s.into_async_read())) - .map_err(std::io::Error::other) - } - }) - .try_flatten(); - let staged = self - .inner - .cas - .stage(StreamReader::new(Box::pin(chunks)), max) - .await?; - // The first chunk joined the local blob cache in case it was - // the whole blob (`append_chunk`). It is not, and nothing - // reads it as it is again. - if let (Some(cache), Some(first)) = (self.inner.store.cache(), parts.first()) { - cache.evict(&first.blob_sha256).await; - } - if staged.size as i64 != upload.size - || !digest.matches(&staged.sha256, &staged.sha512_base64) - { - self.drop_upload(upload.id).await; - return Err(mismatch()); - } - let stored = self.inner.cas.commit(staged).await?; - tx = pool.begin().await?; - if oci_dao::lock_upload(&mut tx, upload.id, t.repo.id, &t.image) - .await? - .is_none_or(|u| u.size != upload.size) - { - return Err(OciError::upload_unknown()); - } - oci_dao::insert_link_ingested( - &mut tx, - t.repo.id, - &t.image, - digest.as_str(), - &stored.sha256, - stored.size as i64, - stored.ingest_token, - ) - .await - .map_err(OciError::from) - .and_then(require_linked)?; - oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; - tx.commit().await?; - return Ok(blob_created(&ctx.target.name, digest.as_str())); - } - } - tx.commit().await?; - Ok(blob_created(&ctx.target.name, digest.as_str())) - } - - /// End a failed upload session (best effort). - async fn drop_upload(&self, id: Uuid) { - let result = async { - let mut tx = self.inner.pool.begin().await?; - oci_dao::delete_upload(&mut tx, id, self.grace()).await?; - tx.commit().await - } - .await; - if let Err(e) = result { - tracing::warn!(error = %e, upload = %id, "registry: dropping a failed upload failed"); - } - } - - // ── Manifests ─────────────────────────────────────────────────────────── - - pub(crate) async fn put_manifest( - &self, - ctx: &RequestCtx, - reference: &str, - req: Request, - ) -> OciResult { - let t = ctx.serving(); - let reference = parse_reference(reference)?; - let mut tags: Vec = Vec::new(); - if let Reference::Tag(tag) = &reference { - tags.push(tag.clone()); - } - for (key, value) in - url::form_urlencoded::parse(req.uri().query().unwrap_or_default().as_bytes()) - { - if key == "tag" { - if !reference::valid_tag(&value) { - return Err(OciError::manifest_invalid(format!("invalid tag {value:?}"))); - } - if !tags.iter().any(|t| *t == value) { - tags.push(value.into_owned()); - } - } - } - let content_type = req - .headers() - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .map(str::to_string); - let body = read_body(req.into_body(), MAX_MANIFEST_BYTES).await?; - let parsed = manifest::parse(&body, content_type.as_deref()).map_err(manifest_error)?; - let digest = match &reference { - Reference::Digest(d) => { - if Digest::of(d.algorithm(), &body) != *d { - return Err(OciError::digest_invalid( - "the manifest does not match the digest it was pushed as", - )); - } - d.clone() - } - Reference::Tag(_) => Digest::sha256_of(&body), - }; - - self.commit_manifest(t, &ctx.caller.principal, &digest, &body, &parsed, &tags) - .await?; - let subject = parsed.subject.as_ref().map(|s| s.digest.to_string()); - - let mut response = StatusCode::CREATED.into_response(); - let headers = response.headers_mut(); - headers.insert( - header::LOCATION, - header_value(&location(&ctx.target.name, &format!("manifests/{digest}"))), - ); - headers.insert("docker-content-digest", header_value(digest.as_str())); - headers.insert(header::CONTENT_LENGTH, HeaderValue::from(0)); - if let Some(subject) = subject { - headers.insert("oci-subject", header_value(&subject)); - } - // A push by digest with `?tag=` parameters names every tag it set. - if matches!(reference, Reference::Digest(_)) { - for tag in &tags { - headers.append("oci-tag", header_value(tag)); - } - } - Ok(response) - } - - /// Store a manifest whose bytes match `digest` and point `tags` at it, - /// as a push does: every blob it names (but non-distributable layers) - /// must be linked into the image and, for an index, every child - /// manifest must exist; tags follow the repository's immutability; the - /// change runs past the publish policy of `principal`. A manifest the - /// image already holds is left as it is (its tags are still set). - pub(crate) async fn commit_manifest( - &self, - t: &Target, - principal: &Principal, - digest: &Digest, - body: &Bytes, - parsed: &ParsedManifest, - tags: &[String], - ) -> OciResult { - // What must exist before the manifest may: its blobs (except layers - // that live elsewhere by definition) and, for an index, its children. - let required_blobs: Vec = parsed - .blob_references() - .into_iter() - .filter(|d| !media::is_non_distributable(&d.media_type)) - .map(|d| d.digest.to_string()) - .collect(); - if required_blobs.iter().any(|d| d == EMPTY_JSON_DIGEST) { - self.ensure_empty_json(t).await?; - } - let children: Vec = parsed - .manifest_references() - .into_iter() - .map(|d| d.digest.to_string()) - .collect(); - let summary = self.manifest_summary(t, parsed).await; - - let staged = self - .inner - .cas - .stage(Cursor::new(body.to_vec()), MAX_MANIFEST_BYTES as u64) - .await?; - let stored = self.inner.cas.commit(staged).await?; - - let mut tx = self.inner.pool.begin().await?; - let linked: HashSet = - oci_dao::lock_links(&mut tx, t.repo.id, &t.image, &required_blobs) - .await? - .into_iter() - .collect(); - if let Some(missing) = required_blobs.iter().find(|d| !linked.contains(*d)) { - return Err(OciError::new( - StatusCode::BAD_REQUEST, - Code::ManifestBlobUnknown, - "the manifest references a blob unknown to this repository", - ) - .with_detail(serde_json::json!({ "digest": missing }))); - } - if !children.is_empty() { - let present: HashSet = - oci_dao::existing_manifests(&mut tx, t.repo.id, &t.image, &children) - .await? - .into_iter() - .collect(); - if let Some(missing) = children.iter().find(|d| !present.contains(*d)) { - return Err(OciError::new( - StatusCode::BAD_REQUEST, - Code::ManifestBlobUnknown, - "the index references a manifest unknown to this repository", - ) - .with_detail(serde_json::json!({ "digest": missing }))); - } - } - let blob_refs: Vec = parsed - .blob_references() - .into_iter() - .map(|d| d.digest.to_string()) - .collect(); - let subject = parsed.subject.as_ref().map(|s| s.digest.to_string()); - let (manifest_id, created) = oci_dao::insert_manifest( - &mut tx, - NewManifest { - repository_id: t.repo.id, - image: &t.image, - digest: digest.as_str(), - blob_sha256: &stored.sha256, - media_type: &parsed.media_type, - artifact_type: parsed.artifact_type.as_deref(), - subject_digest: subject.as_deref(), - size: body.len() as i64, - summary: &summary, - blob_refs: &blob_refs, - manifest_refs: &children, - }, - stored.ingest_token, - ) - .await?; - let immutable = t.repo.immutable_tags(); - let mut written = Vec::with_capacity(tags.len()); - for tag in tags { - let may_move = !immutable || (tag == "latest" && t.repo.latest_mutable()); - match oci_dao::set_tag(&mut tx, t.repo.id, &t.image, tag, manifest_id, may_move).await? - { - TagWrite::Refused => { - return Err(OciError::denied(format!( - "tag {tag:?} already exists and tags are immutable in this repository" - )) - .with_detail(serde_json::json!({ "tag": tag }))); - } - write @ (TagWrite::Created | TagWrite::Moved { .. } | TagWrite::Unchanged) => { - written.push((tag.clone(), write)); - } - } - } - let version = tags.first().cloned().unwrap_or_else(|| digest.to_string()); - self.record_mutation(principal, t, PackageMutation::Publish, version, &mut tx) - .await?; - tx.commit().await?; - Ok(CommittedManifest { - id: manifest_id, - created, - tags: written, - }) - } - - /// Make sure the OCI empty JSON blob (`{}`) is linked into the image: - /// clients commonly reference it as an artifact's config without - /// uploading it. - async fn ensure_empty_json(&self, t: &Target) -> OciResult<()> { - let pool = &self.inner.pool; - if oci_dao::find_link(pool, t.repo.id, &t.image, EMPTY_JSON_DIGEST) - .await? - .is_some() - { - return Ok(()); - } - let stored = self - .inner - .cas - .store(Cursor::new(EMPTY_JSON.to_vec()), EMPTY_JSON.len() as u64) - .await?; - let mut tx = pool.begin().await?; - oci_dao::insert_link_ingested( - &mut tx, - t.repo.id, - &t.image, - EMPTY_JSON_DIGEST, - &stored.sha256, - stored.size as i64, - stored.ingest_token, - ) - .await - .map_err(OciError::from) - .and_then(require_linked)?; - tx.commit().await?; - Ok(()) - } - - /// Display data of a manifest: platforms, layer total, annotations. - pub(crate) async fn manifest_summary( - &self, - t: &Target, - parsed: &ParsedManifest, - ) -> serde_json::Value { - let mut summary = serde_json::Map::new(); - match parsed.kind { - ManifestKind::Index => { - let platforms: Vec = parsed - .manifests - .iter() - .filter_map(|m| m.platform.as_ref()) - .filter(|p| !p.architecture.is_empty() && p.architecture != "unknown") - .map(|p| p.label()) - .collect(); - summary.insert("platforms".into(), serde_json::json!(platforms)); - } - ManifestKind::Image => { - summary.insert( - "layers_size".into(), - serde_json::json!(parsed.layers_size()), - ); - summary.insert("layers".into(), serde_json::json!(parsed.layers.len())); - if let Some(config) = &parsed.config { - summary.insert( - "config_media_type".into(), - serde_json::json!(config.media_type), - ); - summary.insert( - "config_digest".into(), - serde_json::json!(config.digest.as_str()), - ); - if is_image_config(&config.media_type) - && config.size <= MAX_CONFIG_SUMMARY_BYTES - { - if let Some(platform) = self.config_platform(t, &config.digest).await { - summary.insert("platform".into(), serde_json::json!(platform.label())); - } - } - } - } - } - if let Some(annotations) = &parsed.annotations { - // Kept for the referrers API; bounded so one manifest cannot - // bloat every listing. - let value = serde_json::Value::Object(annotations.clone()); - if value.to_string().len() <= 16 * 1024 { - summary.insert("annotations".into(), value); - } - } - serde_json::Value::Object(summary) - } - - /// The platform of an image config blob linked into the image. - async fn config_platform(&self, t: &Target, digest: &Digest) -> Option { - let link = oci_dao::find_link(&self.inner.pool, t.repo.id, &t.image, digest.as_str()) - .await - .ok()??; - let bytes = self - .read_manifest_bytes(BlobRef { - sha256: &link.blob_sha256, - s3_key: &link.s3_key, - size: link.size.max(0) as u64, - }) - .await - .ok()?; - config_platform(&bytes) - } - - pub(crate) async fn delete_manifest( - &self, - ctx: &RequestCtx, - reference: &str, - ) -> OciResult { - let t = ctx.serving(); - let reference = parse_reference(reference)?; - let mut tx = self.inner.pool.begin().await?; - let deleted = match &reference { - Reference::Tag(tag) => oci_dao::delete_tag(&mut tx, t.repo.id, &t.image, tag).await?, - Reference::Digest(digest) => { - oci_dao::delete_manifest( - &mut tx, - t.repo.id, - &t.image, - digest.as_str(), - self.grace(), - ) - .await? - } - }; - if !deleted { - return Err(OciError::manifest_unknown(reference.as_str())); - } - let version = reference.as_str().to_string(); - self.record_mutation( - &ctx.caller.principal, - t, - PackageMutation::Delete, - version, - &mut tx, - ) - .await?; - tx.commit().await?; - Ok(StatusCode::ACCEPTED.into_response()) - } - - pub(crate) async fn delete_blob(&self, ctx: &RequestCtx, digest: &str) -> OciResult { - let t = ctx.serving(); - let digest = parse_digest(digest)?; - let mut tx = self.inner.pool.begin().await?; - // Lock first, check after: a manifest naming the blob that commits - // while this waits for the lock is seen by the check. - if !oci_dao::lock_link(&mut tx, t.repo.id, &t.image, digest.as_str()).await? { - return Err(OciError::blob_unknown(digest.as_str())); - } - if oci_dao::link_referenced(&mut *tx, t.repo.id, &t.image, digest.as_str()).await? { - return Err(OciError::new( - StatusCode::CONFLICT, - Code::Denied, - "the blob is referenced by a manifest of this image; delete the manifest first", - ) - .with_detail(serde_json::json!({ "digest": digest.as_str() }))); - } - let deleted = - oci_dao::delete_link(&mut tx, t.repo.id, &t.image, digest.as_str(), self.grace()) - .await?; - if !deleted { - return Err(OciError::blob_unknown(digest.as_str())); - } - let version = digest.to_string(); - self.record_mutation( - &ctx.caller.principal, - t, - PackageMutation::Delete, - version, - &mut tx, - ) - .await?; - tx.commit().await?; - Ok(StatusCode::ACCEPTED.into_response()) - } - - /// Run a change of an image past the repository's publish policy in the - /// change's transaction: a workload's pushes and deletes go to the - /// trusted-publishing ledger, and fail when its realm or role changed - /// since the request was authenticated. - async fn record_mutation( - &self, - principal: &Principal, - t: &Target, - action: PackageMutation, - version: String, - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - ) -> OciResult<()> { - let Some(recorder) = &self.inner.recorder else { - return Ok(()); - }; - let context = PublishAuthorizationContext { - principal: *principal, - action, - package_id: t.image.clone(), - package_version: version, - }; - recorder - .for_repository(&t.repo) - .record_in_transaction(&context, tx) - .await - .map_err(|e| match e { - WriteError::ForbiddenPackage { .. } => { - OciError::denied("the workload identity may no longer change this repository") - } - other => OciError::internal("publish record", other), - }) - } - - /// Store a manifest fetched from a proxy's upstream (no reference - /// checks: its blobs are fetched on demand) and point `tag` at it. - pub(crate) async fn store_proxied_manifest( - &self, - t: &Target, - digest: &Digest, - parsed: &ParsedManifest, - body: &Bytes, - tag: Option<&str>, - ) -> OciResult { - let children: Vec = parsed - .manifest_references() - .into_iter() - .map(|d| d.digest.to_string()) - .collect(); - let blob_refs: Vec = parsed - .blob_references() - .into_iter() - .map(|d| d.digest.to_string()) - .collect(); - let summary = self.manifest_summary(t, parsed).await; - let staged = self - .inner - .cas - .stage(Cursor::new(body.to_vec()), MAX_MANIFEST_BYTES as u64) - .await?; - let stored = self.inner.cas.commit(staged).await?; - let subject = parsed.subject.as_ref().map(|s| s.digest.to_string()); - let mut tx = self.inner.pool.begin().await?; - crate::proxy::still_upstream(&mut tx, &t.repo).await?; - let (manifest_id, _) = oci_dao::insert_manifest( - &mut tx, - NewManifest { - repository_id: t.repo.id, - image: &t.image, - digest: digest.as_str(), - blob_sha256: &stored.sha256, - media_type: &parsed.media_type, - artifact_type: parsed.artifact_type.as_deref(), - subject_digest: subject.as_deref(), - size: body.len() as i64, - summary: &summary, - blob_refs: &blob_refs, - manifest_refs: &children, - }, - stored.ingest_token, - ) - .await?; - if let Some(tag) = tag { - oci_dao::set_tag(&mut tx, t.repo.id, &t.image, tag, manifest_id, true).await?; - } - tx.commit().await?; - oci_dao::find_manifest(&self.inner.pool, t.repo.id, &t.image, digest.as_str()) - .await? - .ok_or_else(|| OciError::internal("proxy manifest", "stored manifest vanished")) - } -} - -/// A manifest parse failure as the client sees it. -fn manifest_error(e: ManifestError) -> OciError { - match e { - ManifestError::UnsupportedMediaType(m) => { - OciError::manifest_invalid(format!("unsupported manifest media type {m:?}")) - } - other => OciError::manifest_invalid(other.to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn content_ranges() { - let mut h = HeaderMap::new(); - assert_eq!(content_range(&h).unwrap(), None); - h.insert(header::CONTENT_RANGE, HeaderValue::from_static("0-99")); - assert_eq!(content_range(&h).unwrap(), Some((0, 99))); - h.insert( - header::CONTENT_RANGE, - HeaderValue::from_static("bytes 100-199/*"), - ); - assert_eq!(content_range(&h).unwrap(), Some((100, 199))); - h.insert(header::CONTENT_RANGE, HeaderValue::from_static("9-1")); - assert!(content_range(&h).is_err()); - } - - #[test] - fn upload_ranges() { - assert_eq!(upload_range(0), "0-0"); - assert_eq!(upload_range(1), "0-0"); - assert_eq!(upload_range(100), "0-99"); - } -} diff --git a/crates/formats/docker/src/service.rs b/crates/formats/docker/src/service.rs deleted file mode 100644 index 0ffabb69..00000000 --- a/crates/formats/docker/src/service.rs +++ /dev/null @@ -1,880 +0,0 @@ -//! The registry service: request routing, caller resolution and -//! authorization, shared by the pull, push and token handlers. - -use std::sync::Arc; - -use axum::body::Body; -use axum::http::{header, HeaderMap, HeaderValue, Method, Request, StatusCode}; -use axum::response::{IntoResponse, Response}; -use forge_cache::Cache; -use forge_core::auth::authz::Authz; -use forge_core::auth::{AuthMethod, Principal}; -use forge_core::authz::Level; -use forge_core::net::upstream::UpstreamHttp; -use forge_core::repo::PublishPolicy; -use forge_db::model::{RepoKind, Repository}; -use forge_db::{oci_dao, repository_dao}; -use forge_storage::{BlobStore, Cas}; -use sqlx::PgPool; - -use crate::error::{Code, OciError, OciResult}; -use crate::path::{self, Endpoint}; -use crate::reference; -use crate::token::{self, Claims, TokenKeys}; - -/// Tunables of the registry (the server's `[docker]` section and friends). -#[derive(Debug, Clone)] -pub struct RegistryConfig { - /// `server.public_base_url`, without a trailing `/`. - pub public_base_url: String, - /// Largest blob (layer) a proxy downloads from its upstream. - pub max_blob_bytes: u64, - /// Largest blob a push to a hosted repository may carry when the - /// repository has no upload limit of its own - /// ([`Repository::max_upload_bytes`]). - pub default_max_upload_bytes: u64, - /// Lifetime of a registry Bearer token. - pub token_ttl_seconds: u64, - /// Outbox grace of released blob references. - pub delete_grace: chrono::Duration, - /// `groups.max_nesting_depth`. - pub max_group_depth: u32, - /// `docker.unreferenced_blob_ttl_seconds`: a blob link no manifest names - /// is released after this long unused. - pub unreferenced_blob_ttl: chrono::Duration, -} - -impl RegistryConfig { - /// How often a pull (or `HEAD`) of a blob records a use of its link: a - /// tenth of the unreferenced-blob TTL, at most hourly. A link a client - /// confirmed then stays at least nine tenths of the TTL, whereas an - /// hourly record with the smallest TTL (an hour) could leave it none. - pub fn link_touch_interval(&self) -> chrono::Duration { - (self.unreferenced_blob_ttl / 10).min(chrono::Duration::hours(1)) - } -} - -/// Builds the request-scoped policy that records a workload's publishes -/// (the trusted-publishing ledger) in the manifest transaction. -pub trait PublishRecorder: Send + Sync { - fn for_repository(&self, repository: &Repository) -> Arc; -} - -/// Whether the workload principal of a registry token still stands. The -/// server answers it from its realms: like a workload key, a token stands -/// only while its realm is enabled with the configuration it was verified -/// under, and, when it was minted with a workload key, while that key has -/// not ended. -#[async_trait::async_trait] -pub trait WorkloadCheck: Send + Sync { - async fn still_valid(&self, principal: &Principal) -> Result; -} - -/// The Docker / OCI registry: everything under `/v2`. -#[derive(Clone)] -pub struct Registry { - pub(crate) inner: Arc, -} - -pub(crate) struct Inner { - pub pool: PgPool, - pub cas: Cas, - pub store: BlobStore, - pub cache: Cache, - pub authz: Authz, - pub http: UpstreamHttp, - pub envelope: Option>, - pub config: RegistryConfig, - pub keys: TokenKeys, - pub recorder: Option>, - pub workloads: Option>, -} - -/// Parts the registry is built from. -pub struct RegistryParts { - pub pool: PgPool, - pub cas: Cas, - pub store: BlobStore, - pub cache: Cache, - pub authz: Authz, - pub http: UpstreamHttp, - pub envelope: Option>, - pub config: RegistryConfig, - pub keys: TokenKeys, - pub recorder: Option>, - pub workloads: Option>, -} - -impl Registry { - pub fn new(parts: RegistryParts) -> Self { - Registry { - inner: Arc::new(Inner { - pool: parts.pool, - cas: parts.cas, - store: parts.store, - cache: parts.cache, - authz: parts.authz, - http: parts.http, - envelope: parts.envelope, - config: parts.config, - keys: parts.keys, - recorder: parts.recorder, - workloads: parts.workloads, - }), - } - } - - pub fn config(&self) -> &RegistryConfig { - &self.inner.config - } - - /// Answer one `/v2` request. `principal` is what the authentication - /// middleware resolved (registry Bearer tokens are resolved here). - pub async fn handle(&self, principal: Principal, req: Request) -> Response { - let mut response = match self.route(principal, req).await { - Ok(response) => response, - Err(error) => error.into_response(), - }; - let headers = response.headers_mut(); - headers.insert( - "docker-distribution-api-version", - HeaderValue::from_static("registry/2.0"), - ); - harden(headers); - response - } - - async fn route(&self, principal: Principal, req: Request) -> OciResult { - let Some(endpoint) = path::parse(req.uri().path()) else { - return Err(OciError::new( - StatusCode::NOT_FOUND, - Code::NameUnknown, - "not a registry endpoint", - )); - }; - let site = self.site(request_host(&req)).await?; - let caller = self.caller(principal, req.headers(), &site).await?; - match endpoint { - Endpoint::Base => self.version_check(&caller, &site), - Endpoint::Token => { - if req.method() != Method::GET { - return Err(OciError::unsupported( - "the token service answers GET with Basic credentials", - )); - } - self.token(&caller, &site, req.uri().query().unwrap_or_default()) - .await - } - Endpoint::Catalog => Err(OciError::new( - StatusCode::NOT_FOUND, - Code::Unsupported, - "the catalog API is not supported; browse images in the web UI", - )), - endpoint => self.repository_request(caller, site, endpoint, req).await, - } - } - - async fn repository_request( - &self, - caller: Caller, - site: Site, - endpoint: Endpoint, - req: Request, - ) -> OciResult { - let name = endpoint.name().unwrap_or_default().to_string(); - let action = action_of(&endpoint, req.method()); - let target = match self.resolve(&site, &name).await? { - Some(target) => target, - None => { - // An unknown repository is answered like one the caller may - // not use; only a global admin learns that it does not exist. - if self.inner.authz.is_global_admin(&caller.principal).await { - return Err(OciError::name_unknown(&name)); - } - return Err(self.refusal(&caller, &site, &name, action, Decision::Deny)); - } - }; - let write_target = if action == Level::Read { - let decision = self.decide(&caller, &target, Level::Read).await; - if decision != Decision::Allow { - return Err(self.refusal(&caller, &site, &name, action, decision)); - } - None - } else { - Some(self.write_target(&caller, &site, &target, action).await?) - }; - // An offline hosted or group repository serves nothing; an offline - // proxy serves what it cached. - let serving = write_target.as_ref().unwrap_or(&target); - if !serving.repo.online && serving.repo.kind() != RepoKind::Proxy - || !target.repo.online && target.repo.kind() == RepoKind::Group - { - return Err(OciError::unavailable("repository is offline")); - } - let ctx = RequestCtx { - caller, - site, - target, - write_target, - }; - let method = req.method().clone(); - let response = match (&endpoint, &method) { - (Endpoint::Manifest { reference, .. }, &Method::GET | &Method::HEAD) => { - self.get_manifest(&ctx, reference, method == Method::HEAD, req.headers()) - .await - } - (Endpoint::Manifest { reference, .. }, &Method::PUT) => { - self.put_manifest(&ctx, reference, req).await - } - (Endpoint::Manifest { reference, .. }, &Method::DELETE) => { - self.delete_manifest(&ctx, reference).await - } - (Endpoint::Blob { digest, .. }, &Method::GET | &Method::HEAD) => { - self.get_blob(&ctx, digest, method == Method::HEAD, req.headers()) - .await - } - (Endpoint::Blob { digest, .. }, &Method::DELETE) => { - self.delete_blob(&ctx, digest).await - } - (Endpoint::UploadStart { .. }, &Method::POST) => self.start_upload(&ctx, req).await, - (Endpoint::Upload { id, .. }, &Method::GET) => self.upload_status(&ctx, id).await, - (Endpoint::Upload { id, .. }, &Method::PATCH) => self.patch_upload(&ctx, id, req).await, - (Endpoint::Upload { id, .. }, &Method::PUT) => self.finish_upload(&ctx, id, req).await, - (Endpoint::Upload { id, .. }, &Method::DELETE) => self.cancel_upload(&ctx, id).await, - (Endpoint::TagsList { .. }, &Method::GET) => { - self.tags_list(&ctx, req.uri().query().unwrap_or_default()) - .await - } - (Endpoint::Referrers { digest, .. }, &Method::GET) => { - self.referrers(&ctx, digest, req.uri().query().unwrap_or_default()) - .await - } - _ => Err(OciError::unsupported("method not allowed on this endpoint")), - }; - if action != Level::Read { - let status = match &response { - Ok(r) => r.status().as_u16(), - Err(e) => e.status.as_u16(), - }; - tracing::info!( - target: "forge::audit", - %method, - repo = %ctx.serving().repo.name, - image = %ctx.target.image, - endpoint = %endpoint_label(&endpoint), - status, - principal = %ctx.caller.principal.audit_label(), - "image mutation" - ); - } - response - } - - // ── Site and caller ───────────────────────────────────────────────────── - - /// Which host the request reached and the token realm and service that - /// go with it. A host that names a docker repository (`config.host`) - /// serves that repository at the registry root. - async fn site(&self, host: Option) -> OciResult { - let public = &self.inner.config.public_base_url; - let (scheme, public_host) = registry_origin(public); - let host = host.map(|h| h.to_ascii_lowercase()); - let host_repo = match &host { - Some(h) if *h != public_host => { - let bare = strip_port(h); - oci_dao::find_repository_by_host(&self.inner.pool, bare).await? - } - _ => None, - }; - let (service, realm) = match (&host, &host_repo) { - (Some(h), Some(_)) => (h.clone(), format!("{scheme}://{h}/v2/token")), - _ => ( - public_host.clone(), - format!("{scheme}://{public_host}/v2/token"), - ), - }; - Ok(Site { - service, - realm, - issuer: token::issuer(public), - host_repo, - }) - } - - /// The effective caller: a registry Bearer token's principal, else the - /// middleware's. - async fn caller( - &self, - principal: Principal, - headers: &HeaderMap, - site: &Site, - ) -> OciResult { - let Some(bearer) = bearer_token(headers) else { - return Ok(Caller { - principal, - claims: None, - }); - }; - if !token::looks_like_registry_token(bearer) { - // Another bearer (a workload identity token the middleware has - // already resolved, or nothing it knows). - return Ok(Caller { - principal, - claims: None, - }); - } - let invalid = || { - OciError::new( - StatusCode::UNAUTHORIZED, - Code::Unauthorized, - "invalid or expired registry token", - ) - .with_header( - header::WWW_AUTHENTICATE, - site.challenge(None, Some("invalid_token")), - ) - }; - let claims = token::verify(&self.inner.keys, bearer, &site.issuer, &site.service) - .ok_or_else(invalid)?; - let principal = claims.fp.to_principal(principal).ok_or_else(invalid)?; - let principal = self - .refresh_principal(principal) - .await? - .ok_or_else(invalid)?; - Ok(Caller { - principal, - claims: Some(claims), - }) - } - - /// Re-read what a token's principal rests on: a user's account must still - /// be active (and the API token it was minted from live); a workload's - /// realm and key must still stand ([`WorkloadCheck`]). Cached briefly per - /// process. - async fn refresh_principal(&self, principal: Principal) -> OciResult> { - if let Principal::Workload { - realm_id, - config_sha256, - key_id, - .. - } = principal - { - let Some(check) = self.inner.workloads.clone() else { - return Ok(Some(principal)); - }; - let key = format!( - "{realm_id}:{}:{}", - hex::encode(config_sha256), - key_id.map_or_else(|| "token".to_string(), |id| id.to_string()) - ); - let live: Result = self - .inner - .cache - .get_or_try_insert_local( - "oci:workload", - &key, - std::time::Duration::from_secs(10), - || async move { check.still_valid(&principal).await }, - ) - .await; - return match live { - Ok(true) => Ok(Some(principal)), - Ok(false) => Ok(None), - Err(e) => Err(OciError::internal("workload lookup", e)), - }; - } - let Principal::User { - user_id, - token_id, - method, - .. - } = principal - else { - return Ok(Some(principal)); - }; - let key = format!("{user_id}:{}", token_id.unwrap_or(0)); - let pool = self.inner.pool.clone(); - let state: Result = self - .inner - .cache - .get_or_try_insert_local( - "oci:principal", - &key, - std::time::Duration::from_secs(10), - || async move { - oci_dao::live_principal(&pool, user_id, token_id) - .await - .map_err(|e| e.to_string()) - }, - ) - .await; - match state { - Ok(true) => Ok(Some(Principal::User { - user_id, - token_id, - method, - })), - Ok(false) => Ok(None), - Err(e) => Err(OciError::internal("principal lookup", e)), - } - } - - // ── Names and authorization ───────────────────────────────────────────── - - /// The repository and image a `` addresses: on a repository's own - /// host the whole name is the image; otherwise the first component names - /// the Forge repository and the rest the image. `None` for an unknown (or - /// non-docker) repository. - pub(crate) async fn resolve(&self, site: &Site, name: &str) -> OciResult> { - if !reference::valid_name(name) { - return Err(OciError::name_invalid(name)); - } - if let Some(repo) = &site.host_repo { - return Ok(Some(Target { - repo: repo.clone(), - image: name.to_string(), - name: name.to_string(), - })); - } - let Some((repo_name, image)) = name.split_once('/') else { - return Err(OciError::name_invalid(name).with_detail(serde_json::json!({ - "name": name, - "hint": "use /", - }))); - }; - let repo = repository_dao::find_repository_by_name(&self.inner.pool, repo_name).await?; - Ok(repo.filter(Repository::is_docker).map(|repo| Target { - repo, - image: image.to_string(), - name: name.to_string(), - })) - } - - /// Whether the caller may `action` the target's image: the repository - /// permission and its content selectors, then a registry token's scope - /// (checked against the name the client used). - pub(crate) async fn decide(&self, caller: &Caller, target: &Target, action: Level) -> Decision { - if !self - .rbac_allows(&caller.principal, &target.repo, &target.image, action) - .await - { - return Decision::Deny; - } - match &caller.claims { - Some(claims) if !claims_allow(claims, &target.name, action) => Decision::NeedsScope, - _ => Decision::Allow, - } - } - - /// The RBAC part of [`Self::decide`]. - pub(crate) async fn rbac_allows( - &self, - principal: &Principal, - repo: &Repository, - image: &str, - action: Level, - ) -> bool { - let authz = &self.inner.authz; - let allowed = match action { - Level::Read => authz.can_read(principal, repo).await, - Level::Write => authz.can_write(principal, repo).await, - Level::Delete => authz.can_delete(principal, repo).await, - }; - allowed - && authz - .package_filter_for(principal, repo, action) - .await - .allows(image) - } - - /// Where a push or delete addressed to `target` lands (the repository - /// itself, or a group's writable member), after checking the caller may - /// do it there. - async fn write_target( - &self, - caller: &Caller, - site: &Site, - target: &Target, - action: Level, - ) -> OciResult { - if !mutation_credential(&caller.principal) { - return Err(self.refusal(caller, site, &target.name, action, Decision::Deny)); - } - let landing = match target.repo.kind() { - RepoKind::Hosted => target.clone(), - RepoKind::Proxy | RepoKind::Group => { - let decision = self.decide(caller, target, Level::Read).await; - if decision != Decision::Allow { - return Err(self.refusal(caller, site, &target.name, Level::Read, decision)); - } - let member = match target.repo.kind() { - RepoKind::Group => self.writable_member(&target.repo).await?, - _ => None, - }; - match member { - Some(member) => Target { - repo: member, - image: target.image.clone(), - name: target.name.clone(), - }, - None if target.repo.kind() == RepoKind::Proxy => { - return Err(OciError::unsupported( - "a proxy repository is read-only; push to a hosted repository", - )) - } - None => { - return Err(OciError::unsupported( - "this group has no writable member; push to a hosted repository", - )) - } - } - } - }; - let decision = if self - .rbac_allows(&caller.principal, &landing.repo, &landing.image, action) - .await - { - match &caller.claims { - Some(claims) if !claims_allow(claims, &target.name, action) => Decision::NeedsScope, - _ => Decision::Allow, - } - } else { - Decision::Deny - }; - if decision != Decision::Allow { - return Err(self.refusal(caller, site, &target.name, action, decision)); - } - Ok(landing) - } - - /// A group's writable member, if configured and a hosted docker member. - pub(crate) async fn writable_member( - &self, - group: &Repository, - ) -> OciResult> { - let Some(id) = group.writable_member_id() else { - return Ok(None); - }; - let members = repository_dao::group_members(&self.inner.pool, group.id).await?; - Ok(members - .into_iter() - .map(|(member, _)| member) - .find(|m| m.id == id && m.is_docker() && m.kind() == RepoKind::Hosted)) - } - - /// The refusal of `action` on `name`: a challenge (401) when credentials - /// or a token for the scope could help, `DENIED` (403) for an - /// authenticated caller the permission is missing for. - pub(crate) fn refusal( - &self, - caller: &Caller, - site: &Site, - name: &str, - action: Level, - decision: Decision, - ) -> OciError { - let authenticated = mutation_credential(&caller.principal); - let error = caller.claims.as_ref().map(|_| "insufficient_scope"); - match decision { - Decision::NeedsScope => self.challenge_error(site, name, action, error), - Decision::Deny if !authenticated => self.challenge_error(site, name, action, error), - _ => OciError::denied("requested access to the resource is denied") - .with_detail(serde_json::json!({ "name": name, "action": scope_actions(action) })), - } - } - - fn challenge_error( - &self, - site: &Site, - name: &str, - action: Level, - error: Option<&str>, - ) -> OciError { - let scope = format!("repository:{name}:{}", scope_actions(action)); - OciError::new( - StatusCode::UNAUTHORIZED, - Code::Unauthorized, - "authentication required", - ) - .with_detail(serde_json::json!({ "name": name, "action": scope_actions(action) })) - .with_header( - header::WWW_AUTHENTICATE, - site.challenge(Some(&scope), error), - ) - } - - fn version_check(&self, caller: &Caller, site: &Site) -> OciResult { - let signed_in = caller.claims.is_some() - || matches!( - caller.principal, - Principal::User { method, .. } if method != AuthMethod::Session - ) - || matches!(caller.principal, Principal::Workload { .. }); - if signed_in { - return Ok(( - StatusCode::OK, - [(header::CONTENT_TYPE, "application/json")], - "{}", - ) - .into_response()); - } - Err(OciError::new( - StatusCode::UNAUTHORIZED, - Code::Unauthorized, - "authentication required", - ) - .with_header(header::WWW_AUTHENTICATE, site.challenge(None, None))) - } -} - -/// The scheme and `host[:port]` of `public_base_url`: where the registry -/// lives. Docker clients always talk to `/v2/` at the root of a host, so a -/// path in the URL plays no part. -pub fn registry_origin(public_base_url: &str) -> (&'static str, String) { - let scheme = if public_base_url.starts_with("https://") { - "https" - } else { - "http" - }; - let host = url::Url::parse(public_base_url) - .ok() - .and_then(|u| { - u.host_str().map(|h| match u.port() { - Some(p) => format!("{h}:{p}"), - None => h.to_string(), - }) - }) - .unwrap_or_default() - .to_ascii_lowercase(); - (scheme, host) -} - -/// The host a request reached (`Host`, or the HTTP/2 authority). -fn request_host(req: &Request) -> Option { - req.headers() - .get(header::HOST) - .and_then(|h| h.to_str().ok()) - .map(str::to_string) - .or_else(|| req.uri().authority().map(|a| a.to_string())) - .filter(|h| !h.is_empty()) -} - -fn strip_port(host: &str) -> &str { - if host.starts_with('[') { - return host - .split_once(']') - .map_or(host, |(h, _)| h.trim_start_matches('[')); - } - host.rsplit_once(':') - .filter(|(_, port)| port.bytes().all(|b| b.is_ascii_digit())) - .map_or(host, |(h, _)| h) -} - -/// The `Authorization: Bearer` value, if any. -pub(crate) fn bearer_token(headers: &HeaderMap) -> Option<&str> { - let value = headers.get(header::AUTHORIZATION)?.to_str().ok()?; - let (scheme, token) = value.split_once(' ')?; - scheme - .eq_ignore_ascii_case("bearer") - .then(|| token.trim()) - .filter(|t| !t.is_empty()) -} - -/// Whether the principal is a credential a push may come from: an API -/// token, a password (Basic or a token minted from one) or a workload — but -/// never a browser session, which a cross-site page could ride. -pub(crate) fn mutation_credential(principal: &Principal) -> bool { - match principal { - Principal::User { method, .. } => *method != AuthMethod::Session, - Principal::Workload { .. } => true, - _ => false, - } -} - -/// The token-scope actions a request of an access level needs. -fn scope_actions(action: Level) -> &'static str { - match action { - Level::Read => "pull", - Level::Write => "pull,push", - Level::Delete => "delete", - } -} - -fn claims_allow(claims: &Claims, name: &str, action: Level) -> bool { - match action { - Level::Read => claims.grants(name, "pull"), - Level::Write => claims.grants(name, "push"), - Level::Delete => claims.grants(name, "delete"), - } -} - -/// The access level a request needs. -fn action_of(endpoint: &Endpoint, method: &Method) -> Level { - match (endpoint, method) { - (Endpoint::UploadStart { .. } | Endpoint::Upload { .. }, _) => Level::Write, - (_, &Method::PUT | &Method::POST | &Method::PATCH) => Level::Write, - (_, &Method::DELETE) => Level::Delete, - _ => Level::Read, - } -} - -fn endpoint_label(endpoint: &Endpoint) -> String { - match endpoint { - Endpoint::Manifest { reference, .. } => format!("manifests/{reference}"), - Endpoint::Blob { digest, .. } => format!("blobs/{digest}"), - Endpoint::UploadStart { .. } => "blobs/uploads/".into(), - Endpoint::Upload { id, .. } => format!("blobs/uploads/{id}"), - Endpoint::TagsList { .. } => "tags/list".into(), - Endpoint::Referrers { digest, .. } => format!("referrers/{digest}"), - Endpoint::Base | Endpoint::Token | Endpoint::Catalog => String::new(), - } -} - -/// Browser hardening of registry responses: content (layers, manifests, -/// artifacts) is never rendered, sniffed or framed. -fn harden(headers: &mut HeaderMap) { - headers.insert( - header::CONTENT_SECURITY_POLICY, - HeaderValue::from_static("default-src 'none'; sandbox"), - ); - headers.insert( - header::X_CONTENT_TYPE_OPTIONS, - HeaderValue::from_static("nosniff"), - ); -} - -/// An authorization decision. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum Decision { - Allow, - /// RBAC allows it, but the registry token presented does not cover it: - /// the client should fetch a token for this scope. - NeedsScope, - Deny, -} - -/// The host a request reached and its token realm. -#[derive(Debug, Clone)] -pub(crate) struct Site { - /// Token `service` and audience: the host the registry was reached on. - pub service: String, - /// The token endpoint URL. - pub realm: String, - pub issuer: String, - /// The repository this host serves (host-based routing). - pub host_repo: Option, -} - -impl Site { - /// A `WWW-Authenticate: Bearer` challenge. - pub fn challenge(&self, scope: Option<&str>, error: Option<&str>) -> HeaderValue { - let mut value = format!( - "Bearer realm=\"{}\",service=\"{}\"", - self.realm, self.service - ); - if let Some(scope) = scope { - value.push_str(&format!(",scope=\"{scope}\"")); - } - if let Some(error) = error { - value.push_str(&format!(",error=\"{error}\"")); - } - HeaderValue::from_str(&value).unwrap_or_else(|_| HeaderValue::from_static("Bearer")) - } -} - -/// The resolved caller of a request. -#[derive(Debug, Clone)] -pub(crate) struct Caller { - pub principal: Principal, - /// The registry token presented, if any. - pub claims: Option, -} - -/// A repository and image a request addresses. -#[derive(Debug, Clone)] -pub(crate) struct Target { - pub repo: Repository, - pub image: String, - /// The `` as the client wrote it (what `Location` headers and - /// token scopes use). - pub name: String, -} - -/// Everything a repository endpoint handler needs. -pub(crate) struct RequestCtx { - pub caller: Caller, - pub site: Site, - /// The repository the request addressed. - pub target: Target, - /// Where a push or delete lands (`None` for pulls). - pub write_target: Option, -} - -impl RequestCtx { - /// The repository writes go to (the addressed one for pulls). - pub fn serving(&self) -> &Target { - self.write_target.as_ref().unwrap_or(&self.target) - } -} - -/// The `Location` path of a resource of `name`. -pub(crate) fn location(name: &str, rest: &str) -> String { - format!("/v2/{name}/{rest}") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn host_ports_are_stripped() { - assert_eq!(strip_port("docker.example.com:443"), "docker.example.com"); - assert_eq!(strip_port("docker.example.com"), "docker.example.com"); - assert_eq!(strip_port("[::1]:5000"), "::1"); - } - - #[test] - fn actions_by_endpoint() { - let m = Endpoint::Manifest { - name: "a/b".into(), - reference: "x".into(), - }; - assert_eq!(action_of(&m, &Method::GET), Level::Read); - assert_eq!(action_of(&m, &Method::HEAD), Level::Read); - assert_eq!(action_of(&m, &Method::PUT), Level::Write); - assert_eq!(action_of(&m, &Method::DELETE), Level::Delete); - let u = Endpoint::Upload { - name: "a/b".into(), - id: "x".into(), - }; - assert_eq!(action_of(&u, &Method::GET), Level::Write); - assert_eq!(action_of(&u, &Method::DELETE), Level::Write); - } - - #[test] - fn bearer_values() { - let mut h = HeaderMap::new(); - h.insert( - header::AUTHORIZATION, - HeaderValue::from_static("Bearer abc"), - ); - assert_eq!(bearer_token(&h), Some("abc")); - h.insert(header::AUTHORIZATION, HeaderValue::from_static("Basic abc")); - assert_eq!(bearer_token(&h), None); - } - - #[test] - fn sessions_never_mutate() { - assert!(!mutation_credential(&Principal::User { - user_id: 1, - token_id: None, - method: AuthMethod::Session - })); - assert!(mutation_credential(&Principal::User { - user_id: 1, - token_id: None, - method: AuthMethod::Password - })); - assert!(!mutation_credential(&Principal::Unauthenticated)); - } -} diff --git a/crates/formats/docker/src/upstream.rs b/crates/formats/docker/src/upstream.rs new file mode 100644 index 00000000..a29cd1de --- /dev/null +++ b/crates/formats/docker/src/upstream.rs @@ -0,0 +1,479 @@ +//! The upstream side of proxy repositories: what to ask an upstream +//! registry (Docker Hub, GHCR, Quay, MCR or any distribution-spec +//! registry) for, and what its answers say. +//! +//! * Docker Hub's implicit `library/` namespace applies to one-component +//! names (`nginx` is `library/nginx`), so both spellings share one cache +//! ([`UpstreamOp::ImageName`]). +//! * A manifest's and a blob's URLs ([`UpstreamOp::ImageManifest`], +//! [`UpstreamOp::ImageBlob`]): the host sends them, answers the +//! upstream's token challenge, verifies and caches what comes back. +//! * A tag list follows the upstream's own pages (`Link: rel="next"`, back +//! to the upstream only) to fill one page of Forge's; an upstream that +//! refuses a large `n` (400) is asked again without one. +//! * Referrers come from the upstream's referrers API, where it has one. + +use std::collections::BTreeSet; + +use forge_plugin_api::images::{ImageUrl, ManifestInfo, Reference, TagPage}; +use forge_plugin_api::{ + Fetch, FetchResult, Fetched, FormatError, Freshness, Remote, UpstreamAnswer, UpstreamOp, + UpstreamStep, +}; +use serde_json::json; + +use crate::digest::Digest; +use crate::manifest::media; +use crate::reference; + +/// Hosts of Docker Hub's registry, whose one-component names live in +/// `library/`. +const DOCKER_HUB_HOSTS: &[&str] = &[ + "registry-1.docker.io", + "index.docker.io", + "docker.io", + "registry.hub.docker.com", +]; + +/// Pages of an upstream's tag list followed to fill one page of Forge's: an +/// upstream that pages in small steps is listed further on the client's +/// next request. +pub const MAX_UPSTREAM_TAG_PAGES: usize = 10; + +/// The upstream side of a proxy: one step of `op`. +pub fn step( + remote: &Remote, + op: &UpstreamOp, + fetched: &[FetchResult], +) -> Result { + let base = upstream_base(&remote.url); + let done = |answer| Ok(UpstreamStep::Done(answer)); + match op { + UpstreamOp::ImageName(image) => { + done(UpstreamAnswer::ImageName(proxied_image(&remote.url, image))) + } + UpstreamOp::ImageManifest { image, reference } => { + let reference = match reference { + Reference::Tag(tag) if !crate::reference::valid_tag(tag) => { + return Err(FormatError::InvalidVersion(format!("invalid tag {tag:?}"))) + } + Reference::Tag(tag) => tag, + Reference::Digest(digest) => digest, + }; + done(UpstreamAnswer::ImageUrl(ImageUrl { + url: format!("{base}/v2/{image}/manifests/{reference}"), + accept: Some(media::ACCEPT_ALL.join(", ")), + scope: Some(scope(image)), + })) + } + UpstreamOp::ImageBlob { image, digest } => done(UpstreamAnswer::ImageUrl(ImageUrl { + url: format!("{base}/v2/{image}/blobs/{digest}"), + accept: None, + scope: Some(scope(image)), + })), + UpstreamOp::ImageTags { image, last, n } => { + tags(&base, image, last.as_deref(), *n as usize, fetched) + } + UpstreamOp::ImageReferrers { image, digest } => referrers(&base, image, digest, fetched), + _ => Err(FormatError::Internal( + "Docker proxies have only image operations".into(), + )), + } +} + +/// The upstream base URL of a proxy: `remote_url` without a trailing `/` +/// (and without a `/v2` suffix an admin may have pasted). +pub fn upstream_base(remote_url: &str) -> String { + let base = remote_url.trim_end_matches('/'); + base.strip_suffix("/v2").unwrap_or(base).to_string() +} + +fn is_docker_hub(remote_url: &str) -> bool { + url::Url::parse(remote_url) + .ok() + .and_then(|u| u.host_str().map(str::to_ascii_lowercase)) + .is_some_and(|h| DOCKER_HUB_HOSTS.contains(&h.as_str())) +} + +/// The image name a proxy of `remote_url` stores and asks its upstream for. +pub fn proxied_image(remote_url: &str, image: &str) -> String { + if is_docker_hub(remote_url) && !image.contains('/') { + format!("library/{image}") + } else { + image.to_string() + } +} + +/// The token scope reading `image` needs. +fn scope(image: &str) -> String { + format!("repository:{image}:pull") +} + +fn get(url: String, image: &str, accept: Option<&str>) -> Fetch { + Fetch { + accept: accept.map(str::to_string), + scope: Some(scope(image)), + ..Fetch::new(url, Freshness::Metadata) + } +} + +/// Up to `limit` of the upstream's tags of `image` after `last`: one more +/// is asked for, to know whether there are more, and the upstream's own +/// pages are followed for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An +/// upstream that refuses so large an `n` (400, as registries that cap it +/// answer) is asked again without one: its own page size and links take +/// over. The steps so far are in `fetched`. +fn tags( + base: &str, + image: &str, + last: Option<&str>, + limit: usize, + fetched: &[FetchResult], +) -> Result { + let first = |n: Option| { + let mut query = url::form_urlencoded::Serializer::new(String::new()); + if let Some(n) = n { + query.append_pair("n", &n.to_string()); + } + if let Some(last) = last { + query.append_pair("last", last); + } + let query = query.finish(); + let url = format!("{base}/v2/{image}/tags/list"); + let url = if query.is_empty() { + url + } else { + format!("{url}?{query}") + }; + get(url, image, None) + }; + if fetched.is_empty() { + return Ok(UpstreamStep::Fetch(vec![first(Some( + limit.saturating_add(1), + ))])); + } + let mut tags: BTreeSet = BTreeSet::new(); + let mut pages = 0; + let mut left_out = false; + for (i, result) in fetched.iter().enumerate() { + let latest = i + 1 == fetched.len(); + if pages == 0 { + if result.outcome == Fetched::NotFound { + return Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(None))); + } + if i == 0 && result.status == Some(400) { + if latest { + return Ok(UpstreamStep::Fetch(vec![first(None)])); + } + continue; + } + } + let body = match &result.outcome { + Fetched::Found(body) => body, + Fetched::NotFound => { + return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( + "the upstream lost a page of the tag list".into(), + ))) + } + Fetched::Failed(reason) => { + return Ok(UpstreamStep::Done(UpstreamAnswer::Failed(reason.clone()))) + } + }; + pages += 1; + #[derive(serde::Deserialize)] + struct TagList { + #[serde(default)] + tags: Option>, + } + let Ok(list) = serde_json::from_slice::(body) else { + return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( + "the upstream served an invalid tag list".into(), + ))); + }; + tags.extend( + list.tags + .unwrap_or_default() + .into_iter() + .filter(|t| reference::valid_tag(t)) + .filter(|t| last.is_none_or(|l| t.as_bytes() > l.as_bytes())), + ); + if tags.len() > limit { + break; + } + if latest { + match next_link(result, base) { + Some(next) if pages < MAX_UPSTREAM_TAG_PAGES => { + return Ok(UpstreamStep::Fetch(vec![get(next, image, None)])) + } + Some(_) => left_out = true, + None => {} + } + } + } + let more = left_out || tags.len() > limit; + Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(Some( + TagPage { + tags: tags.into_iter().take(limit).collect(), + more, + }, + )))) +} + +/// The `next` page a tag list's `Link` header names (RFC 8288), resolved +/// against the page's URL: only a link back to the upstream itself is +/// followed, never one to another host. +fn next_link(result: &FetchResult, base: &str) -> Option { + let page = url::Url::parse(&result.fetch.url).ok()?; + let base = url::Url::parse(base).ok()?; + result + .headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("link")) + .flat_map(|(_, value)| value.split(',')) + .find_map(|link| { + let mut parts = link.split(';'); + let target = parts.next()?.trim().strip_prefix('<')?.strip_suffix('>')?; + let next = parts.any(|param| { + let param = param.trim().replace(' ', ""); + param.eq_ignore_ascii_case("rel=\"next\"") || param.eq_ignore_ascii_case("rel=next") + }); + if !next { + return None; + } + // Both as the URL parser writes them (lowercase host, no + // default port). + let url = page.join(target).ok()?; + url.as_str() + .strip_prefix(base.as_str().trim_end_matches('/')) + .filter(|path| path.starts_with("/v2/")) + .map(|_| url.to_string()) + }) +} + +/// The upstream's referrers of `digest`; `None` when it has no referrers +/// API (clients fall back to the tag scheme, which a proxy serves like any +/// tag) or it failed. +fn referrers( + base: &str, + image: &str, + digest: &str, + fetched: &[FetchResult], +) -> Result { + let Some(result) = fetched.first() else { + return Ok(UpstreamStep::Fetch(vec![get( + format!("{base}/v2/{image}/referrers/{digest}"), + image, + Some(media::OCI_INDEX), + )])); + }; + let none = || Ok(UpstreamStep::Done(UpstreamAnswer::ImageReferrers(None))); + let Fetched::Found(body) = &result.outcome else { + return none(); + }; + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase")] + struct Desc { + media_type: Option, + digest: String, + size: i64, + artifact_type: Option, + annotations: Option, + } + #[derive(serde::Deserialize)] + struct Index { + #[serde(default)] + manifests: Vec, + } + let Ok(index) = serde_json::from_slice::(body) else { + return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( + "the upstream served an invalid referrers list".into(), + ))); + }; + let found = index + .manifests + .into_iter() + .filter(|d| Digest::parse(&d.digest).is_ok() && d.size >= 0) + .map(|d| ManifestInfo { + digest: d.digest, + media_type: d.media_type.unwrap_or_else(|| media::OCI_MANIFEST.into()), + size: d.size as u64, + artifact_type: d.artifact_type, + subject: Some(digest.to_string()), + summary: d + .annotations + .map(|annotations| json!({ "annotations": annotations }).to_string()), + }) + .collect(); + Ok(UpstreamStep::Done(UpstreamAnswer::ImageReferrers(Some( + found, + )))) +} + +#[cfg(test)] +mod tests { + use forge_plugin_api::testing::converse_answered; + + use super::*; + use crate::Docker; + + fn remote(url: &str) -> Remote { + Remote { + url: url.into(), + index: None, + } + } + + #[test] + fn docker_hub_library_names() { + let hub = "https://registry-1.docker.io"; + assert_eq!(proxied_image(hub, "nginx"), "library/nginx"); + assert_eq!(proxied_image(hub, "bitnami/nginx"), "bitnami/nginx"); + assert_eq!(proxied_image("https://ghcr.io", "nginx"), "nginx"); + assert_eq!(upstream_base("https://ghcr.io/v2/"), "https://ghcr.io"); + assert_eq!( + upstream_base("https://r.example/prefix/"), + "https://r.example/prefix" + ); + } + + #[test] + fn manifest_and_blob_urls() { + let op = UpstreamOp::ImageManifest { + image: "library/nginx".into(), + reference: Reference::Tag("1.27".into()), + }; + let UpstreamStep::Done(UpstreamAnswer::ImageUrl(url)) = + step(&remote("https://registry-1.docker.io/v2/"), &op, &[]).unwrap() + else { + panic!("expected a URL"); + }; + assert_eq!( + url.url, + "https://registry-1.docker.io/v2/library/nginx/manifests/1.27" + ); + assert!(url.accept.unwrap().contains(media::OCI_INDEX)); + assert_eq!(url.scope.as_deref(), Some("repository:library/nginx:pull")); + } + + #[test] + fn tag_lists_follow_the_upstreams_pages() { + let base = "http://up.test"; + let mut pages = vec![ + ( + vec!["a", "b"], + Some("; rel=\"next\""), + ), + ( + vec!["c", "d"], + Some("; rel=\"next\""), + ), + ] + .into_iter(); + let op = UpstreamOp::ImageTags { + image: "team/app".into(), + last: None, + n: 10, + }; + let conversed = converse_answered(&Docker, &remote(base), &op, |fetch| { + let (tags, link) = pages.next().expect("no more pages"); + FetchResult { + fetch: fetch.clone(), + outcome: Fetched::Found(serde_json::to_vec(&json!({ "tags": tags })).unwrap()), + status: Some(200), + headers: link + .map(|l| vec![("link".to_string(), l.to_string())]) + .unwrap_or_default(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageTags(Some(TagPage { + tags: vec!["a".into(), "b".into(), "c".into(), "d".into()], + more: false, + })) + ); + assert_eq!( + conversed.fetches[0].url, + "http://up.test/v2/team/app/tags/list?n=11" + ); + assert_eq!( + conversed.fetches[1].url, + "http://up.test/v2/team/app/tags/list?n=2&last=b" + ); + assert_eq!(conversed.fetches.len(), 2, "no link off the upstream"); + } + + #[test] + fn an_upstream_refusing_a_large_n_is_asked_without_one() { + let op = UpstreamOp::ImageTags { + image: "team/app".into(), + last: Some("a".into()), + n: 1, + }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + let sized = fetch.url.contains("n="); + FetchResult { + fetch: fetch.clone(), + outcome: if sized { + Fetched::Failed("400".into()) + } else { + Fetched::Found(br#"{"tags":["a","b","c"]}"#.to_vec()) + }, + status: Some(if sized { 400 } else { 200 }), + headers: Vec::new(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageTags(Some(TagPage { + tags: vec!["b".into()], + more: true, + })) + ); + assert_eq!( + conversed.fetches[1].url, + "http://up.test/v2/team/app/tags/list?last=a" + ); + } + + #[test] + fn an_unknown_image_has_no_tags_and_referrers_need_an_api() { + let op = UpstreamOp::ImageTags { + image: "team/app".into(), + last: None, + n: 5, + }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + FetchResult { + fetch: fetch.clone(), + outcome: Fetched::NotFound, + status: Some(404), + headers: Vec::new(), + } + }) + .unwrap(); + assert_eq!(conversed.answer, UpstreamAnswer::ImageTags(None)); + + let digest = format!("sha256:{}", "ab".repeat(32)); + let op = UpstreamOp::ImageReferrers { + image: "team/app".into(), + digest: digest.clone(), + }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + FetchResult { + fetch: fetch.clone(), + outcome: Fetched::NotFound, + status: Some(404), + headers: Vec::new(), + } + }) + .unwrap(); + assert_eq!(conversed.answer, UpstreamAnswer::ImageReferrers(None)); + assert_eq!( + conversed.fetches[0].accept.as_deref(), + Some(media::OCI_INDEX) + ); + } +} diff --git a/crates/formats/npm/src/proxy.rs b/crates/formats/npm/src/proxy.rs index 5bfbf502..e406cf5c 100644 --- a/crates/formats/npm/src/proxy.rs +++ b/crates/formats/npm/src/proxy.rs @@ -163,6 +163,11 @@ pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> Up UpstreamOp::Search(query) => search(remote, query, fetched), // A registry holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), + UpstreamOp::ImageName(_) + | UpstreamOp::ImageManifest { .. } + | UpstreamOp::ImageBlob { .. } + | UpstreamOp::ImageTags { .. } + | UpstreamOp::ImageReferrers { .. } => failed("a registry of packages holds no images"), } } diff --git a/crates/formats/nuget/src/upstream.rs b/crates/formats/nuget/src/upstream.rs index 23a46a05..a2f3f04c 100644 --- a/crates/formats/nuget/src/upstream.rs +++ b/crates/formats/nuget/src/upstream.rs @@ -141,6 +141,11 @@ pub(crate) fn step( UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), // NuGet has no tags. UpstreamOp::Tags(_) => done(UpstreamAnswer::Tags(None)), + UpstreamOp::ImageName(_) + | UpstreamOp::ImageManifest { .. } + | UpstreamOp::ImageBlob { .. } + | UpstreamOp::ImageTags { .. } + | UpstreamOp::ImageReferrers { .. } => failed("a feed holds no images"), }) } diff --git a/crates/plugin-api/src/images.rs b/crates/plugin-api/src/images.rs new file mode 100644 index 00000000..70a5c15c --- /dev/null +++ b/crates/plugin-api/src/images.rs @@ -0,0 +1,491 @@ +//! Images formats: what a format whose descriptor says +//! [`Storage::Images`](crate::Storage::Images) (Docker / OCI) and the host's +//! image store exchange. +//! +//! An image repository holds images by name (`team/app`): content-addressed +//! blobs and manifests linked into each image, tags that point at +//! manifests, and upload sessions. The format owns the client protocol: the +//! grammar of its paths, names, references and digests, what a manifest +//! names, its error answers and the URLs of an upstream registry. The host +//! owns the store, the registry's authentication (its token service) and +//! authorization, groups, proxies' caching and retention. +//! +//! A request reaches an images format in three steps: +//! +//! 1. [`FormatPlugin::image_path`] splits the request's path below the +//! format's mount into the name the client wrote and the endpoint, and +//! says what the request needs ([`Permission`]). +//! 2. The host resolves the name to a repository and an image (by path: +//! `/`; on a repository's own host: the whole name), +//! authorizes the request, picks where a change lands (a group's writable +//! member) and refuses what it must, as an [`ImageError`] the format +//! renders ([`FormatPlugin::image_error`]). +//! 3. [`FormatPlugin::handle`] gets the request with the endpoint as its +//! path and the resolved [`ImageName`] as [`Request::image`], and answers +//! as for any format: a response, queries to load ([`ImageQuery`]), or an +//! action ([`Action::ServeBlob`], [`Action::ServeManifest`]). +//! +//! Every answer to an image query is a result: a query that failed comes +//! back to [`FormatPlugin::render`] as its [`ImageError`], for the format to +//! answer in its protocol's words. +//! +//! Digests cross the contract in their canonical text form, +//! `sha256:<64 lowercase hex>` or `sha512:<128 lowercase hex>`, as the +//! format validated them; the host verifies content against them and +//! refuses any other form a format names. +//! +//! [`FormatPlugin::image_path`]: crate::FormatPlugin::image_path +//! [`FormatPlugin::image_error`]: crate::FormatPlugin::image_error +//! [`FormatPlugin::handle`]: crate::FormatPlugin::handle +//! [`FormatPlugin::render`]: crate::FormatPlugin::render +//! [`Request::image`]: crate::Request::image +//! [`Action::ServeBlob`]: crate::Action::ServeBlob +//! [`Action::ServeManifest`]: crate::Action::ServeManifest + +/// What a request to an images format needs of the caller +/// ([`ImagePath::permission`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Permission { + /// Read the image: pull manifests, blobs, tags and referrers. + Read, + /// Change the image: push blobs and manifests, and every request of an + /// upload session. + Write, + /// Delete from the image. + Delete, +} + +/// A request path an images format split ([`FormatPlugin::image_path`]). +/// +/// [`FormatPlugin::image_path`]: crate::FormatPlugin::image_path +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImagePath { + /// The name the client wrote (`docker-hosted/team/app`, or `team/app` + /// on a repository's own host), in the format's name grammar. + pub name: String, + /// The rest of the path (`/manifests/latest`): the endpoint, which + /// [`FormatPlugin::handle`] gets as the request's path. + /// + /// [`FormatPlugin::handle`]: crate::FormatPlugin::handle + pub endpoint: String, + /// What the request needs. + pub permission: Permission, +} + +/// The image a request to an images format is for ([`Request::image`]): +/// the name the format split off its path, as the host resolved it. +/// +/// [`Request::image`]: crate::Request::image +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageName { + /// The image within the repository (`team/app`): what queries name. + pub image: String, + /// The name as the client wrote it (`docker-hosted/team/app`): what + /// the format's `Location` headers name. + pub name: String, +} + +/// A manifest's reference: a tag or a digest. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Reference { + Tag(String), + Digest(String), +} + +impl Reference { + /// The tag or the digest. + pub fn as_str(&self) -> &str { + match self { + Reference::Tag(tag) => tag, + Reference::Digest(digest) => digest, + } + } +} + +/// The inclusive byte range of an upload chunk (`Content-Range`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkRange { + pub start: u64, + /// Inclusive. + pub end: u64, +} + +/// Where a blob may be mounted from instead of uploaded +/// ([`ImageQuery::StartUpload`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Mount { + /// The digest of the blob. + pub digest: String, + /// The name, as the client wrote it, of the image to mount it from; the + /// host resolves it as a request's name and mounts only from an image + /// the caller may read. `None`: the image itself (a client confirming + /// it holds the blob). + pub from: Option, +} + +/// What an images format asks of the host's image store. Each names the +/// image of the request ([`ImageName::image`]); the upload queries name the +/// upload session by the id the host gave it. +/// +/// Reads run against the repository the request addressed: a group answers +/// from its members by priority (only those the caller may read; the +/// dependency-confusion guards apply), a proxy from its cache and its +/// upstream. Changes (the upload queries, [`ImageQuery::PutManifest`] and +/// the deletes) run against where the host lands them, the repository +/// itself or a group's writable member; each is asked alone in its round, +/// at most once per request for those reading the request's body +/// ([`ImageQuery::AppendUpload`], [`ImageQuery::FinishUpload`], +/// [`ImageQuery::PutManifest`]), and only by +/// a request authorized for [`Permission::Write`] (the deletes: +/// [`Permission::Delete`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageQuery { + /// A manifest of the image ([`ImageAnswer::Manifest`]). + Manifest { image: String, reference: Reference }, + /// A blob of the image ([`ImageAnswer::Blob`]). A manifest is content + /// too: one the image holds answers when no blob of its digest does. + Blob { image: String, digest: String }, + /// Up to `n` of the image's tags after `last`, in byte order + /// ([`ImageAnswer::Tags`]). + Tags { + image: String, + last: Option, + n: u32, + }, + /// The manifests whose subject is `digest`, those of `artifact_type` + /// only when one is given ([`ImageAnswer::Referrers`]). + Referrers { + image: String, + digest: String, + artifact_type: Option, + }, + /// Start an upload session, or mount a blob without one when it can be + /// ([`ImageAnswer::Started`]). + StartUpload { image: String, mount: Option }, + /// How far an upload session got ([`ImageAnswer::Upload`]). + UploadStatus(String), + /// Append the request body to an upload session, at `range` when the + /// client gave one ([`ImageAnswer::Upload`]). + AppendUpload { + id: String, + range: Option, + }, + /// Append the request body (it may be empty), then end the session and + /// link its content into the image as `digest`, which it must match + /// ([`ImageAnswer::Blob`]). + FinishUpload { + id: String, + digest: String, + range: Option, + }, + /// End an upload session without a blob ([`ImageAnswer::Done`]). + CancelUpload(String), + /// Store the request body as a manifest of the image, under `reference` + /// and `tags` ([`ImageAnswer::Stored`]). The host stages it (at most + /// [`MAX_MANIFEST_BYTES`]), reads it with + /// [`FormatPlugin::parse_manifest`], checks that what it names is in + /// the image and that it matches a digest reference, and points the + /// tags at it as the repository's tag immutability allows. + /// + /// [`FormatPlugin::parse_manifest`]: crate::FormatPlugin::parse_manifest + PutManifest { + image: String, + reference: Reference, + tags: Vec, + }, + /// Delete a tag, or a manifest by digest ([`ImageAnswer::Done`]). + DeleteManifest { image: String, reference: Reference }, + /// Unlink a blob no manifest of the image names ([`ImageAnswer::Done`]). + DeleteBlob { image: String, digest: String }, +} + +impl ImageQuery { + /// Whether the query changes the repository (it is asked alone in its + /// round). + pub fn is_change(&self) -> bool { + !matches!( + self, + ImageQuery::Manifest { .. } + | ImageQuery::Blob { .. } + | ImageQuery::Tags { .. } + | ImageQuery::Referrers { .. } + ) + } + + /// The image the query names, if it names one. + pub fn image(&self) -> Option<&str> { + match self { + ImageQuery::Manifest { image, .. } + | ImageQuery::Blob { image, .. } + | ImageQuery::Tags { image, .. } + | ImageQuery::Referrers { image, .. } + | ImageQuery::StartUpload { image, .. } + | ImageQuery::PutManifest { image, .. } + | ImageQuery::DeleteManifest { image, .. } + | ImageQuery::DeleteBlob { image, .. } => Some(image), + ImageQuery::UploadStatus(_) + | ImageQuery::AppendUpload { .. } + | ImageQuery::FinishUpload { .. } + | ImageQuery::CancelUpload(_) => None, + } + } +} + +/// The largest manifest the host stores or proxies: the spec asks +/// registries to accept at least 4 MiB. +pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; + +/// The largest blob the host lends to +/// [`FormatPlugin::artifact_files`](crate::FormatPlugin::artifact_files), +/// as stored. +pub const MAX_FILES_BYTES: u64 = 16 * 1024 * 1024; + +/// A stored manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManifestInfo { + pub digest: String, + pub media_type: String, + pub size: u64, + /// What [`ParsedManifest::artifact_type`] said. + pub artifact_type: Option, + /// The digest of the manifest it refers to. + pub subject: Option, + /// What [`ParsedManifest::summary`] said, completed with the config's + /// part ([`FormatPlugin::config_summary`]) once the host held the config. + /// + /// [`FormatPlugin::config_summary`]: crate::FormatPlugin::config_summary + pub summary: Option, +} + +/// A stored blob. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlobInfo { + pub digest: String, + pub size: u64, +} + +/// A page of an image's tags. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TagPage { + /// In byte order. + pub tags: Vec, + /// There are tags after these the page leaves out. + pub more: bool, +} + +/// An upload session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UploadState { + /// Its id, for the session's URL. + pub id: String, + /// The bytes received so far. + pub size: u64, +} + +/// What [`ImageQuery::StartUpload`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Started { + /// A session to upload the blob in. + Session(UploadState), + /// The blob is in the image, mounted without an upload. + Mounted(BlobInfo), +} + +/// A manifest [`ImageQuery::PutManifest`] stored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredManifest { + pub digest: String, + /// The digest of the manifest it refers to. + pub subject: Option, +} + +/// The answer to an [`ImageQuery`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageAnswer { + Manifest(Result, ImageError>), + /// The answer to [`ImageQuery::Blob`] and [`ImageQuery::FinishUpload`] + /// (always `Some` once finished). + Blob(Result, ImageError>), + /// `None`: the repository does not know the image. + Tags(Result, ImageError>), + Referrers(Result, ImageError>), + Started(Result), + /// The answer to [`ImageQuery::UploadStatus`] and + /// [`ImageQuery::AppendUpload`]. + Upload(Result), + Stored(Result), + /// The answer to [`ImageQuery::CancelUpload`] and the deletes. + Done(Result<(), ImageError>), +} + +/// Why the host refused a request to an images format, or a query failed: +/// what the format answers in its own words +/// ([`FormatPlugin::image_error`]). The text some carry is the host's, +/// for the client. +/// +/// [`FormatPlugin::image_error`]: crate::FormatPlugin::image_error +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageError { + // ── The request ──────────────────────────────────────────────────────── + /// The path names no endpoint. + NotAnEndpoint, + /// The name is not one the host can resolve (`hint`: how to name one). + NameInvalid { name: String, hint: Option }, + /// No repository goes by the name (said only to a global admin). + NameUnknown { name: String }, + /// Credentials are needed, or a registry token with the scope `name` + /// and `permission` (401). The host adds its challenge. + AuthenticationRequired { scope: Option<(String, Permission)> }, + /// The registry token presented is not valid (401). + InvalidToken, + /// The credentials presented are not valid (401). + InvalidCredentials, + /// The caller may not do this with `name` (403). + Denied { + name: String, + permission: Permission, + }, + /// What the host does not do: the catalog API, a token request it will + /// not answer, a push to a repository that takes none. + Unsupported { status: u16, message: String }, + /// The repository is offline. + Offline, + + // ── The store ────────────────────────────────────────────────────────── + /// The image has no blob of the digest. + BlobUnknown { digest: String }, + /// The image has no such manifest or tag. + ManifestUnknown { reference: String }, + /// The image has no such upload session. + UploadUnknown, + /// The content does not match its digest. + DigestMismatch, + /// A chunk does not continue the upload: it starts elsewhere (or + /// another chunk landed first). `size`: the bytes received so far. + ChunkOutOfOrder { size: u64 }, + /// A chunk is not as long as its range says. `size`: the bytes received + /// so far. + ChunkLength { size: u64 }, + /// The content is larger than the host takes. + TooLarge { limit: u64 }, + /// The request body could not be read (the client aborted, or stalled). + BodyUnreadable, + /// [`FormatPlugin::parse_manifest`] refused the manifest: its text. + /// + /// [`FormatPlugin::parse_manifest`]: crate::FormatPlugin::parse_manifest + ManifestInvalid(String), + /// A manifest names a blob (or, `manifest`, a child manifest) the image + /// does not hold. + ManifestBlobUnknown { digest: String, manifest: bool }, + /// The tag exists and the repository's tags are immutable. + TagImmutable { tag: String }, + /// A manifest of the image names the blob. + BlobReferenced { digest: String }, + /// The workload identity behind the request may no longer change the + /// repository. + NoLongerAllowed, + /// A proxy's upstream rate-limits it. + RateLimited, + /// The host cannot answer now (storage, an upstream, the staging area): + /// what to tell the client. Worth a retry. + Unavailable(String), + /// The host failed; the details are in its log. + Internal, +} + +/// What a manifest names and says ([`FormatPlugin::parse_manifest`]). +/// +/// [`FormatPlugin::parse_manifest`]: crate::FormatPlugin::parse_manifest +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedManifest { + /// Its media type: what it is served as. + pub media_type: String, + /// The blobs it names (an image's config and layers), each once. + pub blobs: Vec, + /// The digests of the manifests it names (an index's children), each + /// once. A push needs them in the image. + pub manifests: Vec, + /// The digest of the manifest it refers to (a signature's, an SBOM's). + pub subject: Option, + /// What the referrers API reports it as. + pub artifact_type: Option, + /// The blob whose content completes the summary (an image's config, for + /// its platform; a chart's, for its metadata), when it is small enough + /// to read: the host reads it once the image holds it and merges what + /// [`FormatPlugin::config_summary`] makes of it into the summary. + /// + /// [`FormatPlugin::config_summary`]: crate::FormatPlugin::config_summary + pub config: Option, + /// The blob holding the files the web UI shows of the artifact (a Helm + /// chart's archive), whatever its size: the host lends it to + /// [`FormatPlugin::artifact_files`] when it holds it and it is at most + /// [`MAX_FILES_BYTES`]. A manifest that names one is a chart to the web + /// UI, whose metadata is the summary's `chart` object. + /// + /// [`FormatPlugin::artifact_files`]: crate::FormatPlugin::artifact_files + pub files: Option, + /// A JSON object for display (platforms, sizes, annotations), stored + /// with the manifest and handed back with it ([`ManifestInfo::summary`]). + /// When it names a [`Self::config`], it names its digest under + /// `config_digest`: a proxy completes the summaries naming a config it + /// fetches later. + pub summary: String, +} + +/// A blob a manifest names. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ManifestBlob { + pub digest: String, + pub size: u64, + /// Whether a push needs it in the image: `false` for content that lives + /// elsewhere by definition (a non-distributable layer). + pub required: bool, + /// Its content, when the format knows it (the OCI empty JSON, `{}`): + /// the host links it into the image itself when a push names it. + pub known: Option>, +} + +/// The blob that completes a manifest's summary ([`ParsedManifest::config`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConfigBlob { + pub digest: String, + pub media_type: String, + pub size: u64, +} + +/// A text file of an artifact the web UI shows. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TextFile { + /// Its name in the artifact (`README.md`, `values.yaml`). + pub name: String, + /// Its text (invalid UTF-8 replaced), cut where the format bounds it. + pub text: String, + /// Whether the text was cut. + pub truncated: bool, +} + +/// What [`FormatPlugin::artifact_files`] read from an artifact. +/// +/// [`FormatPlugin::artifact_files`]: crate::FormatPlugin::artifact_files +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ArtifactFiles { + /// Its README. + pub readme: Option, + /// Its default values (a Helm chart's `values.yaml`). + pub values: Option, +} + +/// An upstream registry request an images format names +/// ([`UpstreamAnswer::ImageUrl`]): a manifest's or a blob's. +/// +/// [`UpstreamAnswer::ImageUrl`]: crate::UpstreamAnswer::ImageUrl +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageUrl { + /// On the origin of the remote's URL. + pub url: String, + /// The `Accept` header to send (a manifest's media types). + pub accept: Option, + /// The scope of the token to ask the upstream's token service for when + /// it answers 401 with a Bearer challenge that names none. + pub scope: Option, +} diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 533bfab5..b57cb61b 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -7,17 +7,22 @@ //! package bytes — so a plugin is small and never sees a credential, a //! storage key, or a whole package in memory. //! -//! A format stores package versions, or files under paths (raw): its -//! [`Descriptor`] says which (`storage`). A files format has no identity -//! rules and no packages to read: it keeps the defaults of those methods and -//! implements the protocol over the host's file store ([`Query::File`] … -//! [`Query::Remove`], [`Action::ServeFile`]). +//! A format stores package versions, files under paths (raw), or images +//! (Docker / OCI): its [`Descriptor`] says which (`storage`). A files format +//! has no identity rules and no packages to read: it keeps the defaults of +//! those methods and implements the protocol over the host's file store +//! ([`Query::File`] … [`Query::Remove`], [`Action::ServeFile`]). An images +//! format implements it over the host's image store ([`images`]): it splits +//! request paths ([`FormatPlugin::image_path`]), reads manifests +//! ([`FormatPlugin::parse_manifest`]) and answers in its own words what the +//! host refused ([`FormatPlugin::image_error`]). //! //! A plugin is a crate under `crates/formats//` implementing //! [`FormatPlugin`], describing itself with a [`Descriptor`], linked into //! the server and called directly (NuGet, raw). pub mod files; +pub mod images; mod types; #[cfg(feature = "testing")] @@ -29,6 +34,8 @@ use std::path::Path; pub use types::*; +use images::{ArtifactFiles, ImageError, ImagePath, ParsedManifest}; + /// A package the host staged, lent to [`FormatPlugin::parse_package`]. pub trait PackageSource: Read + Seek {} @@ -220,4 +227,56 @@ pub trait FormatPlugin: Send + Sync + 'static { "this format has no proxy repositories".into(), )) } + + /// Images formats: split the path of a request below the format's mount + /// (`/docker-hosted/team/app/manifests/latest`) into the name the + /// client wrote and the endpoint, with what the request needs. The host + /// resolves the name and authorizes the request with it before + /// [`Self::handle`]; an error is answered with [`Self::image_error`]. + /// The default knows no endpoint. + fn image_path(&self, method: &str, path: &str) -> Result { + let _ = (method, path); + Err(ImageError::NotAnEndpoint) + } + + /// Images formats: the response to a request the host refused, or to + /// what failed while it served it outside a query ([`ImageError`]). + /// The host adds its own headers (a Bearer challenge). The default is + /// a plain-text error. + fn image_error(&self, error: &ImageError) -> Response { + Response::text(500, &format!("{error:?}")) + } + + /// Images formats: what a manifest names and says. `content_type` is + /// the one it was pushed or fetched with, if any. The host stages a + /// pushed manifest (at most [`images::MAX_MANIFEST_BYTES`]) and checks + /// what it names before it commits it; a proxy's manifests are read the + /// same way. The default reads none. + fn parse_manifest( + &self, + manifest: &[u8], + content_type: Option<&str>, + ) -> Result { + let _ = (manifest, content_type); + Err(FormatError::MalformedManifest( + "this format stores no images".into(), + )) + } + + /// Images formats: the part of a manifest's summary its config blob + /// ([`ParsedManifest::config`], of `media_type`) holds, as a JSON object + /// the host merges into the summary (an image's platform, a chart's + /// metadata). The default has none. + fn config_summary(&self, media_type: &str, config: &[u8]) -> Option { + let _ = (media_type, config); + None + } + + /// Images formats: the files the web UI shows of an artifact, read + /// from the blob [`ParsedManifest::files`] names, which the host lends + /// positioned at its start. The default reads none. + fn artifact_files(&self, blob: &mut dyn PackageSource) -> ArtifactFiles { + let _ = blob; + ArtifactFiles::default() + } } diff --git a/crates/plugin-api/src/testing.rs b/crates/plugin-api/src/testing.rs index e5904eb2..f5fd2eff 100644 --- a/crates/plugin-api/src/testing.rs +++ b/crates/plugin-api/src/testing.rs @@ -151,6 +151,10 @@ pub const MAX_UPSTREAM_ROUNDS: usize = 4; /// The most fetches of one upstream conversation (as the host). pub const MAX_UPSTREAM_FETCHES: usize = 256; +/// The most rounds of one upstream conversation of an images format (as +/// the host): an upstream's tag list may take a page per round. +pub const MAX_IMAGE_UPSTREAM_ROUNDS: usize = 12; + /// What came of an upstream conversation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Conversed { @@ -172,8 +176,31 @@ pub fn converse( op: &UpstreamOp, mut fetch: impl FnMut(&Fetch) -> Fetched, ) -> Result { + converse_answered(plugin, remote, op, |want| FetchResult { + fetch: want.clone(), + outcome: fetch(want), + status: None, + headers: Vec::new(), + }) +} + +/// [`converse`], with `fetch` answering each fetch whole: its outcome, and +/// for an images format the status and `link` headers of the upstream's +/// answer. An images format's conversation may take +/// [`MAX_IMAGE_UPSTREAM_ROUNDS`] rounds. +pub fn converse_answered( + plugin: &dyn FormatPlugin, + remote: &Remote, + op: &UpstreamOp, + mut fetch: impl FnMut(&Fetch) -> FetchResult, +) -> Result { + let rounds = if plugin.descriptor().storage == crate::Storage::Images { + MAX_IMAGE_UPSTREAM_ROUNDS + } else { + MAX_UPSTREAM_ROUNDS + }; let mut fetched: Vec = Vec::new(); - for _ in 0..MAX_UPSTREAM_ROUNDS { + for _ in 0..rounds { match plugin.upstream(remote, op, &fetched)? { UpstreamStep::Done(answer) => { return Ok(Conversed { @@ -190,16 +217,14 @@ pub fn converse( ))); } for want in wanted { - let outcome = fetch(&want); - fetched.push(FetchResult { - fetch: want, - outcome, - }); + let mut answered = fetch(&want); + answered.fetch = want; + fetched.push(answered); } } } } Err(FormatError::Internal(format!( - "needed more than {MAX_UPSTREAM_ROUNDS} rounds" + "needed more than {rounds} rounds" ))) } diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 184eeb7d..b6107504 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -2,6 +2,10 @@ use std::fmt; +use crate::images::{ + ImageAnswer, ImageName, ImageQuery, ImageUrl, ManifestInfo, Reference, TagPage, +}; + /// A dependency on another package of the same format. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Dependency { @@ -122,7 +126,9 @@ pub struct Descriptor { /// One line on what the format serves. pub description: &'static str, /// Where its repositories are served, `{mount}/{repository}/…`: - /// [`DEFAULT_MOUNT`], or `/` and one lowercase word of its own. + /// [`DEFAULT_MOUNT`], or `/` and one lowercase word of its own. An + /// images format's is [`IMAGES_MOUNT`]: its clients name an image + /// `{host}/{repository}/{image}` and ask for it under `/v2`. pub mount: &'static str, /// The repository kinds it supports, each once. `Proxy` only when the /// format implements [`FormatPlugin::upstream`]. @@ -153,6 +159,10 @@ pub enum UpstreamToken { /// Where a format's repositories are served unless it names another mount. pub const DEFAULT_MOUNT: &str = "/repository"; +/// The mount of an images format ([`Storage::Images`]): the registry API's +/// root, where every distribution-spec client asks. +pub const IMAGES_MOUNT: &str = "/v2"; + /// What a format's repositories hold ([`Descriptor::storage`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum Storage { @@ -162,14 +172,18 @@ pub enum Storage { Packages, /// Files under paths, with their digests (raw). Files, + /// Images: blobs and manifests by digest, tags, upload sessions + /// (Docker / OCI; see [`crate::images`]). + Images, } impl Storage { - /// The name the admin API uses: `packages` or `files`. + /// The name the admin API uses: `packages`, `files` or `images`. pub fn as_str(self) -> &'static str { match self { Storage::Packages => "packages", Storage::Files => "files", + Storage::Images => "images", } } } @@ -237,11 +251,18 @@ pub struct Request { pub query: String, /// The request headers the host passes on (lowercase names): accept, /// accept-language, content-type, if-match, if-modified-since, - /// if-none-match, user-agent. Never credentials or cookies. + /// if-none-match, user-agent, and to an images format content-range. + /// Never credentials or cookies. pub headers: Vec<(String, String)>, /// The request body, once the plugin asked for it ([`Action::ReadBody`]); /// `None` before. A `Content-Encoding: gzip` body is inflated. pub body: Option>, + /// Images formats: the image the request is for, as the host resolved + /// the name [`FormatPlugin::image_path`] split off the path; the path + /// is then the endpoint. `None` for every other format. + /// + /// [`FormatPlugin::image_path`]: crate::FormatPlugin::image_path + pub image: Option, } impl Request { @@ -255,6 +276,7 @@ impl Request { query: query.to_string(), headers: Vec::new(), body: None, + image: None, } } @@ -264,6 +286,16 @@ impl Request { self } + /// The request for `image` (images formats), the client having named + /// it `name`. + pub fn with_image(mut self, image: &str, name: &str) -> Self { + self.image = Some(ImageName { + image: image.to_string(), + name: name.to_string(), + }); + self + } + /// Add a header (`name` lowercase). pub fn with_header(mut self, name: &str, value: &str) -> Self { self.headers @@ -597,13 +629,24 @@ pub enum Query { Store(Vec), /// Delete the file at a path. Remove(String), + /// What an images format asks of the image store ([`crate::images`]). + Image(ImageQuery), } impl Query { /// Whether the query changes the repository (it is asked alone in its /// round). pub fn is_change(&self) -> bool { - matches!(self, Query::Form | Query::Store(_) | Query::Remove(_)) + match self { + Query::Form | Query::Store(_) | Query::Remove(_) => true, + Query::Image(query) => query.is_change(), + _ => false, + } + } + + /// Whether the query is about images. + pub fn is_images(&self) -> bool { + matches!(self, Query::Image(_)) } /// Whether the query is about files rather than packages. @@ -703,6 +746,8 @@ pub enum Answer { Stored(Vec), /// Whether there was a file to delete. Removed(bool), + /// The answer to [`Query::Image`]. + Image(ImageAnswer), } /// A query with its answer. @@ -945,6 +990,16 @@ pub enum Action { /// requests and one range, sets its content type and the repository's /// content disposition, and counts the download. ServeFile(String), + /// Images formats: serve a blob of the image that a + /// [`ImageQuery::Blob`] of this request found (read requests). The host + /// streams it with its `Docker-Content-Digest` and `ETag`, answers one + /// `Range`, and records the pull. + ServeBlob { image: String, digest: String }, + /// Images formats: serve a manifest of the image that a + /// [`ImageQuery::Manifest`] of this request found (read requests), as + /// its media type, with its `Docker-Content-Digest` and `ETag`, and + /// record the pull. + ServeManifest { image: String, reference: Reference }, } /// The upstream a proxy repository mirrors ([`FormatPlugin::upstream`]). @@ -993,6 +1048,31 @@ pub enum UpstreamOp { /// caches it as the file, and re-validates it with a conditional request /// once the repository's maximum age has passed. File(String), + /// Images formats: the name the upstream knows an image by, which a + /// proxy caches it under (Docker Hub's `library/` namespace). The other + /// image operations get this name. + ImageName(String), + /// Images formats: where to ask for a manifest of an image + /// ([`UpstreamAnswer::ImageUrl`]). The host sends a `HEAD` to + /// re-validate a tag it cached and a `GET` for the manifest, checks it + /// against its digest (the reference's, else the upstream's + /// `Docker-Content-Digest`), reads it with `parse_manifest`, and caches + /// it. + ImageManifest { image: String, reference: Reference }, + /// Images formats: where to ask for a blob of an image + /// ([`UpstreamAnswer::ImageUrl`]). The host downloads it, checks it + /// against its digest and caches it. + ImageBlob { image: String, digest: String }, + /// Images formats: up to `n` of the upstream's tags of an image after + /// `last`, in byte order ([`UpstreamAnswer::ImageTags`]). + ImageTags { + image: String, + last: Option, + n: u32, + }, + /// Images formats: the upstream's referrers of a manifest + /// ([`UpstreamAnswer::ImageReferrers`]). + ImageReferrers { image: String, digest: String }, } /// How long the host may reuse a fetched document. @@ -1021,12 +1101,20 @@ pub enum Freshness { /// The host sends the repository's credentials only to the origin of /// [`Remote::url`], bounds the response's size and time, inflates a /// gzip-encoded body, and refuses link-local and cloud-metadata targets. +/// +/// An images format's fetches go to the origin of [`Remote::url`] only; the +/// host answers an upstream registry's Bearer challenge with a token from +/// the registry's token service, and does not keep what it fetched. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Fetch { pub url: String, pub freshness: Freshness, - /// The `Accept` header to send, if any (npm's abbreviated metadata). + /// The `Accept` header to send, if any (npm's abbreviated metadata, an + /// images format's manifest media types). pub accept: Option, + /// Images formats: the scope of the token to ask the upstream's token + /// service for when it answers 401 with a Bearer challenge naming none. + pub scope: Option, } impl Fetch { @@ -1036,6 +1124,7 @@ impl Fetch { url: url.into(), freshness, accept: None, + scope: None, } } @@ -1063,6 +1152,11 @@ pub enum Fetched { pub struct FetchResult { pub fetch: Fetch, pub outcome: Fetched, + /// Images formats: the status of the upstream's answer, when one came. + pub status: Option, + /// Images formats: the `link` headers of the upstream's answer (a page + /// of a listing names the next one there). + pub headers: Vec<(String, String)>, } /// The upstream's metadata of one version. @@ -1153,6 +1247,16 @@ pub enum UpstreamAnswer { /// The file's URL, on an origin a [`Fetch`] may reach; `None`: the /// upstream cannot have the path. File(Option), + /// The answer to [`UpstreamOp::ImageName`]. + ImageName(String), + /// The answer to [`UpstreamOp::ImageManifest`] and + /// [`UpstreamOp::ImageBlob`]. + ImageUrl(ImageUrl), + /// `None`: the upstream does not know the image. + ImageTags(Option), + /// `None`: the upstream offers no referrers API (the host lists the + /// referrers its cache holds). + ImageReferrers(Option>), /// The upstream answered something unusable: the host serves what it /// has cached. Failed(String), diff --git a/crates/plugin/src/descriptor.rs b/crates/plugin/src/descriptor.rs index eefbb462..eda5db0c 100644 --- a/crates/plugin/src/descriptor.rs +++ b/crates/plugin/src/descriptor.rs @@ -1,26 +1,19 @@ //! What a format's [`Descriptor`] may say: a well-formed id that no other //! format and none of the server's names takes, a mount the server leaves -//! free, its repository kinds once each, plain asset names and content +//! free (an images format's: the registry's, [`IMAGES_MOUNT`], and only one +//! format's), its repository kinds once each, plain asset names and content //! types. A test over the server's built-in formats //! (`plugins::builtin`) checks them, so a bad descriptor fails `cargo test`. -use forge_plugin_api::{Assets, Descriptor}; +use forge_plugin_api::{Assets, Descriptor, Storage, IMAGES_MOUNT}; -/// Format ids a plugin can never take: the built-in Docker registry's, and -/// names the server routes or reserves. -const RESERVED_IDS: &[&str] = &[ - "docker", - "v2", - "v3", - "admin", - "browse", - "auth", - "repository", -]; +/// Format ids a plugin can never take: names the server routes or +/// reserves. +const RESERVED_IDS: &[&str] = &["v2", "v3", "admin", "browse", "auth", "repository"]; /// Mounts a plugin can never take: the server's own routes, the web UI's /// pages (the first segment of every route in `web/src/routes.ts`) and the -/// Docker registry. +/// registry's, which only an images format takes ([`IMAGES_MOUNT`]). const RESERVED_MOUNTS: &[&str] = &[ "admin", "api", @@ -45,13 +38,22 @@ const RESERVED_MOUNTS: &[&str] = &[ ]; /// Check the descriptors of the formats one server holds: each on its own -/// ([`check_one`]), and no two with the same id. +/// ([`check_one`]), no two with the same id, and at most one images format +/// (the registry serves one). pub fn check(formats: &[&Descriptor]) -> Result<(), String> { for (i, format) in formats.iter().enumerate() { check_one(format)?; - if formats[..i].iter().any(|other| other.id == format.id) { + let before = &formats[..i]; + if before.iter().any(|other| other.id == format.id) { return Err(format!("two formats have the id {:?}", format.id)); } + let images = |d: &Descriptor| d.storage == Storage::Images; + if images(format) && before.iter().any(|other| images(other)) { + return Err(format!( + "format {:?}: only one format may store images", + format.id + )); + } } Ok(()) } @@ -63,7 +65,15 @@ pub fn check_one(descriptor: &Descriptor) -> Result<(), String> { if descriptor.name.trim().is_empty() { return Err(format!("format {id:?}: the name must not be empty")); } - validate_mount(descriptor.mount).map_err(|e| format!("format {id:?}: {e}"))?; + if descriptor.storage == Storage::Images { + if descriptor.mount != IMAGES_MOUNT { + return Err(format!( + "format {id:?}: an images format's mount is {IMAGES_MOUNT:?}" + )); + } + } else { + validate_mount(descriptor.mount).map_err(|e| format!("format {id:?}: {e}"))?; + } let kinds = descriptor.kinds; if kinds.is_empty() { return Err(format!("format {id:?}: kinds must not be empty")); @@ -195,7 +205,6 @@ mod tests { "Helm", "1helm", "helm_charts", - "docker", "v2", "v3", "admin", @@ -237,6 +246,29 @@ mod tests { } } + #[test] + fn an_images_format_takes_the_registrys_mount_alone() { + let images = Descriptor { + id: "oci", + mount: IMAGES_MOUNT, + storage: Storage::Images, + ..GOOD + }; + assert_eq!(check(&[&GOOD, &images]), Ok(())); + // Its mount is the registry's, and only its. + let elsewhere = Descriptor { + mount: "/oci", + ..images + }; + assert!(check(&[&elsewhere]).is_err()); + let second = Descriptor { + id: "oci-two", + ..images + }; + let err = check(&[&images, &second]).unwrap_err(); + assert!(err.contains("only one format may store images"), "{err}"); + } + #[test] fn refuses_invalid_asset_names_and_types() { let with = |assets: Assets| Descriptor { assets, ..GOOD }; diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index dac732ed..064d2cdf 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -249,6 +249,10 @@ impl Format for PluginFormat { self.storage == Storage::Files } + fn stores_images(&self) -> bool { + self.storage == Storage::Images + } + fn package_asset_path(&self) -> &str { self.assets.package } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 62789037..ae2df727 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -21,10 +21,9 @@ forge-cache = { path = "../cache" } forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } -# Docker / OCI, still native: the registry under /v2 (crate::registry). +# The built-in format plugins (crates/formats/docker, crates/formats/nuget, +# crates/formats/npm, crates/formats/raw), linked in. forge-format-docker = { path = "../formats/docker" } -# The built-in format plugins (crates/formats/nuget, crates/formats/npm, -# crates/formats/raw), linked in. forge-format-nuget = { path = "../formats/nuget" } forge-format-npm = { path = "../formats/npm" } forge-format-raw = { path = "../formats/raw" } @@ -81,6 +80,8 @@ bytes = "1" uuid = { workspace = true } # Registry token signing key generation. rand = { workspace = true } +# Registry Bearer tokens (HS256), the same crate the OIDC verifier uses. +jsonwebtoken = { workspace = true } # Outgoing email (invitations): SMTP submission. lettre = { workspace = true } diff --git a/crates/server/src/admin/formats.rs b/crates/server/src/admin/formats.rs index 043ace17..f6db3fde 100644 --- a/crates/server/src/admin/formats.rs +++ b/crates/server/src/admin/formats.rs @@ -1,9 +1,10 @@ //! `GET /admin/formats` — the package formats this server can create -//! repositories of: the format plugins (NuGet, raw files) and Docker. +//! repositories of: the format plugins (Docker / OCI, NuGet, raw files), +//! each as its descriptor describes it. use axum::extract::State; use axum::response::{IntoResponse, Json, Response}; -use forge_db::model::DOCKER_FORMAT; +use forge_plugin::api::Storage; use serde::Serialize; use crate::state::AppState; @@ -37,8 +38,8 @@ pub struct FormatView { pub max_upload_bytes_limit: Option, } -/// `GET /admin/formats`: NuGet first, then Docker, then the other formats -/// (the other plugins, raw files among them) by id. +/// `GET /admin/formats`: NuGet first, then the images formats (Docker), +/// then the other formats (raw files among them), each group by id. pub async fn list(State(state): State) -> Response { let kinds = |id: &str| state.formats.kinds(id).unwrap_or_default(); let ceiling = state.config.max_repository_upload_bytes(); @@ -52,27 +53,19 @@ pub async fn list(State(state): State) -> Response { mount: descriptor.mount.to_string(), version: Some(descriptor.version.to_string()), description: Some(descriptor.description.to_string()), - default_max_upload_bytes: state.config.default_upload_limit(plugin.id()), + default_max_upload_bytes: state.config.default_upload_limit(descriptor.storage), max_upload_bytes_limit: ceiling, } }; let mut plugins: Vec<_> = state.formats.plugins().collect(); - plugins.sort_by_key(|p| p.id()); - let (nuget, others): (Vec<_>, Vec<_>) = plugins.into_iter().partition(|p| p.id() == "nuget"); - let mut formats: Vec = nuget.into_iter().map(|p| view(p)).collect(); - formats.push(FormatView { - id: DOCKER_FORMAT.to_string(), - name: "Docker / OCI".into(), - kinds: kinds(DOCKER_FORMAT), - storage: "images", - mount: "/v2".into(), - version: None, - description: None, - default_max_upload_bytes: state.config.default_upload_limit(DOCKER_FORMAT), - max_upload_bytes_limit: ceiling, + plugins.sort_by_key(|p| { + let group = match p.descriptor().storage { + _ if p.id() == "nuget" => 0, + Storage::Images => 1, + Storage::Packages | Storage::Files => 2, + }; + (group, p.id()) }); - let mut others: Vec = others.into_iter().map(|p| view(p)).collect(); - others.sort_by(|a, b| a.id.cmp(&b.id)); - formats.extend(others); + let formats: Vec = plugins.into_iter().map(|p| view(p)).collect(); Json(formats).into_response() } diff --git a/crates/server/src/admin/imports.rs b/crates/server/src/admin/imports.rs index d76a53b4..7fd3b40e 100644 --- a/crates/server/src/admin/imports.rs +++ b/crates/server/src/admin/imports.rs @@ -38,7 +38,7 @@ use chrono::{DateTime, Utc}; use forge_core::auth::principal::Principal; use forge_core::net::ssrf::validate_url; use forge_db::import_dao::{self, NewImport, NexusImport}; -use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT}; +use forge_db::model::{RepoKind, Repository}; use forge_db::{repository_dao, secret_dao}; use serde::{Deserialize, Serialize}; @@ -350,12 +350,12 @@ pub async fn create( if !(1..=16).contains(&body.concurrency) { return bad_request("concurrency must be between 1 and 16"); } - let docker = repo.format == DOCKER_FORMAT; + let images = state.formats.stores_images(&repo.format); let registry_url = match body .registry_url .map(|u| u.trim().trim_end_matches('/').to_string()) { - Some(_) if !docker => { + Some(_) if !images => { return bad_request("registry_url applies to Docker repositories only") } Some(u) if u.is_empty() => None, @@ -399,7 +399,7 @@ pub async fn create( } Err(e) => return preflight_error("listing the Nexus repositories", e), } - if docker { + if images { let registry = registry_url .clone() .unwrap_or_else(|| default_registry_url(&url, &source_repository)); diff --git a/crates/server/src/admin/proxy_cache.rs b/crates/server/src/admin/proxy_cache.rs index 4f004057..4628f1e5 100644 --- a/crates/server/src/admin/proxy_cache.rs +++ b/crates/server/src/admin/proxy_cache.rs @@ -80,11 +80,12 @@ pub async fn invalidate( if repo.kind() != RepoKind::Proxy { return bad_request("only a proxy repository caches what an upstream answered"); } - let docker = repo.is_docker(); + let stores_images = state.formats.stores_images(&repo.format); let stores_files = state.formats.stores_files(&repo.format); - // The package as the proxy stores it: a docker image under Docker Hub's - // `library/` spelling, any other id lowercased. + // The package as the proxy stores it: an image under the name its + // upstream knows it by (Docker Hub's `library/` spelling), any other id + // lowercased. let package = match body.and_then(|b| b.0.package) { None => None, Some(p) if stores_files => { @@ -98,11 +99,15 @@ pub async fn invalidate( Err(e) => return bad_request(format!("package: {e}")), } } - Some(p) if docker => { - if !forge_format_docker::reference::valid_name(&p) { + Some(p) if stores_images => { + let format = match state.formats.for_repository(&repo) { + Ok(format) => format, + Err(e) => return format_unavailable(e), + }; + if format.validate_id(&p).is_err() { return bad_request("package is not a valid image name"); } - Some(forge_format_docker::proxied_image(&repo, &p)) + Some(state.registry().proxied_image(&repo, &p)) } Some(p) => { let format = match state.formats.for_repository(&repo) { @@ -117,7 +122,15 @@ pub async fn invalidate( } }; - match forget(&state, &repo, docker, stores_files, package.as_deref()).await { + match forget( + &state, + &repo, + stores_images, + stores_files, + package.as_deref(), + ) + .await + { Ok(Some(invalidated)) => { tracing::info!( repository = %repo.name, @@ -140,7 +153,7 @@ pub async fn invalidate( async fn forget( state: &AppState, repo: &forge_db::model::Repository, - docker: bool, + stores_images: bool, stores_files: bool, package: Option<&str>, ) -> sqlx::Result> { @@ -158,7 +171,7 @@ async fn forget( } else { raw_dao::expire_file_check(&mut *tx, repo.id, prefix).await? }); - } else if docker { + } else if stores_images { invalidated.tags_expired = oci_dao::expire_tags(&mut tx, repo.id, package).await?; } else { invalidated.not_found_cleared = diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index 323d98aa..8e4c82cc 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -23,12 +23,13 @@ use forge_core::format::Format; use forge_core::net::ssrf::validate_url; use forge_db::model::{ DeploymentPolicy, RepoKind, Repository, CONTENT_DISPOSITION_KEY, DEPLOYMENT_POLICY_KEY, - DOCKER_FORMAT, DOCKER_HOST_KEY, IMMUTABLE_TAGS_KEY, LATEST_MUTABLE_KEY, MAX_AGE_SECONDS_KEY, + DOCKER_HOST_KEY, IMMUTABLE_TAGS_KEY, LATEST_MUTABLE_KEY, MAX_AGE_SECONDS_KEY, MAX_UPLOAD_BYTES_KEY, NUGET_FORMAT, PREFER_HOSTED_KEY, RESERVED_PREFIXES_KEY, WRITABLE_MEMBER_ID_KEY, }; use forge_db::repository_dao::{self, NewRepository, RepositoryUpdate}; use forge_db::{component_dao, oci_dao, raw_dao}; +use forge_plugin::api::Storage; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -87,8 +88,9 @@ pub struct RepoView { impl RepoView { /// The view of `r`, naming its writable member as `names` (repository - /// names by id) does; `stores_files` when its format holds files. - fn new(r: &Repository, names: &HashMap, stores_files: bool) -> Self { + /// names by id) does; its format's repositories hold `storage`. + fn new(r: &Repository, names: &HashMap, storage: Storage) -> Self { + let stores_files = storage == Storage::Files; RepoView { id: r.id, name: r.name.clone(), @@ -113,7 +115,7 @@ impl RepoView { .writable_member_id() .and_then(|id| names.get(&id).cloned()), max_upload_bytes: r.max_upload_bytes(), - deployment_policy: (!r.is_docker() && r.kind() == RepoKind::Hosted) + deployment_policy: (storage != Storage::Images && r.kind() == RepoKind::Hosted) .then(|| deployment_policy_of(r).as_str()), content_disposition: stores_files.then(|| { if r.inline_content() { @@ -124,13 +126,7 @@ impl RepoView { }), max_age_seconds: (stores_files && r.kind() == RepoKind::Proxy) .then(|| r.max_age_seconds()), - storage: if r.is_docker() { - "images" - } else if stores_files { - "files" - } else { - "packages" - }, + storage: storage.as_str(), } } } @@ -143,11 +139,7 @@ async fn view(state: &AppState, r: &Repository) -> sqlx::Result { names.insert(id, member.name); } } - Ok(RepoView::new( - r, - &names, - state.formats.stores_files(&r.format), - )) + Ok(RepoView::new(r, &names, state.storage(&r.format))) } /// Create-repository request body. Unknown fields are rejected: a misspelled @@ -399,16 +391,11 @@ const MAX_PACKAGE_DELAY_SECONDS: i32 = 365 * 24 * 60 * 60; const MAX_RESERVED_PREFIXES: usize = 100; /// Validate and normalize a group's reserved id prefixes: each is at most -/// 100 bytes, an optional trailing `*` dropped, deduplicated. A docker -/// group's are image-name prefixes (`A-Z a-z 0-9 . _ - /`, a namespace such -/// as `acme/`), stored lowercase; a package group's are what its format -/// makes of them ([`Format::reserved_prefix`]: NuGet's `A-Z a-z 0-9 . _ -`, -/// lowercase; npm's `@acme` is the scope `@acme/`). -fn normalize_prefixes( - raw: &[String], - docker: bool, - format: Option<&dyn Format>, -) -> Result, String> { +/// 100 bytes, an optional trailing `*` dropped, deduplicated, and what the +/// group's format makes of it ([`Format::reserved_prefix`]: NuGet's +/// `A-Z a-z 0-9 . _ -`, lowercase; npm's `@acme` is the scope `@acme/`; +/// Docker's image-name prefixes may hold a namespace, `acme/`). +fn normalize_prefixes(raw: &[String], format: Option<&dyn Format>) -> Result, String> { if raw.len() > MAX_RESERVED_PREFIXES { return Err(format!( "at most {MAX_RESERVED_PREFIXES} reserved_prefixes are allowed" @@ -423,7 +410,7 @@ fn normalize_prefixes( "reserved_prefixes: {prefix:?} must be 1-100 bytes long" )); } - let prefix = match format.filter(|_| !docker) { + let prefix = match format { Some(format) => format.reserved_prefix(prefix).map_err(|e| match e { forge_core::domain::FormatError::InvalidPackage(m) => { format!("reserved_prefixes: {m}") @@ -431,21 +418,13 @@ fn normalize_prefixes( e => format!("reserved_prefixes: {e}"), })?, None => { - let valid = prefix.bytes().all(|b| { - b.is_ascii_alphanumeric() - || matches!(b, b'.' | b'_' | b'-') - || (docker && b == b'/') - }); + let valid = prefix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); if !valid { - return Err(if docker { - format!( - "reserved_prefixes: {prefix:?} is not an image name prefix (letters, digits, '.', '_', '-', '/')" - ) - } else { - format!( - "reserved_prefixes: {prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" - ) - }); + return Err(format!( + "reserved_prefixes: {prefix:?} is not a package id prefix (letters, digits, '.', '_', '-')" + )); } prefix.to_ascii_lowercase() } @@ -580,7 +559,7 @@ pub async fn list(State(state): State) -> Response { repos.iter().map(|r| (r.id, r.name.clone())).collect(); let views: Vec = repos .iter() - .map(|r| RepoView::new(r, &names, state.formats.stores_files(&r.format))) + .map(|r| RepoView::new(r, &names, state.storage(&r.format))) .collect(); Json(views).into_response() } @@ -627,14 +606,15 @@ pub async fn create(State(state): State, Json(body): Json) body.format )); } - let docker = body.format == DOCKER_FORMAT; - let files = state.formats.stores_files(&body.format); + let storage = state.storage(&body.format); + let images = storage == Storage::Images; + let files = storage == Storage::Files; // A package format plugin's (NuGet's, npm's): package versions. - let package = !files && state.formats.plugin(&body.format).is_some(); + let package = storage == Storage::Packages && state.formats.plugin(&body.format).is_some(); let parsed_kind = RepoKind::from(kind.as_str()); let docker_config = match docker_options( &state, - docker, + images.then_some(body.format.as_str()), package.then_some(body.format.as_str()), parsed_kind, &body.name, @@ -651,10 +631,10 @@ pub async fn create(State(state): State, Json(body): Json) Ok(config) => config, Err(resp) => return resp, }; - if docker && body.package_delay_seconds != 0 { + if images && body.package_delay_seconds != 0 { return bad_request("package_delay_seconds is not supported for docker repositories"); } - if docker && body.allow_redeploy { + if images && body.allow_redeploy { return bad_request( "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); @@ -725,7 +705,7 @@ pub async fn create(State(state): State, Json(body): Json) let reserved_prefixes = match body .reserved_prefixes .as_deref() - .map(|p| normalize_prefixes(p, docker, state.formats.get(&body.format).as_deref())) + .map(|p| normalize_prefixes(p, state.formats.get(&body.format).as_deref())) { Some(Err(msg)) => return bad_request(&msg), Some(Ok(prefixes)) => Some(prefixes), @@ -846,14 +826,15 @@ impl DockerOptions { /// Validate the docker options against the repository's format and kind /// and merge them into `existing` (its stored config object). Returns the -/// config object to store. +/// config object to store. They are the image store's: only a repository +/// of an images format (`images`: its format, Docker) takes them. /// /// A group of a package format (`writable`: its format) takes a writable /// member too: where its clients' publishes and changes land. #[allow(clippy::result_large_err)] async fn docker_options( state: &AppState, - docker: bool, + images: Option<&str>, writable: Option<&str>, kind: RepoKind, name: &str, @@ -861,7 +842,7 @@ async fn docker_options( existing: &serde_json::Map, ) -> Result, Response> { let mut config = existing.clone(); - if !docker { + let Some(images) = images else { let writable_only = options.host.is_none() && options.immutable_tags.is_none() && options.latest_mutable.is_none(); @@ -872,8 +853,15 @@ async fn docker_options( } return writable_member_option(state, writable, kind, options.writable_member, config) .await; - } - if !forge_format_docker::reference::valid_name_component(name) { + }; + // Path-based routing names an image `/`: the + // repository's name must be a valid one (a single component, as a + // repository name has no `/`). + let valid_name = state + .formats + .get(images) + .is_some_and(|format| format.validate_id(name).is_ok()); + if !valid_name { return Err(bad_request(format!( "invalid docker repository name {name:?}: image names start with it, so use \ lowercase letters and digits, separated by '.', '_', '__' or '-'" @@ -903,14 +891,7 @@ async fn docker_options( if let Some(latest) = options.latest_mutable { config.insert(LATEST_MUTABLE_KEY.into(), json!(latest)); } - writable_member_option( - state, - Some(DOCKER_FORMAT), - kind, - options.writable_member, - config, - ) - .await + writable_member_option(state, Some(images), kind, options.writable_member, config).await } /// Merge a group's `writable_member` (a hosted repository of `format`; @@ -1146,22 +1127,23 @@ pub async fn update( } } - let docker = existing.is_docker(); + let storage = state.storage(&existing.format); + let images = storage == Storage::Images; let final_name = body.name.clone().unwrap_or_else(|| existing.name.clone()); - if docker && body.package_delay_seconds.is_some_and(|d| d != 0) { + if images && body.package_delay_seconds.is_some_and(|d| d != 0) { return bad_request("package_delay_seconds is not supported for docker repositories"); } - if docker && body.allow_redeploy == Some(true) { + if images && body.allow_redeploy == Some(true) { return bad_request( "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); } - let files = state.formats.stores_files(&existing.format); - let package = !files && state.formats.plugin(&existing.format).is_some(); + let files = storage == Storage::Files; + let package = storage == Storage::Packages && state.formats.plugin(&existing.format).is_some(); let stored_config = existing.config.as_object().cloned().unwrap_or_default(); let config = match docker_options( &state, - docker, + images.then_some(existing.format.as_str()), package.then_some(existing.format.as_str()), existing.kind(), &final_name, @@ -1233,14 +1215,11 @@ pub async fn update( if existing.kind() != RepoKind::Group { return bad_request("reserved_prefixes is only valid for group repositories"); } - let prefixes = match normalize_prefixes( - prefixes, - docker, - state.formats.get(&existing.format).as_deref(), - ) { - Ok(prefixes) => prefixes, - Err(msg) => return bad_request(&msg), - }; + let prefixes = + match normalize_prefixes(prefixes, state.formats.get(&existing.format).as_deref()) { + Ok(prefixes) => prefixes, + Err(msg) => return bad_request(&msg), + }; if !config.is_object() { config = json!({}); } @@ -1365,7 +1344,7 @@ pub async fn update( // inserting commits before the purge (which removes it), and one // inserting later finds the new upstream and caches nothing // (`oci_dao::lock_proxy_upstream`). - if existing.is_docker() { + if images { match repository_dao::lock_repository(&mut *tx, updated.id).await { Ok(_) => {} Err(e) => return db_error(e), @@ -1493,7 +1472,7 @@ pub async fn delete(State(state): State, Path(name): Path) -> match delete_repository_locked(&state, existing.id, existing.kind(), grace).await { Ok(RepoDelete::Deleted) => StatusCode::NO_CONTENT.into_response(), Ok(RepoDelete::NotFound) => (StatusCode::NOT_FOUND, "repository not found").into_response(), - Ok(RepoDelete::HasPackages) if existing.is_docker() => ( + Ok(RepoDelete::HasPackages) if state.formats.stores_images(&existing.format) => ( StatusCode::CONFLICT, "repository still has images; delete their manifests first", ) @@ -1629,7 +1608,7 @@ pub async fn hard_delete_package( ) .into_response(); } - if repo.is_docker() { + if state.formats.stores_images(&repo.format) { return ( StatusCode::BAD_REQUEST, "delete image manifests and tags through the registry API (DELETE /v2//manifests/)", diff --git a/crates/server/src/admin/retention.rs b/crates/server/src/admin/retention.rs index 8cb5f583..093975a3 100644 --- a/crates/server/src/admin/retention.rs +++ b/crates/server/src/admin/retention.rs @@ -163,7 +163,7 @@ pub async fn put( return bad_request(format!("invalid retention policy: {e}")); } serde_json::to_value(&policy) - } else if repo.is_docker() { + } else if state.formats.stores_images(&repo.format) { let policy: ImageRetentionPolicy = match serde_json::from_value(body.policy) { Ok(p) => p, Err(e) => return unprocessable(e), @@ -330,7 +330,10 @@ pub async fn preview( return Json(PreviewView { decisions: Vec::new(), proxy_decisions: Vec::new(), - image_decisions: repo.is_docker().then(ImageDecisions::default), + image_decisions: state + .formats + .stores_images(&repo.format) + .then(ImageDecisions::default), file_decisions: state.formats.stores_files(&repo.format).then(Vec::new), counts: PreviewCounts::default(), }) @@ -339,7 +342,7 @@ pub async fn preview( Err(e) => return db_error(e), }, }; - if repo.is_docker() { + if state.formats.stores_images(&repo.format) { return preview_images(&state, &repo, policy_json, source, at).await; } if state.formats.stores_files(&repo.format) { diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index 3220d3a6..c538916d 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -151,13 +151,20 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // Forge's own OIDC issuer (discovery + JWKS), public: what an IdP // verifies the client assertions of a `federated_credential` realm // against. 404 until forge has signing keys. - .merge(crate::auth::issuer_router()) - // The Docker / OCI registry: clients always talk to `/v2/` at the - // host root; the registry resolves repositories from the name (or - // the host) itself. - .route("/v2", any(crate::registry::registry)) - .route("/v2/", any(crate::registry::registry)) - .route("/v2/{*rest}", any(crate::registry::registry)); + .merge(crate::auth::issuer_router()); + // The registry of the images format (Docker / OCI) under its mount, + // `/v2`: clients always talk to `/v2/` at the host root; the registry + // resolves repositories from the name (or the host) itself. + let images = state.formats.images_plugin(); + if let Some(mount) = images.as_ref().map(|p| p.descriptor().mount) { + app = app + .route(mount, any(crate::registry::registry)) + .route(&format!("{mount}/"), any(crate::registry::registry)) + .route( + &format!("{mount}/{{*rest}}"), + any(crate::registry::registry), + ); + } // Repositories, each served by its format's plugin under the format's // mount (`crate::plugins`; NuGet at `/v3`, raw files at `/raw`). `any` covers // GET/HEAD/PUT/POST/DELETE; the dispatcher authorizes per method. @@ -183,6 +190,7 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { let hsts = state.cookie_secure; let public_base_url = state.config.public_base_url(); + let images = images.map(|p| p.implementation().clone()); let server = state.config.server.clone(); let app = app .layer(axum::middleware::from_fn_with_state( @@ -192,7 +200,14 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // Caching and transport policy for every response, including the // ones the principal middleware answers itself. .layer(axum::middleware::from_fn(move |req, next| { - response_policy(hsts, feeds.clone(), public_base_url.clone(), req, next) + response_policy( + hsts, + feeds.clone(), + public_base_url.clone(), + images.clone(), + req, + next, + ) })); // A stalled upload or a handler that never answers must not pin a task // and a connection forever (header reads are bounded in `serve`). @@ -274,6 +289,7 @@ async fn response_policy( hsts: bool, feeds: std::sync::Arc>, public_base_url: String, + images: Option>, req: axum::extract::Request, next: axum::middleware::Next, ) -> axum::response::Response { @@ -281,15 +297,18 @@ async fn response_policy( let under = |prefix: &str| crate::web_ui::is_under(path, prefix); let api = under("/admin") || under("/auth") || under("/browse") || under("/api"); let feed = feeds.iter().any(|mount| under(mount)); - let registry = under("/v2"); + let registry = images + .as_ref() + .filter(|images| under(images.descriptor().mount)); let headers = req.headers(); let credentials = headers.contains_key(header::AUTHORIZATION) || headers.contains_key("x-nuget-apikey") || headers.contains_key(header::COOKIE); let mut resp = next.run(req).await; - if registry { - resp = crate::registry::amend_registry_rejection(&public_base_url, resp); + if let Some(images) = registry { + resp = crate::registry::amend_registry_rejection(&public_base_url, images.as_ref(), resp); } + let registry = registry.is_some(); let out = resp.headers_mut(); if api || ((feed || registry) && credentials) { out.insert( diff --git a/crates/server/src/browse.rs b/crates/server/src/browse.rs index 8f4ea550..5f4d64de 100644 --- a/crates/server/src/browse.rs +++ b/crates/server/src/browse.rs @@ -49,6 +49,7 @@ use forge_core::auth::principal::Principal; use forge_core::repo::SearchParams; use forge_db::model::{RepoKind, Repository}; use forge_db::{component_dao, repository_dao}; +use forge_plugin::api::Storage; use serde::{Deserialize, Serialize}; use crate::admin::db_error; @@ -92,8 +93,8 @@ pub struct BrowseRepo { pub format: String, /// The address clients use for the repository (`{public_base_url}{mount}/{name}`, /// for raw `{public_base_url}/raw/{name}`), for the UI's client - /// snippets. Absent for docker repositories, whose registry - /// `/browse/images` names, and for a format not installed here. + /// snippets. Absent for an images format's (Docker's) repositories, whose + /// registry `/browse/images` names, and for a format not installed here. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, /// What the repository holds: `packages`, `files` (a files format: raw, @@ -142,16 +143,18 @@ pub async fn list_repos( kind: repo.kind.clone(), online: repo.online, format: repo.format.clone(), - url: state.formats.plugin(&repo.format).map(|plugin| { - crate::plugins::repository_url(&state, plugin.descriptor().mount, &repo.name) - }), - storage: if repo.is_docker() { - "images" - } else if state.formats.stores_files(&repo.format) { - "files" - } else { - "packages" - }, + url: state + .formats + .plugin(&repo.format) + .filter(|plugin| plugin.descriptor().storage != Storage::Images) + .map(|plugin| { + crate::plugins::repository_url( + &state, + plugin.descriptor().mount, + &repo.name, + ) + }), + storage: state.storage(&repo.format).as_str(), }); } } @@ -250,7 +253,7 @@ pub async fn search_packages( } // Package search covers the package formats; images are listed by // /browse/images, files (raw) by their repository's listings. - if repo.is_docker() || state.formats.stores_files(&repo.format) { + if state.storage(&repo.format) != Storage::Packages { continue; } if state.authz.can_read(&principal, repo).await { @@ -423,9 +426,9 @@ pub async fn package_detail( } let repo = match repository_dao::find_repository_by_name(&state.pool, &repo_name).await { - // A docker repository holds images (`/browse/images`) and a files - // (raw) one files (its listings), not packages. - Ok(Some(r)) if !r.is_docker() && !state.formats.stores_files(&r.format) => r, + // An images (Docker) repository holds images (`/browse/images`) and + // a files (raw) one files (its listings), not packages. + Ok(Some(r)) if state.storage(&r.format) == Storage::Packages => r, // An unknown name answers like an unreadable repository (no name // enumeration), except to a global admin. Ok(_) if state.authz.is_global_admin(&principal).await => { diff --git a/crates/server/src/browse_images.rs b/crates/server/src/browse_images.rs index e3b484bb..03071f33 100644 --- a/crates/server/src/browse_images.rs +++ b/crates/server/src/browse_images.rs @@ -1,5 +1,7 @@ -//! Browse API of docker repositories: the images of a repository and an -//! image's tags and manifests (digests, sizes, platforms), for the SPA. +//! Browse API of the repositories of an images format (Docker): the images +//! of a repository and an image's tags and manifests (digests, sizes, +//! platforms), for the SPA. It reads the image store and the summaries the +//! format made of the manifests. //! //! The same rules as the package browse apply: a signed-in user, per-repo //! read authorization, the caller's content selectors on image names, and a @@ -14,9 +16,12 @@ //! with the `docker pull` name. //! * `GET /browse/chart?repo=&image=&digest=` //! → a Helm chart pushed as an artifact (`helm push`): its `Chart.yaml` -//! metadata, README and default values, read from the blobs Forge holds. +//! metadata, README and default values, read from the blobs Forge holds +//! with the format ([`FormatPlugin::parse_manifest`], +//! [`FormatPlugin::config_summary`], [`FormatPlugin::artifact_files`]). use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; use axum::extract::{Path, Query, State}; use axum::http::{header, StatusCode}; @@ -25,16 +30,24 @@ use chrono::{DateTime, Utc}; use forge_core::auth::principal::Principal; use forge_db::model::{RepoKind, Repository}; use forge_db::{oci_dao, repository_dao}; -use forge_format_docker::chart; -use forge_format_docker::manifest::{self, ManifestKind, MAX_MANIFEST_BYTES}; +use forge_plugin::api::images::{ArtifactFiles, TextFile, MAX_FILES_BYTES, MAX_MANIFEST_BYTES}; +use forge_plugin::FormatPlugin; use forge_storage::BlobRef; use serde::{Deserialize, Serialize}; use crate::admin::db_error; use crate::dispatch::{offline_response, offline_serves_nothing}; +use crate::images::digest::ContentDigest; use crate::middleware::principal_of; use crate::state::AppState; +/// The largest config blob read for a chart's metadata. +const MAX_CONFIG_BYTES: u64 = 1024 * 1024; + +/// The summary key under which the format keeps a chart's metadata +/// ([`FormatPlugin::config_summary`]). +const CHART_KEY: &str = "chart"; + /// Most images considered per repository when listing. const MAX_IMAGES: i64 = 5000; const DEFAULT_TAKE: usize = 50; @@ -119,7 +132,7 @@ pub struct ImageDetail { /// The registry host of the server (`host[:port]` of `public_base_url`). fn registry_host(state: &AppState) -> String { - forge_format_docker::registry_origin(&state.config.public_base_url()).1 + crate::images::registry_origin(&state.config.public_base_url()).1 } /// Where `repo`'s images are pulled from: its own host, or @@ -134,17 +147,18 @@ fn pull_prefix(state: &AppState, repo: &Repository) -> (String, String) { } } -/// The readable docker repository `name`, or the response to send. -async fn readable_docker_repo( +/// The readable repository `name` of an images format, with the format, or +/// the response to send. +async fn readable_images_repo( state: &AppState, principal: &Principal, name: &str, -) -> Result { +) -> Result<(Repository, Arc), Response> { if !matches!(principal, Principal::User { .. }) { return Err((StatusCode::UNAUTHORIZED, "authentication required").into_response()); } let repo = match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(r)) if r.is_docker() => r, + Ok(Some(r)) if state.formats.stores_images(&r.format) => r, Ok(_) if state.authz.is_global_admin(principal).await => { return Err((StatusCode::NOT_FOUND, "repository not found").into_response()) } @@ -159,11 +173,15 @@ async fn readable_docker_repo( if offline_serves_nothing(&repo) { return Err(offline_response()); } - Ok(repo) + let Some(plugin) = state.formats.plugin(&repo.format) else { + return Err((StatusCode::NOT_FOUND, "repository not found").into_response()); + }; + let format = plugin.implementation().clone(); + Ok((repo, format)) } /// The repositories whose images `repo` shows the caller, in priority order: -/// itself, or a group's readable docker members (nested groups expanded). +/// itself, or a group's readable members (nested groups expanded). async fn sources( state: &AppState, principal: &Principal, @@ -186,7 +204,10 @@ async fn sources( for (member, _) in members { // An offline hosted or group member serves nothing through the // group; an offline proxy still serves its cache. - if !member.is_docker() || offline_serves_nothing(&member) || !seen.insert(member.id) { + if member.format != repo.format + || offline_serves_nothing(&member) + || !seen.insert(member.id) + { continue; } if !state.authz.can_read(principal, &member).await { @@ -210,8 +231,8 @@ pub async fn list_images( req: axum::extract::Request, ) -> Response { let principal = principal_of(&req); - let repo = match readable_docker_repo(&state, &principal, &query.repo).await { - Ok(repo) => repo, + let (repo, _) = match readable_images_repo(&state, &principal, &query.repo).await { + Ok(found) => found, Err(resp) => return resp, }; let sources = match sources(&state, &principal, &repo).await { @@ -262,11 +283,11 @@ pub async fn image_detail( req: axum::extract::Request, ) -> Response { let principal = principal_of(&req); - let repo = match readable_docker_repo(&state, &principal, &repo_name).await { - Ok(repo) => repo, + let (repo, format) = match readable_images_repo(&state, &principal, &repo_name).await { + Ok(found) => found, Err(resp) => return resp, }; - if !forge_format_docker::reference::valid_name(&image) { + if format.validate_id(&image).is_err() { return (StatusCode::NOT_FOUND, "image not found").into_response(); } let sources = match sources(&state, &principal, &repo).await { @@ -404,12 +425,12 @@ pub struct ChartQuery { pub struct ChartFileView { pub name: String, pub text: String, - /// Whether the text was cut at [`chart::MAX_FILE_TEXT`] bytes. + /// Whether the text was cut (the format bounds it). pub truncated: bool, } -impl From for ChartFileView { - fn from(file: chart::ChartFile) -> Self { +impl From for ChartFileView { + fn from(file: TextFile) -> Self { Self { name: file.name, text: file.text, @@ -436,12 +457,6 @@ pub struct ChartDetail { pub source: Option, } -/// The chart archive layer's media types: Helm's, and the plain one of the -/// experimental OCI support of Helm before 3.8. -fn is_chart_layer(media_type: &str) -> bool { - media_type == chart::media::CONTENT || media_type == "application/tar+gzip" -} - /// Why a stored blob could not be read. enum ReadFailure { Missing, @@ -455,13 +470,13 @@ async fn read_linked( repo: &Repository, image: &str, digest: &str, - max: i64, + max: u64, ) -> Result, ReadFailure> { let link = oci_dao::find_link(&state.pool, repo.id, image, digest) .await .map_err(|e| ReadFailure::Response(db_error(e)))? .ok_or(ReadFailure::Missing)?; - if link.size > max { + if link.size.max(0) as u64 > max { return Err(ReadFailure::Missing); } read_blob(state, &link.blob_sha256, &link.s3_key, link.size) @@ -515,15 +530,13 @@ pub async fn chart_detail( req: axum::extract::Request, ) -> Response { let principal = principal_of(&req); - let repo = match readable_docker_repo(&state, &principal, &query.repo).await { - Ok(repo) => repo, + let (repo, format) = match readable_images_repo(&state, &principal, &query.repo).await { + Ok(found) => found, Err(resp) => return resp, }; let not_found = || (StatusCode::NOT_FOUND, "chart not found").into_response(); let image = query.image; - if !forge_format_docker::reference::valid_name(&image) - || forge_format_docker::digest::Digest::parse(&query.digest).is_err() - { + if format.validate_id(&image).is_err() || ContentDigest::parse(&query.digest).is_none() { return not_found(); } let sources = match sources(&state, &principal, &repo).await { @@ -561,60 +574,46 @@ pub async fn chart_detail( Ok(bytes) => bytes, Err(resp) => return resp, }; - let Ok(parsed) = manifest::parse(&bytes, Some(&row.media_type)) else { + // A chart is an artifact with files to show (its archive). + let Ok(parsed) = format.parse_manifest(&bytes, Some(&row.media_type)) else { return not_found(); }; - let config = parsed.config.as_ref(); - let is_chart = parsed.kind == ManifestKind::Image - && (config.is_some_and(|c| c.media_type == chart::media::CONFIG) - || parsed.artifact_type.as_deref() == Some(chart::media::CONFIG)); - if !is_chart { + let Some(archive) = parsed.files else { return not_found(); - } + }; let mut metadata = None; - if let Some(config) = config.filter(|c| c.media_type == chart::media::CONFIG) { - match read_linked( - &state, - &source, - &image, - config.digest.as_str(), - chart::MAX_CONFIG_BYTES, - ) - .await - { - Ok(bytes) => metadata = chart::metadata(&bytes), + if let Some(config) = parsed.config { + match read_linked(&state, &source, &image, &config.digest, MAX_CONFIG_BYTES).await { + Ok(bytes) => { + metadata = format + .config_summary(&config.media_type, &bytes) + .and_then(|part| serde_json::from_str::(&part).ok()) + .and_then(|mut part| part.get_mut(CHART_KEY).map(serde_json::Value::take)); + } Err(ReadFailure::Missing) => {} Err(ReadFailure::Response(resp)) => return resp, } } - let mut files = chart::ChartFiles::default(); + let mut files = ArtifactFiles::default(); let mut archive_stored = false; - if let Some(layer) = parsed.layers.iter().find(|l| is_chart_layer(&l.media_type)) { - match read_linked( - &state, - &source, - &image, - layer.digest.as_str(), - chart::MAX_ARCHIVE_BYTES, - ) - .await - { - Ok(archive) => { - archive_stored = true; - files = match tokio::task::spawn_blocking(move || chart::read_files(&archive)).await - { - Ok(files) => files, - Err(e) => { - tracing::error!(error = %e, "browse: reading a chart archive failed"); - return (StatusCode::INTERNAL_SERVER_ERROR, "internal error") - .into_response(); - } - }; - } - Err(ReadFailure::Missing) => {} - Err(ReadFailure::Response(resp)) => return resp, + match read_linked(&state, &source, &image, &archive.digest, MAX_FILES_BYTES).await { + Ok(archive) => { + archive_stored = true; + files = match tokio::task::spawn_blocking(move || { + format.artifact_files(&mut std::io::Cursor::new(archive)) + }) + .await + { + Ok(files) => files, + Err(e) => { + tracing::error!(error = %e, "browse: reading a chart archive failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(); + } + }; } + Err(ReadFailure::Missing) => {} + Err(ReadFailure::Response(resp)) => return resp, } let group = repo.kind() == RepoKind::Group; Json(ChartDetail { diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 9c4e1473..9076f5db 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -24,6 +24,7 @@ use std::time::Duration; use figment::providers::{Env, Format, Serialized, Toml}; use figment::Figment; +use forge_plugin::api::Storage; use ipnet::IpNet; use serde::{Deserialize, Serialize}; @@ -2347,12 +2348,12 @@ impl Config { fn largest_staged_item_and_setting(&self) -> (u64, &'static str) { [ ( - self.default_upload_limit(forge_db::model::NUGET_FORMAT), + self.default_upload_limit(Storage::Packages), "the default upload limit of package repositories", ), (self.proxy.max_package_bytes, "proxy.max_package_bytes"), ( - self.default_upload_limit(forge_db::model::DOCKER_FORMAT), + self.default_upload_limit(Storage::Images), "the default upload limit of docker repositories", ), (self.docker_max_blob_bytes(), "docker.max_blob_bytes"), @@ -2393,15 +2394,15 @@ impl Config { .unwrap_or_else(|| self.docker_default(DEFAULT_DOCKER_MAX_BLOB_BYTES)) } - /// The upload limit of a hosted repository of `format` without one of - /// its own: for docker [`DEFAULT_DOCKER_MAX_UPLOAD_BYTES`] per blob (as - /// far as an explicit staging budget allows, [`Self::docker_default`]), - /// for a package format [`DEFAULT_MAX_UPLOAD_BYTES`]. - pub fn default_upload_limit(&self, format: &str) -> u64 { - if format == forge_db::model::DOCKER_FORMAT { - self.docker_default(DEFAULT_DOCKER_MAX_UPLOAD_BYTES) - } else { - DEFAULT_MAX_UPLOAD_BYTES + /// The upload limit of a hosted repository of a format of `storage` + /// without one of its own: for images (docker) + /// [`DEFAULT_DOCKER_MAX_UPLOAD_BYTES`] per blob (as far as an explicit + /// staging budget allows, [`Self::docker_default`]), for packages and + /// files [`DEFAULT_MAX_UPLOAD_BYTES`]. + pub fn default_upload_limit(&self, storage: Storage) -> u64 { + match storage { + Storage::Images => self.docker_default(DEFAULT_DOCKER_MAX_UPLOAD_BYTES), + Storage::Packages | Storage::Files => DEFAULT_MAX_UPLOAD_BYTES, } } @@ -2634,7 +2635,7 @@ impl Config { .to_string(), ); } - let docker_push = self.default_upload_limit(forge_db::model::DOCKER_FORMAT); + let docker_push = self.default_upload_limit(Storage::Images); if docker_push < DEFAULT_DOCKER_MAX_UPLOAD_BYTES || (self.docker.max_blob_bytes.is_none() && self.docker_max_blob_bytes() < DEFAULT_DOCKER_MAX_BLOB_BYTES) @@ -4178,7 +4179,7 @@ mod tests { + "\n[proxy]\nmax_package_bytes = 268435456\n"; let cfg = Config::from_figment(fig_from_toml(&budget)).expect("a small budget starts"); let fits = 768 * MIB - cfg.nuget.max_nuspec_bytes; - assert_eq!(cfg.default_upload_limit("docker"), fits); + assert_eq!(cfg.default_upload_limit(Storage::Images), fits); assert_eq!(cfg.docker_max_blob_bytes(), fits); assert_eq!(cfg.max_repository_upload_bytes(), Some(fits)); let warnings = cfg.deployment_warnings(); @@ -4217,7 +4218,7 @@ mod tests { ] { let cfg = Config::from_figment(fig_from_toml(&toml)).expect("valid"); assert_eq!( - cfg.default_upload_limit("docker"), + cfg.default_upload_limit(Storage::Images), DEFAULT_DOCKER_MAX_UPLOAD_BYTES ); assert_eq!(cfg.docker_max_blob_bytes(), DEFAULT_DOCKER_MAX_BLOB_BYTES); @@ -4233,9 +4234,12 @@ mod tests { #[test] fn default_upload_limits_and_the_repository_ceiling() { let cfg = Config::from_figment(fig_from_toml(MINIMAL)).expect("valid"); - assert_eq!(cfg.default_upload_limit("nuget"), DEFAULT_MAX_UPLOAD_BYTES); assert_eq!( - cfg.default_upload_limit("docker"), + cfg.default_upload_limit(Storage::Packages), + DEFAULT_MAX_UPLOAD_BYTES + ); + assert_eq!( + cfg.default_upload_limit(Storage::Images), DEFAULT_DOCKER_MAX_UPLOAD_BYTES ); // A repository may hold up to what a push can stage. diff --git a/crates/formats/docker/src/auth.rs b/crates/server/src/images/auth.rs similarity index 87% rename from crates/formats/docker/src/auth.rs rename to crates/server/src/images/auth.rs index bad7e5f1..79cf0f21 100644 --- a/crates/formats/docker/src/auth.rs +++ b/crates/server/src/images/auth.rs @@ -12,10 +12,11 @@ use axum::response::{IntoResponse, Response}; use forge_core::auth::Principal; use forge_core::authz::Level; use forge_db::model::RepoKind; +use forge_plugin::api::images::ImageError; -use crate::error::{Code, OciError, OciResult}; -use crate::service::{mutation_credential, Caller, Registry, Site}; -use crate::token::{self, Access, Claims, TokenPrincipal}; +use super::dispatch::{mutation_credential, Caller, Site}; +use super::token::{self, Access, Claims, TokenPrincipal}; +use super::{Failure, ImgResult, Registry}; impl Registry { pub(crate) async fn token( @@ -23,7 +24,7 @@ impl Registry { caller: &Caller, site: &Site, query: &str, - ) -> OciResult { + ) -> ImgResult { // A registry token is never a credential for another one: it would // let a short-lived token renew itself forever. let principal = if caller.claims.is_some() { @@ -38,9 +39,8 @@ impl Registry { "scope" => { for scope in token::parse_scopes(&value) { if !token::add_scope(&mut scopes, scope) { - return Err(OciError::new( - StatusCode::BAD_REQUEST, - Code::Unsupported, + return Err(Failure::unsupported( + 400, format!( "a token request may name at most {} repositories", token::MAX_SCOPES @@ -56,15 +56,12 @@ impl Registry { let token_principal = TokenPrincipal::of(&principal); if account.is_some() && token_principal == TokenPrincipal::Anonymous { // `docker login` must fail without valid credentials. - return Err(OciError::new( - StatusCode::UNAUTHORIZED, - Code::Unauthorized, - "authentication required", - ) - .with_header( - header::WWW_AUTHENTICATE, - HeaderValue::from_static("Basic realm=\"Forge\""), - )); + return Err( + Failure::new(ImageError::AuthenticationRequired { scope: None }).with_header( + header::WWW_AUTHENTICATE, + HeaderValue::from_static("Basic realm=\"Forge\""), + ), + ); } let mut access = Vec::new(); @@ -102,7 +99,7 @@ impl Registry { fp: token_principal, }; let jwt = token::issue(&self.inner.keys, &claims) - .map_err(|e| OciError::internal("token signing", e))?; + .map_err(|e| Failure::internal("token signing", e))?; if matches!(claims.fp, TokenPrincipal::Anonymous) { tracing::debug!(scopes = ?claims.access, "registry: anonymous token issued"); } else { @@ -131,7 +128,8 @@ impl Registry { } /// The subset of `requested` actions the principal may perform on the - /// repository `name`. An unknown repository grants nothing. + /// repository `name`. An unknown repository, or a name not in the + /// format's grammar, grants nothing. async fn granted( &self, principal: &Principal, @@ -139,6 +137,9 @@ impl Registry { name: &str, requested: &[String], ) -> Vec { + if self.plugin().validate_id(name).is_err() { + return Vec::new(); + } let Ok(Some(target)) = self.resolve(site, name).await else { return Vec::new(); }; diff --git a/crates/server/src/images/digest.rs b/crates/server/src/images/digest.rs new file mode 100644 index 00000000..89192e3a --- /dev/null +++ b/crates/server/src/images/digest.rs @@ -0,0 +1,138 @@ +//! The digests the store verifies content against: `sha256:` and +//! `sha512:`, in the canonical form the format names them in. The CAS +//! hashes everything it stages with SHA-256 and SHA-512 at once, and is +//! keyed by SHA-256: a `sha512:` digest names the same CAS blob as its +//! `sha256:` twin, it is only verified against the other hash. + +use base64::Engine; +use sha2::{Digest as _, Sha256, Sha512}; + +use crate::images::{Failure, ImgResult}; + +/// A content digest of a supported algorithm. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ContentDigest { + text: String, + sha512: bool, +} + +impl ContentDigest { + /// `sha256:<64 lowercase hex>` or `sha512:<128 lowercase hex>`. + pub fn parse(text: &str) -> Option { + let (sha512, hex) = match text.split_once(':')? { + ("sha256", hex) if hex.len() == 64 => (false, hex), + ("sha512", hex) if hex.len() == 128 => (true, hex), + _ => return None, + }; + hex.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + .then(|| ContentDigest { + text: text.to_string(), + sha512, + }) + } + + /// A digest the format named: one of the store's, or the format broke + /// the contract. + pub fn named(text: &str) -> ImgResult { + Self::parse(text).ok_or_else(|| { + Failure::internal( + "image digest", + format!("the images format named {text:?} as a digest"), + ) + }) + } + + /// The SHA-256 digest of `bytes`. + pub fn sha256_of(bytes: &[u8]) -> Self { + ContentDigest { + text: format!("sha256:{}", hex::encode(Sha256::digest(bytes))), + sha512: false, + } + } + + pub fn as_str(&self) -> &str { + &self.text + } + + /// The hex part after the colon. + pub fn hex(&self) -> &str { + &self.text[self.text.find(':').map_or(0, |i| i + 1)..] + } + + /// The CAS key of the content, for a `sha256:` digest. + pub fn sha256_hex(&self) -> Option<&str> { + (!self.sha512).then(|| self.hex()) + } + + /// Whether this digest names content with the given CAS hashes (the + /// lowercase hex SHA-256 and the standard-base64 SHA-512 the CAS + /// computes while staging). + pub fn matches(&self, sha256_hex: &str, sha512_base64: &str) -> bool { + if self.sha512 { + base64::engine::general_purpose::STANDARD + .decode(sha512_base64) + .is_ok_and(|raw| hex::encode(raw) == self.hex()) + } else { + self.hex() == sha256_hex + } + } + + /// Whether this digest names `bytes`. + pub fn matches_bytes(&self, bytes: &[u8]) -> bool { + if self.sha512 { + hex::encode(Sha512::digest(bytes)) == self.hex() + } else { + hex::encode(Sha256::digest(bytes)) == self.hex() + } + } +} + +impl std::fmt::Display for ContentDigest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_the_canonical_forms_only() { + let empty = ContentDigest::sha256_of(b""); + assert_eq!( + empty.as_str(), + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!(ContentDigest::parse(empty.as_str()), Some(empty.clone())); + let sha512 = format!("sha512:{}", hex::encode(Sha512::digest(b""))); + assert!(ContentDigest::parse(&sha512).is_some()); + for bad in [ + "", + "sha256:", + "sha256:abc", + "md5:d41d8cd98f00b204e9800998ecf8427e", + &empty.as_str().to_uppercase(), + &format!("{}/", &empty.as_str()[..70]), + ] { + assert_eq!(ContentDigest::parse(bad), None, "{bad:?}"); + } + } + + #[test] + fn matches_cas_hashes() { + let sha256 = hex::encode(Sha256::digest(b"hello")); + let sha512 = base64::engine::general_purpose::STANDARD.encode(Sha512::digest(b"hello")); + let digest256 = ContentDigest::sha256_of(b"hello"); + let digest512 = + ContentDigest::parse(&format!("sha512:{}", hex::encode(Sha512::digest(b"hello")))) + .unwrap(); + assert!(digest256.matches(&sha256, &sha512)); + assert!(digest512.matches(&sha256, &sha512)); + assert!(digest512.matches_bytes(b"hello")); + assert!(!ContentDigest::sha256_of(b"world").matches(&sha256, &sha512)); + assert_eq!(digest256.sha256_hex(), Some(sha256.as_str())); + assert_eq!(digest512.sha256_hex(), None); + } +} diff --git a/crates/server/src/images/dispatch.rs b/crates/server/src/images/dispatch.rs new file mode 100644 index 00000000..7278d999 --- /dev/null +++ b/crates/server/src/images/dispatch.rs @@ -0,0 +1,1173 @@ +//! A `/v2` request: the host's part of it, then the conversation with the +//! images format. +//! +//! The host answers `/v2/` (the version check), `/v2/token` (the token +//! service, [`super::auth`]) and `/v2/_catalog` (refused) itself. Any other +//! path goes to the format ([`FormatPlugin::image_path`]), which splits off +//! the name the client wrote and says what the request needs; the host +//! resolves the name (by path, `/`, or on a repository's +//! own host, `config.host`), authorizes it (RBAC and package selectors on +//! the image, a registry token's scope), picks where a change lands (a +//! group's writable member) and refuses an offline repository. Then the +//! format's `handle` and `render` decide the response, round by round, +//! with the image queries this module answers from the store and the +//! actions it performs (serving a blob or a manifest). What the host +//! refuses, the format answers in its words +//! ([`FormatPlugin::image_error`]), with the host's challenge. +//! +//! Every answer carries `Docker-Distribution-API-Version` and the +//! registry's hardening; a response the format renders keeps only the +//! headers in [`PASSED_RESPONSE_HEADERS`], `location` and `link` only +//! within `/v2/`. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; + +use axum::body::Body; +use axum::http::{header, HeaderMap, HeaderValue, Method, Request, StatusCode}; +use axum::response::{IntoResponse, Response}; +use forge_core::auth::{AuthMethod, Principal}; +use forge_core::authz::Level; +use forge_db::model::{RepoKind, Repository}; +use forge_db::{oci_dao, repository_dao}; +use forge_plugin::api::images::{ + ImageAnswer, ImageError, ImageName, ImagePath, ImageQuery, Permission, Reference, +}; +use forge_plugin::api::{self as protocol, Action, Answer, Loaded, Query}; +use forge_plugin::FormatPlugin; + +use super::digest::ContentDigest; +use super::pull::{BlobHit, ManifestHit}; +use super::token::{self, Claims}; +use super::{Failure, ImgResult, Registry}; + +/// The most rounds of queries one request may load. +const MAX_ROUNDS: usize = 4; + +/// Request headers the format sees. Never credentials or cookies. +pub const PASSED_REQUEST_HEADERS: &[&str] = + &["accept", "content-range", "content-type", "user-agent"]; + +/// Response headers the format may set. `location` and `link` only within +/// `/v2/`. +pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ + "content-type", + "docker-content-digest", + "docker-upload-uuid", + "link", + "location", + "oci-filters-applied", + "oci-subject", + "oci-tag", + "range", + "retry-after", +]; + +impl Registry { + /// Answer one `/v2` request. `principal` is what the authentication + /// middleware resolved (registry Bearer tokens are resolved here). + pub async fn handle(&self, principal: Principal, req: Request) -> Response { + let mut response = match self.route(principal, req).await { + Ok(response) => response, + Err(failure) => self.failure_response(failure), + }; + let headers = response.headers_mut(); + headers.insert( + "docker-distribution-api-version", + HeaderValue::from_static("registry/2.0"), + ); + harden(headers); + response + } + + /// What the host refused or failed, as the format answers it, with the + /// host's own headers (a challenge). + pub fn failure_response(&self, failure: Failure) -> Response { + let mut response = self.respond(self.plugin().image_error(&failure.error)); + for (name, value) in failure.headers { + response.headers_mut().insert(name, value); + } + response + } + + /// A response the format rendered ([`rendered`]). + pub(crate) fn respond(&self, response: protocol::Response) -> Response { + rendered(response) + } + + async fn route(&self, principal: Principal, req: Request) -> ImgResult { + let mount = self.plugin().descriptor().mount; + let rest = req + .uri() + .path() + .strip_prefix(mount) + .unwrap_or_default() + .to_string(); + // The host's own endpoints; anything else is the format's to split. + let path = match rest.as_str() { + "" | "/" | "/token" | "/_catalog" => None, + _ => Some(self.plugin().image_path(req.method().as_str(), &rest)?), + }; + let site = self.site(request_host(&req)).await?; + let caller = self.caller(principal, req.headers(), &site).await?; + match (rest.as_str(), path) { + (_, Some(path)) => self.repository_request(caller, site, path, req).await, + ("/token", None) => { + if req.method() != Method::GET { + return Err(Failure::unsupported( + 405, + "the token service answers GET with Basic credentials", + )); + } + self.token(&caller, &site, req.uri().query().unwrap_or_default()) + .await + } + ("/_catalog", None) => Err(Failure::unsupported( + 404, + "the catalog API is not supported; browse images in the web UI", + )), + _ => self.version_check(&caller, &site), + } + } + + async fn repository_request( + &self, + caller: Caller, + site: Site, + path: ImagePath, + req: Request, + ) -> ImgResult { + let name = path.name; + let permission = path.permission; + let action = rbac(permission); + let target = match self.resolve(&site, &name).await? { + Some(target) => target, + None => { + // An unknown repository is answered like one the caller may + // not use; only a global admin learns that it does not exist. + if self.inner.authz.is_global_admin(&caller.principal).await { + return Err(Failure::new(ImageError::NameUnknown { name })); + } + return Err(self.refusal(&caller, &site, &name, permission, Decision::Deny)); + } + }; + let write_target = if action == Level::Read { + let decision = self.decide(&caller, &target, Level::Read).await; + if decision != Decision::Allow { + return Err(self.refusal(&caller, &site, &name, permission, decision)); + } + None + } else { + Some( + self.write_target(&caller, &site, &target, permission) + .await?, + ) + }; + // An offline hosted or group repository serves nothing; an offline + // proxy serves what it cached. + let serving = write_target.as_ref().unwrap_or(&target); + if !serving.repo.online && serving.repo.kind() != RepoKind::Proxy + || !target.repo.online && target.repo.kind() == RepoKind::Group + { + return Err(Failure::new(ImageError::Offline)); + } + let ctx = RequestCtx { + caller, + site, + target, + write_target, + }; + let method = req.method().clone(); + let request = protocol::Request { + method: method.as_str().to_string(), + path: path.endpoint.clone(), + query: req.uri().query().unwrap_or_default().to_string(), + headers: req + .headers() + .iter() + .filter(|(name, _)| PASSED_REQUEST_HEADERS.contains(&name.as_str())) + .filter_map(|(name, value)| { + Some((name.to_string(), value.to_str().ok()?.to_string())) + }) + .collect(), + // The format reads no bodies: the host streams uploads into the + // store. + body: None, + image: Some(ImageName { + image: ctx.target.image.clone(), + name: ctx.target.name.clone(), + }), + }; + let conversation = Conversation { + reg: self, + ctx: &ctx, + permission, + head: method == Method::HEAD, + headers: req.headers().clone(), + body: Mutex::new(Some(req.into_body())), + manifests: Mutex::new(HashMap::new()), + blobs: Mutex::new(HashMap::new()), + }; + let response = conversation.run(request).await; + if action != Level::Read { + tracing::info!( + target: "forge::audit", + %method, + repo = %ctx.serving().repo.name, + image = %ctx.target.image, + endpoint = %path.endpoint.trim_start_matches('/'), + status = response.status().as_u16(), + principal = %ctx.caller.principal.audit_label(), + "image mutation" + ); + } + Ok(response) + } + + // ── Site and caller ───────────────────────────────────────────────────── + + /// Which host the request reached and the token realm and service that + /// go with it. A host that names a repository of the images format + /// (`config.host`) serves that repository at the registry root. + pub(crate) async fn site(&self, host: Option) -> ImgResult { + let public = &self.inner.config.public_base_url; + let (scheme, public_host) = registry_origin(public); + let host = host.map(|h| h.to_ascii_lowercase()); + let host_repo = match &host { + Some(h) if *h != public_host => { + let bare = strip_port(h); + oci_dao::find_repository_by_host(&self.inner.pool, bare, self.format()).await? + } + _ => None, + }; + let (service, realm) = match (&host, &host_repo) { + (Some(h), Some(_)) => (h.clone(), format!("{scheme}://{h}/v2/token")), + _ => ( + public_host.clone(), + format!("{scheme}://{public_host}/v2/token"), + ), + }; + Ok(Site { + service, + realm, + issuer: token::issuer(public), + host_repo, + }) + } + + /// The effective caller: a registry Bearer token's principal, else the + /// middleware's. + async fn caller( + &self, + principal: Principal, + headers: &HeaderMap, + site: &Site, + ) -> ImgResult { + let Some(bearer) = bearer_token(headers) else { + return Ok(Caller { + principal, + claims: None, + }); + }; + if !token::looks_like_registry_token(bearer) { + // Another bearer (a workload identity token the middleware has + // already resolved, or nothing it knows). + return Ok(Caller { + principal, + claims: None, + }); + } + let invalid = || { + Failure::new(ImageError::InvalidToken).with_header( + header::WWW_AUTHENTICATE, + site.challenge(None, Some("invalid_token")), + ) + }; + let claims = token::verify(&self.inner.keys, bearer, &site.issuer, &site.service) + .ok_or_else(invalid)?; + let principal = claims.fp.to_principal(principal).ok_or_else(invalid)?; + let principal = self + .refresh_principal(principal) + .await? + .ok_or_else(invalid)?; + Ok(Caller { + principal, + claims: Some(claims), + }) + } + + /// Re-read what a token's principal rests on: a user's account must still + /// be active (and the API token it was minted from live); a workload's + /// realm and key must still stand ([`super::WorkloadCheck`]). Cached + /// briefly per process. + async fn refresh_principal(&self, principal: Principal) -> ImgResult> { + if let Principal::Workload { + realm_id, + config_sha256, + key_id, + .. + } = principal + { + let Some(check) = self.inner.workloads.clone() else { + return Ok(Some(principal)); + }; + let key = format!( + "{realm_id}:{}:{}", + hex::encode(config_sha256), + key_id.map_or_else(|| "token".to_string(), |id| id.to_string()) + ); + let live: Result = self + .inner + .cache + .get_or_try_insert_local( + "oci:workload", + &key, + std::time::Duration::from_secs(10), + || async move { check.still_valid(&principal).await }, + ) + .await; + return match live { + Ok(true) => Ok(Some(principal)), + Ok(false) => Ok(None), + Err(e) => Err(Failure::internal("workload lookup", e)), + }; + } + let Principal::User { + user_id, + token_id, + method, + .. + } = principal + else { + return Ok(Some(principal)); + }; + let key = format!("{user_id}:{}", token_id.unwrap_or(0)); + let pool = self.inner.pool.clone(); + let state: Result = self + .inner + .cache + .get_or_try_insert_local( + "oci:principal", + &key, + std::time::Duration::from_secs(10), + || async move { + oci_dao::live_principal(&pool, user_id, token_id) + .await + .map_err(|e| e.to_string()) + }, + ) + .await; + match state { + Ok(true) => Ok(Some(Principal::User { + user_id, + token_id, + method, + })), + Ok(false) => Ok(None), + Err(e) => Err(Failure::internal("principal lookup", e)), + } + } + + // ── Names and authorization ───────────────────────────────────────────── + + /// The repository and image a `` addresses: on a repository's own + /// host the whole name is the image; otherwise the first component names + /// the Forge repository and the rest the image. `None` for an unknown + /// repository, or one of another format. The name is in the format's + /// grammar already. + pub(crate) async fn resolve(&self, site: &Site, name: &str) -> ImgResult> { + if let Some(repo) = &site.host_repo { + return Ok(Some(Target { + repo: repo.clone(), + image: name.to_string(), + name: name.to_string(), + })); + } + let Some((repo_name, image)) = name.split_once('/') else { + return Err(Failure::new(ImageError::NameInvalid { + name: name.to_string(), + hint: Some("use /".into()), + })); + }; + let repo = repository_dao::find_repository_by_name(&self.inner.pool, repo_name).await?; + Ok(repo.filter(|r| self.serves(r)).map(|repo| Target { + repo, + image: image.to_string(), + name: name.to_string(), + })) + } + + /// Whether the caller may `action` the target's image: the repository + /// permission and its content selectors, then a registry token's scope + /// (checked against the name the client used). + pub(crate) async fn decide(&self, caller: &Caller, target: &Target, action: Level) -> Decision { + if !self + .rbac_allows(&caller.principal, &target.repo, &target.image, action) + .await + { + return Decision::Deny; + } + match &caller.claims { + Some(claims) if !claims_allow(claims, &target.name, action) => Decision::NeedsScope, + _ => Decision::Allow, + } + } + + /// The RBAC part of [`Self::decide`]. + pub(crate) async fn rbac_allows( + &self, + principal: &Principal, + repo: &Repository, + image: &str, + action: Level, + ) -> bool { + let authz = &self.inner.authz; + let allowed = match action { + Level::Read => authz.can_read(principal, repo).await, + Level::Write => authz.can_write(principal, repo).await, + Level::Delete => authz.can_delete(principal, repo).await, + }; + allowed + && authz + .package_filter_for(principal, repo, action) + .await + .allows(image) + } + + /// Where a push or delete addressed to `target` lands (the repository + /// itself, or a group's writable member), after checking the caller may + /// do it there. + async fn write_target( + &self, + caller: &Caller, + site: &Site, + target: &Target, + permission: Permission, + ) -> ImgResult { + let action = rbac(permission); + if !mutation_credential(&caller.principal) { + return Err(self.refusal(caller, site, &target.name, permission, Decision::Deny)); + } + let landing = match target.repo.kind() { + RepoKind::Hosted => target.clone(), + RepoKind::Proxy | RepoKind::Group => { + let decision = self.decide(caller, target, Level::Read).await; + if decision != Decision::Allow { + return Err(self.refusal( + caller, + site, + &target.name, + Permission::Read, + decision, + )); + } + let member = match target.repo.kind() { + RepoKind::Group => self.writable_member(&target.repo).await?, + _ => None, + }; + match member { + Some(member) => Target { + repo: member, + image: target.image.clone(), + name: target.name.clone(), + }, + None if target.repo.kind() == RepoKind::Proxy => { + return Err(Failure::unsupported( + 405, + "a proxy repository is read-only; push to a hosted repository", + )) + } + None => { + return Err(Failure::unsupported( + 405, + "this group has no writable member; push to a hosted repository", + )) + } + } + } + }; + let decision = if self + .rbac_allows(&caller.principal, &landing.repo, &landing.image, action) + .await + { + match &caller.claims { + Some(claims) if !claims_allow(claims, &target.name, action) => Decision::NeedsScope, + _ => Decision::Allow, + } + } else { + Decision::Deny + }; + if decision != Decision::Allow { + return Err(self.refusal(caller, site, &target.name, permission, decision)); + } + Ok(landing) + } + + /// A group's writable member, if configured and a hosted member of the + /// images format. + pub(crate) async fn writable_member( + &self, + group: &Repository, + ) -> ImgResult> { + let Some(id) = group.writable_member_id() else { + return Ok(None); + }; + let members = repository_dao::group_members(&self.inner.pool, group.id).await?; + Ok(members + .into_iter() + .map(|(member, _)| member) + .find(|m| m.id == id && self.serves(m) && m.kind() == RepoKind::Hosted)) + } + + /// The refusal of `permission` on `name`: a challenge (401) when + /// credentials or a token for the scope could help, a denial (403) for + /// an authenticated caller the permission is missing for. + pub(crate) fn refusal( + &self, + caller: &Caller, + site: &Site, + name: &str, + permission: Permission, + decision: Decision, + ) -> Failure { + let authenticated = mutation_credential(&caller.principal); + let error = caller.claims.as_ref().map(|_| "insufficient_scope"); + match decision { + Decision::NeedsScope => challenge(site, name, permission, error), + Decision::Deny if !authenticated => challenge(site, name, permission, error), + _ => Failure::new(ImageError::Denied { + name: name.to_string(), + permission, + }), + } + } + + fn version_check(&self, caller: &Caller, site: &Site) -> ImgResult { + let signed_in = caller.claims.is_some() + || matches!( + caller.principal, + Principal::User { method, .. } if method != AuthMethod::Session + ) + || matches!(caller.principal, Principal::Workload { .. }); + if signed_in { + return Ok(( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + "{}", + ) + .into_response()); + } + Err( + Failure::new(ImageError::AuthenticationRequired { scope: None }) + .with_header(header::WWW_AUTHENTICATE, site.challenge(None, None)), + ) + } +} + +/// A challenge for a token with the scope `permission` on `name`. +fn challenge(site: &Site, name: &str, permission: Permission, error: Option<&str>) -> Failure { + let scope = format!("repository:{name}:{}", scope_actions(permission)); + Failure::new(ImageError::AuthenticationRequired { + scope: Some((name.to_string(), permission)), + }) + .with_header( + header::WWW_AUTHENTICATE, + site.challenge(Some(&scope), error), + ) +} + +/// One request's conversation with the format: the queries it asks for, +/// answered from the store, and the actions it ends with. +struct Conversation<'a> { + reg: &'a Registry, + ctx: &'a RequestCtx, + /// What the request was authorized for. + permission: Permission, + head: bool, + /// The request's headers, for a blob's `Range` and a manifest's type. + headers: HeaderMap, + /// The request body, until a query reads it. + body: Mutex>, + /// The manifests queries found, for serving one of them. + manifests: Mutex>, + /// The blobs queries found, by digest, for serving one of them. + blobs: Mutex>, +} + +impl Conversation<'_> { + /// Run a protocol call of the format off the async workers. A format + /// that panics answers 500. + async fn call( + &self, + f: impl FnOnce(&dyn FormatPlugin) -> R + Send + 'static, + ) -> Result { + let plugin = Arc::clone(&self.reg.inner.plugin); + tokio::task::spawn_blocking(move || f(plugin.as_ref())) + .await + .map_err(|e| { + tracing::error!(error = %e, "images format failed"); + self.reg + .failure_response(Failure::new(ImageError::Internal)) + }) + } + + async fn run(&self, request: protocol::Request) -> Response { + let handled = { + let (target, request) = (self.target(), request.clone()); + self.call(move |plugin| plugin.handle(&target, &request)) + .await + }; + let mut action = match handled { + Ok(action) => action, + Err(response) => return response, + }; + let mut data: Vec = Vec::new(); + for _ in 0..MAX_ROUNDS { + let Action::Load(queries) = action else { + return self.act(action).await; + }; + if let Err(response) = self.check_round(&queries) { + return response; + } + for query in queries { + let Query::Image(image_query) = &query else { + return self.misused("a query of another storage"); + }; + let answer = match self.answer(image_query).await { + Ok(answer) => answer, + Err(response) => return response, + }; + data.push(Loaded { + query, + answer: Answer::Image(answer), + }); + } + let (target, req) = (self.target(), request.clone()); + let rendered = self + .call(move |plugin| { + let rendered = plugin.render(&target, &req, &data); + (rendered, data) + }) + .await; + match rendered { + Ok((next, loaded)) => { + data = loaded; + action = next; + } + Err(response) => return response, + } + } + if !matches!(action, Action::Load(_)) { + return self.act(action).await; + } + tracing::warn!("images format asked for more rounds than one request may load"); + self.reg + .failure_response(Failure::new(ImageError::Internal)) + } + + /// The repository as the format sees it: the one the request addressed. + fn target(&self) -> protocol::Repository { + let repo = &self.ctx.target.repo; + protocol::Repository { + name: repo.name.clone(), + kind: match repo.kind() { + RepoKind::Hosted => protocol::RepositoryKind::Hosted, + RepoKind::Proxy => protocol::RepositoryKind::Proxy, + RepoKind::Group => protocol::RepositoryKind::Group, + }, + base_url: format!( + "{}{}", + self.reg.inner.config.public_base_url, + self.reg.plugin().descriptor().mount + ), + inline: false, + // A group's reserved prefixes are the host's to apply + // (`super::group`). + reserved_prefixes: Vec::new(), + } + } + + /// The format asked for something the contract does not allow: a bug in + /// it, answered with a server error. + fn misused(&self, what: &str) -> Response { + tracing::error!(repo = %self.ctx.target.repo.name, "images format asked for {what}"); + self.reg + .failure_response(Failure::new(ImageError::Internal)) + } + + /// Whether a round of queries keeps to the contract: image queries, of + /// the request's image, and a change alone in its round, on a request + /// authorized for it. + fn check_round(&self, queries: &[Query]) -> Result<(), Response> { + if queries.is_empty() { + return Err(self.misused("an empty round")); + } + if queries.len() > 1 && queries.iter().any(Query::is_change) { + return Err(self.misused("a change with other queries in its round")); + } + for query in queries { + let Query::Image(query) = query else { + return Err(self.misused("a query of another storage")); + }; + if query + .image() + .is_some_and(|image| image != self.ctx.target.image) + { + return Err(self.misused("a query of another image")); + } + let needs = match query { + ImageQuery::DeleteManifest { .. } | ImageQuery::DeleteBlob { .. } => { + Some(Permission::Delete) + } + query if query.is_change() => Some(Permission::Write), + _ => None, + }; + if needs.is_some_and(|needs| needs != self.permission) + || needs.is_some() && self.ctx.write_target.is_none() + { + return Err(self.misused("a change the request was not authorized for")); + } + } + Ok(()) + } + + /// The request body, for the one query that reads it. + fn take_body(&self) -> Result { + self.body + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| self.misused("the request body twice")) + } + + /// A digest the format named, as the store verifies content against it. + fn digest(&self, digest: &str) -> Result { + ContentDigest::named(digest).map_err(|failure| self.reg.failure_response(failure)) + } + + /// The answer to one query, or the response that ends the request + /// instead. + async fn answer(&self, query: &ImageQuery) -> Result { + let reg = self.reg; + let ctx = self.ctx; + let t = &ctx.target; + let landing = ctx.serving(); + let principal = &ctx.caller.principal; + let error = |failure: Failure| failure.error; + Ok(match query { + ImageQuery::Manifest { reference, .. } => { + if let Reference::Digest(digest) = reference { + self.digest(digest)?; + } + let found = reg + .manifest_from(&ctx.caller, &t.repo, &t.image, reference, 0) + .await; + if let Ok(Some(hit)) = &found { + self.manifests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(reference.clone(), hit.clone()); + } + ImageAnswer::Manifest(found.map(|hit| hit.map(|h| h.info())).map_err(error)) + } + ImageQuery::Blob { digest, .. } => { + let digest = self.digest(digest)?; + let found = reg + .blob_from(&ctx.caller, &t.repo, &t.image, &digest, 0) + .await; + if let Ok(Some(hit)) = &found { + self.blobs + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(digest.to_string(), hit.clone()); + } + ImageAnswer::Blob(found.map(|hit| hit.map(|h| h.info())).map_err(error)) + } + ImageQuery::Tags { last, n, .. } => ImageAnswer::Tags( + reg.tags_from( + &ctx.caller, + &t.repo, + &t.image, + last.as_deref(), + *n as usize, + 0, + ) + .await + .map_err(error), + ), + ImageQuery::Referrers { + digest, + artifact_type, + .. + } => { + let digest = self.digest(digest)?; + let found = reg + .referrers_from(&ctx.caller, &t.repo, &t.image, &digest, 0) + .await + .map(|mut found| { + if let Some(filter) = artifact_type { + found.retain(|r| r.artifact_type.as_deref() == Some(filter.as_str())); + } + found + }); + ImageAnswer::Referrers(found.map_err(error)) + } + ImageQuery::StartUpload { mount, .. } => { + let mount = match mount { + Some(mount) => Some((self.digest(&mount.digest)?, mount.from.as_deref())), + None => None, + }; + ImageAnswer::Started(reg.start_upload(ctx, landing, mount).await.map_err(error)) + } + ImageQuery::UploadStatus(id) => { + ImageAnswer::Upload(reg.upload_status(landing, id).await.map_err(error)) + } + ImageQuery::AppendUpload { id, range } => { + let body = self.take_body()?; + ImageAnswer::Upload( + reg.append_upload(landing, id, *range, body) + .await + .map_err(error), + ) + } + ImageQuery::FinishUpload { id, digest, range } => { + let digest = self.digest(digest)?; + let body = self.take_body()?; + ImageAnswer::Blob( + reg.finish_upload(landing, id, &digest, *range, body) + .await + .map(Some) + .map_err(error), + ) + } + ImageQuery::CancelUpload(id) => { + ImageAnswer::Done(reg.cancel_upload(landing, id).await.map_err(error)) + } + ImageQuery::PutManifest { + reference, tags, .. + } => { + if let Reference::Digest(digest) = reference { + self.digest(digest)?; + } + let body = self.take_body()?; + let content_type = self + .headers + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + ImageAnswer::Stored( + reg.put_manifest(principal, landing, reference, tags, content_type, body) + .await + .map_err(error), + ) + } + ImageQuery::DeleteManifest { reference, .. } => { + if let Reference::Digest(digest) = reference { + self.digest(digest)?; + } + ImageAnswer::Done( + reg.delete_manifest(principal, landing, reference) + .await + .map_err(error), + ) + } + ImageQuery::DeleteBlob { digest, .. } => { + let digest = self.digest(digest)?; + ImageAnswer::Done( + reg.delete_blob(principal, landing, &digest) + .await + .map_err(error), + ) + } + }) + } + + /// Do what the format decided, other than loading data. + async fn act(&self, action: Action) -> Response { + match action { + Action::Respond(response) => self.reg.respond(response), + Action::ServeBlob { digest, .. } => { + let found = self + .blobs + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&digest) + .cloned(); + match found { + Some(hit) if self.permission == Permission::Read => self + .reg + .serve_blob(&hit, self.head, &self.headers) + .await + .unwrap_or_else(|failure| self.reg.failure_response(failure)), + _ => self.misused("serving a blob no query found"), + } + } + Action::ServeManifest { reference, .. } => { + let found = self + .manifests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&reference) + .cloned(); + match found { + Some(hit) if self.permission == Permission::Read => self + .reg + .serve_manifest(&hit, self.head) + .await + .unwrap_or_else(|failure| self.reg.failure_response(failure)), + _ => self.misused("serving a manifest no query found"), + } + } + _ => self.misused("an action of another storage"), + } + } +} + +/// A response the images format rendered, keeping only the headers it may +/// set: [`PASSED_RESPONSE_HEADERS`], `location` and `link` only within +/// `/v2/`. +pub(crate) fn rendered(rendered: protocol::Response) -> Response { + let status = StatusCode::from_u16(rendered.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + // The format's `content-type`, none of the host's. + let mut response = Response::new(Body::from(rendered.body)); + *response.status_mut() = status; + let headers = response.headers_mut(); + for (name, value) in &rendered.headers { + let name = name.to_ascii_lowercase(); + if !PASSED_RESPONSE_HEADERS.contains(&name.as_str()) { + continue; + } + let inside = match name.as_str() { + "location" => value.starts_with("/v2/"), + "link" => value.trim_start().starts_with(" true, + }; + if !inside { + tracing::warn!(header = %name, "images format: a header outside the registry dropped"); + continue; + } + let (Ok(name), Ok(value)) = ( + header::HeaderName::try_from(name), + HeaderValue::from_str(value), + ) else { + continue; + }; + headers.append(name, value); + } + response +} + +/// The scheme and `host[:port]` of `public_base_url`: where the registry +/// lives. Docker clients always talk to `/v2/` at the root of a host, so a +/// path in the URL plays no part. +pub fn registry_origin(public_base_url: &str) -> (&'static str, String) { + let scheme = if public_base_url.starts_with("https://") { + "https" + } else { + "http" + }; + let host = url::Url::parse(public_base_url) + .ok() + .and_then(|u| { + u.host_str().map(|h| match u.port() { + Some(p) => format!("{h}:{p}"), + None => h.to_string(), + }) + }) + .unwrap_or_default() + .to_ascii_lowercase(); + (scheme, host) +} + +/// The host a request reached (`Host`, or the HTTP/2 authority). +fn request_host(req: &Request) -> Option { + req.headers() + .get(header::HOST) + .and_then(|h| h.to_str().ok()) + .map(str::to_string) + .or_else(|| req.uri().authority().map(|a| a.to_string())) + .filter(|h| !h.is_empty()) +} + +fn strip_port(host: &str) -> &str { + if host.starts_with('[') { + return host + .split_once(']') + .map_or(host, |(h, _)| h.trim_start_matches('[')); + } + host.rsplit_once(':') + .filter(|(_, port)| port.bytes().all(|b| b.is_ascii_digit())) + .map_or(host, |(h, _)| h) +} + +/// The `Authorization: Bearer` value, if any. +pub(crate) fn bearer_token(headers: &HeaderMap) -> Option<&str> { + let value = headers.get(header::AUTHORIZATION)?.to_str().ok()?; + let (scheme, token) = value.split_once(' ')?; + scheme + .eq_ignore_ascii_case("bearer") + .then(|| token.trim()) + .filter(|t| !t.is_empty()) +} + +/// Whether the principal is a credential a push may come from: an API +/// token, a password (Basic or a token minted from one) or a workload — but +/// never a browser session, which a cross-site page could ride. +pub(crate) fn mutation_credential(principal: &Principal) -> bool { + match principal { + Principal::User { method, .. } => *method != AuthMethod::Session, + Principal::Workload { .. } => true, + _ => false, + } +} + +/// The RBAC action a permission is. +pub(crate) fn rbac(permission: Permission) -> Level { + match permission { + Permission::Read => Level::Read, + Permission::Write => Level::Write, + Permission::Delete => Level::Delete, + } +} + +/// The token-scope actions a permission needs. +fn scope_actions(permission: Permission) -> &'static str { + match permission { + Permission::Read => "pull", + Permission::Write => "pull,push", + Permission::Delete => "delete", + } +} + +fn claims_allow(claims: &Claims, name: &str, action: Level) -> bool { + match action { + Level::Read => claims.grants(name, "pull"), + Level::Write => claims.grants(name, "push"), + Level::Delete => claims.grants(name, "delete"), + } +} + +/// Browser hardening of registry responses: content (layers, manifests, +/// artifacts) is never rendered, sniffed or framed. +fn harden(headers: &mut HeaderMap) { + headers.insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static("default-src 'none'; sandbox"), + ); + headers.insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); +} + +/// An authorization decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Decision { + Allow, + /// RBAC allows it, but the registry token presented does not cover it: + /// the client should fetch a token for this scope. + NeedsScope, + Deny, +} + +/// The host a request reached and its token realm. +#[derive(Debug, Clone)] +pub(crate) struct Site { + /// Token `service` and audience: the host the registry was reached on. + pub service: String, + /// The token endpoint URL. + pub realm: String, + pub issuer: String, + /// The repository this host serves (host-based routing). + pub host_repo: Option, +} + +impl Site { + /// A `WWW-Authenticate: Bearer` challenge. + pub fn challenge(&self, scope: Option<&str>, error: Option<&str>) -> HeaderValue { + let mut value = format!( + "Bearer realm=\"{}\",service=\"{}\"", + self.realm, self.service + ); + if let Some(scope) = scope { + value.push_str(&format!(",scope=\"{scope}\"")); + } + if let Some(error) = error { + value.push_str(&format!(",error=\"{error}\"")); + } + HeaderValue::from_str(&value).unwrap_or_else(|_| HeaderValue::from_static("Bearer")) + } +} + +/// The resolved caller of a request. +#[derive(Debug, Clone)] +pub(crate) struct Caller { + pub principal: Principal, + /// The registry token presented, if any. + pub claims: Option, +} + +/// A repository and image a request addresses. +#[derive(Debug, Clone)] +pub(crate) struct Target { + pub repo: Repository, + pub image: String, + /// The `` as the client wrote it (what token scopes use). + pub name: String, +} + +/// Everything a repository request needs. +pub(crate) struct RequestCtx { + pub caller: Caller, + pub site: Site, + /// The repository the request addressed. + pub target: Target, + /// Where a push or delete lands (`None` for pulls). + pub write_target: Option, +} + +impl RequestCtx { + /// The repository writes go to (the addressed one for pulls). + pub fn serving(&self) -> &Target { + self.write_target.as_ref().unwrap_or(&self.target) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn host_ports_are_stripped() { + assert_eq!(strip_port("docker.example.com:443"), "docker.example.com"); + assert_eq!(strip_port("docker.example.com"), "docker.example.com"); + assert_eq!(strip_port("[::1]:5000"), "::1"); + } + + #[test] + fn bearer_values() { + let mut h = HeaderMap::new(); + h.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer abc"), + ); + assert_eq!(bearer_token(&h), Some("abc")); + h.insert(header::AUTHORIZATION, HeaderValue::from_static("Basic abc")); + assert_eq!(bearer_token(&h), None); + } + + #[test] + fn sessions_never_mutate() { + assert!(!mutation_credential(&Principal::User { + user_id: 1, + token_id: None, + method: AuthMethod::Session + })); + assert!(mutation_credential(&Principal::User { + user_id: 1, + token_id: None, + method: AuthMethod::Password + })); + assert!(!mutation_credential(&Principal::Unauthenticated)); + } +} diff --git a/crates/formats/docker/src/gc.rs b/crates/server/src/images/gc.rs similarity index 100% rename from crates/formats/docker/src/gc.rs rename to crates/server/src/images/gc.rs diff --git a/crates/formats/docker/src/group.rs b/crates/server/src/images/group.rs similarity index 90% rename from crates/formats/docker/src/group.rs rename to crates/server/src/images/group.rs index 2b5e910d..82789865 100644 --- a/crates/formats/docker/src/group.rs +++ b/crates/server/src/images/group.rs @@ -19,11 +19,12 @@ use forge_core::authz::Level; use forge_db::model::{RepoKind, Repository}; use forge_db::{oci_dao, repository_dao}; -use crate::digest::Digest; -use crate::error::{OciError, OciResult}; -use crate::pull::{BlobHit, ManifestHit, ReferrerDescriptor, TagPage}; -use crate::reference::Reference; -use crate::service::{Caller, Registry}; +use forge_plugin::api::images::{ManifestInfo, Reference, TagPage}; + +use super::digest::ContentDigest; +use super::dispatch::Caller; +use super::pull::{BlobHit, ManifestHit}; +use super::{Failure, ImgResult, Registry}; /// A leaf member serving a group pull, in priority order. struct Leaf { @@ -48,7 +49,7 @@ impl Registry { group: &Repository, image: &str, depth: u32, - ) -> OciResult> { + ) -> ImgResult> { let mut leaves = Vec::new(); let mut owners = HashSet::new(); let mut path = HashSet::from([group.id]); @@ -103,7 +104,7 @@ impl Registry { leaves: &'a mut Vec, owners: &'a mut HashSet, walk: Walk, - ) -> futures_util::future::BoxFuture<'a, OciResult<()>> { + ) -> futures_util::future::BoxFuture<'a, ImgResult<()>> { Box::pin(async move { if depth >= self.inner.config.max_group_depth.max(1) { return Ok(()); @@ -115,7 +116,7 @@ impl Registry { .into_iter() .collect(); for (member, _priority) in members { - if !member.is_docker() || path.contains(&member.id) { + if !self.serves(&member) || path.contains(&member.id) { continue; } let serving = walk.serving @@ -174,8 +175,8 @@ impl Registry { image: &str, reference: &Reference, depth: u32, - ) -> OciResult> { - let mut failure: Option = None; + ) -> ImgResult> { + let mut failure: Option = None; for member in self.serving_members(caller, group, image, depth).await? { match self .manifest_from(caller, &member, image, reference, depth + 1) @@ -184,7 +185,7 @@ impl Registry { Ok(Some(hit)) => return Ok(Some(hit)), Ok(None) => {} Err(e) => { - tracing::warn!(group = %group.name, member = %member.name, error = %e.message, "registry group: member failed"); + tracing::warn!(group = %group.name, member = %member.name, error = %e.message(), "registry group: member failed"); failure = Some(e); } } @@ -197,10 +198,10 @@ impl Registry { caller: &Caller, group: &Repository, image: &str, - digest: &Digest, + digest: &ContentDigest, depth: u32, - ) -> OciResult> { - let mut failure: Option = None; + ) -> ImgResult> { + let mut failure: Option = None; for member in self.serving_members(caller, group, image, depth).await? { match self .blob_from(caller, &member, image, digest, depth + 1) @@ -209,7 +210,7 @@ impl Registry { Ok(Some(hit)) => return Ok(Some(hit)), Ok(None) => {} Err(e) => { - tracing::warn!(group = %group.name, member = %member.name, error = %e.message, "registry group: member failed"); + tracing::warn!(group = %group.name, member = %member.name, error = %e.message(), "registry group: member failed"); failure = Some(e); } } @@ -225,9 +226,9 @@ impl Registry { last: Option<&str>, limit: usize, depth: u32, - ) -> OciResult> { + ) -> ImgResult> { let mut known = false; - let mut failure: Option = None; + let mut failure: Option = None; let mut tags: Vec = Vec::new(); // The smallest last tag of a member page that left tags out: the // merged page ends there, since the member's next page may hold @@ -272,10 +273,10 @@ impl Registry { caller: &Caller, group: &Repository, image: &str, - subject: &Digest, + subject: &ContentDigest, depth: u32, - ) -> OciResult> { - let mut out: Vec = Vec::new(); + ) -> ImgResult> { + let mut out: Vec = Vec::new(); let mut seen = HashSet::new(); for member in self.serving_members(caller, group, image, depth).await? { match self @@ -290,7 +291,7 @@ impl Registry { } } Err(e) => { - tracing::warn!(group = %group.name, member = %member.name, error = %e.message, "registry group: member failed"); + tracing::warn!(group = %group.name, member = %member.name, error = %e.message(), "registry group: member failed"); } } } diff --git a/crates/formats/docker/src/import.rs b/crates/server/src/images/import.rs similarity index 77% rename from crates/formats/docker/src/import.rs rename to crates/server/src/images/import.rs index 84a8f554..88e4435c 100644 --- a/crates/formats/docker/src/import.rs +++ b/crates/server/src/images/import.rs @@ -21,6 +21,9 @@ //! * What the source recorded is kept: its publish time becomes the tag's //! push time and the new manifest's creation time (retention's age rules), //! its last download the manifest's last pull. +//! +//! The images format names the source's URLs and reads its manifests, as +//! for a proxy's upstream. use std::sync::Arc; @@ -32,16 +35,18 @@ use forge_core::net::upstream::RequestClass; use forge_core::repo::proxy::Credentials; use forge_db::model::{RepoKind, Repository}; use forge_db::oci_dao::{self, LinkOutcome, TagWrite}; +use forge_plugin::api::images::{ + ImageError, ManifestBlob, ParsedManifest, Reference, MAX_MANIFEST_BYTES, +}; +use forge_plugin::api::UpstreamOp; use futures_util::TryStreamExt; use reqwest::Method; -use crate::digest::{Algorithm, Digest}; -use crate::error::{Code, OciError, OciResult}; -use crate::manifest::{self, media, Descriptor, ParsedManifest, EMPTY_JSON_DIGEST}; -use crate::proxy::{content_type, read_limited, upstream_base, Upstream}; -use crate::push::require_linked; -use crate::reference; -use crate::service::{Registry, Target}; +use super::digest::ContentDigest; +use super::dispatch::Target; +use super::proxy::{content_type, read_limited, Upstream}; +use super::push::require_linked; +use super::{Failure, ImgResult, Registry}; /// How deep indexes may nest below the copied manifest. const MAX_DEPTH: usize = 4; @@ -55,21 +60,21 @@ pub struct ImportSource { } impl ImportSource { - /// The registry API at `base_url` (`/v2` is appended), with Basic - /// `credentials` for it and for the token service its challenge names. + /// The registry API at `base_url`, with Basic `credentials` for it and + /// for the token service its challenge names. pub fn new(base_url: &str, credentials: Option, owner: String) -> Self { ImportSource { - base: upstream_base(base_url), + base: base_url.to_string(), credentials: credentials.map(Arc::new), owner, } } - fn upstream<'a>(&'a self, registry: &'a Registry) -> Upstream<'a> { + fn upstream<'a>(&self, registry: &'a Registry) -> Upstream<'a> { Upstream { - reg: registry, + http: ®istry.inner.http, owner: self.owner.clone(), - base: self.base.clone(), + remote: self.base.clone(), creds: self.credentials.clone(), } } @@ -127,7 +132,7 @@ pub struct ImportedImage { /// A manifest fetched from the source and checked against its digest. struct Fetched { - digest: Digest, + digest: ContentDigest, body: Bytes, parsed: ParsedManifest, } @@ -144,11 +149,8 @@ impl Registry { tag: &str, expected: Option<&str>, options: &ImageImportOptions, - ) -> OciResult { + ) -> ImgResult { let t = self.import_target(repo, image, &options.principal).await?; - if !reference::valid_tag(tag) { - return Err(OciError::manifest_invalid(format!("invalid tag {tag:?}"))); - } let pool = &self.inner.pool; let existing = oci_dao::find_tag(pool, repo.id, image, tag) .await? @@ -179,7 +181,9 @@ impl Registry { } let upstream = source.upstream(self); - let fetched = self.fetch_manifest(&upstream, image, tag, None).await?; + let fetched = self + .fetch_manifest(&upstream, image, &Reference::Tag(tag.to_string()), None) + .await?; let digest = fetched.digest.to_string(); let decided = |outcome| { Ok(ImportedImage { @@ -256,10 +260,13 @@ impl Registry { image: &str, digest: &str, options: &ImageImportOptions, - ) -> OciResult { + ) -> ImgResult { let t = self.import_target(repo, image, &options.principal).await?; - let digest = Digest::parse(digest) - .map_err(|e| OciError::digest_invalid(format!("invalid digest {digest:?}: {e}")))?; + let digest = ContentDigest::parse(digest).ok_or_else(|| { + Failure::new(ImageError::ManifestInvalid(format!( + "invalid digest {digest:?}" + ))) + })?; let pool = &self.inner.pool; // Held already: a referrer copied before, or a manifest an imported // tag brought along (an index's child), which is no referrer. @@ -276,7 +283,12 @@ impl Registry { } let upstream = source.upstream(self); let fetched = self - .fetch_manifest(&upstream, image, digest.as_str(), Some(&digest)) + .fetch_manifest( + &upstream, + image, + &Reference::Digest(digest.to_string()), + Some(&digest), + ) .await?; let mut bytes = fetched.body.len() as u64; let decided = |outcome| { @@ -292,7 +304,7 @@ impl Registry { if options.dry_run { return decided(ImageOutcome::Planned { replaces: false }); } - if oci_dao::find_manifest(pool, repo.id, image, subject.digest.as_str()) + if oci_dao::find_manifest(pool, repo.id, image, subject) .await? .is_none() { @@ -337,26 +349,33 @@ impl Registry { }) } - /// The image `image` of `repo` as an import target: a hosted docker - /// repository the import's administrator may push the image to. + /// The image `image` of `repo` as an import target: a hosted repository + /// of the images format the import's administrator may push the image + /// to. async fn import_target( &self, repo: &Repository, image: &str, principal: &Principal, - ) -> OciResult { - if !repo.is_docker() || repo.kind() != RepoKind::Hosted { - return Err(OciError::unsupported( + ) -> ImgResult { + if !self.serves(repo) || repo.kind() != RepoKind::Hosted { + return Err(Failure::unsupported( + 405, "images are imported into a hosted docker repository", )); } - if !reference::valid_name(image) { - return Err(OciError::name_invalid(image)); + if self.plugin().validate_id(image).is_err() { + return Err(Failure::new(ImageError::NameInvalid { + name: image.to_string(), + hint: None, + })); } if !self.rbac_allows(principal, repo, image, Level::Write).await { - return Err(OciError::denied(format!( - "the import may not push the image {image:?}" - ))); + return Err(Failure::new(ImageError::Denied { + name: image.to_string(), + permission: forge_plugin::api::images::Permission::Write, + }) + .with_text(format!("the import may not push the image {image:?}"))); } Ok(Target { repo: repo.clone(), @@ -375,27 +394,28 @@ impl Registry { fetched: &Fetched, options: &ImageImportOptions, depth: usize, - ) -> OciResult { + ) -> ImgResult { let pool = &self.inner.pool; let mut bytes = 0; - for child in fetched.parsed.manifest_references() { - if oci_dao::find_manifest(pool, t.repo.id, &t.image, child.digest.as_str()) + for child in &fetched.parsed.manifests { + if oci_dao::find_manifest(pool, t.repo.id, &t.image, child) .await? .is_some() { continue; } if depth >= MAX_DEPTH { - return Err(OciError::manifest_invalid( - "the source's indexes nest too deep", - )); + return Err(Failure::new(ImageError::ManifestInvalid( + "the source's indexes nest too deep".into(), + ))); } + let digest = ContentDigest::named(child)?; let child = self .fetch_manifest( upstream, &t.image, - child.digest.as_str(), - Some(&child.digest), + &Reference::Digest(child.clone()), + Some(&digest), ) .await?; bytes += child.body.len() as u64; @@ -423,12 +443,10 @@ impl Registry { ) .await?; } - for blob in fetched.parsed.blob_references() { - // Non-distributable layers live elsewhere by definition; the - // empty JSON blob is linked by `commit_manifest` itself. - if media::is_non_distributable(&blob.media_type) - || blob.digest.as_str() == EMPTY_JSON_DIGEST - { + for blob in &fetched.parsed.blobs { + // Content that lives elsewhere by definition, and content the + // format knows (linked by `commit_manifest` itself). + if !blob.required || blob.known.is_some() { continue; } bytes += self.copy_blob(t, upstream, blob).await?; @@ -442,10 +460,10 @@ impl Registry { &self, t: &Target, upstream: &Upstream<'_>, - blob: &Descriptor, - ) -> OciResult { + blob: &ManifestBlob, + ) -> ImgResult { let pool = &self.inner.pool; - let digest = &blob.digest; + let digest = ContentDigest::named(&blob.digest)?; if oci_dao::find_link(pool, t.repo.id, &t.image, digest.as_str()) .await? .is_some() @@ -454,9 +472,9 @@ impl Registry { } // Another image of this repository holds it: its content is the // digest's, and the repository's readers can already read it. - if digest.algorithm() == Algorithm::Sha256 { + if let Some(sha256) = digest.sha256_hex() { if let Some((_, size)) = - oci_dao::blob_linked_in_repository(pool, t.repo.id, digest.hex()).await? + oci_dao::blob_linked_in_repository(pool, t.repo.id, sha256).await? { let mut tx = pool.begin().await?; match oci_dao::insert_link_existing( @@ -464,7 +482,7 @@ impl Registry { t.repo.id, &t.image, digest.as_str(), - digest.hex(), + sha256, size, ) .await? @@ -478,21 +496,22 @@ impl Registry { } } + let url = self.image_url( + &upstream.remote, + &UpstreamOp::ImageBlob { + image: t.image.clone(), + digest: digest.to_string(), + }, + )?; let response = upstream - .send( - Method::GET, - &format!("/v2/{}/blobs/{digest}", t.image), - &format!("repository:{}:pull", t.image), - None, - RequestClass::Download, - ) + .send_url(Method::GET, &url, RequestClass::Download) .await?; if !response.status().is_success() { return Err(source_status(response.status(), digest.as_str())); } let max = self.blob_limit(&t.repo); if response.content_length().is_some_and(|l| l > max) { - return Err(OciError::too_large(max)); + return Err(Failure::new(ImageError::TooLarge { limit: max })); } let body = response.bytes_stream().map_err(std::io::Error::other); let staged = self @@ -501,7 +520,7 @@ impl Registry { .stage(tokio_util::io::StreamReader::new(body), max) .await?; if !digest.matches(&staged.sha256, &staged.sha512_base64) { - return Err(OciError::unavailable(format!( + return Err(Failure::unavailable(format!( "the source served a blob that does not match its digest {digest}" ))); } @@ -518,7 +537,7 @@ impl Registry { stored.ingest_token, ) .await - .map_err(OciError::from) + .map_err(Failure::from) .and_then(require_linked)?; tx.commit().await?; Ok(size) @@ -532,53 +551,59 @@ impl Registry { &self, upstream: &Upstream<'_>, image: &str, - reference: &str, - expected: Option<&Digest>, - ) -> OciResult { - let accept = media::ACCEPT_ALL.join(", "); + reference: &Reference, + expected: Option<&ContentDigest>, + ) -> ImgResult { + let what = reference.as_str(); + let url = self.image_url( + &upstream.remote, + &UpstreamOp::ImageManifest { + image: image.to_string(), + reference: reference.clone(), + }, + )?; let response = upstream - .send( - Method::GET, - &format!("/v2/{image}/manifests/{reference}"), - &format!("repository:{image}:pull"), - Some(&accept), - RequestClass::Metadata, - ) + .send_url(Method::GET, &url, RequestClass::Metadata) .await?; if !response.status().is_success() { - return Err(source_status(response.status(), reference)); + return Err(source_status(response.status(), what)); } let announced = response .headers() .get("docker-content-digest") .and_then(|v| v.to_str().ok()) - .and_then(|v| Digest::parse(v).ok()); + .and_then(ContentDigest::parse); let content_type = content_type(&response); - let body = read_limited(response, manifest::MAX_MANIFEST_BYTES).await?; + let body = read_limited(response, MAX_MANIFEST_BYTES).await?; let mismatch = || { - OciError::unavailable(format!( - "the source served a manifest for {image}:{reference} that does not match its digest" + Failure::unavailable(format!( + "the source served a manifest for {image}:{what} that does not match its digest" )) }; let digest = match expected { Some(expected) => { - if Digest::of(expected.algorithm(), &body) != *expected { + if !expected.matches_bytes(&body) { return Err(mismatch()); } expected.clone() } None => { if let Some(announced) = &announced { - if Digest::of(announced.algorithm(), &body) != *announced { + if !announced.matches_bytes(&body) { return Err(mismatch()); } } - Digest::sha256_of(&body) + ContentDigest::sha256_of(&body) } }; - let parsed = manifest::parse(&body, content_type.as_deref()).map_err(|e| { - OciError::manifest_invalid(format!("the source's manifest {image}:{reference}: {e}")) - })?; + let parsed = self + .parse_manifest(&body, content_type.as_deref()) + .map_err(|e| { + Failure::new(ImageError::ManifestInvalid(format!( + "the source's manifest {image}:{what}: {}", + e.message() + ))) + })?; Ok(Fetched { digest, body, @@ -588,22 +613,18 @@ impl Registry { } /// The error of a source answer that is not a success. -fn source_status(status: reqwest::StatusCode, what: &str) -> OciError { +fn source_status(status: reqwest::StatusCode, what: &str) -> Failure { match status.as_u16() { - 404 => OciError::new( - axum::http::StatusCode::NOT_FOUND, - Code::ManifestUnknown, - format!("the source does not have {what}"), - ), - 401 | 403 => OciError::unavailable(format!( + 404 => Failure::new(ImageError::ManifestUnknown { + reference: what.to_string(), + }) + .with_text(format!("the source does not have {what}")), + 401 | 403 => Failure::unavailable(format!( "the source registry refused the import's credentials ({status}) for {what}" )), - 429 => OciError::new( - axum::http::StatusCode::TOO_MANY_REQUESTS, - Code::TooManyRequests, - "the source registry rate-limits the import; retry later", - ), - _ => OciError::unavailable(format!("the source registry answered {status} for {what}")), + 429 => Failure::new(ImageError::RateLimited) + .with_text("the source registry rate-limits the import; retry later"), + _ => Failure::unavailable(format!("the source registry answered {status} for {what}")), } } diff --git a/crates/server/src/images/mod.rs b/crates/server/src/images/mod.rs new file mode 100644 index 00000000..89d4fbbd --- /dev/null +++ b/crates/server/src/images/mod.rs @@ -0,0 +1,389 @@ +//! The image store: the repositories of a format whose descriptor says +//! `Storage::Images` (Docker / OCI, `crates/formats/docker`) hold images, +//! with hosted, proxy and group repositories over Forge's CAS. The format +//! owns the client protocol under `/v2`; this module is everything else: +//! +//! * [`dispatch`]: the `/v2` request, routed and authorized here, then a +//! conversation with the format ([`forge_plugin::api::images`]): the +//! site (by path, or a repository's own host, `config.host`), the caller +//! (registry Bearer tokens), RBAC, package selectors and token scopes on +//! the image, where a change lands (a group's writable member), the +//! offline mode, the queries and actions the format asks for, and the +//! response hardening; +//! * [`token`], [`auth`]: the registry's Bearer tokens and the token service +//! (`/v2/token`); +//! * [`push`], [`pull`], [`group`], [`proxy`]: upload sessions, chunks and +//! cross-repository mounts, manifest commits and their references, +//! immutable tags, the trusted-publishing ledger; manifests, blobs, tags +//! and referrers of hosted, proxy and group repositories; an upstream +//! registry's token exchange and caching; +//! * [`gc`]: abandoned upload sessions and unreferenced blob links; +//! * [`import`]: copying images from another registry (a Nexus hosted +//! repository) into a hosted repository, through the same store. + +pub mod auth; +pub mod digest; +pub mod dispatch; +pub mod gc; +mod group; +pub mod import; +mod proxy; +mod pull; +mod push; +pub mod token; + +use std::sync::Arc; + +use axum::http::{HeaderName, HeaderValue}; +use forge_cache::Cache; +use forge_core::auth::authz::Authz; +use forge_core::auth::Principal; +use forge_core::net::upstream::UpstreamHttp; +use forge_core::repo::PublishPolicy; +use forge_db::model::Repository; +use forge_plugin::api::images::ImageError; +use forge_plugin::FormatPlugin; +use forge_storage::{BlobStore, Cas}; +use sqlx::PgPool; + +pub use dispatch::registry_origin; +use token::TokenKeys; + +/// Tunables of the registry (the server's `[docker]` section and friends). +#[derive(Debug, Clone)] +pub struct RegistryConfig { + /// `server.public_base_url`, without a trailing `/`. + pub public_base_url: String, + /// Largest blob (layer) a proxy downloads from its upstream. + pub max_blob_bytes: u64, + /// Largest blob a push to a hosted repository may carry when the + /// repository has no upload limit of its own + /// ([`Repository::max_upload_bytes`]). + pub default_max_upload_bytes: u64, + /// Lifetime of a registry Bearer token. + pub token_ttl_seconds: u64, + /// Outbox grace of released blob references. + pub delete_grace: chrono::Duration, + /// `groups.max_nesting_depth`. + pub max_group_depth: u32, + /// `docker.unreferenced_blob_ttl_seconds`: a blob link no manifest names + /// is released after this long unused. + pub unreferenced_blob_ttl: chrono::Duration, +} + +impl RegistryConfig { + /// How often a pull (or `HEAD`) of a blob records a use of its link: a + /// tenth of the unreferenced-blob TTL, at most hourly. A link a client + /// confirmed then stays at least nine tenths of the TTL, whereas an + /// hourly record with the smallest TTL (an hour) could leave it none. + pub fn link_touch_interval(&self) -> chrono::Duration { + (self.unreferenced_blob_ttl / 10).min(chrono::Duration::hours(1)) + } +} + +/// Builds the request-scoped policy that records a workload's publishes +/// (the trusted-publishing ledger) in the manifest transaction. +pub trait PublishRecorder: Send + Sync { + fn for_repository(&self, repository: &Repository) -> Arc; +} + +/// Whether the workload principal of a registry token still stands. The +/// server answers it from its realms: like a workload key, a token stands +/// only while its realm is enabled with the configuration it was verified +/// under, and, when it was minted with a workload key, while that key has +/// not ended. +#[async_trait::async_trait] +pub trait WorkloadCheck: Send + Sync { + async fn still_valid(&self, principal: &Principal) -> Result; +} + +/// The registry: everything under the images format's mount (`/v2`). +#[derive(Clone)] +pub struct Registry { + pub(crate) inner: Arc, +} + +pub(crate) struct Inner { + pub pool: PgPool, + pub cas: Cas, + pub store: BlobStore, + pub cache: Cache, + pub authz: Authz, + pub http: UpstreamHttp, + pub envelope: Option>, + pub config: RegistryConfig, + pub keys: TokenKeys, + pub recorder: Option>, + pub workloads: Option>, + /// The images format: the protocol. + pub plugin: Arc, +} + +/// Parts the registry is built from. +pub struct RegistryParts { + pub pool: PgPool, + pub cas: Cas, + pub store: BlobStore, + pub cache: Cache, + pub authz: Authz, + pub http: UpstreamHttp, + pub envelope: Option>, + pub config: RegistryConfig, + pub keys: TokenKeys, + pub recorder: Option>, + pub workloads: Option>, + pub plugin: Arc, +} + +impl Registry { + pub fn new(parts: RegistryParts) -> Self { + Registry { + inner: Arc::new(Inner { + pool: parts.pool, + cas: parts.cas, + store: parts.store, + cache: parts.cache, + authz: parts.authz, + http: parts.http, + envelope: parts.envelope, + config: parts.config, + keys: parts.keys, + recorder: parts.recorder, + workloads: parts.workloads, + plugin: parts.plugin, + }), + } + } + + pub fn config(&self) -> &RegistryConfig { + &self.inner.config + } + + /// The images format. + pub(crate) fn plugin(&self) -> &dyn FormatPlugin { + self.inner.plugin.as_ref() + } + + /// The images format's id: `repository.format` of its repositories. + pub(crate) fn format(&self) -> &'static str { + self.inner.plugin.descriptor().id + } + + /// Whether `repo` is a repository of the images format. + pub(crate) fn serves(&self, repo: &Repository) -> bool { + repo.format == self.format() + } +} + +/// What went wrong serving an image request: what the format answers +/// ([`ImageError`], [`FormatPlugin::image_error`]), the headers the host +/// adds to it (a Bearer challenge), and what to tell an administrator (an +/// import's report, the log). +#[derive(Debug, Clone)] +pub struct Failure { + pub error: ImageError, + pub headers: Vec<(HeaderName, HeaderValue)>, + /// Said instead of the error's own description. + text: Option, +} + +/// Result alias of the image store. +pub type ImgResult = Result; + +impl Failure { + pub fn new(error: ImageError) -> Self { + Failure { + error, + headers: Vec::new(), + text: None, + } + } + + /// Say `text` of it instead of the error's own description. + pub fn with_text(mut self, text: impl Into) -> Self { + self.text = Some(text.into()); + self + } + + pub fn with_header(mut self, name: HeaderName, value: HeaderValue) -> Self { + self.headers.push((name, value)); + self + } + + /// The service cannot answer now (staging full, upstream down, offline + /// repository): the client may retry. + pub fn unavailable(message: impl Into) -> Self { + Failure::new(ImageError::Unavailable(message.into())) + } + + /// What the host does not do (405, or `status`). + pub fn unsupported(status: u16, message: impl Into) -> Self { + Failure::new(ImageError::Unsupported { + status, + message: message.into(), + }) + } + + /// An internal failure; the details go to the log, not the client. + pub fn internal(context: &str, error: impl std::fmt::Display) -> Self { + tracing::error!(error = %error, "registry: {context}"); + Failure::new(ImageError::Internal) + } + + /// A failed read from the object store (through the local blob cache): + /// unavailable while S3 cannot be reached, a server error otherwise. + pub(crate) fn storage_read(context: &str, error: forge_storage::S3Error) -> Self { + if error.is_unavailable() { + tracing::warn!(%error, "registry: {context}: the object store is unavailable"); + Failure::object_store_unavailable() + } else { + Failure::internal(context, error) + } + } + + fn object_store_unavailable() -> Self { + Failure::unavailable("the object store is unavailable; retry later") + } + + /// What went wrong, for an administrator. + pub fn message(&self) -> String { + if let Some(text) = &self.text { + return text.clone(); + } + match &self.error { + ImageError::NotAnEndpoint => "not a registry endpoint".into(), + ImageError::NameInvalid { name, .. } => format!("invalid image name {name:?}"), + ImageError::NameUnknown { name } => format!("no repository named by {name:?}"), + ImageError::AuthenticationRequired { .. } => "authentication required".into(), + ImageError::InvalidToken => "invalid or expired registry token".into(), + ImageError::InvalidCredentials => "invalid credentials".into(), + ImageError::Denied { name, .. } => format!("access to {name:?} denied"), + ImageError::Unsupported { message, .. } + | ImageError::ManifestInvalid(message) + | ImageError::Unavailable(message) => message.clone(), + ImageError::Offline => "the repository is offline".into(), + ImageError::BlobUnknown { digest } => format!("blob {digest} unknown"), + ImageError::ManifestUnknown { reference } => format!("manifest {reference} unknown"), + ImageError::UploadUnknown => "upload unknown".into(), + ImageError::DigestMismatch => "the content does not match its digest".into(), + ImageError::ChunkOutOfOrder { .. } => "the chunk does not continue the upload".into(), + ImageError::ChunkLength { .. } => { + "the chunk length does not match its Content-Range".into() + } + ImageError::TooLarge { limit } => { + format!("content exceeds the limit of {limit} bytes") + } + ImageError::BodyUnreadable => "the request body could not be read".into(), + ImageError::ManifestBlobUnknown { digest, manifest } => format!( + "the manifest references a {} the repository does not hold: {digest}", + if *manifest { "manifest" } else { "blob" } + ), + ImageError::TagImmutable { tag } => { + format!("tag {tag:?} already exists and tags are immutable in this repository") + } + ImageError::BlobReferenced { digest } => { + format!("the blob {digest} is referenced by a manifest") + } + ImageError::NoLongerAllowed => { + "the workload identity may no longer change this repository".into() + } + ImageError::RateLimited => "the upstream registry rate-limits; retry later".into(), + ImageError::Internal => "internal error".into(), + } + } +} + +impl From for Failure { + fn from(error: ImageError) -> Self { + Failure::new(error) + } +} + +impl From for Failure { + fn from(e: sqlx::Error) -> Self { + Failure::internal("database error", e) + } +} + +/// A request body that could not be read: the client aborted it, or it +/// stalled past the idle timeout. Travels inside the I/O error staging the +/// body fails with. +#[derive(Debug)] +pub(crate) struct RequestBodyError(pub axum::Error); + +impl std::fmt::Display for RequestBodyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "reading the request body: {}", self.0) + } +} + +impl std::error::Error for RequestBodyError {} + +impl From for Failure { + fn from(e: forge_storage::CasError) -> Self { + match e { + forge_storage::CasError::TooLarge { limit } => { + Failure::new(ImageError::TooLarge { limit }) + } + forge_storage::CasError::StagingFull { .. } => { + Failure::unavailable("temporary storage is full; retry later") + } + e if e.is_unavailable() => { + tracing::warn!(error = %e, "registry: the object store is unavailable"); + Failure::object_store_unavailable() + } + forge_storage::CasError::Io(e) + if e.get_ref() + .is_some_and(|inner| inner.is::()) => + { + tracing::debug!(error = %e, "registry: the upload body could not be read"); + Failure::new(ImageError::BodyUnreadable) + } + other => Failure::internal("storage error", other), + } + } +} + +/// A header value from text Forge produced or validated. +pub(crate) fn header_value(s: &str) -> HeaderValue { + HeaderValue::from_str(s).unwrap_or_else(|_| HeaderValue::from_static("invalid")) +} + +#[cfg(test)] +mod tests { + use forge_storage::{CasError, S3Error}; + + use super::*; + + fn down() -> S3Error { + S3Error::Unavailable("connection refused".into()) + } + + #[test] + fn an_unavailable_object_store_is_worth_a_retry() { + // Also when a stream read from S3 failed while it was staged (the + // finish of a chunked upload). + for e in [ + CasError::S3(down()), + CasError::Io(std::io::Error::other(down())), + ] { + assert!(matches!(Failure::from(e).error, ImageError::Unavailable(_))); + } + let e = Failure::storage_read("blob read", down()); + assert!(matches!(e.error, ImageError::Unavailable(_))); + let e = Failure::storage_read("blob read", S3Error::S3("access denied".into())); + assert_eq!(e.error, ImageError::Internal); + } + + #[test] + fn an_unreadable_request_body_is_the_clients_error() { + let aborted = std::io::Error::other(RequestBodyError(axum::Error::new( + "connection reset by peer", + ))); + let e = Failure::from(CasError::Io(aborted)); + assert_eq!(e.error, ImageError::BodyUnreadable); + // Staging's own I/O errors stay server errors. + let e = Failure::from(CasError::Io(std::io::Error::other("disk full"))); + assert_eq!(e.error, ImageError::Internal); + } +} diff --git a/crates/formats/docker/src/proxy.rs b/crates/server/src/images/proxy.rs similarity index 64% rename from crates/formats/docker/src/proxy.rs rename to crates/server/src/images/proxy.rs index 1cfa4bf2..abf8d053 100644 --- a/crates/formats/docker/src/proxy.rs +++ b/crates/server/src/images/proxy.rs @@ -9,47 +9,41 @@ //! tag is served. //! * Multi-arch indexes are cached as they are; their child manifests are //! fetched when a client asks for them. -//! * Docker Hub's implicit `library/` namespace applies to one-component -//! names (`nginx` is `library/nginx`), locally as well, so both spellings -//! share one cache. +//! * The images format names what to ask the upstream for +//! ([`UpstreamOp`]): the name the upstream knows an image by (Docker Hub's +//! `library/` namespace, so both spellings share one cache), a manifest's +//! and a blob's URL, and it reads the upstream's tag lists and referrers. //! //! Upstream requests go through the shared [`UpstreamHttp`] client (metadata -//! guard, timeouts, redirects without credentials). Docker's token flow is -//! followed: a `401` with a `Bearer` challenge sends the upstream credentials -//! to the token realm that challenge names, and the token is cached until it -//! expires. +//! guard, timeouts, redirects without credentials), to the origin of the +//! repository's `remote_url` only. Docker's token flow is followed: a `401` +//! with a `Bearer` challenge sends the upstream credentials to the token +//! realm that challenge names, and the token is cached until it expires. -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::{Duration, Instant}; -use axum::http::StatusCode; use bytes::Bytes; -use forge_core::net::upstream::{Origin, RequestClass}; +use forge_core::net::upstream::{Origin, RequestClass, UpstreamHttp}; use forge_core::repo::proxy::{upstream_credentials, Credentials}; use forge_db::model::Repository; use forge_db::oci_dao::{self, LinkOutcome}; +use forge_plugin::api::images::ImageError; +use forge_plugin::api::images::{ImageUrl, ManifestInfo, Reference, TagPage, MAX_MANIFEST_BYTES}; +use forge_plugin::api::{ + FetchResult, Fetched, FormatError, Remote, UpstreamAnswer, UpstreamOp, UpstreamStep, +}; use forge_storage::BlobRef; use futures_util::{StreamExt, TryStreamExt}; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, WWW_AUTHENTICATE}; use reqwest::Method; -use crate::digest::Digest; -use crate::error::{Code, OciError, OciResult}; -use crate::manifest::{self, media, MAX_MANIFEST_BYTES}; -use crate::pull::{BlobHit, ManifestHit, ReferrerDescriptor, TagPage}; -use crate::push::require_linked; -use crate::reference::{self, Reference}; -use crate::service::{Registry, Target}; - -/// Hosts of Docker Hub's registry, whose one-component names live in -/// `library/`. -const DOCKER_HUB_HOSTS: &[&str] = &[ - "registry-1.docker.io", - "index.docker.io", - "docker.io", - "registry.hub.docker.com", -]; +use super::digest::ContentDigest; +use super::dispatch::Target; +use super::pull::{BlobHit, ManifestHit}; +use super::push::{merge, require_linked}; +use super::{Failure, ImgResult, Registry}; /// How long a tag is served without asking the upstream, unless the /// repository sets `metadata_ttl_seconds`. @@ -59,16 +53,15 @@ const DEFAULT_TAG_TTL_SECS: i64 = 600; /// sets `not_found_ttl_seconds`. const DEFAULT_NOT_FOUND_TTL_SECS: u64 = 60; -/// Largest blob read as a possible image config. +/// Largest blob read as a possible config. const MAX_CONFIG_BYTES: i64 = 1024 * 1024; /// Largest upstream tag list or referrers document read. const MAX_LISTING_BYTES: usize = 16 * 1024 * 1024; -/// Pages of an upstream's tag list followed (`Link: rel="next"`) to fill -/// one page of Forge's: an upstream that pages in small steps is listed -/// further on the client's next request. -const MAX_UPSTREAM_TAG_PAGES: usize = 10; +/// The most rounds of one upstream conversation of the images format: an +/// upstream's tag list may take a page per round. +const MAX_UPSTREAM_ROUNDS: usize = 12; /// Cached upstream registry tokens: key → (token, expiry). Bounded: the key /// names an image a client chose, and the token and its lifetime are the @@ -132,44 +125,6 @@ fn cache_token(key: String, token: String, ttl: Duration) { tokens.insert(key, (token, now + ttl.min(MAX_TOKEN_TTL))); } -/// The upstream base URL of a proxy: `remote_url` without a trailing `/` -/// (and without a `/v2` suffix an admin may have pasted). -pub(crate) fn upstream_base(remote_url: &str) -> String { - let base = remote_url.trim_end_matches('/'); - base.strip_suffix("/v2").unwrap_or(base).to_string() -} - -/// The `next` page a response's `Link` header names (RFC 8288), as a path -/// under `base`, resolved against the response's URL: only a link back to -/// the upstream itself is followed, never one to another host. -fn next_link(response: &reqwest::Response, base: &str) -> Option { - response - .headers() - .get_all(reqwest::header::LINK) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .find_map(|link| { - let mut parts = link.split(';'); - let target = parts.next()?.trim().strip_prefix('<')?.strip_suffix('>')?; - let next = parts.any(|param| { - let param = param.trim().replace(' ', ""); - param.eq_ignore_ascii_case("rel=\"next\"") || param.eq_ignore_ascii_case("rel=next") - }); - if !next { - return None; - } - let url = response.url().join(target).ok()?; - // Both as the URL parser writes them (lowercase host, no - // default port). - let base = url::Url::parse(base).ok()?; - url.as_str() - .strip_prefix(base.as_str().trim_end_matches('/')) - .filter(|path| path.starts_with("/v2/")) - .map(str::to_string) - }) -} - /// Inside a transaction that caches what `repo`'s upstream served: lock the /// repository row and make sure the proxy still points at the upstream the /// content came from ([`oci_dao::lock_proxy_upstream`]). A proxy repointed @@ -178,31 +133,16 @@ fn next_link(response: &reqwest::Response, base: &str) -> Option { pub(crate) async fn still_upstream( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, repo: &Repository, -) -> OciResult<()> { +) -> ImgResult<()> { if oci_dao::lock_proxy_upstream(tx, repo.id, repo.remote_url.as_deref()).await? { Ok(()) } else { - Err(OciError::unavailable( + Err(Failure::unavailable( "the proxy was pointed at another upstream while this was fetched; retry", )) } } -fn is_docker_hub(remote_url: &str) -> bool { - url::Url::parse(remote_url) - .ok() - .and_then(|u| u.host_str().map(str::to_ascii_lowercase)) - .is_some_and(|h| DOCKER_HUB_HOSTS.contains(&h.as_str())) -} - -/// The image name a proxy stores and asks its upstream for. -pub fn proxied_image(repo: &Repository, image: &str) -> String { - match &repo.remote_url { - Some(remote) if is_docker_hub(remote) && !image.contains('/') => format!("library/{image}"), - _ => image.to_string(), - } -} - /// A parsed `WWW-Authenticate` challenge. #[derive(Debug, Clone, PartialEq, Eq)] enum Challenge { @@ -250,42 +190,38 @@ fn parse_challenge(value: &str) -> Option { } /// Read at most `limit` bytes of an upstream body. -pub(crate) async fn read_limited(response: reqwest::Response, limit: usize) -> OciResult { +pub(crate) async fn read_limited(response: reqwest::Response, limit: usize) -> ImgResult { if response.content_length().is_some_and(|l| l > limit as u64) { - return Err(OciError::unavailable("the upstream response is too large")); + return Err(Failure::unavailable("the upstream response is too large")); } let mut stream = response.bytes_stream(); let mut out = Vec::new(); while let Some(chunk) = stream.next().await { let chunk = chunk.map_err(|e| upstream_unreachable(&e))?; if out.len() + chunk.len() > limit { - return Err(OciError::unavailable("the upstream response is too large")); + return Err(Failure::unavailable("the upstream response is too large")); } out.extend_from_slice(&chunk); } Ok(Bytes::from(out)) } -fn upstream_unreachable(e: &dyn std::fmt::Display) -> OciError { +fn upstream_unreachable(e: &dyn std::fmt::Display) -> Failure { tracing::warn!(error = %e, "registry proxy: upstream request failed"); - OciError::unavailable("the upstream registry is unreachable") + Failure::unavailable("the upstream registry is unreachable") } /// The error a non-success upstream status turns into. -fn upstream_status(status: reqwest::StatusCode) -> OciError { +fn upstream_status(status: reqwest::StatusCode) -> Failure { match status.as_u16() { - 429 => OciError::new( - StatusCode::TOO_MANY_REQUESTS, - Code::TooManyRequests, - "the upstream registry rate-limits this proxy; retry later", - ), + 429 => Failure::new(ImageError::RateLimited), 401 | 403 => { tracing::warn!(%status, "registry proxy: upstream refused the request"); - OciError::unavailable("the upstream registry refused the proxy's credentials") + Failure::unavailable("the upstream registry refused the proxy's credentials") } _ => { tracing::warn!(%status, "registry proxy: upstream answered with an error"); - OciError::unavailable("the upstream registry answered with an error") + Failure::unavailable("the upstream registry answered with an error") } } } @@ -293,16 +229,25 @@ fn upstream_status(status: reqwest::StatusCode) -> OciError { /// An upstream registry (a proxy repository's, or an import's source), for /// one request. pub(crate) struct Upstream<'a> { - pub(crate) reg: &'a Registry, + pub(crate) http: &'a UpstreamHttp, /// Who the cached tokens belong to: a proxy repository's id, an import. pub(crate) owner: String, - pub(crate) base: String, + /// The upstream's URL, as configured: requests go to its origin only. + pub(crate) remote: String, pub(crate) creds: Option>, } impl Upstream<'_> { fn token_key(&self, scope: &str) -> String { - format!("{}:{}:{scope}", self.owner, self.base) + format!("{}:{}:{scope}", self.owner, self.remote) + } + + /// What the images format sees of the upstream. + pub(crate) fn as_remote(&self) -> Remote { + Remote { + url: self.remote.clone(), + index: None, + } } async fn attempt( @@ -313,7 +258,7 @@ impl Upstream<'_> { class: RequestClass, bearer: Option<&str>, basic: bool, - ) -> OciResult { + ) -> ImgResult { let mut headers = HeaderMap::new(); if let Some(accept) = accept { if let Ok(v) = HeaderValue::from_str(accept) { @@ -330,36 +275,36 @@ impl Upstream<'_> { } else { None }; - self.reg - .inner - .http + self.http .send(method, url, class, headers, auth.as_ref()) .await .map_err(|e| upstream_unreachable(&e)) } - /// Send a request for `path` (`/v2/…`), answering the upstream's - /// authentication challenge once. + /// Send a request to `url`, answering the upstream's authentication + /// challenge once, with a token for its scope (else `scope`). Only to + /// the origin of the upstream's URL: the images format names no other. pub(crate) async fn send( &self, method: Method, - path: &str, - scope: &str, + url: &str, + scope: Option<&str>, accept: Option<&str>, class: RequestClass, - ) -> OciResult { - let url = format!("{}{path}", self.base); + ) -> ImgResult { + let origin = Origin::of(&self.remote); + if origin.is_none() || Origin::of(url) != origin { + tracing::warn!(%url, remote = %self.remote, "images format named an URL off its upstream; not sent"); + return Err(Failure::internal( + "upstream request", + format!("{url} is not on the upstream's origin"), + )); + } + let scope = scope.unwrap_or_default(); let key = self.token_key(scope); let cached = cached_token(&key); let response = self - .attempt( - method.clone(), - &url, - accept, - class, - cached.as_deref(), - false, - ) + .attempt(method.clone(), url, accept, class, cached.as_deref(), false) .await?; if response.status() != reqwest::StatusCode::UNAUTHORIZED { return Ok(response); @@ -367,7 +312,7 @@ impl Upstream<'_> { // Only the upstream itself names where its credentials go: a 401 // from a host it redirected to (a CDN, an object store) is answered // as the failure it is, never followed to the realm it names. - if Origin::of(response.url().as_str()) != Origin::of(&self.base) { + if Origin::of(response.url().as_str()) != origin { return Ok(response); } let challenge = response @@ -384,11 +329,11 @@ impl Upstream<'_> { let scope = challenged.unwrap_or_else(|| scope.to_string()); let token = self.fetch_token(&realm, service.as_deref(), &scope).await?; cache_token(key, token.0.clone(), token.1); - self.attempt(method, &url, accept, class, Some(&token.0), false) + self.attempt(method, url, accept, class, Some(&token.0), false) .await } Some(Challenge::Basic) if self.creds.is_some() => { - self.attempt(method, &url, accept, class, None, true).await + self.attempt(method, url, accept, class, None, true).await } _ => Ok(response), } @@ -402,14 +347,14 @@ impl Upstream<'_> { realm: &str, service: Option<&str>, scope: &str, - ) -> OciResult<(String, Duration)> { + ) -> ImgResult<(String, Duration)> { let parsed = url::Url::parse(realm) - .map_err(|_| OciError::unavailable("the upstream named an invalid token realm"))?; - let upstream_https = self.base.starts_with("https://"); + .map_err(|_| Failure::unavailable("the upstream named an invalid token realm"))?; + let upstream_https = self.remote.starts_with("https://"); if !matches!(parsed.scheme(), "https" | "http") || (upstream_https && parsed.scheme() != "https") { - return Err(OciError::unavailable( + return Err(Failure::unavailable( "the upstream named a token realm Forge will not send credentials to", )); } @@ -419,15 +364,15 @@ impl Upstream<'_> { if let Some(service) = service { q.append_pair("service", service); } - q.append_pair("scope", scope); + if !scope.is_empty() { + q.append_pair("scope", scope); + } } let auth = match (&self.creds, Origin::of(realm)) { (Some(creds), Some(origin)) => Some(creds.auth_for(origin)), _ => None, }; let response = self - .reg - .inner .http .send( Method::GET, @@ -449,12 +394,12 @@ impl Upstream<'_> { expires_in: Option, } let parsed: TokenResponse = serde_json::from_slice(&body) - .map_err(|_| OciError::unavailable("the upstream token service answered nonsense"))?; + .map_err(|_| Failure::unavailable("the upstream token service answered nonsense"))?; let token = parsed .token .or(parsed.access_token) .filter(|t| !t.is_empty()) - .ok_or_else(|| OciError::unavailable("the upstream token service sent no token"))?; + .ok_or_else(|| Failure::unavailable("the upstream token service sent no token"))?; let ttl = parsed .expires_in .unwrap_or(60) @@ -463,25 +408,174 @@ impl Upstream<'_> { .max(10); Ok((token, Duration::from_secs(ttl))) } + + /// Send the request the images format named for a manifest or a blob. + pub(crate) async fn send_url( + &self, + method: Method, + url: &ImageUrl, + class: RequestClass, + ) -> ImgResult { + self.send( + method, + &url.url, + url.scope.as_deref(), + url.accept.as_deref(), + class, + ) + .await + } + + /// Run an upstream conversation of the images format: what it answers + /// at once, or after the fetches it asks for (sent to the upstream's + /// origin, with its token exchange, not kept). + pub(crate) async fn converse( + &self, + reg: &Registry, + op: &UpstreamOp, + ) -> ImgResult { + let remote = self.as_remote(); + let mut fetched: Vec = Vec::new(); + for _ in 0..MAX_UPSTREAM_ROUNDS { + let step = reg + .plugin() + .upstream(&remote, op, &fetched) + .map_err(|e| format_failure("upstream", e))?; + let wanted = match step { + UpstreamStep::Done(answer) => return Ok(answer), + UpstreamStep::Fetch(wanted) => wanted, + }; + if wanted.is_empty() { + return Err(Failure::internal( + "upstream", + "the images format asked for no fetch", + )); + } + for fetch in wanted { + let response = self + .send( + Method::GET, + &fetch.url, + fetch.scope.as_deref(), + fetch.accept.as_deref(), + RequestClass::Metadata, + ) + .await; + let result = match response { + Ok(response) => { + let status = response.status(); + let headers: Vec<(String, String)> = response + .headers() + .get_all(reqwest::header::LINK) + .iter() + .filter_map(|v| v.to_str().ok()) + .map(|v| ("link".to_string(), v.to_string())) + .collect(); + let outcome = if status == reqwest::StatusCode::NOT_FOUND { + Fetched::NotFound + } else if !status.is_success() { + Fetched::Failed(upstream_status(status).message()) + } else { + match read_limited(response, MAX_LISTING_BYTES).await { + Ok(body) => Fetched::Found(body.to_vec()), + Err(failure) => Fetched::Failed(failure.message()), + } + }; + FetchResult { + fetch, + outcome, + status: Some(status.as_u16()), + headers, + } + } + Err(failure) => FetchResult { + fetch, + outcome: Fetched::Failed(failure.message()), + status: None, + headers: Vec::new(), + }, + }; + fetched.push(result); + } + } + Err(Failure::internal( + "upstream", + format!("the images format needed more than {MAX_UPSTREAM_ROUNDS} rounds"), + )) + } +} + +/// A format error of the upstream side: a reference it does not take (an +/// import's tag), else the format's bug. +fn format_failure(context: &str, e: FormatError) -> Failure { + match e { + FormatError::InvalidVersion(m) | FormatError::InvalidId(m) => { + Failure::new(ImageError::ManifestInvalid(m)) + } + other => Failure::internal(context, other), + } } impl Registry { - async fn upstream<'a>(&'a self, repo: &'a Repository) -> OciResult> { + pub(crate) async fn upstream<'a>(&'a self, repo: &Repository) -> ImgResult> { let remote = repo .remote_url .as_deref() - .ok_or_else(|| OciError::internal("proxy", "proxy repository without remote_url"))?; + .ok_or_else(|| Failure::internal("proxy", "proxy repository without remote_url"))?; let creds = upstream_credentials(&self.inner.pool, self.inner.envelope.as_ref(), repo) .await - .map_err(|e| OciError::internal("proxy credentials", e))?; + .map_err(|e| Failure::internal("proxy credentials", e))?; Ok(Upstream { - reg: self, + http: &self.inner.http, owner: repo.id.to_string(), - base: upstream_base(remote), + remote: remote.to_string(), creds: creds.map(Arc::new), }) } + /// The images format's answer to an operation it answers without + /// fetching anything (a name, a URL), for the upstream at `remote`. + pub(crate) fn upstream_answer( + &self, + remote: &str, + op: &UpstreamOp, + ) -> ImgResult { + let remote = Remote { + url: remote.to_string(), + index: None, + }; + match self.plugin().upstream(&remote, op, &[]) { + Ok(UpstreamStep::Done(answer)) => Ok(answer), + Ok(UpstreamStep::Fetch(_)) => Err(Failure::internal( + "upstream", + "the images format asked for a fetch to name a URL", + )), + Err(e) => Err(format_failure("upstream", e)), + } + } + + /// Where the upstream at `remote` has the manifest or blob `op` names. + pub(crate) fn image_url(&self, remote: &str, op: &UpstreamOp) -> ImgResult { + match self.upstream_answer(remote, op)? { + UpstreamAnswer::ImageUrl(url) => Ok(url), + other => Err(Failure::internal( + "upstream", + format!("the images format answered {other:?} for a URL"), + )), + } + } + + /// The image name a proxy stores and asks its upstream for. + pub(crate) fn proxied_image(&self, repo: &Repository, image: &str) -> String { + let Some(remote) = &repo.remote_url else { + return image.to_string(); + }; + match self.upstream_answer(remote, &UpstreamOp::ImageName(image.to_string())) { + Ok(UpstreamAnswer::ImageName(name)) => name, + _ => image.to_string(), + } + } + /// Keyed by the repository's `updated_at` too, so an administrator's /// cache invalidation (which bumps it) forgets the upstream's "not found" /// answers on every instance. @@ -531,35 +625,33 @@ impl Registry { repo: &Repository, image: &str, reference: &Reference, - ) -> OciResult> { - let local = proxied_image(repo, image); + ) -> ImgResult> { + let local = self.proxied_image(repo, image); let target = Target { repo: repo.clone(), image: local.clone(), name: local.clone(), }; let pool = &self.inner.pool; - let accept = media::ACCEPT_ALL.join(", "); - let scope = format!("repository:{local}:pull"); match reference { Reference::Digest(digest) => { - if let Some(row) = - oci_dao::find_manifest(pool, repo.id, &local, digest.as_str()).await? - { + if let Some(row) = oci_dao::find_manifest(pool, repo.id, &local, digest).await? { return Ok(Some(ManifestHit::from_row(row))); } - if !repo.online || self.negative_cached(repo, &local, digest.as_str()).await { + if !repo.online || self.negative_cached(repo, &local, digest).await { return Ok(None); } + let digest = ContentDigest::named(digest)?; let upstream = self.upstream(repo).await?; + let url = self.image_url( + &upstream.remote, + &UpstreamOp::ImageManifest { + image: local.clone(), + reference: reference.clone(), + }, + )?; let response = upstream - .send( - Method::GET, - &format!("/v2/{local}/manifests/{digest}"), - &scope, - Some(&accept), - RequestClass::Metadata, - ) + .send_url(Method::GET, &url, RequestClass::Metadata) .await?; if response.status() == reqwest::StatusCode::NOT_FOUND { self.remember_not_found(repo, &local, digest.as_str()).await; @@ -570,17 +662,17 @@ impl Registry { } let content_type = content_type(&response); let body = read_limited(response, MAX_MANIFEST_BYTES).await?; - if Digest::of(digest.algorithm(), &body) != *digest { + if !digest.matches_bytes(&body) { tracing::warn!(repo = %repo.name, image = %local, %digest, "registry proxy: upstream manifest does not match its digest"); - return Err(OciError::unavailable( + return Err(Failure::unavailable( "the upstream served a manifest that does not match its digest", )); } - let parsed = manifest::parse(&body, content_type.as_deref()).map_err(|_| { - OciError::unavailable("the upstream served an invalid manifest") - })?; + let parsed = self + .parse_manifest(&body, content_type.as_deref()) + .map_err(|_| Failure::unavailable("the upstream served an invalid manifest"))?; let row = self - .store_proxied_manifest(&target, digest, &parsed, &body, None) + .store_proxied_manifest(&target, &digest, &parsed, &body, None) .await?; let mut hit = ManifestHit::from_row(row); hit.bytes = Some(body); @@ -604,14 +696,14 @@ impl Registry { return Ok(None); } match self - .revalidate_tag(repo, &target, tag, &scope, &accept, cached.as_ref()) + .revalidate_tag(repo, &target, reference, tag, cached.as_ref()) .await { Ok(found) => Ok(found), Err(e) => match cached { Some(c) => { tracing::warn!( - repo = %repo.name, image = %local, %tag, error = %e.message, + repo = %repo.name, image = %local, %tag, error = %e.message(), "registry proxy: upstream unavailable; serving the cached tag" ); Ok(Some(ManifestHit::from_row(c.manifest))) @@ -628,23 +720,22 @@ impl Registry { &self, repo: &Repository, target: &Target, + reference: &Reference, tag: &str, - scope: &str, - accept: &str, cached: Option<&oci_dao::TaggedManifest>, - ) -> OciResult> { + ) -> ImgResult> { let local = &target.image; let upstream = self.upstream(repo).await?; - let path = format!("/v2/{local}/manifests/{tag}"); + let url = self.image_url( + &upstream.remote, + &UpstreamOp::ImageManifest { + image: local.clone(), + reference: reference.clone(), + }, + )?; if let Some(c) = cached { let head = upstream - .send( - Method::HEAD, - &path, - scope, - Some(accept), - RequestClass::Metadata, - ) + .send_url(Method::HEAD, &url, RequestClass::Metadata) .await?; if head.status() == reqwest::StatusCode::NOT_FOUND { self.remember_not_found(repo, local, tag).await; @@ -669,13 +760,7 @@ impl Registry { // usefully: fetch the manifest. } let response = upstream - .send( - Method::GET, - &path, - scope, - Some(accept), - RequestClass::Metadata, - ) + .send_url(Method::GET, &url, RequestClass::Metadata) .await?; if response.status() == reqwest::StatusCode::NOT_FOUND { self.remember_not_found(repo, local, tag).await; @@ -688,20 +773,21 @@ impl Registry { .headers() .get("docker-content-digest") .and_then(|v| v.to_str().ok()) - .and_then(|d| Digest::parse(d).ok()); + .and_then(ContentDigest::parse); let content_type = content_type(&response); let body = read_limited(response, MAX_MANIFEST_BYTES).await?; if let Some(declared) = &declared { - if Digest::of(declared.algorithm(), &body) != *declared { + if !declared.matches_bytes(&body) { tracing::warn!(repo = %repo.name, image = %local, %tag, "registry proxy: upstream manifest does not match its Docker-Content-Digest"); - return Err(OciError::unavailable( + return Err(Failure::unavailable( "the upstream served a manifest that does not match its digest", )); } } - let digest = Digest::sha256_of(&body); - let parsed = manifest::parse(&body, content_type.as_deref()) - .map_err(|_| OciError::unavailable("the upstream served an invalid manifest"))?; + let digest = ContentDigest::sha256_of(&body); + let parsed = self + .parse_manifest(&body, content_type.as_deref()) + .map_err(|_| Failure::unavailable("the upstream served an invalid manifest"))?; let row = self .store_proxied_manifest(target, &digest, &parsed, &body, Some(tag)) .await?; @@ -714,9 +800,9 @@ impl Registry { &self, repo: &Repository, image: &str, - digest: &Digest, - ) -> OciResult> { - let local = proxied_image(repo, image); + digest: &ContentDigest, + ) -> ImgResult> { + let local = self.proxied_image(repo, image); if let Some(hit) = self.hosted_blob(repo, &local, digest).await? { return Ok(Some(hit)); } @@ -729,8 +815,13 @@ impl Registry { return Ok(Some(hit)); } let upstream = self.upstream(repo).await?; - let scope = format!("repository:{local}:pull"); - let path = format!("/v2/{local}/blobs/{digest}"); + let url = self.image_url( + &upstream.remote, + &UpstreamOp::ImageBlob { + image: local.clone(), + digest: digest.to_string(), + }, + )?; let pool = &self.inner.pool; // A layer another image of this proxy already holds (served by this @@ -738,12 +829,12 @@ impl Registry { // upstream confirms this image has it, without downloading it again. // Anything else in the CAS is fetched and verified like a new blob: // the upstream's HEAD must never reach content it did not serve. - if digest.algorithm() == crate::digest::Algorithm::Sha256 { + if let Some(sha256) = digest.sha256_hex() { if let Some((_, size)) = - oci_dao::blob_linked_in_repository(pool, repo.id, digest.hex()).await? + oci_dao::blob_linked_in_repository(pool, repo.id, sha256).await? { let head = upstream - .send(Method::HEAD, &path, &scope, None, RequestClass::Metadata) + .send_url(Method::HEAD, &url, RequestClass::Metadata) .await?; if head.status().is_success() { let mut tx = pool.begin().await?; @@ -753,7 +844,7 @@ impl Registry { repo.id, &local, digest.as_str(), - digest.hex(), + sha256, size, ) .await? @@ -763,7 +854,7 @@ impl Registry { tx.commit().await?; let hit = self.hosted_blob(repo, &local, digest).await?; if let Some(hit) = &hit { - self.backfill_platform( + self.complete_summaries( repo, &local, digest, @@ -784,7 +875,7 @@ impl Registry { } let response = upstream - .send(Method::GET, &path, &scope, None, RequestClass::Download) + .send_url(Method::GET, &url, RequestClass::Download) .await?; if response.status() == reqwest::StatusCode::NOT_FOUND { self.remember_not_found(repo, &local, digest.as_str()).await; @@ -795,7 +886,7 @@ impl Registry { } let max = self.inner.config.max_blob_bytes; if response.content_length().is_some_and(|l| l > max) { - return Err(OciError::too_large(max)); + return Err(Failure::new(ImageError::TooLarge { limit: max })); } let body = response.bytes_stream().map_err(std::io::Error::other); let staged = self @@ -805,11 +896,11 @@ impl Registry { .await?; if !digest.matches(&staged.sha256, &staged.sha512_base64) { tracing::warn!(repo = %repo.name, image = %local, %digest, "registry proxy: upstream blob does not match its digest"); - return Err(OciError::unavailable( + return Err(Failure::unavailable( "the upstream served a blob that does not match its digest", )); } - // A small blob may be an image config (read while it is at hand). + // A small blob may be a config (read while it is at hand). let small = if staged.size <= MAX_CONFIG_BYTES as u64 { tokio::fs::read(&staged.temp_path).await.ok() } else { @@ -828,24 +919,25 @@ impl Registry { stored.ingest_token, ) .await - .map_err(OciError::from) + .map_err(Failure::from) .and_then(require_linked)?; tx.commit().await?; if let Some(bytes) = small { - self.backfill_platform(repo, &local, digest, bytes.len() as i64, None, Some(bytes)) + self.complete_summaries(repo, &local, digest, bytes.len() as i64, None, Some(bytes)) .await; } self.hosted_blob(repo, &local, digest).await } /// A proxy caches a manifest before its config: once the config blob - /// `digest` is cached, its platform completes the summaries of the - /// image's manifests that name it. Best effort. - async fn backfill_platform( + /// `digest` is cached, what it holds (an image's platform, a chart's + /// metadata) completes the summaries of the image's manifests that name + /// it as their config. Best effort. + async fn complete_summaries( &self, repo: &Repository, image: &str, - digest: &Digest, + digest: &ContentDigest, size: i64, stored: Option>, bytes: Option>, @@ -854,14 +946,15 @@ impl Registry { return; } let pool = &self.inner.pool; - match oci_dao::config_needs_platform(pool, repo.id, image, digest.as_str()).await { - Ok(true) => {} - Ok(false) => return, - Err(e) => { - tracing::warn!(error = %e, "registry proxy: platform lookup failed"); - return; - } - } + let media_type = + match oci_dao::config_media_type(pool, repo.id, image, digest.as_str()).await { + Ok(Some(media_type)) => media_type, + Ok(None) => return, + Err(e) => { + tracing::warn!(error = %e, "registry proxy: summary lookup failed"); + return; + } + }; let bytes = match (bytes, stored) { (Some(bytes), _) => bytes, (None, Some(blob)) => match self.read_manifest_bytes(blob).await { @@ -870,14 +963,17 @@ impl Registry { }, (None, None) => return, }; - let Some(platform) = manifest::config_platform(&bytes) else { + let Some(part) = self.config_summary_part(&media_type, &bytes) else { return; }; + let keys: Vec = part.keys().cloned().collect(); + let mut merged = serde_json::json!({}); + merge(&mut merged, part); if let Err(e) = - oci_dao::set_config_platform(pool, repo.id, image, digest.as_str(), &platform.label()) + oci_dao::complete_config_summary(pool, repo.id, image, digest.as_str(), &merged, &keys) .await { - tracing::warn!(error = %e, "registry proxy: recording an image platform failed"); + tracing::warn!(error = %e, "registry proxy: completing manifest summaries failed"); } } @@ -887,14 +983,14 @@ impl Registry { image: &str, last: Option<&str>, limit: usize, - ) -> OciResult> { - let local = proxied_image(repo, image); + ) -> ImgResult> { + let local = self.proxied_image(repo, image); if repo.online { match self.upstream_tags(repo, &local, last, limit).await { Ok(Some(tags)) => return Ok(Some(tags)), Ok(None) => {} Err(e) => tracing::warn!( - repo = %repo.name, image = %local, error = %e.message, + repo = %repo.name, image = %local, error = %e.message(), "registry proxy: upstream tag list unavailable; listing cached tags" ), } @@ -902,104 +998,44 @@ impl Registry { self.local_tags(repo, &local, last, limit).await } - /// Up to `limit` of the upstream's tags of `local` after `last`. Asks - /// for one more than that, to know whether there are more, and follows - /// the upstream's own pages (`Link: rel="next"`, back to the upstream - /// only) for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An upstream that - /// refuses so large an `n` (400, as registries that cap it answer) is - /// asked again without one: its own page size and links take over. + /// Up to `limit` of the upstream's tags of `local` after `last`, as the + /// images format reads them. async fn upstream_tags( &self, repo: &Repository, local: &str, last: Option<&str>, limit: usize, - ) -> OciResult> { + ) -> ImgResult> { let upstream = self.upstream(repo).await?; - let first = |n: Option| { - let mut query = url::form_urlencoded::Serializer::new(String::new()); - if let Some(n) = n { - query.append_pair("n", &n.to_string()); - } - if let Some(last) = last { - query.append_pair("last", last); - } - let query = query.finish(); - if query.is_empty() { - format!("/v2/{local}/tags/list") - } else { - format!("/v2/{local}/tags/list?{query}") - } + let op = UpstreamOp::ImageTags { + image: local.to_string(), + last: last.map(str::to_string), + n: u32::try_from(limit).unwrap_or(u32::MAX), }; - let scope = format!("repository:{local}:pull"); - let mut tags: BTreeSet = BTreeSet::new(); - let mut path = Some(first(Some(limit.saturating_add(1)))); - let mut sized = true; - let mut pages = 0; - let mut left_out = false; - while let Some(current) = path.take() { - let response = upstream - .send(Method::GET, ¤t, &scope, None, RequestClass::Metadata) - .await?; - if pages == 0 { - if response.status() == reqwest::StatusCode::NOT_FOUND { - return Ok(None); - } - if response.status() == reqwest::StatusCode::BAD_REQUEST && sized { - sized = false; - path = Some(first(None)); - continue; - } - } - if !response.status().is_success() { - return Err(upstream_status(response.status())); - } - pages += 1; - let next = next_link(&response, &upstream.base); - #[derive(serde::Deserialize)] - struct TagList { - #[serde(default)] - tags: Option>, - } - let body = read_limited(response, MAX_LISTING_BYTES).await?; - let list: TagList = serde_json::from_slice(&body) - .map_err(|_| OciError::unavailable("the upstream served an invalid tag list"))?; - tags.extend( - list.tags - .unwrap_or_default() - .into_iter() - .filter(|t| reference::valid_tag(t)) - .filter(|t| last.is_none_or(|l| t.as_bytes() > l.as_bytes())), - ); - if tags.len() > limit { - break; - } - match next { - Some(next) if pages < MAX_UPSTREAM_TAG_PAGES => path = Some(next), - Some(_) => left_out = true, - None => {} - } + match upstream.converse(self, &op).await? { + UpstreamAnswer::ImageTags(page) => Ok(page), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "upstream tags", + format!("the images format answered {other:?}"), + )), } - let more = left_out || tags.len() > limit; - Ok(Some(TagPage { - tags: tags.into_iter().take(limit).collect(), - more, - })) } pub(crate) async fn proxy_referrers( &self, repo: &Repository, image: &str, - subject: &Digest, - ) -> OciResult> { - let local = proxied_image(repo, image); + subject: &ContentDigest, + ) -> ImgResult> { + let local = self.proxied_image(repo, image); if repo.online { match self.upstream_referrers(repo, &local, subject).await { Ok(Some(referrers)) => return Ok(referrers), Ok(None) => {} Err(e) => tracing::warn!( - repo = %repo.name, image = %local, error = %e.message, + repo = %repo.name, image = %local, error = %e.message(), "registry proxy: upstream referrers unavailable; listing cached ones" ), } @@ -1011,54 +1047,21 @@ impl Registry { &self, repo: &Repository, local: &str, - subject: &Digest, - ) -> OciResult>> { + subject: &ContentDigest, + ) -> ImgResult>> { let upstream = self.upstream(repo).await?; - let response = upstream - .send( - Method::GET, - &format!("/v2/{local}/referrers/{subject}"), - &format!("repository:{local}:pull"), - Some(media::OCI_INDEX), - RequestClass::Metadata, - ) - .await?; - if !response.status().is_success() { - // 404: the upstream has no referrers API (clients fall back to - // the tag scheme, which a proxy serves like any tag). - return Ok(None); - } - #[derive(serde::Deserialize)] - #[serde(rename_all = "camelCase")] - struct Desc { - media_type: Option, - digest: String, - size: i64, - artifact_type: Option, - annotations: Option, - } - #[derive(serde::Deserialize)] - struct Index { - #[serde(default)] - manifests: Vec, + let op = UpstreamOp::ImageReferrers { + image: local.to_string(), + digest: subject.to_string(), + }; + match upstream.converse(self, &op).await? { + UpstreamAnswer::ImageReferrers(found) => Ok(found), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "upstream referrers", + format!("the images format answered {other:?}"), + )), } - let body = read_limited(response, MAX_LISTING_BYTES).await?; - let index: Index = serde_json::from_slice(&body) - .map_err(|_| OciError::unavailable("the upstream served an invalid referrers list"))?; - Ok(Some( - index - .manifests - .into_iter() - .filter(|d| Digest::parse(&d.digest).is_ok() && d.size >= 0) - .map(|d| ReferrerDescriptor { - media_type: d.media_type.unwrap_or_else(|| media::OCI_MANIFEST.into()), - digest: d.digest, - size: d.size, - artifact_type: d.artifact_type, - annotations: d.annotations, - }) - .collect(), - )) } } @@ -1118,41 +1121,4 @@ mod tests { assert_eq!(parse_challenge("Negotiate"), None); assert_eq!(parse_challenge("Bearer service=\"x\""), None); } - - #[test] - fn docker_hub_library_names() { - let mut repo = test_repo(); - repo.remote_url = Some("https://registry-1.docker.io".into()); - assert_eq!(proxied_image(&repo, "nginx"), "library/nginx"); - assert_eq!(proxied_image(&repo, "bitnami/nginx"), "bitnami/nginx"); - repo.remote_url = Some("https://ghcr.io".into()); - assert_eq!(proxied_image(&repo, "nginx"), "nginx"); - assert_eq!(upstream_base("https://ghcr.io/v2/"), "https://ghcr.io"); - assert_eq!( - upstream_base("https://r.example/prefix/"), - "https://r.example/prefix" - ); - } - - fn test_repo() -> Repository { - Repository { - id: 1, - name: "hub".into(), - format: "docker".into(), - kind: "proxy".into(), - anonymous_read: false, - online: true, - config: serde_json::json!({}), - remote_url: None, - package_delay_seconds: 0, - upstream_username: None, - upstream_password: None, - upstream_password_enc: None, - metadata_ttl_seconds: None, - not_found_ttl_seconds: None, - allow_redeploy: false, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - } - } } diff --git a/crates/formats/docker/src/pull.rs b/crates/server/src/images/pull.rs similarity index 62% rename from crates/formats/docker/src/pull.rs rename to crates/server/src/images/pull.rs index 1b3f44a4..d2f0dc70 100644 --- a/crates/formats/docker/src/pull.rs +++ b/crates/server/src/images/pull.rs @@ -1,5 +1,6 @@ -//! Pulls: manifests, blobs, tag lists and referrers, served from hosted, -//! proxy and group repositories alike. +//! Pulls: manifests, blobs, tag lists and referrers, read from hosted, +//! proxy and group repositories alike, and serving a manifest or a blob the +//! format asked for. use axum::body::Body; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; @@ -7,28 +8,13 @@ use axum::response::{IntoResponse, Response}; use bytes::Bytes; use forge_db::model::{RepoKind, Repository}; use forge_db::oci_dao::{self, ManifestRow}; +use forge_plugin::api::images::{BlobInfo, ManifestInfo, Reference, TagPage, MAX_MANIFEST_BYTES}; use forge_storage::BlobRef; use futures_util::future::BoxFuture; -use crate::digest::{Digest, DigestError}; -use crate::error::{Code, OciError, OciResult}; -use crate::manifest::{media, MAX_MANIFEST_BYTES}; -use crate::reference::{Reference, ReferenceError}; -use crate::service::{location, Caller, Registry, RequestCtx}; - -/// Tags returned by one `tags/list` page when the client sets no `n`. -const DEFAULT_TAG_PAGE: usize = 10_000; - -/// One page of an image's tags: the tags after `last`, in byte order, at -/// most as many as asked for. -#[derive(Debug, Clone, Default)] -pub(crate) struct TagPage { - pub tags: Vec, - /// The source has tags after these that the page leaves out: the full - /// page did not hold them, or the source could not list them all at - /// once (a proxy's upstream that pages its tags). - pub more: bool, -} +use super::digest::ContentDigest; +use super::dispatch::Caller; +use super::{header_value, Failure, ImgResult, Registry}; /// A manifest found for a pull. #[derive(Debug, Clone)] @@ -38,6 +24,9 @@ pub(crate) struct ManifestHit { pub digest: String, pub media_type: String, pub size: i64, + pub artifact_type: Option, + pub subject_digest: Option, + pub summary: serde_json::Value, pub blob_sha256: String, pub s3_key: String, /// The bytes, when they are already in memory (a proxy just fetched @@ -61,11 +50,30 @@ impl ManifestHit { digest: row.digest, media_type: row.media_type, size: row.size, + artifact_type: row.artifact_type, + subject_digest: row.subject_digest, + summary: row.summary, blob_sha256: row.blob_sha256, s3_key: row.s3_key, bytes: None, } } + + /// What the format learns of it. + pub fn info(&self) -> ManifestInfo { + ManifestInfo { + digest: self.digest.clone(), + media_type: self.media_type.clone(), + size: self.size.max(0) as u64, + artifact_type: self.artifact_type.clone(), + subject: self.subject_digest.clone(), + summary: self + .summary + .as_object() + .filter(|s| !s.is_empty()) + .map(|_| self.summary.to_string()), + } + } } /// A blob found for a pull. @@ -91,58 +99,16 @@ impl BlobHit { size: self.size.max(0) as u64, } } -} - -/// A referrer descriptor. -#[derive(Debug, Clone, serde::Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ReferrerDescriptor { - pub media_type: String, - pub digest: String, - pub size: i64, - #[serde(skip_serializing_if = "Option::is_none")] - pub artifact_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub annotations: Option, -} -impl ReferrerDescriptor { - pub fn from_row(row: &ManifestRow) -> Self { - ReferrerDescriptor { - media_type: row.media_type.clone(), - digest: row.digest.clone(), - size: row.size, - artifact_type: row.artifact_type.clone(), - annotations: row - .summary - .get("annotations") - .filter(|a| a.as_object().is_some_and(|o| !o.is_empty())) - .cloned(), + /// What the format learns of it. + pub fn info(&self) -> BlobInfo { + BlobInfo { + digest: self.digest.clone(), + size: self.size.max(0) as u64, } } } -pub(crate) fn parse_digest(raw: &str) -> OciResult { - Digest::parse(raw).map_err(|e| match e { - DigestError::Invalid(_) => OciError::digest_invalid(format!("invalid digest {raw:?}")), - DigestError::Unsupported(_) => OciError::new( - StatusCode::BAD_REQUEST, - Code::DigestInvalid, - format!("unsupported digest algorithm in {raw:?}: use sha256 or sha512"), - ), - }) -} - -pub(crate) fn parse_reference(raw: &str) -> OciResult { - Reference::parse(raw).map_err(|e| match e { - ReferenceError::InvalidTag => OciError::manifest_invalid(format!("invalid tag {raw:?}")) - .with_detail(serde_json::json!({ "reference": raw })), - ReferenceError::Digest(_) => parse_digest(raw) - .err() - .unwrap_or_else(|| OciError::digest_invalid(format!("invalid digest {raw:?}"))), - }) -} - impl Registry { // ── Source dispatch ───────────────────────────────────────────────────── @@ -154,7 +120,7 @@ impl Registry { image: &'a str, reference: &'a Reference, depth: u32, - ) -> BoxFuture<'a, OciResult>> { + ) -> BoxFuture<'a, ImgResult>> { Box::pin(async move { match repo.kind() { RepoKind::Hosted => self.hosted_manifest(repo, image, reference).await, @@ -173,9 +139,9 @@ impl Registry { caller: &'a Caller, repo: &'a Repository, image: &'a str, - digest: &'a Digest, + digest: &'a ContentDigest, depth: u32, - ) -> BoxFuture<'a, OciResult>> { + ) -> BoxFuture<'a, ImgResult>> { Box::pin(async move { match repo.kind() { RepoKind::Hosted => self.hosted_blob(repo, image, digest).await, @@ -195,7 +161,7 @@ impl Registry { last: Option<&'a str>, limit: usize, depth: u32, - ) -> BoxFuture<'a, OciResult>> { + ) -> BoxFuture<'a, ImgResult>> { Box::pin(async move { match repo.kind() { RepoKind::Hosted => self.local_tags(repo, image, last, limit).await, @@ -214,9 +180,9 @@ impl Registry { caller: &'a Caller, repo: &'a Repository, image: &'a str, - subject: &'a Digest, + subject: &'a ContentDigest, depth: u32, - ) -> BoxFuture<'a, OciResult>> { + ) -> BoxFuture<'a, ImgResult>> { Box::pin(async move { match repo.kind() { RepoKind::Hosted => self.local_referrers(repo, image, subject).await, @@ -236,14 +202,14 @@ impl Registry { repo: &Repository, image: &str, reference: &Reference, - ) -> OciResult> { + ) -> ImgResult> { let pool = &self.inner.pool; let row = match reference { Reference::Tag(tag) => oci_dao::find_tag(pool, repo.id, image, tag) .await? .map(|t| t.manifest), Reference::Digest(digest) => { - oci_dao::find_manifest(pool, repo.id, image, digest.as_str()).await? + oci_dao::find_manifest(pool, repo.id, image, digest).await? } }; Ok(row.map(ManifestHit::from_row)) @@ -253,8 +219,8 @@ impl Registry { &self, repo: &Repository, image: &str, - digest: &Digest, - ) -> OciResult> { + digest: &ContentDigest, + ) -> ImgResult> { let pool = &self.inner.pool; if let Some(link) = oci_dao::find_link(pool, repo.id, image, digest.as_str()).await? { return Ok(Some(BlobHit { @@ -290,7 +256,7 @@ impl Registry { image: &str, last: Option<&str>, limit: usize, - ) -> OciResult> { + ) -> ImgResult> { let pool = &self.inner.pool; let asked = i64::try_from(limit.saturating_add(1)).unwrap_or(i64::MAX); let mut tags = oci_dao::list_tags(pool, repo.id, image, last, asked).await?; @@ -307,18 +273,21 @@ impl Registry { &self, repo: &Repository, image: &str, - subject: &Digest, - ) -> OciResult> { + subject: &ContentDigest, + ) -> ImgResult> { let rows = oci_dao::list_referrers(&self.inner.pool, repo.id, image, subject.as_str()).await?; - Ok(rows.iter().map(ReferrerDescriptor::from_row).collect()) + Ok(rows + .into_iter() + .map(|row| ManifestHit::from_row(row).info()) + .collect()) } - /// Read a stored manifest's (or image config's) bytes, through the - /// local blob cache. - pub(crate) async fn read_manifest_bytes(&self, blob: BlobRef<'_>) -> OciResult { + /// Read a stored manifest's (or a config's) bytes, through the local + /// blob cache. + pub(crate) async fn read_manifest_bytes(&self, blob: BlobRef<'_>) -> ImgResult { let too_large = || { - OciError::internal( + Failure::internal( "manifest read", format!("stored manifest {} is too large", blob.s3_key), ) @@ -331,9 +300,9 @@ impl Registry { .store .read_all(blob) .await - .map_err(|e| OciError::storage_read("manifest read", e))?; + .map_err(|e| Failure::storage_read("manifest read", e))?; if bytes.len() as u64 != blob.size { - return Err(OciError::internal( + return Err(Failure::internal( "manifest read", format!( "stored manifest {} has {} bytes, expected {}", @@ -346,28 +315,14 @@ impl Registry { Ok(Bytes::from(bytes)) } - // ── Handlers ──────────────────────────────────────────────────────────── + // ── Serving ───────────────────────────────────────────────────────────── - pub(crate) async fn get_manifest( + /// Serve a manifest the format asked for, as its media type. + pub(crate) async fn serve_manifest( &self, - ctx: &RequestCtx, - reference: &str, + hit: &ManifestHit, head: bool, - _headers: &HeaderMap, - ) -> OciResult { - let reference = parse_reference(reference).map_err(|e| { - // A malformed reference names nothing that can exist. - if e.code == Code::ManifestInvalid { - OciError::manifest_unknown(reference) - } else { - e - } - })?; - let t = &ctx.target; - let hit = self - .manifest_from(&ctx.caller, &t.repo, &t.image, &reference, 0) - .await? - .ok_or_else(|| OciError::manifest_unknown(reference.as_str()))?; + ) -> ImgResult { let mut response = if head { Response::new(Body::empty()) } else { @@ -392,19 +347,14 @@ impl Registry { Ok(response) } - pub(crate) async fn get_blob( + /// Serve a blob the format asked for, with one `Range`, and record the + /// pull of its link. + pub(crate) async fn serve_blob( &self, - ctx: &RequestCtx, - digest: &str, + hit: &BlobHit, head: bool, headers: &HeaderMap, - ) -> OciResult { - let digest = parse_digest(digest)?; - let t = &ctx.target; - let hit = self - .blob_from(&ctx.caller, &t.repo, &t.image, &digest, 0) - .await? - .ok_or_else(|| OciError::blob_unknown(digest.as_str()))?; + ) -> ImgResult { if hit.linked { if let Err(e) = oci_dao::touch_link( &self.inner.pool, @@ -444,7 +394,7 @@ impl Registry { .store .open(hit.blob(), range.map(|(start, end)| start..end + 1)) .await - .map_err(|e| OciError::storage_read("blob read", e))?; + .map_err(|e| Failure::storage_read("blob read", e))?; match range { Some((start, end)) => ( StatusCode::PARTIAL_CONTENT, @@ -473,99 +423,6 @@ impl Registry { } Ok(response) } - - pub(crate) async fn tags_list(&self, ctx: &RequestCtx, query: &str) -> OciResult { - let mut n: Option = None; - let mut last: Option = None; - for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "n" => { - n = Some(value.parse().map_err(|_| { - OciError::new( - StatusCode::BAD_REQUEST, - Code::Unsupported, - format!("invalid n {value:?}"), - ) - })?) - } - "last" => last = Some(value.into_owned()), - _ => {} - } - } - let page = n.unwrap_or(DEFAULT_TAG_PAGE).min(DEFAULT_TAG_PAGE); - let t = &ctx.target; - let listed = if page == 0 { - Some(TagPage::default()) - } else { - self.tags_from(&ctx.caller, &t.repo, &t.image, last.as_deref(), page, 0) - .await? - }; - let Some(TagPage { tags, more }) = listed else { - return Err(OciError::name_unknown(&t.name)); - }; - let body = serde_json::json!({ "name": t.name, "tags": tags }); - let mut response = ( - StatusCode::OK, - [(header::CONTENT_TYPE, "application/json")], - body.to_string(), - ) - .into_response(); - if more { - if let Some(last) = tags.last() { - let next: String = url::form_urlencoded::Serializer::new(String::new()) - .append_pair("n", &page.to_string()) - .append_pair("last", last) - .finish(); - let link = format!("<{}?{next}>; rel=\"next\"", location(&t.name, "tags/list")); - response - .headers_mut() - .insert(header::LINK, header_value(&link)); - } - } - Ok(response) - } - - pub(crate) async fn referrers( - &self, - ctx: &RequestCtx, - digest: &str, - query: &str, - ) -> OciResult { - let subject = parse_digest(digest)?; - let artifact_type = url::form_urlencoded::parse(query.as_bytes()) - .find(|(k, _)| k == "artifactType") - .map(|(_, v)| v.into_owned()); - let t = &ctx.target; - let mut referrers = self - .referrers_from(&ctx.caller, &t.repo, &t.image, &subject, 0) - .await?; - if let Some(filter) = &artifact_type { - referrers.retain(|r| r.artifact_type.as_deref() == Some(filter.as_str())); - } - let body = serde_json::json!({ - "schemaVersion": 2, - "mediaType": media::OCI_INDEX, - "manifests": referrers, - }); - let mut response = ( - StatusCode::OK, - [(header::CONTENT_TYPE, media::OCI_INDEX)], - body.to_string(), - ) - .into_response(); - if artifact_type.is_some() { - response.headers_mut().insert( - "oci-filters-applied", - HeaderValue::from_static("artifactType"), - ); - } - Ok(response) - } -} - -/// A header value from text Forge produced or validated. -pub(crate) fn header_value(s: &str) -> HeaderValue { - HeaderValue::from_str(s).unwrap_or_else(|_| HeaderValue::from_static("invalid")) } /// One `bytes=` range of a `size`-byte object, as inclusive offsets. `None` diff --git a/crates/server/src/images/push.rs b/crates/server/src/images/push.rs new file mode 100644 index 00000000..674bfd98 --- /dev/null +++ b/crates/server/src/images/push.rs @@ -0,0 +1,854 @@ +//! Pushes and deletes: blob uploads (monolithic, chunked, cross-repository +//! mounts), manifests and tags. +//! +//! Upload chunks are CAS blobs held by `oci_upload_part` rows, so a session +//! survives on any instance. When an upload consists of one chunk (what +//! docker, podman and containerd send, and what a monolithic upload is) that +//! chunk already is the blob and is linked as is; only multi-chunk uploads +//! are assembled into a new blob. + +use std::collections::HashSet; +use std::io::Cursor; + +use axum::body::Body; +use bytes::Bytes; +use forge_core::auth::Principal; +use forge_core::authz::Level; +use forge_core::repo::{PackageMutation, PublishAuthorizationContext, WriteError}; +use forge_db::model::{RepoKind, Repository}; +use forge_db::oci_dao::{self, LinkOutcome, NewManifest, TagWrite, UploadRow}; +use forge_plugin::api::images::{ + BlobInfo, ChunkRange, ImageError, ParsedManifest, Reference, Started, StoredManifest, + UploadState, MAX_MANIFEST_BYTES, +}; +use forge_plugin::api::FormatError; +use futures_util::{StreamExt, TryStreamExt}; +use tokio_util::io::StreamReader; +use uuid::Uuid; + +use super::digest::ContentDigest; +use super::dispatch::{Decision, RequestCtx, Target}; +use super::{Failure, ImgResult, Registry, RequestBodyError}; + +/// A manifest [`Registry::commit_manifest`] stored. +pub(crate) struct CommittedManifest { + pub id: i64, + /// Whether the image did not hold the manifest before. + pub created: bool, + /// What became of each tag. + pub tags: Vec<(String, TagWrite)>, +} + +/// Refuse to commit a link whose blob turned out to be gone or being +/// deleted: dropping the transaction rolls the link row back, so no link +/// ever exists without its reference. +pub(crate) fn require_linked(outcome: LinkOutcome) -> ImgResult<()> { + match outcome { + LinkOutcome::BlobUnavailable => Err(Failure::unavailable( + "the blob is being removed from storage; retry the upload", + )), + LinkOutcome::Inserted | LinkOutcome::AlreadyLinked => Ok(()), + } +} + +/// A request body as an `AsyncRead`. A body that cannot be read fails with a +/// [`RequestBodyError`] inside the I/O error, which is the client's. +fn body_reader(body: Body) -> impl tokio::io::AsyncRead + Unpin + Send { + StreamReader::new( + body.into_data_stream() + .map_err(|e| std::io::Error::other(RequestBodyError(e))), + ) +} + +/// Read a whole (small) request body, refusing more than `limit` bytes. +async fn read_body(body: Body, limit: usize) -> ImgResult { + let mut stream = body.into_data_stream(); + let mut out = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| { + tracing::debug!(error = %e, "registry: the manifest body could not be read"); + Failure::new(ImageError::BodyUnreadable) + })?; + if out.len() + chunk.len() > limit { + return Err(Failure::new(ImageError::TooLarge { + limit: limit as u64, + })); + } + out.extend_from_slice(&chunk); + } + Ok(Bytes::from(out)) +} + +/// An upload session id from the URL. +fn upload_id(raw: &str) -> ImgResult { + Uuid::parse_str(raw).map_err(|_| Failure::new(ImageError::UploadUnknown)) +} + +/// An upload session as the format learns of it. +fn upload_state(upload: &UploadRow) -> UploadState { + UploadState { + id: upload.id.to_string(), + size: upload.size.max(0) as u64, + } +} + +impl Registry { + fn grace(&self) -> chrono::Duration { + self.inner.config.delete_grace + } + + /// The largest blob a push to `repo` may carry: its own upload limit, + /// else the registry's default. + pub(crate) fn blob_limit(&self, repo: &Repository) -> u64 { + repo.max_upload_bytes() + .unwrap_or(self.inner.config.default_max_upload_bytes) + } + + // ── Uploads ───────────────────────────────────────────────────────────── + + /// Start an upload session in `t`, or mount `mount`'s digest there + /// without one when it can be: from the image `mount` names (one the + /// caller may read), or when the image holds the blob already. + pub(crate) async fn start_upload( + &self, + ctx: &RequestCtx, + t: &Target, + mount: Option<(ContentDigest, Option<&str>)>, + ) -> ImgResult { + if let Some((digest, from)) = mount { + if let Some(size) = self.mount(ctx, t, &digest, from).await? { + return Ok(Started::Mounted(BlobInfo { + digest: digest.to_string(), + size: size.max(0) as u64, + })); + } + // Not mountable: a regular upload session. + } + let id = Uuid::new_v4(); + oci_dao::create_upload(&self.inner.pool, id, t.repo.id, &t.image).await?; + Ok(Started::Session(UploadState { + id: id.to_string(), + size: 0, + })) + } + + /// Link `digest` into the target image without uploading it: from the + /// image `from` the caller may read, or (without `from`) when the image + /// already has it. Its size; `None` when it cannot be mounted. + async fn mount( + &self, + ctx: &RequestCtx, + t: &Target, + digest: &ContentDigest, + from: Option<&str>, + ) -> ImgResult> { + let pool = &self.inner.pool; + if let Some(held) = oci_dao::find_link(pool, t.repo.id, &t.image, digest.as_str()).await? { + // The client is about to name it in a manifest: record the use + // now, or the sweeper may release it first. + oci_dao::touch_link( + pool, + t.repo.id, + &t.image, + digest.as_str(), + chrono::Duration::zero(), + ) + .await?; + return Ok(Some(held.size)); + } + let Some(from) = from else { + return Ok(None); + }; + let Ok(Some(source)) = self.resolve(&ctx.site, from).await else { + return Ok(None); + }; + // What the source would serve: an offline hosted repository serves + // nothing (an offline proxy still serves its cache). + let serves = match source.repo.kind() { + RepoKind::Hosted => source.repo.online, + RepoKind::Proxy => true, + RepoKind::Group => false, + }; + if !serves || self.decide(&ctx.caller, &source, Level::Read).await != Decision::Allow { + return Ok(None); + } + let Some(link) = + oci_dao::find_link(pool, source.repo.id, &source.image, digest.as_str()).await? + else { + return Ok(None); + }; + let mut tx = pool.begin().await?; + match oci_dao::insert_link_existing( + &mut tx, + t.repo.id, + &t.image, + digest.as_str(), + &link.blob_sha256, + link.size, + ) + .await? + { + LinkOutcome::BlobUnavailable => { + tx.rollback().await?; + Ok(None) + } + LinkOutcome::Inserted | LinkOutcome::AlreadyLinked => { + tx.commit().await?; + Ok(Some(link.size)) + } + } + } + + async fn upload_of(&self, t: &Target, id: &str) -> ImgResult { + oci_dao::get_upload(&self.inner.pool, upload_id(id)?, t.repo.id, &t.image) + .await? + .ok_or_else(|| Failure::new(ImageError::UploadUnknown)) + } + + pub(crate) async fn upload_status(&self, t: &Target, id: &str) -> ImgResult { + let upload = self.upload_of(t, id).await?; + Ok(upload_state(&upload)) + } + + pub(crate) async fn cancel_upload(&self, t: &Target, id: &str) -> ImgResult<()> { + let upload = self.upload_of(t, id).await?; + let mut tx = self.inner.pool.begin().await?; + let deleted = oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; + tx.commit().await?; + if !deleted { + return Err(Failure::new(ImageError::UploadUnknown)); + } + Ok(()) + } + + /// Store one chunk of an upload. The chunk must begin where the session + /// ended when the request started; the whole blob may be at most `max` + /// bytes. + async fn append_chunk( + &self, + upload: &UploadRow, + range: Option, + body: Body, + max: u64, + ) -> ImgResult { + let size = upload.size.max(0) as u64; + let out_of_order = |size: i64| { + Failure::new(ImageError::ChunkOutOfOrder { + size: size.max(0) as u64, + }) + }; + if let Some(range) = range { + if range.start != size { + return Err(out_of_order(upload.size)); + } + } + let remaining = max.saturating_sub(size); + let staged = self + .inner + .cas + .stage(body_reader(body), remaining) + .await + .map_err(|e| match e { + forge_storage::CasError::TooLarge { .. } => { + Failure::new(ImageError::TooLarge { limit: max }) + } + other => other.into(), + })?; + if let Some(range) = range { + if staged.size != range.end - range.start + 1 { + return Err(Failure::new(ImageError::ChunkLength { size })); + } + } + if staged.size == 0 { + return Ok(upload.clone()); + } + // The first chunk may be the whole blob (a monolithic upload, or a + // layer streamed in one PATCH, as docker, podman and containerd + // send it), so it joins the local blob cache like any pushed blob. + // Later chunks are only read back once, to assemble the blob: + // caching them would evict content that is downloaded. + let stored = if upload.size == 0 { + self.inner.cas.commit(staged).await? + } else { + self.inner.cas.commit_uncached(staged).await? + }; + let mut tx = self.inner.pool.begin().await?; + let Some(locked) = + oci_dao::lock_upload(&mut tx, upload.id, upload.repository_id, &upload.image).await? + else { + return Err(Failure::new(ImageError::UploadUnknown)); + }; + if locked.size != upload.size { + // Another chunk landed meanwhile. + return Err(out_of_order(locked.size)); + } + oci_dao::add_upload_part( + &mut tx, + &locked, + stored.size as i64, + &stored.sha256, + &stored.sha512_base64, + stored.ingest_token, + ) + .await?; + tx.commit().await?; + Ok(UploadRow { + size: locked.size + stored.size as i64, + updated_at: chrono::Utc::now(), + ..locked + }) + } + + pub(crate) async fn append_upload( + &self, + t: &Target, + id: &str, + range: Option, + body: Body, + ) -> ImgResult { + let upload = self.upload_of(t, id).await?; + let max = self.blob_limit(&t.repo); + let upload = self.append_chunk(&upload, range, body, max).await?; + Ok(upload_state(&upload)) + } + + /// Append the last chunk, then end the session and link its content into + /// the image as `digest`. + pub(crate) async fn finish_upload( + &self, + t: &Target, + id: &str, + digest: &ContentDigest, + range: Option, + body: Body, + ) -> ImgResult { + let upload = self.upload_of(t, id).await?; + let max = self.blob_limit(&t.repo); + let upload = self.append_chunk(&upload, range, body, max).await?; + let pool = &self.inner.pool; + let parts = oci_dao::upload_parts(pool, upload.id).await?; + let mismatch = || Failure::new(ImageError::DigestMismatch); + let finished = BlobInfo { + digest: digest.to_string(), + size: upload.size.max(0) as u64, + }; + + // The blob: the single chunk itself, or the chunks assembled. + let mut tx; + match parts.as_slice() { + [part] => { + if !digest.matches(&part.blob_sha256, &part.sha512_base64) { + self.drop_upload(upload.id).await; + return Err(mismatch()); + } + tx = pool.begin().await?; + if oci_dao::lock_upload(&mut tx, upload.id, t.repo.id, &t.image) + .await? + .is_none_or(|u| u.size != upload.size) + { + return Err(Failure::new(ImageError::UploadUnknown)); + } + // Linked before the chunk's reference is released, so the + // blob never drops to zero references. + if oci_dao::insert_link_existing( + &mut tx, + t.repo.id, + &t.image, + digest.as_str(), + &part.blob_sha256, + part.size, + ) + .await? + == LinkOutcome::BlobUnavailable + { + tx.rollback().await?; + return Err(Failure::internal( + "upload finish", + "the uploaded chunk vanished from the CAS", + )); + } + oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; + } + parts => { + let store = self.inner.store.clone(); + let keys: Vec = parts.iter().map(|p| p.s3_key.clone()).collect(); + let chunks = futures_util::stream::iter(keys) + .then(move |key| { + let store = store.clone(); + async move { + store + .get(&key) + .await + .map(|s| tokio_util::io::ReaderStream::new(s.into_async_read())) + .map_err(std::io::Error::other) + } + }) + .try_flatten(); + let staged = self + .inner + .cas + .stage(StreamReader::new(Box::pin(chunks)), max) + .await?; + // The first chunk joined the local blob cache in case it was + // the whole blob (`append_chunk`). It is not, and nothing + // reads it as it is again. + if let (Some(cache), Some(first)) = (self.inner.store.cache(), parts.first()) { + cache.evict(&first.blob_sha256).await; + } + if staged.size as i64 != upload.size + || !digest.matches(&staged.sha256, &staged.sha512_base64) + { + self.drop_upload(upload.id).await; + return Err(mismatch()); + } + let stored = self.inner.cas.commit(staged).await?; + tx = pool.begin().await?; + if oci_dao::lock_upload(&mut tx, upload.id, t.repo.id, &t.image) + .await? + .is_none_or(|u| u.size != upload.size) + { + return Err(Failure::new(ImageError::UploadUnknown)); + } + oci_dao::insert_link_ingested( + &mut tx, + t.repo.id, + &t.image, + digest.as_str(), + &stored.sha256, + stored.size as i64, + stored.ingest_token, + ) + .await + .map_err(Failure::from) + .and_then(require_linked)?; + oci_dao::delete_upload(&mut tx, upload.id, self.grace()).await?; + } + } + tx.commit().await?; + Ok(finished) + } + + /// End a failed upload session (best effort). + async fn drop_upload(&self, id: Uuid) { + let result = async { + let mut tx = self.inner.pool.begin().await?; + oci_dao::delete_upload(&mut tx, id, self.grace()).await?; + tx.commit().await + } + .await; + if let Err(e) = result { + tracing::warn!(error = %e, upload = %id, "registry: dropping a failed upload failed"); + } + } + + // ── Manifests ─────────────────────────────────────────────────────────── + + /// What the images format reads in `manifest`. + pub(crate) fn parse_manifest( + &self, + manifest: &[u8], + content_type: Option<&str>, + ) -> ImgResult { + self.plugin() + .parse_manifest(manifest, content_type) + .map_err(|e| match e { + FormatError::Internal(m) => Failure::internal("manifest parse", m), + FormatError::InvalidPackage(m) + | FormatError::MissingManifest(m) + | FormatError::MalformedManifest(m) + | FormatError::InvalidVersion(m) + | FormatError::InvalidId(m) => Failure::new(ImageError::ManifestInvalid(m)), + }) + } + + /// Store the request body as a manifest of `t`, under `reference` and + /// `tags`. + pub(crate) async fn put_manifest( + &self, + principal: &Principal, + t: &Target, + reference: &Reference, + tags: &[String], + content_type: Option, + body: Body, + ) -> ImgResult { + let body = read_body(body, MAX_MANIFEST_BYTES).await?; + let parsed = self.parse_manifest(&body, content_type.as_deref())?; + let digest = match reference { + Reference::Digest(d) => { + let digest = ContentDigest::named(d)?; + if !digest.matches_bytes(&body) { + return Err(Failure::new(ImageError::DigestMismatch)); + } + digest + } + Reference::Tag(_) => ContentDigest::sha256_of(&body), + }; + self.commit_manifest(t, principal, &digest, &body, &parsed, tags) + .await?; + Ok(StoredManifest { + digest: digest.to_string(), + subject: parsed.subject.clone(), + }) + } + + /// Store a manifest whose bytes match `digest` and point `tags` at it, + /// as a push does: every blob it needs must be linked into the image + /// (the format names content it knows, such as the empty JSON, which is + /// linked here) and, for an index, every child manifest must exist; + /// tags follow the repository's immutability; the change runs past the + /// publish policy of `principal`. A manifest the image already holds is + /// left as it is (its tags are still set). + pub(crate) async fn commit_manifest( + &self, + t: &Target, + principal: &Principal, + digest: &ContentDigest, + body: &Bytes, + parsed: &ParsedManifest, + tags: &[String], + ) -> ImgResult { + // What must exist before the manifest may: its blobs (except those + // that live elsewhere by definition) and, for an index, its + // children. + let mut required_blobs: Vec = Vec::new(); + for blob in parsed.blobs.iter().filter(|b| b.required) { + if let Some(content) = &blob.known { + self.ensure_known(t, &blob.digest, content).await?; + } + required_blobs.push(blob.digest.clone()); + } + let children = &parsed.manifests; + let summary = self.manifest_summary(t, parsed).await; + + let staged = self + .inner + .cas + .stage(Cursor::new(body.to_vec()), MAX_MANIFEST_BYTES as u64) + .await?; + let stored = self.inner.cas.commit(staged).await?; + + let mut tx = self.inner.pool.begin().await?; + let linked: HashSet = + oci_dao::lock_links(&mut tx, t.repo.id, &t.image, &required_blobs) + .await? + .into_iter() + .collect(); + if let Some(missing) = required_blobs.iter().find(|d| !linked.contains(*d)) { + return Err(Failure::new(ImageError::ManifestBlobUnknown { + digest: missing.clone(), + manifest: false, + })); + } + if !children.is_empty() { + let present: HashSet = + oci_dao::existing_manifests(&mut tx, t.repo.id, &t.image, children) + .await? + .into_iter() + .collect(); + if let Some(missing) = children.iter().find(|d| !present.contains(*d)) { + return Err(Failure::new(ImageError::ManifestBlobUnknown { + digest: missing.clone(), + manifest: true, + })); + } + } + let blob_refs: Vec = parsed.blobs.iter().map(|b| b.digest.clone()).collect(); + let (manifest_id, created) = oci_dao::insert_manifest( + &mut tx, + NewManifest { + repository_id: t.repo.id, + image: &t.image, + digest: digest.as_str(), + blob_sha256: &stored.sha256, + media_type: &parsed.media_type, + artifact_type: parsed.artifact_type.as_deref(), + subject_digest: parsed.subject.as_deref(), + size: body.len() as i64, + summary: &summary, + blob_refs: &blob_refs, + manifest_refs: children, + }, + stored.ingest_token, + ) + .await?; + let immutable = t.repo.immutable_tags(); + let mut written = Vec::with_capacity(tags.len()); + for tag in tags { + let may_move = !immutable || (tag == "latest" && t.repo.latest_mutable()); + match oci_dao::set_tag(&mut tx, t.repo.id, &t.image, tag, manifest_id, may_move).await? + { + TagWrite::Refused => { + return Err(Failure::new(ImageError::TagImmutable { tag: tag.clone() })); + } + write @ (TagWrite::Created | TagWrite::Moved { .. } | TagWrite::Unchanged) => { + written.push((tag.clone(), write)); + } + } + } + let version = tags.first().cloned().unwrap_or_else(|| digest.to_string()); + self.record_mutation(principal, t, PackageMutation::Publish, version, &mut tx) + .await?; + tx.commit().await?; + Ok(CommittedManifest { + id: manifest_id, + created, + tags: written, + }) + } + + /// Make sure the blob `digest`, whose content the format knows, is linked + /// into the image: clients commonly reference the OCI empty JSON (`{}`) + /// as an artifact's config without uploading it. + async fn ensure_known(&self, t: &Target, digest: &str, content: &[u8]) -> ImgResult<()> { + let pool = &self.inner.pool; + if oci_dao::find_link(pool, t.repo.id, &t.image, digest) + .await? + .is_some() + { + return Ok(()); + } + let digest = ContentDigest::named(digest)?; + if !digest.matches_bytes(content) { + return Err(Failure::internal( + "known blob", + format!("the images format named content that is not {digest}"), + )); + } + let stored = self + .inner + .cas + .store(Cursor::new(content.to_vec()), content.len() as u64) + .await?; + let mut tx = pool.begin().await?; + oci_dao::insert_link_ingested( + &mut tx, + t.repo.id, + &t.image, + digest.as_str(), + &stored.sha256, + stored.size as i64, + stored.ingest_token, + ) + .await + .map_err(Failure::from) + .and_then(require_linked)?; + tx.commit().await?; + Ok(()) + } + + /// The display summary of a manifest: what the format made of it, and + /// the part its config holds once the image holds the config. + pub(crate) async fn manifest_summary( + &self, + t: &Target, + parsed: &ParsedManifest, + ) -> serde_json::Value { + let mut summary = match serde_json::from_str::(&parsed.summary) { + Ok(summary @ serde_json::Value::Object(_)) => summary, + _ => { + tracing::warn!("images format: a manifest summary that is not a JSON object"); + serde_json::json!({}) + } + }; + if let Some(config) = &parsed.config { + if let Some(part) = self + .config_part(t, &config.digest, &config.media_type, config.size) + .await + { + merge(&mut summary, part); + } + } + summary + } + + /// The part of a summary the config `digest` (of `media_type`) holds, + /// if the image holds the config. + pub(crate) async fn config_part( + &self, + t: &Target, + digest: &str, + media_type: &str, + size: u64, + ) -> Option> { + let link = oci_dao::find_link(&self.inner.pool, t.repo.id, &t.image, digest) + .await + .ok()??; + if link.size.max(0) as u64 != size { + return None; + } + let bytes = self + .read_manifest_bytes(forge_storage::BlobRef { + sha256: &link.blob_sha256, + s3_key: &link.s3_key, + size, + }) + .await + .ok()?; + self.config_summary_part(media_type, &bytes) + } + + /// What the format makes of a config's content, as a JSON object. + pub(crate) fn config_summary_part( + &self, + media_type: &str, + config: &[u8], + ) -> Option> { + let part = self.plugin().config_summary(media_type, config)?; + match serde_json::from_str(&part) { + Ok(serde_json::Value::Object(part)) => Some(part), + _ => { + tracing::warn!("images format: a config summary that is not a JSON object"); + None + } + } + } + + pub(crate) async fn delete_manifest( + &self, + principal: &Principal, + t: &Target, + reference: &Reference, + ) -> ImgResult<()> { + let mut tx = self.inner.pool.begin().await?; + let deleted = match reference { + Reference::Tag(tag) => oci_dao::delete_tag(&mut tx, t.repo.id, &t.image, tag).await?, + Reference::Digest(digest) => { + oci_dao::delete_manifest(&mut tx, t.repo.id, &t.image, digest, self.grace()).await? + } + }; + if !deleted { + return Err(Failure::new(ImageError::ManifestUnknown { + reference: reference.as_str().to_string(), + })); + } + let version = reference.as_str().to_string(); + self.record_mutation(principal, t, PackageMutation::Delete, version, &mut tx) + .await?; + tx.commit().await?; + Ok(()) + } + + pub(crate) async fn delete_blob( + &self, + principal: &Principal, + t: &Target, + digest: &ContentDigest, + ) -> ImgResult<()> { + let unknown = || { + Failure::new(ImageError::BlobUnknown { + digest: digest.to_string(), + }) + }; + let mut tx = self.inner.pool.begin().await?; + // Lock first, check after: a manifest naming the blob that commits + // while this waits for the lock is seen by the check. + if !oci_dao::lock_link(&mut tx, t.repo.id, &t.image, digest.as_str()).await? { + return Err(unknown()); + } + if oci_dao::link_referenced(&mut *tx, t.repo.id, &t.image, digest.as_str()).await? { + return Err(Failure::new(ImageError::BlobReferenced { + digest: digest.to_string(), + })); + } + let deleted = + oci_dao::delete_link(&mut tx, t.repo.id, &t.image, digest.as_str(), self.grace()) + .await?; + if !deleted { + return Err(unknown()); + } + let version = digest.to_string(); + self.record_mutation(principal, t, PackageMutation::Delete, version, &mut tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Run a change of an image past the repository's publish policy in the + /// change's transaction: a workload's pushes and deletes go to the + /// trusted-publishing ledger, and fail when its realm or role changed + /// since the request was authenticated. + async fn record_mutation( + &self, + principal: &Principal, + t: &Target, + action: PackageMutation, + version: String, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> ImgResult<()> { + let Some(recorder) = &self.inner.recorder else { + return Ok(()); + }; + let context = PublishAuthorizationContext { + principal: *principal, + action, + package_id: t.image.clone(), + package_version: version, + }; + recorder + .for_repository(&t.repo) + .record_in_transaction(&context, tx) + .await + .map_err(|e| match e { + WriteError::ForbiddenPackage { .. } => Failure::new(ImageError::NoLongerAllowed), + other => Failure::internal("publish record", other), + }) + } + + /// Store a manifest fetched from a proxy's upstream (no reference + /// checks: its blobs are fetched on demand) and point `tag` at it. + pub(crate) async fn store_proxied_manifest( + &self, + t: &Target, + digest: &ContentDigest, + parsed: &ParsedManifest, + body: &Bytes, + tag: Option<&str>, + ) -> ImgResult { + let blob_refs: Vec = parsed.blobs.iter().map(|b| b.digest.clone()).collect(); + let summary = self.manifest_summary(t, parsed).await; + let staged = self + .inner + .cas + .stage(Cursor::new(body.to_vec()), MAX_MANIFEST_BYTES as u64) + .await?; + let stored = self.inner.cas.commit(staged).await?; + let mut tx = self.inner.pool.begin().await?; + super::proxy::still_upstream(&mut tx, &t.repo).await?; + let (manifest_id, _) = oci_dao::insert_manifest( + &mut tx, + NewManifest { + repository_id: t.repo.id, + image: &t.image, + digest: digest.as_str(), + blob_sha256: &stored.sha256, + media_type: &parsed.media_type, + artifact_type: parsed.artifact_type.as_deref(), + subject_digest: parsed.subject.as_deref(), + size: body.len() as i64, + summary: &summary, + blob_refs: &blob_refs, + manifest_refs: &parsed.manifests, + }, + stored.ingest_token, + ) + .await?; + if let Some(tag) = tag { + oci_dao::set_tag(&mut tx, t.repo.id, &t.image, tag, manifest_id, true).await?; + } + tx.commit().await?; + oci_dao::find_manifest(&self.inner.pool, t.repo.id, &t.image, digest.as_str()) + .await? + .ok_or_else(|| Failure::internal("proxy manifest", "stored manifest vanished")) + } +} + +/// Add `part`'s keys to the JSON object `summary`. +pub(crate) fn merge( + summary: &mut serde_json::Value, + part: serde_json::Map, +) { + if let serde_json::Value::Object(summary) = summary { + summary.extend(part); + } +} diff --git a/crates/formats/docker/src/token.rs b/crates/server/src/images/token.rs similarity index 99% rename from crates/formats/docker/src/token.rs rename to crates/server/src/images/token.rs index 5d072d97..5b8c8ba4 100644 --- a/crates/formats/docker/src/token.rs +++ b/crates/server/src/images/token.rs @@ -157,7 +157,7 @@ impl TokenPrincipal { } /// Back to a request principal. Whether the user is still active is - /// re-read by the caller (see [`crate::service`]); `fallback` is the + /// re-read by the caller (see [`super::dispatch`]); `fallback` is the /// request's own principal, used for anonymous tokens. pub fn to_principal(&self, fallback: Principal) -> Option { fn arr(hex_str: &str) -> Option<[u8; 32]> { diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index cd92336e..764b77a3 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -26,6 +26,7 @@ pub mod email; pub mod feed; pub mod files; pub mod health; +pub mod images; pub mod invitations; pub mod me; pub mod me_tokens; diff --git a/crates/server/src/nexus_import/mod.rs b/crates/server/src/nexus_import/mod.rs index b9ddec78..1d8236e8 100644 --- a/crates/server/src/nexus_import/mod.rs +++ b/crates/server/src/nexus_import/mod.rs @@ -25,7 +25,7 @@ //! (`action = "import"`). A version Forge holds with the same SHA-256 //! is skipped without a download; //! * a Docker tag is copied through the Nexus repository's registry API -//! ([`forge_format_docker::Registry::import_tag`]): digests are kept, +//! ([`crate::images::Registry::import_tag`]): digests are kept, //! multi-platform indexes arrive with every child manifest, and a //! referrer is copied once its subject is in Forge; //! * what Nexus recorded is kept: the publish time (`blobCreated`, for a @@ -61,9 +61,8 @@ use forge_core::repo::proxy::Credentials; use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; use forge_crypto::PlaintextSecret; use forge_db::import_dao::{self, ImportItem, NewItem, NexusImport}; -use forge_db::model::{Repository, DOCKER_FORMAT}; +use forge_db::model::Repository; use forge_db::{repository_dao, secret_dao, user_dao}; -use forge_format_docker::import::{ImageImportOptions, ImageOutcome, ImportSource}; use forge_storage::StagedBlob; use futures_util::{FutureExt, StreamExt, TryStreamExt}; use serde_json::Value as JsonValue; @@ -71,11 +70,13 @@ use sha2::Digest as _; use tokio::task::JoinHandle; use uuid::Uuid; +use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; use crate::state::AppState; use nexus::{Listing, Nexus, SourceAsset, SourceItem}; -/// The formats an import copies (Nexus calls them by the same names). -pub const FORMATS: &[&str] = &["nuget", DOCKER_FORMAT]; +/// The formats an import copies, by the names Nexus and Forge share: NuGet +/// packages, and Docker images into the image store. +pub const FORMATS: &[&str] = &["nuget", "docker"]; /// How often an idle instance looks for an import to run. const POLL_INTERVAL: Duration = Duration::from_secs(5); @@ -210,7 +211,7 @@ enum Target { /// Docker, through the registry, from the Nexus repository's registry /// API. Images { - registry: forge_format_docker::Registry, + registry: crate::images::Registry, source: ImportSource, }, } @@ -228,7 +229,8 @@ struct Ctx { } impl Ctx { - fn docker(&self) -> bool { + /// Whether the items are images (Docker). + fn images(&self) -> bool { matches!(self.target, Target::Images { .. }) } } @@ -334,7 +336,7 @@ impl Run<'_> { password.clone(), ); - let target = if repo.format == DOCKER_FORMAT { + let target = if self.state.formats.stores_images(&repo.format) { let registry_url = import.registry_url.clone().unwrap_or_else(|| { default_registry_url(&import.base_url, &import.source_repository) }); @@ -408,10 +410,10 @@ impl Run<'_> { loop { self.check().await?; let page = self.page(ctx, listing, cursor.as_deref()).await?; - let scanned = scanned_items(&ctx.repo.format, listing, &page.items); + let scanned = scanned_items(&ctx.repo.format, ctx.images(), listing, &page.items); let (next_listing, next_cursor, phase) = match page.next { Some(next) => (listing, Some(next), "scan"), - None if ctx.docker() && listing == Listing::Components => { + None if ctx.images() && listing == Listing::Components => { (Listing::Assets, None, "scan") } None => (listing, None, "copy"), @@ -430,7 +432,7 @@ impl Run<'_> { name: &item.name, version: &item.version, size: primary.and_then(|a| a.size), - published_at: primary.and_then(|a| published_at(&ctx.repo.format, a)), + published_at: primary.and_then(|a| published_at(ctx.images(), a)), downloaded_at: primary.and_then(|a| a.downloaded), uploader: primary.and_then(|a| a.uploader.as_deref()), source, @@ -678,7 +680,7 @@ impl Run<'_> { let options = ImportOptions { overwrite, dry_run: false, - published_at: published_at(&ctx.repo.format, asset), + published_at: published_at(ctx.images(), asset), downloaded_at: asset.downloaded, uploader: asset.uploader.clone(), source: ctx.source.clone(), @@ -705,7 +707,7 @@ impl Run<'_> { async fn copy_image( &self, ctx: &Ctx, - registry: &forge_format_docker::Registry, + registry: &crate::images::Registry, from: &ImportSource, kind: &str, source: &SourceItem, @@ -719,7 +721,7 @@ impl Run<'_> { principal: ctx.principal, overwrite: self.import.on_conflict == "overwrite", dry_run: self.import.dry_run, - published_at: asset.and_then(|a| published_at(&ctx.repo.format, a)), + published_at: asset.and_then(|a| published_at(ctx.images(), a)), downloaded_at: asset.and_then(|a| a.downloaded), uploader: asset.and_then(|a| a.uploader.clone()), source: ctx.source.clone(), @@ -743,7 +745,7 @@ impl Run<'_> { ) .await } - .map_err(|e| e.message)?; + .map_err(|e| e.message())?; let bytes = imported.bytes_copied; Ok(match imported.outcome { ImageOutcome::Created => item("imported", imported.digest, bytes), @@ -799,8 +801,8 @@ pub fn default_registry_url(base_url: &str, repository: &str) -> String { /// When Nexus says an asset was published: for a Docker tag when it last /// moved (`lastModified`), for a package when its bytes were stored /// (`blobCreated`). -fn published_at(format: &str, asset: &SourceAsset) -> Option> { - if format == DOCKER_FORMAT { +fn published_at(images: bool, asset: &SourceAsset) -> Option> { + if images { asset.modified.or(asset.created) } else { asset.created.or(asset.modified) @@ -834,8 +836,12 @@ struct Scanned<'a> { /// (Docker) asset listing, the manifests held by digest. Every asset is /// counted once: a package component's in the components listing, a /// Docker repository's (blobs included) in the asset listing. -fn scanned_items<'a>(format: &str, listing: Listing, page: &'a [SourceItem]) -> Scanned<'a> { - let docker = format == DOCKER_FORMAT; +fn scanned_items<'a>( + format: &str, + images: bool, + listing: Listing, + page: &'a [SourceItem], +) -> Scanned<'a> { let mut scanned = Scanned { items: Vec::new(), assets: 0, @@ -844,7 +850,7 @@ fn scanned_items<'a>(format: &str, listing: Listing, page: &'a [SourceItem]) -> for item in page { match listing { Listing::Components => { - let primary = if docker { + let primary = if images { item.assets.iter().find(|a| a.path.contains("/manifests/")) } else { scanned.assets += item.assets.len() as i64; @@ -1060,7 +1066,7 @@ mod tests { ..Default::default() }, ]; - let scanned = scanned_items(DOCKER_FORMAT, Listing::Assets, &page); + let scanned = scanned_items("docker", true, Listing::Assets, &page); assert_eq!(scanned.assets, 3); assert_eq!(scanned.bytes, 120); assert_eq!(scanned.items.len(), 1); diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index b3b35a54..49207544 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -124,14 +124,15 @@ pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ ]; /// The format plugins built into this server: crates under -/// `crates/formats//`, linked in and always installed (NuGet, npm, raw). -/// NuGet reads archives under `nuget_limits` (`[nuget]`), npm requests under -/// `npm_limits` (`[npm]`). +/// `crates/formats//`, linked in and always installed (Docker / OCI, +/// NuGet, npm, raw). NuGet reads archives under `nuget_limits` (`[nuget]`), +/// npm requests under `npm_limits` (`[npm]`). pub fn builtin( nuget_limits: forge_format_nuget::NupkgLimits, npm_limits: forge_format_npm::NpmLimits, ) -> Vec> { vec![ + Plugin::builtin(forge_format_docker::Docker), Plugin::builtin(forge_format_nuget::Nuget::with_limits(nuget_limits)), Plugin::builtin(forge_format_npm::Npm::with_limits(npm_limits)), Plugin::builtin(forge_format_raw::Raw), @@ -299,7 +300,8 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { let Some(plugin) = state.formats.plugin(&repo.format) else { return if state.formats.is_installed(&repo.format) { - // A built-in format has its own mount (Docker: `/v2`). + // A format without a protocol (a test's own) is served under + // no mount. (StatusCode::NOT_FOUND, "repository is not served here").into_response() } else { ( @@ -393,6 +395,7 @@ async fn serve( .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) .collect(), body: None, + image: None, }; let peer = req .extensions() @@ -612,6 +615,10 @@ impl Ctx<'_> { plugin_failed(self.repo, &"a body was asked for twice") } Action::ServeFile(_) => self.plugin_misused("an action of the other storage"), + // Images are the registry's (`crate::images`). + Action::ServeBlob { .. } | Action::ServeManifest { .. } => { + self.plugin_misused("an image action") + } } } @@ -686,6 +693,10 @@ impl Ctx<'_> { /// format's storage only, and a change (`form`, `store`, `remove`) alone /// in its round. fn check_round(&self, queries: &[Query]) -> Result<(), Response> { + // Images are the registry's (`crate::images`). + if queries.iter().any(Query::is_images) { + return Err(self.plugin_misused("an image query")); + } if queries.iter().any(|q| q.is_files() != self.stores_files) { return Err(self.plugin_misused("a query of the other storage")); } @@ -1022,6 +1033,12 @@ impl Ctx<'_> { "a files query of a package format".into(), )) } + // Refused by `check_round`. + Query::Image(_) => { + return Err(RepoError::Storage( + "an image query of a package format".into(), + )) + } }) } @@ -2158,7 +2175,7 @@ mod tests { let descriptors: Vec<_> = plugins.iter().map(|p| p.descriptor()).collect(); assert_eq!(forge_plugin::descriptor::check(&descriptors), Ok(())); let ids: Vec<_> = plugins.iter().map(|p| p.id()).collect(); - assert_eq!(ids, ["nuget", "npm", "raw"]); + assert_eq!(ids, ["docker", "nuget", "npm", "raw"]); for plugin in &plugins { assert_eq!(plugin.descriptor().version, "1.0.0", "{}", plugin.id()); } @@ -2171,9 +2188,10 @@ mod tests { assert_eq!(bearer, ["npm"]); // The check refuses a malformed or reserved id, a second format - // with a built-in's id, and invalid asset names. + // with a built-in's id (Docker's included), a second images format, + // and invalid asset names. let nuget = forge_format_nuget::DESCRIPTOR; - for id in ["NuGet", "n", "docker", "v2"] { + for id in ["NuGet", "n", "v2"] { let renamed = forge_plugin::Descriptor { id, ..nuget }; assert!( forge_plugin::descriptor::check(&[&renamed]).is_err(), @@ -2188,6 +2206,20 @@ mod tests { let mut with_twin = descriptors.clone(); with_twin.push(&twin); assert!(forge_plugin::descriptor::check(&with_twin).is_err()); + let docker_twin = forge_plugin::Descriptor { + id: "docker", + ..nuget + }; + let mut with_twin = descriptors.clone(); + with_twin.push(&docker_twin); + assert!(forge_plugin::descriptor::check(&with_twin).is_err()); + let second_images = forge_plugin::Descriptor { + id: "oci", + ..forge_format_docker::DESCRIPTOR + }; + let mut with_second = descriptors.clone(); + with_second.push(&second_images); + assert!(forge_plugin::descriptor::check(&with_second).is_err()); let bad_assets = forge_plugin::Descriptor { assets: forge_plugin::api::Assets { package: "../package.nupkg", diff --git a/crates/server/src/registry.rs b/crates/server/src/registry.rs index 31a8e59f..c8fc3b98 100644 --- a/crates/server/src/registry.rs +++ b/crates/server/src/registry.rs @@ -1,6 +1,7 @@ -//! The Docker / OCI registry (`/v2`), wired into the server: the route -//! handler, the registry-token signing key, and the trusted-publishing -//! ledger of workload pushes. +//! The registry of the images format (Docker / OCI, `/v2`), wired into the +//! server: the route handler, the registry-token signing key, and the +//! trusted-publishing ledger of workload pushes. The image store and the +//! request's conversation with the format are [`crate::images`]. use std::sync::Arc; @@ -8,7 +9,7 @@ use anyhow::Context; use axum::body::Body; use axum::extract::{Request, State}; use axum::http::{header, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; +use axum::response::Response; use base64::Engine; use forge_core::auth::Principal; use forge_core::repo::PublishPolicy; @@ -16,11 +17,13 @@ use forge_crypto::service::EnvelopeService; use forge_crypto::{ExposeSecret, PlaintextSecret}; use forge_db::model::Repository; use forge_db::{authz_dao, secret_dao, workload_api_key_dao}; -use forge_format_docker::token::TokenKeys; +use forge_plugin::api::images::ImageError; +use forge_plugin::FormatPlugin; use sha2::Digest; use sqlx::PgPool; use crate::config::Config; +use crate::images::token::TokenKeys; use crate::middleware::principal_of; use crate::state::AppState; @@ -36,7 +39,7 @@ pub async fn registry(State(state): State, req: Request) -> Resp /// Workload pushes go to the trusted-publishing ledger. pub(crate) struct LedgerRecorder; -impl forge_format_docker::PublishRecorder for LedgerRecorder { +impl crate::images::PublishRecorder for LedgerRecorder { fn for_repository(&self, repository: &Repository) -> Arc { crate::dispatch::workload_ledger_policy(repository) } @@ -50,7 +53,7 @@ pub(crate) struct WorkloadLiveness { } #[async_trait::async_trait] -impl forge_format_docker::WorkloadCheck for WorkloadLiveness { +impl crate::images::WorkloadCheck for WorkloadLiveness { async fn still_valid(&self, principal: &Principal) -> Result { let Principal::Workload { realm_id, @@ -84,25 +87,21 @@ impl forge_format_docker::WorkloadCheck for WorkloadLiveness { /// A 401 the authentication middleware answered itself (wrong credentials) /// on a `/v2` path, made a registry error: the Bearer challenge clients -/// expect and the distribution-spec error body. -pub(crate) fn amend_registry_rejection(public_base_url: &str, response: Response) -> Response { +/// expect, and the images format's answer to invalid credentials. +pub(crate) fn amend_registry_rejection( + public_base_url: &str, + images: &dyn FormatPlugin, + response: Response, +) -> Response { if response.status() != StatusCode::UNAUTHORIZED || response.headers().contains_key(header::WWW_AUTHENTICATE) { return response; } - let (scheme, service) = forge_format_docker::registry_origin(public_base_url); + let (scheme, service) = crate::images::registry_origin(public_base_url); let challenge = format!("Bearer realm=\"{scheme}://{service}/v2/token\",service=\"{service}\""); - let body = serde_json::json!({ - "errors": [{ "code": "UNAUTHORIZED", "message": "invalid credentials" }] - }) - .to_string(); - let mut amended = ( - StatusCode::UNAUTHORIZED, - [(header::CONTENT_TYPE, "application/json")], - body, - ) - .into_response(); + let mut amended = + crate::images::dispatch::rendered(images.image_error(&ImageError::InvalidCredentials)); if let Ok(value) = HeaderValue::from_str(&challenge) { amended .headers_mut() diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index beb18573..ac848554 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -33,7 +33,7 @@ use forge_core::repo::{ }; use forge_core::retention::FormatResolver; use forge_core::session::SessionService; -use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT}; +use forge_db::model::{RepoKind, Repository}; use forge_storage::{BlobRef, BlobStore, Cas, S3Error}; use sqlx::PgPool; @@ -99,7 +99,7 @@ pub struct AppState { /// install's shared one ([`crate::registry::load_token_keys`] via /// [`AppState::with_registry_keys`]) so every instance accepts every /// instance's tokens. - pub registry_keys: forge_format_docker::token::TokenKeys, + pub registry_keys: crate::images::token::TokenKeys, /// This process among the instances of an install, as per-instance /// reports (the local blob cache's statistics) name it. pub instance: Arc, @@ -140,15 +140,15 @@ impl Instance { /// Format registry: maps a format id (the `repository.format` column) to its /// [`Format`] and its plugin: the built-in format plugins -/// ([`crate::plugins::builtin`]: NuGet, npm, raw) and Docker. +/// ([`crate::plugins::builtin`]: Docker / OCI, NuGet, npm, raw). /// -/// Docker ([`DOCKER_FORMAT`]) is installed but has no [`Format`]: its -/// repositories hold images (manifests and blobs), not package versions, -/// and are served by the registry under `/v2` ([`crate::registry`]), so none -/// of the package seams (parsing, version order, the read ops) apply to -/// them. A files format (raw) has a [`Format`] whose -/// [`Format::stores_files`] says so: its repositories hold files in the -/// file store ([`crate::files`]). +/// A files format (raw) has a [`Format`] whose [`Format::stores_files`] says +/// so: its repositories hold files in the file store ([`crate::files`]). An +/// images format (Docker) has one whose [`Format::stores_images`] says so: +/// its repositories hold images (manifests and blobs) in the image store +/// ([`crate::images`]) and are served under its mount, `/v2`, by the +/// registry, so none of the package seams (parsing, version order, the read +/// ops) apply to them. pub struct FormatRegistry { formats: BTreeMap>, /// Every format's plugin, by format id: its client protocol. @@ -176,12 +176,12 @@ impl FormatRegistry { } /// Add `format` under its [`Format::id`]. An id that is already taken - /// (NuGet's and Docker's included) is refused: a plugin never shadows + /// (a built-in format's included) is refused: a plugin never shadows /// another format. pub fn with_format(mut self, format: Arc) -> Result { let id = format.id().to_string(); if self.is_installed(&id) { - let built_in = Self::is_native(&id) || self.plugins.contains_key(&id); + let built_in = self.plugins.contains_key(&id); return Err(if built_in { format!("format {id:?} is built into this server") } else { @@ -216,15 +216,9 @@ impl FormatRegistry { self.plugins.values() } - /// Whether format `id` is one served natively, without a [`Format`]: - /// Docker. - pub fn is_native(id: &str) -> bool { - id == DOCKER_FORMAT - } - - /// Whether format `id` is installed: Docker, or a format plugin. + /// Whether format `id` is installed. pub fn is_installed(&self, id: &str) -> bool { - Self::is_native(id) || self.formats.contains_key(id) + self.formats.contains_key(id) } /// Whether format `id`'s repositories hold files under paths (raw): it @@ -233,12 +227,30 @@ impl FormatRegistry { self.formats.get(id).is_some_and(|f| f.stores_files()) } - /// The mounts plugin formats are served under, sorted and distinct - /// (`/repository`, and any other a plugin names). + /// Whether format `id`'s repositories hold images (Docker / OCI): it is + /// installed and its descriptor says `Storage::Images`. + pub fn stores_images(&self, id: &str) -> bool { + self.formats.get(id).is_some_and(|f| f.stores_images()) + } + + /// The images format, served under its mount by the registry + /// ([`crate::images`]): the first, by id, whose repositories hold + /// images. The built-in formats have one, Docker. + pub fn images_plugin(&self) -> Option> { + self.plugins + .values() + .find(|p| p.descriptor().storage == forge_plugin::api::Storage::Images) + .cloned() + } + + /// The mounts plugin formats are served under by the plugin dispatcher + /// ([`crate::plugins`]), sorted and distinct (`/repository`, and any + /// other a plugin names). The images format's mount is the registry's. pub fn mounts(&self) -> Vec<&str> { let mut mounts: Vec<&str> = self .plugins .values() + .filter(|p| p.descriptor().storage != forge_plugin::api::Storage::Images) .map(|p| p.descriptor().mount) .collect(); mounts.sort_unstable(); @@ -248,19 +260,13 @@ impl FormatRegistry { /// The installed format ids, sorted. pub fn ids(&self) -> impl Iterator { - let mut ids: Vec<&str> = self.formats.keys().map(String::as_str).collect(); - ids.push(DOCKER_FORMAT); - ids.sort_unstable(); - ids.into_iter() + self.formats.keys().map(String::as_str) } /// The repository kinds format `id` supports, `None` when it is not - /// installed. Docker has every kind; a format plugin serves the kinds its - /// descriptor names (NuGet's, npm's and raw's all three). + /// installed: what its descriptor names (all three for every built-in + /// format: Docker's, NuGet's, npm's and raw's). pub fn kinds(&self, id: &str) -> Option> { - if Self::is_native(id) { - return Some(vec!["hosted", "proxy", "group"]); - } let plugin = self.plugins.get(id)?; Some( plugin @@ -371,7 +377,7 @@ impl AppState { oidc_login_starts, envelope: None, upstream, - registry_keys: forge_format_docker::token::TokenKeys::new(vec![{ + registry_keys: crate::images::token::TokenKeys::new(vec![{ use rand::Rng; let mut key = vec![0u8; 32]; rand::rng().fill_bytes(&mut key); @@ -389,14 +395,19 @@ impl AppState { } /// Use `keys` to sign and verify registry tokens. - pub fn with_registry_keys(mut self, keys: forge_format_docker::token::TokenKeys) -> Self { + pub fn with_registry_keys(mut self, keys: crate::images::token::TokenKeys) -> Self { self.registry_keys = keys; self } - /// The Docker / OCI registry service, wired to this state. - pub fn registry(&self) -> forge_format_docker::Registry { - forge_format_docker::Registry::new(forge_format_docker::RegistryParts { + /// The registry of the images format (Docker / OCI), wired to this + /// state. + pub fn registry(&self) -> crate::images::Registry { + let plugin = self + .formats + .images_plugin() + .expect("the images format is built in"); + crate::images::Registry::new(crate::images::RegistryParts { pool: self.pool.clone(), cas: self.cas.clone(), store: self.blob_store.clone(), @@ -404,10 +415,12 @@ impl AppState { authz: self.authz.clone(), http: self.upstream.clone(), envelope: self.envelope.clone(), - config: forge_format_docker::RegistryConfig { + config: crate::images::RegistryConfig { public_base_url: self.config.public_base_url(), max_blob_bytes: self.config.docker_max_blob_bytes(), - default_max_upload_bytes: self.config.default_upload_limit(DOCKER_FORMAT), + default_max_upload_bytes: self + .config + .default_upload_limit(forge_plugin::api::Storage::Images), token_ttl_seconds: self.config.docker.token_ttl_seconds, delete_grace: self.config.outbox.grace(), max_group_depth: self.config.groups.max_nesting_depth.max(0) as u32, @@ -422,6 +435,7 @@ impl AppState { realms: self.realms.clone(), pool: self.pool.clone(), })), + plugin: plugin.implementation().clone(), }) } @@ -479,7 +493,16 @@ impl AppState { /// ([`Config::default_upload_limit`]). pub fn upload_limit(&self, repo: &Repository) -> u64 { repo.max_upload_bytes() - .unwrap_or_else(|| self.config.default_upload_limit(&repo.format)) + .unwrap_or_else(|| self.config.default_upload_limit(self.storage(&repo.format))) + } + + /// What repositories of format `id` hold (packages when it is not + /// installed). + pub fn storage(&self, id: &str) -> forge_plugin::api::Storage { + self.formats + .plugin(id) + .map(|p| p.descriptor().storage) + .unwrap_or_default() } /// A blob-byte fetcher over the S3 store (serves `BlobFetch`/`PackageBytes`). @@ -700,25 +723,27 @@ mod tests { } #[test] - fn docker_is_installed_without_a_package_format() { + fn the_builtin_formats_and_their_storage() { let registry = FormatRegistry::new(Default::default(), Default::default()); - assert!(registry.is_installed(DOCKER_FORMAT)); - assert!(registry.get(DOCKER_FORMAT).is_none()); assert_eq!( registry.ids().collect::>(), ["docker", "npm", "nuget", "raw"] ); - // Raw is a built-in plugin whose repositories hold files. + // Raw is a built-in plugin whose repositories hold files, Docker one + // whose repositories hold images, served by the registry. assert!(registry.get("raw").is_some()); assert!(registry.stores_files("raw")); assert!(!registry.stores_files("nuget")); - assert!(!registry.stores_files(DOCKER_FORMAT)); + assert!(registry.stores_images("docker")); + assert!(!registry.stores_images("raw")); + assert!(!registry.stores_files("docker")); + assert_eq!(registry.images_plugin().map(|p| p.id()), Some("docker")); assert_eq!( registry.kinds("raw"), Some(vec!["hosted", "proxy", "group"]) ); assert_eq!( - registry.kinds(DOCKER_FORMAT), + registry.kinds("docker"), Some(vec!["hosted", "proxy", "group"]) ); assert_eq!( diff --git a/crates/server/src/workers.rs b/crates/server/src/workers.rs index e1b84140..ff326375 100644 --- a/crates/server/src/workers.rs +++ b/crates/server/src/workers.rs @@ -324,7 +324,7 @@ fn spawn_registry_sweeper( ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { ticker.tick().await; - match forge_format_docker::gc::sweep_once( + match crate::images::gc::sweep_once( &pool, chrono::Duration::seconds(docker.upload_ttl_seconds as i64), chrono::Duration::seconds(docker.unreferenced_blob_ttl_seconds as i64), diff --git a/crates/server/tests/admin.rs b/crates/server/tests/admin.rs index a91ca36c..60c4d588 100644 --- a/crates/server/tests/admin.rs +++ b/crates/server/tests/admin.rs @@ -1570,8 +1570,8 @@ async fn registry_token_key_survives_dek_rotation() { .unwrap(); assert_eq!(stored().await, after, "not replaced"); let now = chrono::Utc::now().timestamp(); - let claims = forge_format_docker::token::Claims { - iss: forge_format_docker::token::issuer("https://forge.example.com"), + let claims = forge_server::images::token::Claims { + iss: forge_server::images::token::issuer("https://forge.example.com"), aud: "forge.example.com".into(), sub: "anonymous".into(), iat: now, @@ -1579,11 +1579,11 @@ async fn registry_token_key_survives_dek_rotation() { exp: now + 300, jti: "j".into(), access: Vec::new(), - fp: forge_format_docker::token::TokenPrincipal::Anonymous, + fp: forge_server::images::token::TokenPrincipal::Anonymous, }; - let token = forge_format_docker::token::issue(&srv.state.registry_keys, &claims).unwrap(); + let token = forge_server::images::token::issue(&srv.state.registry_keys, &claims).unwrap(); assert!( - forge_format_docker::token::verify(&restarted, &token, &claims.iss, &claims.aud).is_some(), + forge_server::images::token::verify(&restarted, &token, &claims.iss, &claims.aud).is_some(), "a restarted instance accepts the running instance's tokens" ); } diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 7095e978..7ef32ce0 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -410,7 +410,7 @@ async fn version_check_challenges_and_tokens_authenticate() { async fn token_requests_name_a_bounded_number_of_repositories() { use base64::Engine; let srv = common::start().await; - let max = forge_format_docker::token::MAX_SCOPES; + let max = forge_server::images::token::MAX_SCOPES; let request = |scopes: Vec| { let mut req = srv .client() @@ -2711,7 +2711,7 @@ async fn deleting_an_image_never_frees_a_layer_another_image_uses() { assert_eq!(ref_count(&srv, &layer).await, Some(2), "one link per image"); let sweep = || { - forge_format_docker::gc::sweep_once( + forge_server::images::gc::sweep_once( &srv.pool, chrono::Duration::zero(), chrono::Duration::zero(), @@ -2878,7 +2878,7 @@ async fn retention_keeps_newest_tags_and_drops_old_untagged_manifests() { assert_eq!(resp.status(), 202); let worker = forge_core::retention::RetentionWorker::new( srv.pool.clone(), - srv.state.formats.get("nuget").expect("NuGet is built in"), + forge_server::state::RegistryResolver(srv.state.formats.clone()), Arc::new(forge_core::repo::NoopCacheInvalidator), forge_core::retention::RetentionConfig::default(), ); diff --git a/docs/changes/2.0.0/447.feature.md b/docs/changes/2.0.0/447.feature.md new file mode 100644 index 00000000..f60afeb5 --- /dev/null +++ b/docs/changes/2.0.0/447.feature.md @@ -0,0 +1,13 @@ +Docker / OCI is a format plugin, `crates/formats/docker`, like NuGet and +raw: the plugin speaks the distribution-spec protocol (paths, names, +manifests, error answers, Helm charts, an upstream registry's URLs), and the +server's image store keeps the images, the registry tokens and the +authorization. `/v2` answers as before. The plugin contract gains a third +storage model for this, images: a plugin whose descriptor says +`Storage::Images` asks the host for manifests, blobs, tags and referrers, +runs upload sessions and stores manifests through it, and serves a blob or +a manifest with `serve-blob` and `serve-manifest`; a proxy's plugin names +the upstream's URLs and the host fetches them, exchanging the upstream's +Bearer tokens. `GET /admin/formats` lists Docker from its descriptor, with +its version and description. See +[docs/format-plugins.md](docs/format-plugins.md#images-formats). diff --git a/docs/docker.md b/docs/docker.md index fc9b5799..4732303e 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -7,10 +7,12 @@ and docker-group repositories do. It speaks the v1.1 that Docker, Podman, containerd, crane, skopeo, oras, Helm and cosign use. CI runs the spec's conformance suite against Forge on every change. -Images live in repositories of `format: "docker"`, a format built into -Forge like NuGet (`GET /admin/formats` lists it; see -[format plugins](format-plugins.md#why-docker-is-not-a-plugin) for why it is -not a plugin). They come in the same three kinds as NuGet repositories: +Images live in repositories of `format: "docker"`, a format plugin built +into Forge like NuGet and raw (`GET /admin/formats` lists it, with `storage: +images`). It is an [images format](format-plugins.md#images-formats): the +plugin, [`crates/formats/docker`](../crates/formats/docker), speaks the +protocol, and Forge's image store keeps the images. They come in the same +three kinds as NuGet repositories: - **hosted**: push and pull your own images; - **proxy**: a pull-through cache of an upstream registry: Docker Hub, GHCR, diff --git a/docs/format-plugins.md b/docs/format-plugins.md index f1c6aec8..66508e99 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -1,7 +1,6 @@ # Format plugins -Forge serves the Docker / OCI registry natively ([docker.md](docker.md)). -Every package format comes from a **format plugin**: a Rust crate that +Every format Forge serves comes from a **format plugin**: a Rust crate that teaches Forge the format's rules, its client protocol and, for proxies, how to read an upstream, by implementing one trait, [`FormatPlugin`](../crates/plugin-api/src/lib.rs). The repository machinery @@ -11,25 +10,29 @@ streaming of package bytes. Every plugin is built in: a crate under `crates/formats//` linked into the server and called directly, like any other Rust code. Plugins need no -configuration. Forge ships three: **NuGet** +configuration. Forge ships four: **Docker / OCI** +([`crates/formats/docker`](../crates/formats/docker), +[Images formats](#images-formats), [docker.md](docker.md)), **NuGet** ([`crates/formats/nuget`](../crates/formats/nuget)), **npm** ([`crates/formats/npm`](../crates/formats/npm), [npm.md](npm.md)) and **raw files** ([`crates/formats/raw`](../crates/formats/raw), -[Files formats](#files-formats)). Helm charts are served by the registry -([docker.md](docker.md#helm-charts)). +[Files formats](#files-formats)). Helm charts are pushed to Docker +repositories ([docker.md](docker.md#helm-charts)). > **Status: preview.** The plugin contract may change in any release. The > plugins build with the server, so the compiler checks each of them against > the contract it is linked with. Contract 0.4.0 added files formats > ([Files formats](#files-formats)), and `render` returns an action, like -> `handle`; and for npm, request bodies, document uploads, changes (tags, -> deprecations), logins and forwards ([npm as a plugin](#npm-as-a-plugin)). +> `handle`; for npm, request bodies, document uploads, changes (tags, +> deprecations), logins and forwards followed ([npm as a plugin](#npm-as-a-plugin)), +> and images formats ([Images formats](#images-formats)). ## Contents - [Layout](#layout) - [How it fits together](#how-it-fits-together) - [Files formats](#files-formats) +- [Images formats](#images-formats) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) - [Security model](#security-model) @@ -66,6 +69,15 @@ crates/formats/npm/ ├── src/publish.rs # the tarball inside a publish document, as it streams ├── src/proxy.rs # proxies: reading another registry (upstream) └── tests/ # the protocol against canned answers + +crates/formats/docker/ +├── Cargo.toml +├── src/lib.rs # DESCRIPTOR (mount "/v2", storage images, kinds hosted/proxy/group) +│ # and impl FormatPlugin for Docker: paths, manifests, charts +├── src/protocol.rs # the distribution-spec API: handle and render +├── src/upstream.rs # proxies: URLs, tag pages and referrers of an upstream registry +├── src/error.rs # the distribution-spec error answers +└── src/manifest.rs, chart.rs, path.rs, reference.rs, digest.rs ``` The pieces: @@ -75,7 +87,7 @@ The pieces: | [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, and, behind its `testing` feature, helpers that run a plugin's protocol against canned answers. Depends on nothing but `std`. | | [`crates/formats/`](../crates/formats) | One crate per plugin, implementing `FormatPlugin`. | | [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin, which describes itself), `PluginFormat`, which adapts it to Forge's `Format` seam, and the checks of a descriptor (`descriptor::check`). | -| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`), the `{mount}/{repo}/...` dispatcher, and the file store files formats keep their files in ([`crates/server/src/files`](../crates/server/src/files)). | +| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`), the `{mount}/{repo}/...` dispatcher, the file store files formats keep their files in ([`crates/server/src/files`](../crates/server/src/files)), and the image store and token service of images formats, under `/v2` ([`crates/server/src/images`](../crates/server/src/images)). | ## How it fits together @@ -230,6 +242,114 @@ names applied; `DELETE` is a `remove`. hundred lines; the file store is [`crates/server/src/files`](../crates/server/src/files). +## Images formats + +A format whose descriptor says `storage: Storage::Images` holds images: +content-addressed blobs and manifests linked into each image (`team/app`), +tags that point at manifests, indexes that name other manifests, referrers, +and uploads in sessions of several requests. [Docker / OCI](docker.md) is +one ([`crates/formats/docker`](../crates/formats/docker)). Its mount is +`/v2` (`IMAGES_MOUNT`), where every registry client asks, so a server holds +one images format at most. + +The split is the one files formats have. The format owns the protocol: the +grammar of paths, names, references and digests, what a manifest names and +says, the error answers, the response headers, and the URLs of an upstream +registry. The host owns the image store +([`crates/server/src/images`](../crates/server/src/images)), which keeps +everything else: + +- `/v2/` (the version check), `/v2/token` (the token service and its signing + keys) and `/v2/_catalog` (refused), answered by the host itself; +- resolving a name to a repository and an image, by path + (`/`) or on a repository's own host (`host`); +- authorization: the Bearer challenge and a token's scopes, RBAC and package + selectors on image names, and where a change lands (a group's writable + member); +- the store: blobs and manifests by digest in the content-addressed store, + tags (immutable ones too), upload sessions and their chunks, mounts from + other images, the references between manifests, and the trusted-publishing + ledger; +- groups, a proxy's caching, retention, garbage collection, and the Nexus + import; +- serving a blob or a manifest, with one `Range`, `Docker-Content-Digest` + and `ETag`, and the registry's response hardening on every answer. + +The plugin never sees credentials, never touches storage and never holds a +blob in memory. A request goes through three steps: + +1. `image_path` splits the path below the mount into the name the client + wrote and the endpoint (`/manifests/latest`), and says what the request + needs: read, write (every request of an upload session included) or + delete. +2. The host resolves the name and authorizes the request. What it refuses + (an unknown name, a missing token, a denied scope, an offline + repository) it hands to the format as an `ImageError`, and `image_error` + renders it, the host adding its Bearer challenge. +3. `handle` gets the request with the endpoint as its path and the resolved + image as `request.image`, and answers as for any format: a response, + queries, or an action. + +| Query | Answer | +| --- | --- | +| `manifest(image, reference)` | A manifest by tag or digest: digest, media type, size, artifact type, subject, the stored summary | +| `blob(image, digest)` | A blob's digest and size; a manifest is content too | +| `tags(image, last, n)` | A page of the image's tags in byte order, and whether more follow | +| `referrers(image, digest, artifact_type)` | The manifests whose subject is a manifest, of one artifact type when one is given | +| `start-upload(image, mount)` | An upload session, or a blob mounted from another image without one | +| `upload-status(id)`, `append-upload(id, range)` | How far a session got, after appending the request body (at `range`, when the client gave one) | +| `finish-upload(id, digest, range)` | Appends the body, ends the session and links its content into the image as `digest` | +| `cancel-upload(id)` | Ends a session without a blob | +| `put-manifest(image, reference, tags)` | Stores the request body as a manifest under `reference` and points `tags` at it | +| `delete-manifest(image, reference)`, `delete-blob(image, digest)` | Deletes a tag or a manifest, or unlinks a blob no manifest names | + +A query that changes the repository comes alone in its round. Every answer +is a result: a query that failed (a blob unknown, a digest that does not +match, a manifest naming a blob the image lacks, a tag that is immutable, an +upload too large) comes back to `render` as its `ImageError`, and the format +answers it with its own error codes and status, which registry clients read. +The actions `serve-blob(image, digest)` and `serve-manifest(image, +reference)` stream what a query of the same request found. A response the +format renders keeps `content-type`, `docker-content-digest`, +`docker-upload-uuid`, `link`, `location`, `oci-filters-applied`, +`oci-subject`, `oci-tag`, `range` and `retry-after`; `location` and `link` +only when they stay under `/v2/`. + +Manifests are the format's to read. The host stages a pushed manifest (at +most 4 MiB), and `parse_manifest` says what it is served as, the blobs and +manifests it names (which the push needs in the image), its subject and +artifact type, and a JSON summary stored with it. A proxy's manifests are +read the same way. When the manifest names a config small enough to read +(an image's, a Helm chart's), the host reads it once the image holds it, +and merges what `config_summary` makes of it (the platform, the chart's +metadata) into the summary; the summary names the config under +`config_digest` and `config_media_type`, so a proxy completes the summaries +of the manifests it cached before their config. The web UI reads +`layers_size`, `platform`, `platforms` and `chart` from a summary, and shows a +chart's README and values with `artifact_files`, over the archive blob the +host lends it (at most 16 MiB). + +A proxy's upstream operations name where things are, and the host fetches +them: + +| Operation | The plugin answers | +| --- | --- | +| `image-name(image)` | The name the upstream knows the image by, which the proxy caches it under (Docker Hub's `library/` namespace) | +| `image-manifest(image, reference)` | The manifest's URL, the `Accept` header to send and the scope of a token for it. The host re-validates a cached tag with a `HEAD`, downloads the manifest, checks it against its digest and reads it with `parse_manifest` | +| `image-blob(image, digest)` | The blob's URL; the host streams it into the store and checks its digest | +| `image-tags(image, last, n)` | A page of the upstream's tags, following the upstream's `Link` pages (a conversation of fetches) | +| `image-referrers(image, digest)` | The upstream's referrers of a manifest | + +An images format's fetches go to the origin of `remote_url` only, carry the +`Accept` header the plugin names, and come back with their status and `Link` +headers. When the upstream answers 401 with a Bearer challenge, the host asks +the realm the challenge names for a token (with the repository's +credentials, if it has any) and sends the request again, as Docker Hub +requires; the token is the host's, cached for its lifetime. + +`testing::converse_answered` runs `upstream` against canned answers with +their status and headers, as the Docker plugin's tests do. + ## Proxy repositories A plugin that lists `proxy` among its kinds implements the upstream side of @@ -299,12 +419,12 @@ cached. ## Using a plugin `GET /admin/formats` lists every format the server can create repositories -of: the plugins (NuGet, raw files) and Docker, each with the repository -kinds it supports, what its repositories hold (`storage`: `packages`, -`files`, or Docker's `images`) and where its repositories are served. The +of: the plugins (Docker, NuGet, npm, raw files), each from its descriptor, with +the repository kinds it supports, what its repositories hold (`storage`: +`packages`, `files` or `images`) and where its repositories are served. The web UI's *New repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream -side (NuGet, npm and raw do). Docker has proxies of its own. +side (all four do). A repository of a plugin's format is created like any other, with the plugin's format id, and served under the plugin's mount (`/repository` @@ -415,6 +535,7 @@ pub const DESCRIPTOR: Descriptor = Descriptor { // `upstream`). kinds: &[RepositoryKind::Hosted, RepositoryKind::Group], storage: Storage::Packages, // or `Storage::Files` (Files formats) + // or `Storage::Images` (Images formats) // How a version's files are stored: `package` and `manifest`, both // application/octet-stream, unless it names its own. assets: Assets::DEFAULT, @@ -442,7 +563,8 @@ impl FormatPlugin for MyFormat { An id is 2–32 lowercase letters, digits and `-`, starting with a letter, and no other format's. A mount is `/` and one lowercase word; the server's own paths (`/admin`, `/auth`, `/browse`, `/v2`, the web UI's pages, …) are -reserved, and plugins may share a mount. List `Proxy` among the kinds only +reserved, and plugins may share a mount. An images format's mount is `/v2`, +and only one format of a server may store images. List `Proxy` among the kinds only when the plugin implements `upstream`. Asset names are plain file names, and their content types `type/subtype`. @@ -479,12 +601,14 @@ rules hold: These are the current limits of the preview, roughly in the order we expect to lift them: -- **Proxies fetch whole documents with GET.** An upstream request carries - no headers but the repository's credentials (Basic, or a Bearer token) and - the `Accept` the fetch names, and its body is read whole up to - `proxy.max_metadata_bytes` before the plugin sees it; only a `forward` - POSTs. An upstream that needs a token exchange (Docker Hub's) or streams a - listing too large to hold needs more from the contract. +- **Proxies fetch whole documents with GET.** A package or files format's + upstream request carries no headers but the repository's credentials + (Basic, or a Bearer token) and the `Accept` the fetch names, and its body + is read whole up to `proxy.max_metadata_bytes` before the plugin sees it; + only a `forward` POSTs. An upstream that streams a listing too large to + hold needs more from the contract. (An images format's fetches get a + Bearer token exchange, and the host streams the blobs and manifests they + name: [Images formats](#images-formats).) - **One package file per version.** Formats that publish several files per version (PyPI wheels per platform, Maven's jar, pom and sources) need multi-asset versions. @@ -541,29 +665,3 @@ Contract 0.4.0 grew for it too, and every piece is general: group go, and reserved prefixes in the format's own words (`@acme` is the scope `@acme/`, `reserved_prefix`). -### Why Docker is not a plugin - -The contract models a repository as package versions, each one file with an -id, a version and a manifest. A container registry is a different model: -content-addressed blobs and manifests linked per image, tags that move, -indexes that refer to other manifests, referrers, and uploads in sessions of -several requests with bodies the plugin would have to see. Its clients log in -with Bearer tokens, read layers with `Range` requests, and expect proxies of -Docker Hub and other registries. Every one of those is on the roadmap above -or outside the contract altogether. - -So Docker is a native format (`crates/formats/docker`, served under `/v2`), -registered next to the plugins: `GET /admin/formats` lists it with all three -repository kinds, the admin API and the web UI create its repositories the -same way, a group holds members of its format only, and repositories, RBAC, -package selectors (on image names), retention and the blob store are the -same as for every other format. - -Raw file repositories ([raw.md](raw.md)) were native for similar reasons -until contract 0.4.0: they hold files under paths, not package versions, -list directories, take several files in one form, always answer ranges and -conditional requests, and their proxies follow redirects and re-validate with -`If-None-Match`. 0.4.0 gave the contract a second storage model, files -([Files formats](#files-formats)), so raw is a plugin now -([`crates/formats/raw`](../crates/formats/raw)) like NuGet, and what was the -raw server module is the host's file store.