From d9d483c3f3c5500f32c3d1d19a436136772b9204 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 23:46:55 +0000 Subject: [PATCH] Raw repositories as a built-in format plugin over a host file store The plugin contract (forge:format@0.4.0) gets a second storage model: a format whose plugin.toml says `storage = "files"` holds files under paths. Its plugin asks the host for files (`file`, `fetch-file`), directory pages (`directory`), receives forms and stores and deletes files (`form`, `store`, `remove`), and serves a file with the `serve-file` action; the upstream side gets a `file` operation that names a path's URL. `render` returns an action like `handle`, and the package rules default to refusing everything, so a files format implements only the protocol. A request path is empty when the URL ends with the repository's name. Raw is now plugins/raw, built in like NuGet: paths, routing by method, checksum sidecars, HTML and JSON listings, Nexus' form fields and upstream URLs. crates/server/src/raw becomes the host's file store (crates/server/src/files): storage, deployment policy, ranges and validators, proxies, groups, listings, retention, the web UI's upload and delete and the admin API's directory delete, for every files format. Host checks for the raw format id become checks for files storage, and /admin/formats, the repositories API and /browse/repos report what a repository holds (`storage`). /raw answers as before: crates/server/tests/raw.rs passes unchanged, and so does .github/scripts/raw-e2e.sh against GitHub. Fixes #431 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Kb5ECL544V6eRz8Va8KRHk --- Cargo.lock | 14 + Cargo.toml | 1 + DEVELOPMENT.md | 5 +- README.md | 4 +- crates/core/src/format.rs | 7 + crates/core/src/repo/proxy.rs | 31 ++ crates/core/src/retention/worker.rs | 7 +- crates/db/src/model.rs | 23 +- .../raw/path.rs => plugin-api/src/files.rs} | 177 ++++-- crates/plugin-api/src/lib.rs | 68 ++- crates/plugin-api/src/testing.rs | 15 +- crates/plugin-api/src/types.rs | 249 ++++++++- crates/plugin/src/format.rs | 14 +- crates/plugin/src/manifest.rs | 63 ++- crates/server/Cargo.toml | 9 +- crates/server/src/admin/formats.rs | 21 +- crates/server/src/admin/members.rs | 14 +- crates/server/src/admin/mod.rs | 2 +- crates/server/src/admin/proxy_cache.rs | 12 +- crates/server/src/admin/repositories.rs | 135 +++-- crates/server/src/admin/retention.rs | 6 +- crates/server/src/app.rs | 22 +- crates/server/src/browse.rs | 34 +- crates/server/src/files/listing.rs | 268 +++++++++ crates/server/src/files/mod.rs | 226 ++++++++ crates/server/src/{raw => files}/proxy.rs | 163 +++--- crates/server/src/{raw => files}/serve.rs | 126 +---- .../src/{raw/upload.rs => files/store.rs} | 420 +++++++-------- crates/server/src/{raw => files}/web.rs | 100 +++- crates/server/src/lib.rs | 2 +- crates/server/src/plugins.rs | 509 ++++++++++++++++-- crates/server/src/raw/listing.rs | 509 ------------------ crates/server/src/raw/mod.rs | 436 --------------- crates/server/src/state.rs | 49 +- crates/server/tests/plugin_contract.rs | 10 +- docs/changes/2.0.0/431.feature.md | 11 + docs/format-plugins.md | 128 ++++- docs/raw.md | 9 +- plugins/README.md | 4 +- plugins/nuget/plugin.toml | 2 +- plugins/nuget/src/lib.rs | 4 +- plugins/nuget/src/protocol.rs | 20 +- plugins/nuget/src/upstream.rs | 2 + plugins/nuget/tests/protocol.rs | 2 + plugins/raw/Cargo.toml | 24 + plugins/raw/plugin.toml | 13 + plugins/raw/src/form.rs | 189 +++++++ plugins/raw/src/lib.rs | 274 ++++++++++ plugins/raw/src/listing.rs | 473 ++++++++++++++++ plugins/raw/src/sidecar.rs | 210 ++++++++ plugins/raw/tests/protocol.rs | 280 ++++++++++ 51 files changed, 3624 insertions(+), 1772 deletions(-) rename crates/{server/src/raw/path.rs => plugin-api/src/files.rs} (59%) create mode 100644 crates/server/src/files/listing.rs create mode 100644 crates/server/src/files/mod.rs rename crates/server/src/{raw => files}/proxy.rs (72%) rename crates/server/src/{raw => files}/serve.rs (72%) rename crates/server/src/{raw/upload.rs => files/store.rs} (53%) rename crates/server/src/{raw => files}/web.rs (64%) delete mode 100644 crates/server/src/raw/listing.rs delete mode 100644 crates/server/src/raw/mod.rs create mode 100644 docs/changes/2.0.0/431.feature.md create mode 100644 plugins/raw/Cargo.toml create mode 100644 plugins/raw/plugin.toml create mode 100644 plugins/raw/src/form.rs create mode 100644 plugins/raw/src/lib.rs create mode 100644 plugins/raw/src/listing.rs create mode 100644 plugins/raw/src/sidecar.rs create mode 100644 plugins/raw/tests/protocol.rs diff --git a/Cargo.lock b/Cargo.lock index 47c0d88a..7e4c4f22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1805,6 +1805,19 @@ dependencies = [ "zip", ] +[[package]] +name = "forge-format-raw" +version = "1.0.0" +dependencies = [ + "base64 0.23.1", + "forge-plugin-api", + "httpdate", + "percent-encoding", + "serde", + "serde_json", + "url", +] + [[package]] name = "forge-oci" version = "1.1.0" @@ -1873,6 +1886,7 @@ dependencies = [ "forge-crypto", "forge-db", "forge-format-nuget", + "forge-format-raw", "forge-oci", "forge-plugin", "forge-storage", diff --git a/Cargo.toml b/Cargo.toml index 63ebd495..6e7e140a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/server", "crates/e2e", "plugins/nuget", + "plugins/raw", "xtask", ] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 14b321a0..c10570de 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -241,8 +241,9 @@ password: `docker login localhost:8421 -u admin@example.com`, then `docker push localhost:8421/docker-hosted/team/app:1`. See [docs/docker.md](docs/docker.md). -Raw repositories (`"format":"raw"`) hold files under arbitrary paths, served -under `/raw/`: `curl -u admin@example.com: --upload-file x.tgz +Raw repositories (`"format":"raw"`, the built-in plugin `plugins/raw`) hold +files under arbitrary paths, served under `/raw/`: `curl -u +admin@example.com: --upload-file x.tgz localhost:8421/raw/binaries/tools/x.tgz`, a proxy of any HTTP(S) base URL (`"remote_url":"https://github.com///releases/download/"`), and groups. See [docs/raw.md](docs/raw.md). diff --git a/README.md b/README.md index 860e9c11..1084a339 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,10 @@ A Cargo workspace of small crates plus the web SPA: | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | | `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | | `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | -| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`), the Docker registry, raw file repositories (`/raw`), `/admin` + `/auth` REST, background workers, static SPA serving. | +| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | -| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`. | +| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `raw`. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | **Single-origin serving.** `forge-server` mounts each format plugin's feed diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index 67df4ee1..cb736d2a 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -46,6 +46,13 @@ pub trait Format: Send + Sync { /// the `repository.format` column. fn id(&self) -> &'static str; + /// Whether the format's repositories hold files under paths (raw) rather + /// than package versions: their contents are the file store's + /// (`raw_file`), and none of the package seams below apply to them. + fn stores_files(&self) -> bool { + false + } + /// Parse an uploaded package into format-neutral [`ParsedUpload`] data. fn parse_upload(&self, file: &mut dyn ReadSeek) -> Result; diff --git a/crates/core/src/repo/proxy.rs b/crates/core/src/repo/proxy.rs index 9dc825a9..bb299030 100644 --- a/crates/core/src/repo/proxy.rs +++ b/crates/core/src/repo/proxy.rs @@ -512,6 +512,16 @@ impl ProxyRepo { ); return Ok((api::UpstreamAnswer::Failed(reason), origins)); } + api::UpstreamStep::Done(api::UpstreamAnswer::File(Some(url))) + if !origins.permits(&url) => + { + tracing::warn!(format = %self.format.id(), repo = %repo.name, %url, "the format answered a file URL on an origin neither the remote_url nor a document fetched from the upstream names; not downloaded"); + let reason = format!( + "upstream file {url}: not downloaded, its origin is neither the \ + remote_url's nor named by a document fetched from the upstream" + ); + return Ok((api::UpstreamAnswer::Failed(reason), origins)); + } api::UpstreamStep::Done(answer) => return Ok((answer, origins)), api::UpstreamStep::Fetch(wanted) => wanted, }; @@ -619,6 +629,27 @@ impl ProxyRepo { } } + /// The upstream URL of the file at `path` of proxy `repo` + /// ([`api::UpstreamOp::File`], a files format), on an origin the + /// upstream points at; `None` when the upstream cannot have it. The + /// file store downloads and re-validates it. + pub async fn file_url( + &self, + repo: &Repository, + path: &str, + ) -> Result, RepoError> { + match self + .converse_resolved(repo, api::UpstreamOp::File(path.to_string())) + .await? + { + api::UpstreamAnswer::File(url) => Ok(url), + api::UpstreamAnswer::Failed(reason) => Err(RepoError::Storage(reason)), + _ => Err(RepoError::Storage( + "upstream protocol answered file with something else".into(), + )), + } + } + /// The upstream index, resolved, for `op` on `repo`. async fn converse_resolved( &self, diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index 949b770e..10798b07 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -226,7 +226,12 @@ impl RetentionWorker { if repo.format == forge_db::model::DOCKER_FORMAT { return self.process_images(repo, kind, run_id).await; } - if repo.format == forge_db::model::RAW_FORMAT { + // A files format's repositories (raw) hold files under paths. + if self + .formats + .resolve(&repo.format) + .is_some_and(|format| format.stores_files()) + { return self.process_files(repo, kind, run_id).await; } diff --git a/crates/db/src/model.rs b/crates/db/src/model.rs index 5f3e6a3b..31a8470a 100644 --- a/crates/db/src/model.rs +++ b/crates/db/src/model.rs @@ -282,9 +282,6 @@ impl Repository { } } -/// `repository.format` of raw (generic file) repositories. -pub const RAW_FORMAT: &str = "raw"; - /// `repository.config` key of [`Repository::deployment_policy`]. pub const DEPLOYMENT_POLICY_KEY: &str = "deployment_policy"; @@ -294,9 +291,9 @@ pub const CONTENT_DISPOSITION_KEY: &str = "content_disposition"; /// `repository.config` key of [`Repository::max_age_seconds`]. pub const MAX_AGE_SECONDS_KEY: &str = "max_age_seconds"; -/// How long a raw proxy serves a cached file before it asks its upstream -/// whether the file changed, when the repository sets no `max_age_seconds`: -/// a day. +/// How long a proxy of files (raw) serves a cached file before it asks its +/// upstream whether the file changed, when the repository sets no +/// `max_age_seconds`: a day. pub const DEFAULT_RAW_MAX_AGE_SECONDS: u64 = 24 * 60 * 60; /// What a hosted raw repository does with an upload to a path it has @@ -333,13 +330,11 @@ impl DeploymentPolicy { } } +/// The options of a repository of a files format (raw): which repositories +/// hold files is the format registry's to say (the server's +/// `FormatRegistry::stores_files`). impl Repository { - /// Whether this is a raw (generic file) repository. - pub fn is_raw(&self) -> bool { - self.format == RAW_FORMAT - } - - /// Hosted raw option (`config.deployment_policy`, default + /// Hosted files option (`config.deployment_policy`, default /// [`DeploymentPolicy::DisableRedeploy`]). pub fn deployment_policy(&self) -> DeploymentPolicy { self.config @@ -349,7 +344,7 @@ impl Repository { .unwrap_or_default() } - /// Raw option (`config.content_disposition = "inline"`, default + /// Files option (`config.content_disposition = "inline"`, default /// `"attachment"`): files are served for display in the browser rather /// than as downloads. pub fn inline_content(&self) -> bool { @@ -359,7 +354,7 @@ impl Repository { == Some("inline") } - /// Raw proxy option (`config.max_age_seconds`): how long a cached file + /// Files proxy option (`config.max_age_seconds`): how long a cached file /// is served before the upstream is asked whether it changed; `0` never /// asks (the upstream's files are immutable). Default /// [`DEFAULT_RAW_MAX_AGE_SECONDS`]. diff --git a/crates/server/src/raw/path.rs b/crates/plugin-api/src/files.rs similarity index 59% rename from crates/server/src/raw/path.rs rename to crates/plugin-api/src/files.rs index 31267441..73f7681c 100644 --- a/crates/server/src/raw/path.rs +++ b/crates/plugin-api/src/files.rs @@ -1,14 +1,20 @@ -//! Paths in raw repositories: what a request names, and how it is stored. +//! Paths in files repositories: what a request names, and how a file is +//! stored. //! //! A stored path is a list of segments joined by `/`, without a leading or -//! trailing `/`: `pdfium/7350/pdfium-linux-x64.tgz`. A request names it -//! percent-encoded after `/raw//`, and is refused when any decoded -//! segment is empty (`//`), `.` or `..` (also spelled `%2e%2e`), holds a `/` -//! (`%2F`), a backslash or a control character, or the path is too long. -//! Nothing is collapsed or resolved: a path that needs normalizing is -//! refused, so there is exactly one spelling of every stored path. +//! trailing `/`: `pdfium/7350/pdfium-linux-x64.tgz`. A directory is `""` (the +//! root) or a path ending in `/`. A request names a path percent-encoded, and +//! is refused when any decoded segment is empty (`//`), `.` or `..` (also +//! spelled `%2e%2e`), holds a `/` (`%2F`), a backslash or a control +//! character, or the path is too long. Nothing is collapsed or resolved: a +//! path that needs normalizing is refused, so there is exactly one spelling +//! of every stored path. +//! +//! The host refuses every path a plugin names that breaks these rules +//! ([`validate_path`], [`validate_directory`]); a plugin parses its requests +//! with [`parse_request`] to refuse them with the reason. -use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS}; +use std::fmt; /// Longest stored path, in bytes. pub const MAX_PATH_BYTES: usize = 1024; @@ -26,26 +32,37 @@ pub enum Target { } /// Why a path is refused. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PathError { - #[error("invalid path: empty segment (`//`)")] EmptySegment, - #[error("invalid path: `.` and `..` segments are not allowed")] DotSegment, - #[error("invalid path: backslashes are not allowed")] Backslash, - #[error("invalid path: an encoded `/` (%2F) is not allowed in a segment")] EncodedSlash, - #[error("invalid path: control characters are not allowed")] Control, - #[error("invalid path: not valid percent-encoded UTF-8")] Encoding, - #[error("invalid path: longer than {MAX_PATH_BYTES} bytes")] TooLong, - #[error("invalid path: a segment is longer than {MAX_SEGMENT_BYTES} bytes")] SegmentTooLong, } +impl fmt::Display for PathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + PathError::EmptySegment => "invalid path: empty segment (`//`)", + PathError::DotSegment => "invalid path: `.` and `..` segments are not allowed", + PathError::Backslash => "invalid path: backslashes are not allowed", + PathError::EncodedSlash => { + "invalid path: an encoded `/` (%2F) is not allowed in a segment" + } + PathError::Control => "invalid path: control characters are not allowed", + PathError::Encoding => "invalid path: not valid percent-encoded UTF-8", + PathError::TooLong => "invalid path: longer than 1024 bytes", + PathError::SegmentTooLong => "invalid path: a segment is longer than 255 bytes", + }) + } +} + +impl std::error::Error for PathError {} + /// Check one decoded segment. pub fn validate_segment(segment: &str) -> Result<(), PathError> { if segment.is_empty() { @@ -69,9 +86,55 @@ pub fn validate_segment(segment: &str) -> Result<(), PathError> { Ok(()) } -/// Parse the percent-encoded rest of a request path (what follows -/// `/raw//`): empty or ending in `/` names a directory, anything else -/// a file. +/// Check a stored file path: segments joined by `/`, no leading or trailing +/// `/`. +pub fn validate_path(path: &str) -> Result<(), PathError> { + if path.len() > MAX_PATH_BYTES { + return Err(PathError::TooLong); + } + path.split('/').try_for_each(validate_segment) +} + +/// Check a directory: `""`, or a file path followed by `/`. +pub fn validate_directory(directory: &str) -> Result<(), PathError> { + if directory.is_empty() { + return Ok(()); + } + match directory.strip_suffix('/') { + Some(path) if directory.len() <= MAX_PATH_BYTES => validate_path(path), + Some(_) => Err(PathError::TooLong), + None => Err(PathError::EmptySegment), + } +} + +/// `%XX`-decode a segment: an escape of two hex digits is decoded, a `%` +/// before anything else stays as it is. +fn decode_segment(segment: &str) -> Result { + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let escaped = (bytes[i] == b'%') + .then(|| segment.get(i + 1..i + 3)) + .flatten() + .and_then(|hex| u8::from_str_radix(hex, 16).ok()); + match escaped { + Some(byte) => { + out.push(byte); + i += 3; + } + None => { + out.push(bytes[i]); + i += 1; + } + } + } + String::from_utf8(out).map_err(|_| PathError::Encoding) +} + +/// Parse a percent-encoded request path below a repository (without its +/// leading `/`): empty or ending in `/` names a directory, anything else a +/// file. pub fn parse_request(encoded: &str) -> Result { if encoded.is_empty() { return Ok(Target::Directory(String::new())); @@ -85,11 +148,9 @@ pub fn parse_request(encoded: &str) -> Result { if raw.contains('\\') { return Err(PathError::Backslash); } - let decoded = percent_decode_str(raw) - .decode_utf8() - .map_err(|_| PathError::Encoding)?; + let decoded = decode_segment(raw)?; validate_segment(&decoded)?; - segments.push(decoded.into_owned()); + segments.push(decoded); } let path = segments.join("/"); if path.len() + usize::from(directory) > MAX_PATH_BYTES { @@ -102,9 +163,9 @@ pub fn parse_request(encoded: &str) -> Result { }) } -/// Normalize a path given unencoded (an upload's file name, a `directory` -/// form field, an admin API parameter): segments separated by `/`, leading -/// and trailing `/` ignored. Empty yields `""`. +/// Normalize a path given unencoded (an upload's file name, a directory form +/// field, an admin API parameter): segments separated by `/`, leading and +/// trailing `/` ignored. Empty yields `""`. pub fn normalize(raw: &str) -> Result { let trimmed = raw.trim_matches('/'); if trimmed.is_empty() { @@ -144,34 +205,30 @@ pub fn join(directory: &str, name: &str) -> Result { Ok(path) } -/// Characters percent-encoded in a URL path segment: everything but the -/// unreserved characters and the sub-delimiters, `:` and `@` that a +/// Whether `byte` is percent-encoded in a URL path segment: everything but +/// the unreserved characters and the sub-delimiters, `:` and `@` that a /// segment may carry as they are. -const SEGMENT: &AsciiSet = &CONTROLS - .add(b' ') - .add(b'"') - .add(b'#') - .add(b'%') - .add(b'/') - .add(b'<') - .add(b'>') - .add(b'?') - .add(b'[') - .add(b'\\') - .add(b']') - .add(b'^') - .add(b'`') - .add(b'{') - .add(b'|') - .add(b'}'); +fn encoded_in_segment(byte: u8) -> bool { + !(0x20..0x7f).contains(&byte) || b" \"#%/<>?[\\]^`{|}".contains(&byte) +} /// `path` percent-encoded for a URL, segment by segment (a trailing `/` /// kept). pub fn encode(path: &str) -> String { - path.split('/') - .map(|segment| utf8_percent_encode(segment, SEGMENT).to_string()) - .collect::>() - .join("/") + let mut out = String::with_capacity(path.len()); + for (i, segment) in path.split('/').enumerate() { + if i > 0 { + out.push('/'); + } + for &byte in segment.as_bytes() { + if encoded_in_segment(byte) { + out.push_str(&format!("%{byte:02X}")); + } else { + out.push(char::from(byte)); + } + } + } + out } /// The last segment of a path (a directory's without its `/`). @@ -212,6 +269,9 @@ mod tests { assert_eq!(parse_request("%C3%A4/x"), Ok(Target::File("ä/x".into()))); assert_eq!(parse_request("..."), Ok(Target::File("...".into()))); assert_eq!(parse_request(".hidden"), Ok(Target::File(".hidden".into()))); + // A `%` without two hex digits is a `%`. + assert_eq!(parse_request("a%zzb"), Ok(Target::File("a%zzb".into()))); + assert_eq!(parse_request("a%2"), Ok(Target::File("a%2".into()))); } #[test] @@ -250,6 +310,21 @@ mod tests { assert_eq!(parse_request(&deep), Err(PathError::TooLong)); } + #[test] + fn stored_paths_and_directories_are_checked() { + assert_eq!(validate_path("a/b.txt"), Ok(())); + assert_eq!(validate_path(""), Err(PathError::EmptySegment)); + assert_eq!(validate_path("/a"), Err(PathError::EmptySegment)); + assert_eq!(validate_path("a/"), Err(PathError::EmptySegment)); + assert_eq!(validate_path("a/../b"), Err(PathError::DotSegment)); + assert_eq!(validate_path(&"a/".repeat(600)), Err(PathError::TooLong)); + assert_eq!(validate_directory(""), Ok(())); + assert_eq!(validate_directory("a/b/"), Ok(())); + assert_eq!(validate_directory("a/b"), Err(PathError::EmptySegment)); + assert_eq!(validate_directory("/"), Err(PathError::EmptySegment)); + assert_eq!(validate_directory("a//"), Err(PathError::EmptySegment)); + } + #[test] fn unencoded_names_normalize_their_slashes_only() { assert_eq!(normalize("/pdfium/7350/"), Ok("pdfium/7350".into())); @@ -286,6 +361,8 @@ mod tests { assert_eq!(parse_request(&encoded), Ok(expected), "{encoded}"); } assert_eq!(encode("a b/x?.txt"), "a%20b/x%3F.txt"); + assert_eq!(encode("ä"), "%C3%A4"); + assert_eq!(encode("a:b@c!$&'()*+,;=-._~"), "a:b@c!$&'()*+,;=-._~"); assert_eq!(file_name("a/b/c.txt"), "c.txt"); assert_eq!(file_name("a/b/"), "b"); assert_eq!(file_name("c.txt"), "c.txt"); diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 68c77a72..1e4921aa 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -7,10 +7,17 @@ //! package bytes — so a plugin is small and never sees a credential, a //! storage key, or a whole package in memory. //! +//! A format stores package versions, or files under paths (raw): its +//! `plugin.toml` says which (`storage`). A files format has no identity +//! rules and no packages to read: it keeps the defaults of those methods and +//! implements the protocol over the host's file store ([`Query::File`] … +//! [`Query::Remove`], [`Action::ServeFile`]). +//! //! A plugin is a crate under `plugins//` implementing [`FormatPlugin`], //! with a `plugin.toml` naming it, linked into the server and called -//! directly (NuGet). +//! directly (NuGet, raw). +pub mod files; mod types; #[cfg(feature = "testing")] @@ -24,7 +31,7 @@ pub use types::*; /// The `forge:format` contract version this crate implements: what a plugin /// built against it declares as `api` in its `plugin.toml`. -pub const API: &str = "0.3.0"; +pub const API: &str = "0.4.0"; /// The `plugin.toml` of the crate this is expanded in: how a built-in plugin /// hands the server its id and name (`pub const PLUGIN_TOML: &str = @@ -49,26 +56,48 @@ impl PackageSource for T {} /// they may take as long as they need; the rules ([`Self::canonical_id`] /// through [`Self::compare_versions`], [`Self::manifest_summary`]) run inline /// and must be quick. +/// +/// A package format implements the rules and [`Self::parse_package`]. A +/// files format has neither ids nor versions nor packages: it keeps their +/// defaults, which refuse every id, version and package. pub trait FormatPlugin: Send + Sync + 'static { /// The canonical form of a package id, used for lookups and uniqueness - /// (e.g. lowercased). - fn canonical_id(&self, id: &str) -> String; + /// (e.g. lowercased). The default keeps the id as it is. + fn canonical_id(&self, id: &str) -> String { + id.to_string() + } /// Whether `id` (canonical) is a well-formed id. The host treats a query /// or an action naming any other id like one naming an unknown package - /// (an empty answer, a 404). - fn validate_id(&self, id: &str) -> Result<(), FormatError>; + /// (an empty answer, a 404). The default accepts none. + fn validate_id(&self, id: &str) -> Result<(), FormatError> { + Err(FormatError::InvalidId(format!( + "{id:?}: this format has no packages" + ))) + } /// The normalized form of a version, used for lookups and uniqueness. - /// Prerelease versions must contain a `-` (the host filters on it). - fn normalize_version(&self, version: &str) -> Result; + /// Prerelease versions must contain a `-` (the host filters on it). The + /// default accepts none. + fn normalize_version(&self, version: &str) -> Result { + Err(FormatError::InvalidVersion(format!( + "{version:?}: this format has no versions" + ))) + } - /// The order of two normalized versions. - fn compare_versions(&self, a: &str, b: &str) -> Ordering; + /// The order of two normalized versions. The default is byte order. + fn compare_versions(&self, a: &str, b: &str) -> Ordering { + a.cmp(b) + } /// Read a package's identity, manifest and dependencies. `package` is - /// positioned at its start. - fn parse_package(&self, package: &mut dyn PackageSource) -> Result; + /// positioned at its start. The default reads none. + fn parse_package(&self, package: &mut dyn PackageSource) -> Result { + let _ = package; + Err(FormatError::InvalidPackage( + "this format stores files, not packages".into(), + )) + } /// [`Self::parse_package`] of a package the host staged at `path`. fn parse_package_file(&self, path: &Path) -> Result { @@ -80,8 +109,11 @@ pub trait FormatPlugin: Send + Sync + 'static { /// A JSON object summarizing `manifest`, stored with the version so that /// rendering never re-reads the manifest ([`PackageInfo::summary`]). /// `None` when there is nothing worth keeping. Every listing hands each - /// version's summary back, so keep it small. - fn manifest_summary(&self, manifest: &[u8]) -> Option; + /// version's summary back, so keep it small. The default keeps none. + fn manifest_summary(&self, manifest: &[u8]) -> Option { + let _ = manifest; + None + } /// What a request is: a response right away, data to load first (then /// [`Self::render`]), or an action only the host can perform. @@ -91,9 +123,11 @@ pub trait FormatPlugin: Send + Sync + 'static { /// write, DELETE delete) before it asks. fn handle(&self, repo: &Repository, req: &Request) -> Action; - /// The response to a request, from the data [`Self::handle`] (and earlier - /// rounds of `render`) asked for — or more data to load. - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered; + /// What to do once the data [`Self::handle`] (and earlier rounds of + /// `render`) asked for is loaded: `data` holds every query so far with + /// its answer. Anything `handle` may answer: a response, more data to + /// load, or an action of the host's. + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action; /// The upstream side of the format's proxy repositories: what to fetch /// from `remote` for `op`, and what came of it. `fetched` holds every diff --git a/crates/plugin-api/src/testing.rs b/crates/plugin-api/src/testing.rs index 2557d765..deaeb877 100644 --- a/crates/plugin-api/src/testing.rs +++ b/crates/plugin-api/src/testing.rs @@ -4,7 +4,8 @@ //! [`FormatPlugin::handle`], answers the queries the plugin asks for with //! `answer`, and calls [`FormatPlugin::render`] round by round, under the //! host's bounds. What only the host can do (a download, a publish, a -//! change) comes back as the action, for the test to inspect. +//! change, serving a file) comes back as the action, for the test to +//! inspect. //! //! [`converse`] does the same for the upstream side of a proxy //! ([`FormatPlugin::upstream`]): it answers every fetch the plugin asks for @@ -18,7 +19,7 @@ //! assert_eq!(served.response().status, 200); //! ``` -use crate::{Action, Answer, FormatPlugin, Loaded, Query, Rendered, Repository, RepositoryKind}; +use crate::{Action, Answer, FormatPlugin, Loaded, Query, Repository, RepositoryKind}; use crate::{Fetch, FetchResult, Fetched, FormatError, Remote, UpstreamAnswer, UpstreamOp}; use crate::{Request, Response, UpstreamStep}; @@ -35,6 +36,7 @@ pub fn repository(kind: RepositoryKind, mount: &str) -> Repository { name: "repo".into(), kind, base_url: format!("https://forge.example.com{mount}/repo"), + inline: false, } } @@ -97,13 +99,8 @@ pub fn serve( }); } match plugin.render(repo, req, &data) { - Rendered::Respond(response) => { - return Served { - outcome: Action::Respond(response), - queries, - } - } - Rendered::Load(more) => pending = more, + Action::Load(more) => pending = more, + outcome => return Served { outcome, queries }, } } Served { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 30c8d5af..f1f7e023 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -87,6 +87,11 @@ pub struct Repository { /// The absolute URL the repository is served at, without a trailing /// slash (e.g. `https://forge.example.com/repository/charts`). pub base_url: String, + /// Files formats: whether the repository's files are displayed by a + /// browser (its `inline` content disposition) rather than downloaded, as + /// [`Action::ServeFile`] serves them. Always `false` for a package + /// format. + pub inline: bool, } /// An HTTP request from a package client. @@ -95,7 +100,8 @@ pub struct Request { /// The method, upper case. pub method: String, /// The path below `base_url`, starting with `/`, still percent-encoded - /// as the client sent it. + /// as the client sent it; empty when the URL ends with the repository's + /// name. pub path: String, /// The query string without its `?`, empty when there is none. pub query: String, @@ -193,8 +199,8 @@ fn percent_decode(s: &str, plus_is_space: bool) -> Option { } /// An HTTP response. The host keeps only the headers it allows (content -/// type, caching and validators, content-disposition, location) and adds its -/// own security headers. +/// type, caching and validators, content-disposition, vary, and location and +/// link within the repository) and adds its own security headers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub status: u16, @@ -208,6 +214,15 @@ impl Response { Response::new(status, "text/plain; charset=utf-8", body) } + /// A response without a body or headers (a 204, a redirect). + pub fn empty(status: u16) -> Self { + Response { + status, + headers: Vec::new(), + body: Vec::new(), + } + } + /// A response with a body of `content_type`. pub fn new(status: u16, content_type: &str, body: impl Into>) -> Self { Response { @@ -256,9 +271,145 @@ pub struct SearchQuery { pub upstream: bool, } -/// Data a plugin can ask for. Package ids are canonical, versions -/// normalized. Everything the caller may not read (package selectors, group -/// members it cannot see) is left out, exactly as for an unknown package. +/// A page of a directory of a files repository ([`Query::Directory`]). +/// +/// A file path is a list of segments joined by `/`, without a leading or +/// trailing `/` (see [`crate::files`]). A directory is `""` (the root) or a +/// path ending in `/`. The host refuses any other path a plugin names. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryQuery { + pub directory: String, + /// Start after this entry, a name in the directory (a subdirectory's ends + /// in `/`); `None` starts at the first. + pub after: Option, + /// The page size: 0 means the default (100), more than 1000 is 1000. + pub limit: u32, +} + +/// A file of a files repository. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileInfo { + pub path: String, + pub size: u64, + /// What the upload declared, else what its name suggests. + pub content_type: String, + /// Lowercase hex. + pub sha256: String, + /// Base64. + pub sha512: String, + /// Lowercase hex. + pub sha1: String, + /// Lowercase hex. + pub md5: String, + /// When its content was last stored (uploaded, or fetched by a proxy): + /// RFC 3339 in UTC. + pub modified: String, + /// When it was last downloaded, to the hour: RFC 3339 in UTC. + pub last_downloaded: Option, +} + +/// One entry of a [`DirectoryPage`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DirectoryEntry { + File(FileInfo), + /// A subdirectory: its path, ending in `/`. + Directory(String), +} + +/// A page of a directory ([`Answer::Directory`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryPage { + /// In byte order of their names, a subdirectory's ending in `/`. + pub entries: Vec, + /// The `after` of the next page; `None` on the last. + pub next: Option, +} + +/// One part of a `multipart/form-data` body ([`Query::Form`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormPart { + /// The part's field name. + pub field: String, + pub content: FormContent, +} + +/// What a [`FormPart`] holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FormContent { + /// A file: received and staged by the host, stored with + /// [`FileSource::Part`] and its index among the form's parts. + File(FormFile), + /// Any other field: its value (UTF-8, at most 4 KiB). + Text(String), +} + +/// A file part of a form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormFile { + /// The file name the part declares. + pub name: String, + /// The content type the part declares. + pub content_type: Option, + pub size: u64, +} + +/// Where a file to store comes from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileSource { + /// The request body. + Body, + /// A file part of the request's form, by its index ([`Query::Form`]). + Part(u32), +} + +/// A file to store ([`Query::Store`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewFile { + pub path: String, + pub source: FileSource, + /// The content type the client declared. The host keeps it unless it + /// says no more than "bytes" (none, `application/octet-stream`, a form + /// encoding); then it takes the one the path's extension suggests. + pub content_type: Option, +} + +/// What storing a file did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoreStatus { + /// The path was new. + Created, + /// The path held other content, which this replaced. + Replaced, + /// The path held this content already. + Unchanged, +} + +/// A file stored ([`Answer::Stored`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredFile { + pub path: String, + pub size: u64, + /// Lowercase hex. + pub sha256: String, + pub status: StoreStatus, +} + +/// What a plugin can ask of the host: data, and in a files repository +/// changes to its files. Package ids are canonical, versions normalized. +/// Everything the caller may not read (package selectors, group members it +/// cannot see) is left out, exactly as for an unknown package or file. +/// +/// A package format asks for packages ([`Query::Versions`] … +/// [`Query::Search`]), a files format for files ([`Query::File`] … +/// [`Query::Remove`]): a query of the other kind fails the request. +/// [`Query::Form`], [`Query::Store`] and [`Query::Remove`] change the +/// repository: each is asked alone in its round, at most once per request +/// for `Form` and `Store`, and only by a request authorized for them (write +/// for `Form` and `Store`, delete for `Remove`) in a hosted repository, with +/// an API token or a workload identity. When one fails (403 not allowed, 405 +/// not hosted or read-only, 409 the path exists and the repository does not +/// allow redeploying it, 413 too large) the host answers with its own +/// status. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Query { /// Every version of a package, ascending. @@ -274,6 +425,49 @@ pub enum Query { Manifest(PackageRef), /// The packages matching a search. Search(SearchQuery), + /// The file stored at a path. A proxy answers with its cached copy, + /// re-validated with its upstream once it is due; a group with its first + /// member, by priority, that has the path. + File(String), + /// [`Query::File`], and a proxy (a group's proxy members too) that has + /// not cached the path asks its upstream for it ([`UpstreamOp::File`]) + /// and caches what it gets. + FetchFile(String), + /// A page of a directory. A proxy lists the files it has cached, a group + /// the merged entries of its members (the first member's file of a + /// name). A subdirectory is listed when the caller may see a file in it. + Directory(DirectoryQuery), + /// Receive the request's `multipart/form-data` body: its file parts are + /// staged (each within the repository's upload limit, at most 100), its + /// other fields read. + Form, + /// Store files, all of them or none. A path that holds a file is replaced + /// only when the repository allows redeploying and the caller may delete; + /// the same content again is accepted as it is. + Store(Vec), + /// Delete the file at a path. + Remove(String), +} + +impl Query { + /// Whether the query changes the repository (it is asked alone in its + /// round). + pub fn is_change(&self) -> bool { + matches!(self, Query::Form | Query::Store(_) | Query::Remove(_)) + } + + /// Whether the query is about files rather than packages. + pub fn is_files(&self) -> bool { + matches!( + self, + Query::File(_) + | Query::FetchFile(_) + | Query::Directory(_) + | Query::Form + | Query::Store(_) + | Query::Remove(_) + ) + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -334,6 +528,16 @@ pub enum Answer { Package(Option), Manifest(Option>), Search(SearchResult), + /// The answer to [`Query::File`] and to [`Query::FetchFile`]. + File(Option), + /// `None`: the directory does not exist (a directory other than the root + /// without an entry the caller may see, asked for without `after`). + Directory(Option), + Form(Vec), + /// What became of each file, in the order of [`Query::Store`]. + Stored(Vec), + /// Whether there was a file to delete. + Removed(bool), } /// A query with its answer. @@ -372,14 +576,17 @@ pub struct ChangeRequest { pub done: Response, } -/// What a request is ([`FormatPlugin::handle`]). +/// What a request is ([`FormatPlugin::handle`]), or what to do once the data +/// it asked for is loaded ([`FormatPlugin::render`]). /// /// [`FormatPlugin::handle`]: crate::FormatPlugin::handle +/// [`FormatPlugin::render`]: crate::FormatPlugin::render #[derive(Debug, Clone, PartialEq, Eq)] pub enum Action { /// Answer with this response. Respond(Response), - /// Load this data, then call `render` with it. + /// Load this data, then call `render` with everything loaded so far (the + /// host bounds the rounds and the number of queries). Load(Vec), /// Serve the stored package of this version, or 404 when the caller /// cannot read it. The host streams it and counts the download. @@ -393,18 +600,12 @@ pub enum Action { Relist(ChangeRequest), /// Remove a version and its bytes (delete permission). Delete(ChangeRequest), -} - -/// What a round of [`FormatPlugin::render`] produced. -/// -/// [`FormatPlugin::render`]: crate::FormatPlugin::render -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Rendered { - /// Answer with this response. - Respond(Response), - /// Load more data, then call `render` again with everything loaded so - /// far (the host bounds the rounds and the number of queries). - Load(Vec), + /// Serve the file at this path (read requests), found as + /// [`Query::FetchFile`] finds it, or 404. The host streams it with its + /// validators (`ETag`, its SHA-256; `Last-Modified`), answers conditional + /// requests and one range, sets its content type and the repository's + /// content disposition, and counts the download. + ServeFile(String), } /// The upstream a proxy repository mirrors ([`FormatPlugin::upstream`]). @@ -446,6 +647,11 @@ pub enum UpstreamOp { Dating(PackageVersions), /// A page of the upstream's search results. Search(SearchQuery), + /// The URL of the file at a path (files formats). The host downloads it, + /// following redirects (its credentials go to the remote's origin only), + /// caches it as the file, and re-validates it with a conditional request + /// once the repository's maximum age has passed. + File(String), } /// How long the host may reuse a fetched document. @@ -568,6 +774,9 @@ pub enum UpstreamAnswer { Metadata(PackageMetadata), /// `None`: the upstream offers no search. Search(Option), + /// The file's URL, on an origin a [`Fetch`] may reach; `None`: the + /// upstream cannot have the path. + File(Option), /// The upstream answered something unusable: the host serves what it /// has cached. Failed(String), diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index f4483cc2..60178b6a 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -8,7 +8,7 @@ use forge_core::domain::{Dependency, DependencyGroup, FormatError, ParsedUpload} use forge_core::format::{Format, ReadSeek}; use forge_plugin_api::{self as api, FormatPlugin}; -use crate::manifest::Assets; +use crate::manifest::{Assets, Storage}; use crate::Plugin; /// The longest version the host takes from a format plugin, in bytes: a @@ -35,6 +35,7 @@ pub struct PluginFormat { id: &'static str, plugin: Arc, assets: Assets, + storage: Storage, } impl PluginFormat { @@ -44,6 +45,7 @@ impl PluginFormat { id: plugin.id(), plugin: plugin.implementation().clone(), assets: plugin.manifest().assets.clone(), + storage: plugin.manifest().storage, } } } @@ -128,6 +130,10 @@ impl Format for PluginFormat { self.id } + fn stores_files(&self) -> bool { + self.storage == Storage::Files + } + fn package_asset_path(&self) -> &str { &self.assets.package } @@ -263,8 +269,8 @@ mod tests { _repo: &api::Repository, _req: &api::Request, _data: &[api::Loaded], - ) -> api::Rendered { - api::Rendered::Respond(api::Response::text(404, "not found")) + ) -> api::Action { + api::Action::Respond(api::Response::text(404, "not found")) } } @@ -284,7 +290,7 @@ mod tests { /// [`Unlimited`] as the host sees it. fn unlimited(normalized: Option<&str>, parsed: api::ParsedPackage) -> PluginFormat { let plugin = Plugin::builtin( - "[plugin]\nid = \"unlimited\"\nname = \"Unlimited\"\nversion = \"0.1.0\"\napi = \"0.3.0\"\n", + "[plugin]\nid = \"unlimited\"\nname = \"Unlimited\"\nversion = \"0.1.0\"\napi = \"0.4.0\"\n", Unlimited { normalized: normalized.map(str::to_string), parsed, diff --git a/crates/plugin/src/manifest.rs b/crates/plugin/src/manifest.rs index b77d8217..1c3a0997 100644 --- a/crates/plugin/src/manifest.rs +++ b/crates/plugin/src/manifest.rs @@ -6,12 +6,14 @@ //! id = "nuget" # the format id repositories are created with //! name = "NuGet" //! version = "1.0.0" # the plugin's own version (informational) -//! api = "0.3.0" # the forge:format contract it is built against +//! api = "0.4.0" # the forge:format contract it is built against //! mount = "/v3" # optional: served at {mount}/{repository}/… //! kinds = ["hosted", "group"] # optional: the repository kinds it supports //! # ("proxy" too, when it implements `upstream`) +//! storage = "packages" # optional: what its repositories hold, +//! # package versions or "files" under paths //! -//! [assets] # optional: how its files are stored +//! [assets] # optional: how a version's files are stored //! package = "package" //! package_type = "application/octet-stream" //! manifest = "manifest" @@ -28,10 +30,9 @@ pub const SUPPORTED_APIS: &[&str] = &[forge_plugin_api::API]; /// Format ids a plugin can never take: the built-in Docker registry's, and /// names the server routes or reserves. (A built-in plugin's id, such as -/// NuGet's, is taken by that plugin: see the server's registry.) +/// NuGet's or raw's, is taken by that plugin: see the server's registry.) const RESERVED_IDS: &[&str] = &[ "docker", - "raw", "v2", "v3", "admin", @@ -58,7 +59,6 @@ const RESERVED_MOUNTS: &[&str] = &[ "login", "manage", "packages", - "raw", "readyz", "realms", "repos", @@ -87,10 +87,33 @@ pub struct Manifest { /// The repository kinds the format supports. `proxy` means the plugin /// implements [`forge_plugin_api::FormatPlugin::upstream`]. pub kinds: Vec, + /// What the format's repositories hold. + pub storage: Storage, /// How a version's files are stored. pub assets: Assets, } +/// What a format's repositories hold (`storage`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Storage { + /// Package versions: an id, a version, a package file and its manifest + /// (NuGet, Helm). + #[default] + Packages, + /// Files under paths, with their digests (raw). + Files, +} + +impl Storage { + /// The name `plugin.toml` and the admin API use. + pub fn as_str(self) -> &'static str { + match self { + Storage::Packages => "packages", + Storage::Files => "files", + } + } +} + /// The `[assets]` table: the names (within a version) and content types of /// a version's package and manifest. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -131,6 +154,7 @@ struct RawPlugin { description: Option, mount: Option, kinds: Option>, + storage: Option, } impl Manifest { @@ -155,6 +179,15 @@ impl Manifest { None => vec![RepositoryKind::Hosted, RepositoryKind::Group], Some(kinds) => parse_kinds(&kinds)?, }; + let storage = match p.storage.as_deref() { + None | Some("packages") => Storage::Packages, + Some("files") => Storage::Files, + Some(other) => { + return Err(PluginError::Manifest(format!( + "unknown storage {other:?} (packages, files)" + ))) + } + }; validate_assets(&raw.assets)?; Ok(Manifest { id: p.id, @@ -164,6 +197,7 @@ impl Manifest { description: p.description, mount, kinds, + storage, assets: raw.assets, }) } @@ -287,7 +321,7 @@ mod tests { id = "helm" name = "Helm charts" version = "0.1.0" - api = "0.3.0" + api = "0.4.0" "#; #[test] @@ -327,8 +361,8 @@ mod tests { let with = |plugin_keys: &str, tables: &str| { let text = GOOD.replace( - "api = \"0.3.0\"", - &format!("api = \"0.3.0\"\n{plugin_keys}"), + "api = \"0.4.0\"", + &format!("api = \"0.4.0\"\n{plugin_keys}"), ); Manifest::parse(&format!("{text}\n{tables}")) }; @@ -354,6 +388,10 @@ mod tests { // A plugin with an upstream side serves proxies. let m = with("kinds = [\"hosted\", \"proxy\"]", "").unwrap(); assert_eq!(m.kinds, [RepositoryKind::Hosted, RepositoryKind::Proxy]); + assert_eq!(m.storage, Storage::Packages); + let m = with("storage = \"files\"", "").unwrap(); + assert_eq!(m.storage, Storage::Files); + assert!(with("storage = \"images\"", "").is_err()); // `[proxy]` (0.2's host-side upstream protocols) is gone. assert!(with("", "[proxy]\nupstream = \"nuget-v3\"\n").is_err()); for kinds in ["[]", "[\"mirror\"]", "[\"hosted\", \"hosted\"]"] { @@ -401,9 +439,12 @@ mod tests { #[test] fn refuses_unknown_apis_and_keys() { - assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"9.0.0\"")).is_err()); - // 0.1.0 was a preview that no release shipped. - assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"0.2.0\"")).is_err()); + assert!(Manifest::parse(&GOOD.replace("api = \"0.4.0\"", "api = \"9.0.0\"")).is_err()); + // Contracts before 0.4.0 had no files. + for old in ["0.2.0", "0.3.0"] { + let text = GOOD.replace("api = \"0.4.0\"", &format!("api = {old:?}")); + assert!(Manifest::parse(&text).is_err(), "{old} accepted"); + } assert!(Manifest::parse(&format!("{GOOD}\nunknown = 1\n")).is_err()); // What a WebAssembly component named is gone with it. assert!(Manifest::parse(&format!("{GOOD}component = \"helm.wasm\"\n")).is_err()); diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 1e6a5a7e..a156baf2 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,8 +22,9 @@ forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } forge-oci = { path = "../format-oci" } -# The built-in format plugin (plugins/nuget), linked in. +# The built-in format plugins (plugins/nuget, plugins/raw), linked in. forge-format-nuget = { path = "../../plugins/nuget" } +forge-format-raw = { path = "../../plugins/raw" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "signal"] } axum = { workspace = true, features = ["multipart"] } @@ -62,9 +63,9 @@ argon2 = { workspace = true, features = ["password-hash"] } tokio-util = { workspace = true } # Format-plugin uploads: the named file part of a multipart body, streamed. multer = "3" -# Raw repositories: the MD5 of uploads (the `.md5` sidecar), file paths in -# URLs, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content types -# guessed from file names. +# The file store: the MD5 of uploads (the `.md5` sidecar), file names in +# headers, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content +# types guessed from file names. md-5 = "0.11" percent-encoding = "2.3" httpdate = "1" diff --git a/crates/server/src/admin/formats.rs b/crates/server/src/admin/formats.rs index 2f26119a..49dc1d30 100644 --- a/crates/server/src/admin/formats.rs +++ b/crates/server/src/admin/formats.rs @@ -1,9 +1,9 @@ //! `GET /admin/formats` — the package formats this server can create -//! repositories of: the format plugins (NuGet), Docker and raw files. +//! repositories of: the format plugins (NuGet, raw files) and Docker. use axum::extract::State; use axum::response::{IntoResponse, Json, Response}; -use forge_db::model::{DOCKER_FORMAT, RAW_FORMAT}; +use forge_db::model::DOCKER_FORMAT; use serde::Serialize; use crate::state::AppState; @@ -17,6 +17,9 @@ pub struct FormatView { pub name: String, /// The repository kinds it supports. pub kinds: Vec<&'static str>, + /// What its repositories hold: `packages` (versions of packages), + /// `files` (files under paths: raw) or `images` (Docker). + pub storage: &'static str, /// Where its repositories are served: `{mount}/{repository}` (for /// Docker, `/v2/{repository}/{image}`, or a repository's own host). pub mount: String, @@ -35,7 +38,7 @@ pub struct FormatView { } /// `GET /admin/formats`: NuGet first, then Docker, then the other formats -/// (the other plugins and raw files) by id. +/// (the other plugins, raw files among them) by id. pub async fn list(State(state): State) -> Response { let kinds = |id: &str| state.formats.kinds(id).unwrap_or_default(); let ceiling = state.config.max_repository_upload_bytes(); @@ -45,6 +48,7 @@ pub async fn list(State(state): State) -> Response { id: plugin.id().to_string(), name: manifest.name.clone(), kinds: kinds(plugin.id()), + storage: manifest.storage.as_str(), mount: manifest.mount.clone(), version: Some(manifest.version.clone()), description: manifest.description.clone(), @@ -60,6 +64,7 @@ pub async fn list(State(state): State) -> Response { id: DOCKER_FORMAT.to_string(), name: "Docker / OCI".into(), kinds: kinds(DOCKER_FORMAT), + storage: "images", mount: "/v2".into(), version: None, description: None, @@ -67,16 +72,6 @@ pub async fn list(State(state): State) -> Response { max_upload_bytes_limit: ceiling, }); let mut others: Vec = others.into_iter().map(|p| view(p)).collect(); - others.push(FormatView { - id: RAW_FORMAT.to_string(), - name: "Raw files".into(), - kinds: kinds(RAW_FORMAT), - mount: crate::raw::MOUNT.into(), - version: None, - description: Some("Arbitrary files under arbitrary paths".into()), - default_max_upload_bytes: state.config.default_upload_limit(RAW_FORMAT), - max_upload_bytes_limit: ceiling, - }); others.sort_by(|a, b| a.id.cmp(&b.id)); formats.extend(others); Json(formats).into_response() diff --git a/crates/server/src/admin/members.rs b/crates/server/src/admin/members.rs index a919b853..1e2be7ba 100644 --- a/crates/server/src/admin/members.rs +++ b/crates/server/src/admin/members.rs @@ -66,14 +66,14 @@ pub struct UpdateMember { /// Refuse to flag a proxy as an owner: it would never own anything, and a /// group whose only flagged member is a proxy would serve its reserved ids /// from nowhere. -fn owner_kind_error(member: &forge_db::model::Repository) -> Option { - if member.is_raw() { - // Owners guard package ids; a raw group serves a path from its first - // member that has it. +fn owner_kind_error(state: &AppState, member: &forge_db::model::Repository) -> Option { + if state.formats.stores_files(&member.format) { + // Owners guard package ids; a group of files (raw) serves a path + // from its first member that has it. return Some( ( StatusCode::BAD_REQUEST, - "raw groups have no owner members: a path is served by the first member that has it", + "groups of files (raw) have no owner members: a path is served by the first member that has it", ) .into_response(), ); @@ -161,7 +161,7 @@ pub async fn add( .into_response(); } if body.owner { - if let Some(resp) = owner_kind_error(&member) { + if let Some(resp) = owner_kind_error(&state, &member) { return resp; } } @@ -215,7 +215,7 @@ pub async fn update( Err(e) => return db_error(e), }; if body.owner { - if let Some(resp) = owner_kind_error(&member) { + if let Some(resp) = owner_kind_error(&state, &member) { return resp; } } diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index 296ad655..5c4f50d7 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -191,7 +191,7 @@ pub fn admin_router(state: AppState) -> Router { ) .route( "/repositories/{name}/files", - axum::routing::delete(crate::raw::web::admin_delete), + axum::routing::delete(crate::files::web::admin_delete), ) .route( "/repositories/{name}/proxy-cache/invalidate", diff --git a/crates/server/src/admin/proxy_cache.rs b/crates/server/src/admin/proxy_cache.rs index a109e2b5..dd043152 100644 --- a/crates/server/src/admin/proxy_cache.rs +++ b/crates/server/src/admin/proxy_cache.rs @@ -81,16 +81,17 @@ pub async fn invalidate( return bad_request("only a proxy repository caches what an upstream answered"); } let docker = repo.is_docker(); + let stores_files = state.formats.stores_files(&repo.format); // The package as the proxy stores it: a docker image under Docker Hub's // `library/` spelling, any other id lowercased. let package = match body.and_then(|b| b.0.package) { None => None, - Some(p) if repo.is_raw() => { + Some(p) if stores_files => { let normalized = if p.ends_with('/') { - crate::raw::path::normalize_directory(&p) + forge_plugin::api::files::normalize_directory(&p) } else { - crate::raw::path::normalize(&p) + forge_plugin::api::files::normalize(&p) }; match normalized { Ok(path) => Some(path), @@ -116,7 +117,7 @@ pub async fn invalidate( } }; - match forget(&state, &repo, docker, package.as_deref()).await { + match forget(&state, &repo, docker, stores_files, package.as_deref()).await { Ok(Some(invalidated)) => { tracing::info!( repository = %repo.name, @@ -140,6 +141,7 @@ async fn forget( state: &AppState, repo: &forge_db::model::Repository, docker: bool, + stores_files: bool, package: Option<&str>, ) -> sqlx::Result> { let mut tx = state.pool.begin().await?; @@ -148,7 +150,7 @@ async fn forget( tags_expired: 0, files_expired: None, }; - if repo.is_raw() { + if stores_files { // A file is the range of its own path; a directory, of its files. let prefix = package.unwrap_or(""); invalidated.files_expired = Some(if prefix.is_empty() || prefix.ends_with('/') { diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index bcc42398..abe23813 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -23,7 +23,7 @@ use forge_core::net::ssrf::validate_url; use forge_db::model::{ DeploymentPolicy, RepoKind, Repository, CONTENT_DISPOSITION_KEY, DEPLOYMENT_POLICY_KEY, DOCKER_FORMAT, DOCKER_HOST_KEY, IMMUTABLE_TAGS_KEY, LATEST_MUTABLE_KEY, MAX_AGE_SECONDS_KEY, - MAX_UPLOAD_BYTES_KEY, NUGET_FORMAT, PREFER_HOSTED_KEY, RAW_FORMAT, RESERVED_PREFIXES_KEY, + MAX_UPLOAD_BYTES_KEY, NUGET_FORMAT, PREFER_HOSTED_KEY, RESERVED_PREFIXES_KEY, WRITABLE_MEMBER_ID_KEY, }; use forge_db::repository_dao::{self, NewRepository, RepositoryUpdate}; @@ -70,19 +70,24 @@ pub struct RepoView { /// Hosted: the largest push it accepts (for docker, the largest blob), /// `None` for the default of its format (`GET /admin/formats`). pub max_upload_bytes: Option, - /// Hosted raw: `allow_redeploy` | `disable_redeploy` | `read_only`. + /// Hosted, of a files format (raw): `allow_redeploy` | + /// `disable_redeploy` | `read_only`. pub deployment_policy: Option<&'static str>, - /// Raw: `attachment` (files download) | `inline` (files display). + /// A files format (raw): `attachment` (files download) | `inline` + /// (files display). pub content_disposition: Option<&'static str>, - /// Raw proxy: seconds a cached file is served before the upstream is - /// asked whether it changed (`0`: never). + /// Proxy of a files format (raw): seconds a cached file is served + /// before the upstream is asked whether it changed (`0`: never). pub max_age_seconds: Option, + /// What the repository holds: `packages`, `files` (a files format: raw) + /// or `images` (Docker). + pub storage: &'static str, } impl RepoView { /// The view of `r`, naming its writable member as `names` (repository - /// names by id) does. - fn new(r: &Repository, names: &HashMap) -> Self { + /// names by id) does; `stores_files` when its format holds files. + fn new(r: &Repository, names: &HashMap, stores_files: bool) -> Self { RepoView { id: r.id, name: r.name.clone(), @@ -107,30 +112,41 @@ impl RepoView { .writable_member_id() .and_then(|id| names.get(&id).cloned()), max_upload_bytes: r.max_upload_bytes(), - deployment_policy: (r.is_raw() && r.kind() == RepoKind::Hosted) + deployment_policy: (stores_files && r.kind() == RepoKind::Hosted) .then(|| r.deployment_policy().as_str()), - content_disposition: r.is_raw().then(|| { + content_disposition: stores_files.then(|| { if r.inline_content() { "inline" } else { "attachment" } }), - max_age_seconds: (r.is_raw() && r.kind() == RepoKind::Proxy) + max_age_seconds: (stores_files && r.kind() == RepoKind::Proxy) .then(|| r.max_age_seconds()), + storage: if r.is_docker() { + "images" + } else if stores_files { + "files" + } else { + "packages" + }, } } } /// The view of `r`, its writable member's current name looked up. -async fn view(pool: &sqlx::PgPool, r: &Repository) -> sqlx::Result { +async fn view(state: &AppState, r: &Repository) -> sqlx::Result { let mut names = HashMap::new(); if let Some(id) = r.writable_member_id() { - if let Some(member) = repository_dao::find_repository_by_id(pool, id).await? { + if let Some(member) = repository_dao::find_repository_by_id(&state.pool, id).await? { names.insert(id, member.name); } } - Ok(RepoView::new(r, &names)) + Ok(RepoView::new( + r, + &names, + state.formats.stores_files(&r.format), + )) } /// Create-repository request body. Unknown fields are rejected: a misspelled @@ -521,7 +537,10 @@ pub async fn list(State(state): State) -> Response { Ok(repos) => { let names: HashMap = repos.iter().map(|r| (r.id, r.name.clone())).collect(); - let views: Vec = repos.iter().map(|r| RepoView::new(r, &names)).collect(); + let views: Vec = repos + .iter() + .map(|r| RepoView::new(r, &names, state.formats.stores_files(&r.format))) + .collect(); Json(views).into_response() } Err(e) => db_error(e), @@ -531,7 +550,7 @@ pub async fn list(State(state): State) -> Response { /// `GET /admin/repositories/{name}` pub async fn get(State(state): State, Path(name): Path) -> Response { match repository_dao::find_repository_by_name(&state.pool, &name).await { - Ok(Some(r)) => match view(&state.pool, &r).await { + Ok(Some(r)) => match view(&state, &r).await { Ok(view) => Json(view).into_response(), Err(e) => db_error(e), }, @@ -595,9 +614,9 @@ pub async fn create(State(state): State, Json(body): Json) "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); } - let raw = body.format == RAW_FORMAT; - if raw { - if let Err(resp) = refuse_for_raw( + let files = state.formats.stores_files(&body.format); + if files { + if let Err(resp) = refuse_for_files( Some(body.package_delay_seconds), Some(body.allow_redeploy), Some(body.metadata_ttl_seconds), @@ -606,10 +625,10 @@ pub async fn create(State(state): State, Json(body): Json) return resp; } } - let docker_config = match raw_options( - raw, + let docker_config = match file_options( + files, parsed_kind, - RawOptions { + FileOptions { deployment_policy: body.deployment_policy.clone(), content_disposition: body.content_disposition.clone(), max_age_seconds: body.max_age_seconds.map(Some), @@ -628,8 +647,8 @@ pub async fn create(State(state): State, Json(body): Json) if let Err(resp) = validate_remote_url(url) { return resp; } - if raw { - if let Err(resp) = validate_raw_remote_url(url) { + if files { + if let Err(resp) = validate_files_remote_url(url) { return resp; } } @@ -740,7 +759,7 @@ pub async fn create(State(state): State, Json(body): Json) } else { repo }; - match view(&state.pool, &repo).await { + match view(&state, &repo).await { Ok(view) => (StatusCode::CREATED, Json(view)).into_response(), Err(e) => db_error(e), } @@ -833,15 +852,15 @@ async fn docker_options( Ok(config) } -/// The raw options of a create or update request (`None`: not sent; for -/// `max_age_seconds`, `Some(None)`: cleared). -struct RawOptions { +/// The options of a files format (raw) in a create or update request +/// (`None`: not sent; for `max_age_seconds`, `Some(None)`: cleared). +struct FileOptions { deployment_policy: Option, content_disposition: Option, max_age_seconds: Option>, } -impl RawOptions { +impl FileOptions { fn any(&self) -> bool { self.deployment_policy.is_some() || self.content_disposition.is_some() @@ -850,21 +869,21 @@ impl RawOptions { } /// Longest accepted `max_age_seconds`: ten years. -const MAX_RAW_MAX_AGE_SECONDS: u64 = 10 * 366 * 24 * 60 * 60; +const MAX_FILE_MAX_AGE_SECONDS: u64 = 10 * 366 * 24 * 60 * 60; -/// Validate the raw options against the repository's format and kind and +/// Validate the files options against the repository's format and kind and /// merge them into `config` (the config object to store). #[allow(clippy::result_large_err)] -fn raw_options( - raw: bool, +fn file_options( + files: bool, kind: RepoKind, - options: RawOptions, + options: FileOptions, mut config: serde_json::Map, ) -> Result, Response> { - if !raw { + if !files { if options.any() { return Err(bad_request( - "deployment_policy, content_disposition and max_age_seconds are only valid for raw repositories", + "deployment_policy, content_disposition and max_age_seconds are only valid for repositories of files (raw)", )); } return Ok(config); @@ -897,9 +916,9 @@ fn raw_options( )); } match max_age { - Some(seconds) if seconds > MAX_RAW_MAX_AGE_SECONDS => { + Some(seconds) if seconds > MAX_FILE_MAX_AGE_SECONDS => { return Err(bad_request(format!( - "max_age_seconds must be at most {MAX_RAW_MAX_AGE_SECONDS} (0: never re-validate)" + "max_age_seconds must be at most {MAX_FILE_MAX_AGE_SECONDS} (0: never re-validate)" ))); } Some(seconds) => { @@ -913,9 +932,9 @@ fn raw_options( Ok(config) } -/// The options of other formats a raw repository refuses. +/// The options of other formats a repository of files (raw) refuses. #[allow(clippy::result_large_err)] -fn refuse_for_raw( +fn refuse_for_files( package_delay_seconds: Option, allow_redeploy: Option, metadata_ttl_seconds: Option>, @@ -923,34 +942,34 @@ fn refuse_for_raw( ) -> Result<(), Response> { if package_delay_seconds.is_some_and(|d| d != 0) { return Err(bad_request( - "package_delay_seconds is not supported for raw repositories", + "package_delay_seconds is not supported for repositories of files (raw)", )); } if allow_redeploy == Some(true) { return Err(bad_request( - "allow_redeploy is a NuGet option; raw repositories have deployment_policy", + "allow_redeploy is a NuGet option; repositories of files (raw) have deployment_policy", )); } if matches!(metadata_ttl_seconds, Some(Some(_))) { return Err(bad_request( - "metadata_ttl_seconds is not used by raw proxies; set max_age_seconds", + "metadata_ttl_seconds is not used by proxies of files (raw); set max_age_seconds", )); } if group_guard { return Err(bad_request( - "a raw group serves a path from its first member that has it; prefer_hosted and reserved_prefixes do not apply", + "a group of files (raw) serves a path from its first member that has it; prefer_hosted and reserved_prefixes do not apply", )); } Ok(()) } -/// A raw proxy's `remote_url` is the base its paths are appended to: no +/// A files proxy's `remote_url` is the base its paths are appended to: no /// query or fragment. #[allow(clippy::result_large_err)] -fn validate_raw_remote_url(url: &str) -> Result<(), Response> { +fn validate_files_remote_url(url: &str) -> Result<(), Response> { match url::Url::parse(url) { Ok(parsed) if parsed.query().is_some() || parsed.fragment().is_some() => Err(bad_request( - "a raw proxy's remote_url is the base URL files are under: no query or fragment", + "a proxy of files' remote_url is the base URL the files are under: no query or fragment", )), _ => Ok(()), } @@ -1016,16 +1035,16 @@ pub async fn update( if let Err(resp) = validate_remote_url(url) { return resp; } - if existing.is_raw() { - if let Err(resp) = validate_raw_remote_url(url) { + if state.formats.stores_files(&existing.format) { + if let Err(resp) = validate_files_remote_url(url) { return resp; } } } } } - if existing.is_raw() { - if let Err(resp) = refuse_for_raw( + if state.formats.stores_files(&existing.format) { + if let Err(resp) = refuse_for_files( body.package_delay_seconds, body.allow_redeploy, body.metadata_ttl_seconds, @@ -1064,10 +1083,10 @@ pub async fn update( Ok(config) => config, Err(resp) => return resp, }; - let mut config = match raw_options( - existing.is_raw(), + let mut config = match file_options( + state.formats.stores_files(&existing.format), existing.kind(), - RawOptions { + FileOptions { deployment_policy: body.deployment_policy.clone(), content_disposition: body.content_disposition.clone(), max_age_seconds: body.max_age_seconds, @@ -1225,8 +1244,8 @@ pub async fn update( return db_error(e); } } - // A raw proxy's files are the previous upstream's: likewise. - if existing.is_raw() { + // A files proxy's files are the previous upstream's: likewise. + if state.formats.stores_files(&existing.format) { match repository_dao::lock_repository(&mut *tx, updated.id).await { Ok(_) => {} Err(e) => return db_error(e), @@ -1293,7 +1312,7 @@ pub async fn update( } else { updated }; - match view(&state.pool, &updated).await { + match view(&state, &updated).await { Ok(repo) => Json(UpdatedRepoView { repo, upstream_credentials_dropped: credentials_dropped, @@ -1347,7 +1366,7 @@ pub async fn delete(State(state): State, Path(name): Path) -> "repository still has images; delete their manifests first", ) .into_response(), - Ok(RepoDelete::HasPackages) if existing.is_raw() => ( + Ok(RepoDelete::HasPackages) if state.formats.stores_files(&existing.format) => ( StatusCode::CONFLICT, "repository still has files; delete them first (DELETE /admin/repositories//files?path=)", ) @@ -1485,7 +1504,7 @@ pub async fn hard_delete_package( ) .into_response(); } - if repo.is_raw() { + if state.formats.stores_files(&repo.format) { return ( StatusCode::BAD_REQUEST, "delete files through DELETE /admin/repositories//files?path=", diff --git a/crates/server/src/admin/retention.rs b/crates/server/src/admin/retention.rs index 979579b8..5a46a745 100644 --- a/crates/server/src/admin/retention.rs +++ b/crates/server/src/admin/retention.rs @@ -131,7 +131,7 @@ pub async fn put( }; // Parsed with its repository's model and stored in its canonical form. - let normalized = if repo.is_raw() { + let normalized = if state.formats.stores_files(&repo.format) { let policy: FileRetentionPolicy = match serde_json::from_value(body.policy) { Ok(p) => p, Err(e) => return unprocessable(e), @@ -255,7 +255,7 @@ pub async fn preview( decisions: Vec::new(), proxy_decisions: Vec::new(), image_decisions: repo.is_docker().then(ImageDecisions::default), - file_decisions: repo.is_raw().then(Vec::new), + file_decisions: state.formats.stores_files(&repo.format).then(Vec::new), counts: PreviewCounts::default(), }) .into_response() @@ -265,7 +265,7 @@ pub async fn preview( if repo.is_docker() { return preview_images(&state, &repo, policy_json, body.and_then(|b| b.0.at)).await; } - if repo.is_raw() { + if state.formats.stores_files(&repo.format) { return preview_files(&state, &repo, policy_json, body.and_then(|b| b.0.at)).await; } let policy: RetentionPolicy = match serde_json::from_value(policy_json) { diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index 7c0bf839..3220d3a6 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -129,14 +129,9 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { .into_iter() .map(str::to_string) .collect(); - // Raw repositories are feeds too, for the response policy. - let feeds: std::sync::Arc> = std::sync::Arc::new( - mounts - .iter() - .cloned() - .chain([crate::raw::MOUNT.to_string()]) - .collect(), - ); + // Every format's mount is a feed, for the response policy (raw's + // `/raw` among them). + let feeds: std::sync::Arc> = std::sync::Arc::new(mounts.clone()); let mut app = Router::new() // Liveness / readiness probes (unauthenticated, dependency-light). .merge(crate::health::health_router()) @@ -162,17 +157,16 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // the host) itself. .route("/v2", any(crate::registry::registry)) .route("/v2/", any(crate::registry::registry)) - .route("/v2/{*rest}", any(crate::registry::registry)) - // Raw (generic file) repositories: `/raw//`. - .route("/raw/{repo}", any(crate::raw::handle)) - .route("/raw/{repo}/", any(crate::raw::handle)) - .route("/raw/{repo}/{*rest}", any(crate::raw::handle)); + .route("/v2/{*rest}", any(crate::registry::registry)); // Repositories, each served by its format's plugin under the format's - // mount (`crate::plugins`; NuGet at `/v3`). `any` covers + // mount (`crate::plugins`; NuGet at `/v3`, raw files at `/raw`). `any` covers // GET/HEAD/PUT/POST/DELETE; the dispatcher authorizes per method. for mount in mounts.iter() { app = app .route(&format!("{mount}/{{repo}}"), any(crate::plugins::dispatch)) + // A wildcard matches no empty rest: the root of a repository + // (a files format's root listing) needs its own route. + .route(&format!("{mount}/{{repo}}/"), any(crate::plugins::dispatch)) .route( &format!("{mount}/{{repo}}/{{*rest}}"), any(crate::plugins::dispatch), diff --git a/crates/server/src/browse.rs b/crates/server/src/browse.rs index 8dd6de66..39023b45 100644 --- a/crates/server/src/browse.rs +++ b/crates/server/src/browse.rs @@ -100,6 +100,9 @@ pub struct BrowseRepo { /// `/browse/images` names, and for a format not installed here. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, + /// What the repository holds: `packages`, `files` (a files format: raw, + /// browsed through its listings) or `images` (docker). + pub storage: &'static str, } /// Query parameters for `GET /browse/repos`. @@ -143,16 +146,15 @@ pub async fn list_repos( kind: repo.kind.clone(), online: repo.online, format: repo.format.clone(), - url: if repo.is_raw() { - Some(crate::plugins::repository_url( - &state, - crate::raw::MOUNT, - &repo.name, - )) + url: state.formats.plugin(&repo.format).map(|plugin| { + crate::plugins::repository_url(&state, &plugin.manifest().mount, &repo.name) + }), + storage: if repo.is_docker() { + "images" + } else if state.formats.stores_files(&repo.format) { + "files" } else { - state.formats.plugin(&repo.format).map(|plugin| { - crate::plugins::repository_url(&state, &plugin.manifest().mount, &repo.name) - }) + "packages" }, }); } @@ -251,8 +253,8 @@ pub async fn search_packages( } } // Package search covers the package formats; images are listed by - // /browse/images, raw files by their repository's listings. - if repo.is_docker() || repo.is_raw() { + // /browse/images, files (raw) by their repository's listings. + if repo.is_docker() || state.formats.stores_files(&repo.format) { continue; } if state.authz.can_read(&principal, repo).await { @@ -413,9 +415,9 @@ pub async fn package_detail( } let repo = match repository_dao::find_repository_by_name(&state.pool, &repo_name).await { - // A docker repository holds images (`/browse/images`) and a raw one - // files (its listings), not packages. - Ok(Some(r)) if !r.is_docker() && !r.is_raw() => r, + // A docker repository holds images (`/browse/images`) and a files + // (raw) one files (its listings), not packages. + Ok(Some(r)) if !r.is_docker() && !state.formats.stores_files(&r.format) => r, // An unknown name answers like an unreadable repository (no name // enumeration), except to a global admin. Ok(_) if state.authz.is_global_admin(&principal).await => { @@ -658,7 +660,7 @@ pub fn browse_router() -> Router { ) .route( "/browse/raw/{repo}/files", - axum::routing::post(crate::raw::web::browse_upload) - .delete(crate::raw::web::browse_delete), + axum::routing::post(crate::files::web::browse_upload) + .delete(crate::files::web::browse_delete), ) } diff --git a/crates/server/src/files/listing.rs b/crates/server/src/files/listing.rs new file mode 100644 index 00000000..b9bd76fd --- /dev/null +++ b/crates/server/src/files/listing.rs @@ -0,0 +1,268 @@ +//! Directory pages of the file store (the `directory` query): what a +//! directory holds, a page at a time, for the format's plugin to render. +//! +//! A page holds at most `limit` entries (default 100, at most 1000) after +//! the entry named by `after`, in byte order of their names, a +//! subdirectory's name ending in `/`; `next` names where the following page +//! starts. A listing walks the index ([`raw_dao::walk_directory`]): a page +//! costs `limit` index probes, however many files are below the directory. +//! +//! Package selectors apply: a file outside the caller's selector is not +//! listed, and a subdirectory is listed only when it holds a file the +//! caller may see (looked for among its first files; a subdirectory none of +//! whose first [`SUBDIRECTORY_SCAN`] files is visible is left out). A +//! group lists the merged entries of the members the caller may read, the +//! highest-priority member's file first; a proxy lists what it has cached. + +use std::collections::BTreeMap; + +use forge_core::authz::PackageFilter; +use forge_db::model::{RepoKind, Repository}; +use forge_db::raw_dao::{self, RawFile}; +use forge_plugin::api::files; + +use super::{FileError, Files}; + +/// Default page size. +pub const DEFAULT_LIMIT: usize = 100; + +/// Largest page size. +pub const MAX_LIMIT: usize = 1000; + +/// How many of a subdirectory's files a listing reads, at most, to find one +/// a selector-restricted caller may see. +pub const SUBDIRECTORY_SCAN: usize = 1000; + +/// One entry of a page. +#[derive(Debug, Clone)] +pub struct Entry { + /// The entry's name in its directory; a subdirectory's ends in `/`. + pub name: String, + pub kind: EntryKind, +} + +#[derive(Debug, Clone)] +pub enum EntryKind { + File(Box), + /// A subdirectory: its path, ending in `/`. + Directory(String), +} + +/// A page of a directory. +#[derive(Debug, Clone)] +pub struct Page { + pub entries: Vec, + /// The `after` of the next page, if there is one. + pub next: Option, +} + +/// Where a page starting after the entry `after` (a name in directory +/// `prefix`, a subdirectory's ending in `/`) resumes: the least path past +/// that entry (past all of a subdirectory's files: `'0'` follows `'/'`; past +/// a file: no path holds a control character). +fn resume_at(prefix: &str, after: Option<&str>) -> Result { + let Some(after) = after.filter(|a| !a.is_empty()) else { + return Ok(prefix.to_string()); + }; + let (name, directory) = match after.strip_suffix('/') { + Some(name) => (name, true), + None => (after, false), + }; + files::validate_segment(name).map_err(|e| FileError::BadRequest(format!("after: {e}")))?; + Ok(if directory { + format!("{prefix}{name}0") + } else { + format!("{prefix}{name}\u{1}") + }) +} + +/// One repository's entries of a page: what it has after `from`, as far as +/// it read (`more`: it has entries past the last one returned, up to and +/// including the entry named `more`, which it has read). +struct Partial { + entries: Vec, + more: Option, +} + +impl Files<'_> { + /// The entries of `prefix` in hosted or proxy `repo` from `from` on, + /// visible through every one of `filters`. + async fn list_local( + &self, + repo: &Repository, + prefix: &str, + from: &str, + limit: usize, + filters: &[PackageFilter], + ) -> Result { + let walked = raw_dao::walk_directory( + &self.state.pool, + repo.id, + prefix, + from, + i64::try_from(limit + 1).unwrap_or(i64::MAX), + ) + .await?; + let more = walked.len() > limit; + let mut entries = Vec::new(); + let mut last = None; + for walked in walked.into_iter().take(limit) { + let rest = &walked.path[prefix.len()..]; + let (entry, key) = match (rest.find('/'), &walked.file) { + (None, Some(file)) => { + let visible = filters.iter().all(|f| f.allows(&file.path)); + let entry = Entry { + name: rest.to_string(), + kind: EntryKind::File(Box::new(file.clone())), + }; + (visible.then_some(entry), rest.to_string()) + } + (Some(i), _) => { + let name = &rest[..i]; + let dir = format!("{prefix}{name}/"); + let visible = self.directory_visible(repo, &dir, filters).await?; + let entry = Entry { + name: format!("{name}/"), + kind: EntryKind::Directory(dir), + }; + (visible.then_some(entry), format!("{name}/")) + } + // A file row vanished between the walk and the join. + (None, None) => (None, rest.to_string()), + }; + entries.extend(entry); + last = Some(key); + } + Ok(Partial { + entries, + more: if more { last } else { None }, + }) + } + + /// Whether directory `dir` of `repo` holds a file visible through every + /// one of `filters`. + async fn directory_visible( + &self, + repo: &Repository, + dir: &str, + filters: &[PackageFilter], + ) -> Result { + if filters.iter().all(|f| matches!(f, PackageFilter::AllowAll)) { + return Ok(true); + } + const BATCH: usize = 200; + let mut after = String::new(); + let mut read = 0; + while read < SUBDIRECTORY_SCAN { + let paths = + raw_dao::paths_under(&self.state.pool, repo.id, dir, &after, BATCH as i64).await?; + if paths.iter().any(|p| filters.iter().all(|f| f.allows(p))) { + return Ok(true); + } + if paths.len() < BATCH { + return Ok(false); + } + read += paths.len(); + after = paths.last().cloned().unwrap_or_default(); + } + Ok(false) + } + + /// The entries of `prefix` in `repo` (any kind) from `from` on. + fn list_in<'a>( + &'a self, + repo: &'a Repository, + prefix: &'a str, + from: &'a str, + limit: usize, + filters: Vec, + depth: u32, + ) -> futures_util::future::BoxFuture<'a, Result> { + Box::pin(async move { + let mut filters = filters; + filters.push(self.state.authz.package_filter(&self.principal, repo).await); + if repo.kind() != RepoKind::Group { + return self.list_local(repo, prefix, from, limit, &filters).await; + } + // A merge of the members' pages, complete up to the first point + // a member stopped reading at. + let mut merged: BTreeMap = BTreeMap::new(); + let mut cutoff: Option = None; + for member in self.serving_members(repo, depth).await? { + let part = self + .list_in(&member, prefix, from, limit, filters.clone(), depth + 1) + .await?; + for entry in part.entries { + merged.entry(entry.name.clone()).or_insert(entry); + } + if let Some(more) = part.more { + if cutoff.as_ref().is_none_or(|c| more < *c) { + cutoff = Some(more); + } + } + } + let mut entries: Vec = merged + .into_values() + .filter(|e| cutoff.as_ref().is_none_or(|c| e.name <= *c)) + .collect(); + let more = if entries.len() > limit { + entries.truncate(limit); + entries.last().map(|e| e.name.clone()) + } else { + cutoff + }; + Ok(Partial { entries, more }) + }) + } + + /// A page of directory `prefix` of `repo` after the entry `after`, of + /// `limit` entries (0: the default), `None` when it does not exist (a + /// directory other than the root without visible entries, asked for + /// without `after`). + pub async fn list( + &self, + repo: &Repository, + prefix: &str, + after: Option<&str>, + limit: usize, + ) -> Result, FileError> { + files::validate_directory(prefix).map_err(|e| { + FileError::Internal(format!("the format named directory {prefix:?}: {e}")) + })?; + let limit = if limit == 0 { + DEFAULT_LIMIT + } else { + limit.min(MAX_LIMIT) + }; + let from = resume_at(prefix, after)?; + let partial = self + .list_in(repo, prefix, &from, limit, Vec::new(), 0) + .await?; + if partial.entries.is_empty() && partial.more.is_none() && !prefix.is_empty() { + // Past the end of an existing directory is an empty page, not a + // missing directory. + if after.is_none() { + return Ok(None); + } + } + Ok(Some(Page { + entries: partial.entries, + next: partial.more, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pages_resume_past_the_named_entry() { + assert_eq!(resume_at("a/", None).unwrap(), "a/"); + assert_eq!(resume_at("a/", Some("")).unwrap(), "a/"); + assert_eq!(resume_at("a/", Some("x.bin")).unwrap(), "a/x.bin\u{1}"); + assert_eq!(resume_at("", Some("dir/")).unwrap(), "dir0"); + assert!(resume_at("a/", Some("../x")).is_err()); + assert!(resume_at("a/", Some("b/c")).is_err()); + } +} diff --git a/crates/server/src/files/mod.rs b/crates/server/src/files/mod.rs new file mode 100644 index 00000000..9debdcc2 --- /dev/null +++ b/crates/server/src/files/mod.rs @@ -0,0 +1,226 @@ +//! The file store: the repositories of a format whose `plugin.toml` says +//! `storage = "files"` (raw) hold files under paths, like Nexus' raw-hosted, +//! raw-proxy and raw-group repositories. The format's plugin owns the +//! protocol (`plugins/raw`, served under its mount by [`crate::plugins`]); +//! this module answers the file queries and actions it asks for: +//! +//! * `file` and `fetch-file`: [`Files::locate`] — the stored file; a proxy's +//! copy, re-validated with its upstream once it is due, or fetched from it +//! ([`proxy`]); a group's first member, by priority, that has the path; +//! * `directory`: a page of a directory ([`listing`]); +//! * `form`, `store` and `remove`: receiving a multipart form, storing files +//! under the deployment policy, deleting a file ([`store`]); +//! * `serve-file`: its bytes, one `Range`, `If-Range`, `If-None-Match` and +//! `If-Modified-Since` ([`serve`]). +//! +//! The web UI's upload and delete and the admin API's directory delete are +//! here too ([`web`]). +//! +//! Paths follow [`forge_plugin::api::files`]: `..`, `.`, empty segments, +//! backslashes and encoded slashes are refused, never resolved, and so is any +//! other path a plugin names. Package selectors match paths (`pdfium/**`). +//! Files are CAS blobs, so the same content under several paths or +//! repositories is stored once (one `raw_file` row, one reference each). + +pub mod listing; +pub mod proxy; +pub mod serve; +pub mod store; +pub mod web; + +use std::collections::HashSet; + +use axum::http::{header, StatusCode}; +use axum::response::{IntoResponse, Response}; +use forge_core::auth::principal::Principal; +use forge_core::authz::PackageFilter; +use forge_db::model::{RepoKind, Repository}; +use forge_db::{raw_dao, repository_dao}; + +use crate::dispatch::offline_serves_nothing; +use crate::state::AppState; +use raw_dao::RawFile; + +/// A request's view of the file store: the server's state and the caller. +pub struct Files<'a> { + pub state: &'a AppState, + pub principal: Principal, +} + +/// Why a file request failed, as its answer. +#[derive(Debug)] +pub enum FileError { + NotFound, + BadRequest(String), + /// A mutation needs an API token or a workload identity. + CredentialRequired, + /// The caller may not do this (repository-level). + Denied, + Forbidden(String), + Conflict(String), + MethodNotAllowed(&'static str), + TooLarge(u64), + Unavailable(String), + BadGateway(String), + Internal(String), +} + +impl FileError { + /// The error's text, for logs. + pub fn message(&self) -> String { + match self { + FileError::NotFound => "not found".into(), + FileError::CredentialRequired => "API token or workload identity required".into(), + FileError::Denied => "forbidden".into(), + FileError::MethodNotAllowed(m) => (*m).into(), + FileError::TooLarge(limit) => format!("larger than {limit} bytes"), + FileError::BadRequest(m) + | FileError::Forbidden(m) + | FileError::Conflict(m) + | FileError::Unavailable(m) + | FileError::BadGateway(m) + | FileError::Internal(m) => m.clone(), + } + } +} + +impl From for FileError { + fn from(e: sqlx::Error) -> Self { + match e { + sqlx::Error::PoolTimedOut => FileError::Unavailable("database busy".into()), + e => FileError::Internal(format!("database: {e}")), + } + } +} + +impl IntoResponse for FileError { + fn into_response(self) -> Response { + match self { + FileError::NotFound => (StatusCode::NOT_FOUND, "not found").into_response(), + FileError::BadRequest(m) => (StatusCode::BAD_REQUEST, m).into_response(), + FileError::CredentialRequired => ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Basic realm=\"Forge\"")], + "API token or workload identity required", + ) + .into_response(), + FileError::Denied => (StatusCode::FORBIDDEN, "forbidden").into_response(), + FileError::Forbidden(m) => (StatusCode::FORBIDDEN, m).into_response(), + FileError::Conflict(m) => (StatusCode::CONFLICT, m).into_response(), + FileError::MethodNotAllowed(m) => ( + StatusCode::METHOD_NOT_ALLOWED, + [(header::ALLOW, "GET, HEAD")], + m, + ) + .into_response(), + FileError::TooLarge(limit) => ( + StatusCode::PAYLOAD_TOO_LARGE, + format!("the upload exceeds the repository's limit of {limit} bytes"), + ) + .into_response(), + FileError::Unavailable(m) => ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "30")], + m, + ) + .into_response(), + FileError::BadGateway(m) => (StatusCode::BAD_GATEWAY, m).into_response(), + FileError::Internal(m) => { + tracing::error!(error = %m, "file store request failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() + } + } + } +} + +impl Files<'_> { + /// Find `path` in `repo` for the caller, through every one of `filters` + /// (a group's, for its members) and `repo`'s own package filter: the + /// stored file (a proxy's re-validated once it is due), else with + /// `fetch` a proxy's upstream file, else (a group) the first member's + /// that has one. A member that fails is skipped while another has the + /// path. + pub fn locate<'a>( + &'a self, + repo: &'a Repository, + path: &'a str, + fetch: bool, + filters: Vec, + depth: u32, + ) -> futures_util::future::BoxFuture<'a, Result, FileError>> { + Box::pin(async move { + let mut filters = filters; + filters.push(self.state.authz.package_filter(&self.principal, repo).await); + if !filters.iter().all(|f| f.allows(path)) { + return Ok(None); + } + match repo.kind() { + RepoKind::Hosted | RepoKind::Proxy => { + let proxy = repo.kind() == RepoKind::Proxy; + if let Some(file) = raw_dao::find(&self.state.pool, repo.id, path).await? { + return Ok(Some(if proxy { + self.fresh(repo, file).await + } else { + file + })); + } + if proxy && fetch { + return self.fetch(repo, path).await; + } + Ok(None) + } + RepoKind::Group => { + let mut failure = None; + for member in self.serving_members(repo, depth).await? { + match self + .locate(&member, path, fetch, filters.clone(), depth + 1) + .await + { + Ok(Some(found)) => return Ok(Some(found)), + Ok(None) => {} + Err(e) => { + tracing::warn!(group = %repo.name, member = %member.name, error = %e.message(), "file group: member failed"); + failure = Some(e); + } + } + } + failure.map_or(Ok(None), Err) + } + } + }) + } + + /// The members of `group` that may serve the caller, highest priority + /// first: repositories of the group's format the caller may read itself + /// (reading a group grants nothing on its members), without offline hosted or group + /// members, and without nested groups past the nesting limit or on a + /// cycle. + pub async fn serving_members( + &self, + group: &Repository, + depth: u32, + ) -> Result, FileError> { + let max_depth = self.state.config.groups.max_nesting_depth.max(1) as u32; + if depth >= max_depth { + return Ok(Vec::new()); + } + let mut seen = HashSet::from([group.id]); + let mut out = Vec::new(); + for (member, _priority) in repository_dao::group_members(&self.state.pool, group.id).await? + { + if member.format != group.format + || !seen.insert(member.id) + || offline_serves_nothing(&member) + { + continue; + } + if member.kind() == RepoKind::Group && depth + 1 >= max_depth { + continue; + } + if self.state.authz.can_read(&self.principal, &member).await { + out.push(member); + } + } + Ok(out) + } +} diff --git a/crates/server/src/raw/proxy.rs b/crates/server/src/files/proxy.rs similarity index 72% rename from crates/server/src/raw/proxy.rs rename to crates/server/src/files/proxy.rs index d6be2ac6..c882064a 100644 --- a/crates/server/src/raw/proxy.rs +++ b/crates/server/src/files/proxy.rs @@ -1,12 +1,14 @@ -//! Raw proxy repositories: files fetched from an HTTP(S) base URL on first -//! request, cached content-addressed, and re-validated against the upstream -//! once they are older than the repository's `max_age_seconds`. +//! Proxy repositories of the file store: files fetched from the upstream on +//! first request, cached content-addressed, and re-validated against the +//! upstream once they are older than the repository's `max_age_seconds`. //! -//! * A file `` is `/`, each segment -//! percent-encoded. Redirects are followed (GitHub release assets -//! redirect to `objects.githubusercontent.com`), every hop through the -//! upstream client's cloud-metadata guard; the upstream credentials go to -//! the origin of `remote_url` only. +//! * The format's plugin names the upstream URL of a path (its upstream +//! side's `file` operation; raw's is `/`, each segment +//! percent-encoded), on an origin the upstream points at. Redirects are +//! followed (GitHub release assets redirect to +//! `objects.githubusercontent.com`), every hop through the upstream +//! client's cloud-metadata guard; the upstream credentials go to the +//! origin of `remote_url` only. //! * Re-validation is a conditional GET (`If-None-Match` with the upstream's //! `ETag`, `If-Modified-Since` with its `Last-Modified`): a `304` keeps //! the copy, new content replaces it. A `max_age_seconds` of `0` never @@ -31,8 +33,8 @@ use forge_db::raw_dao::{self, NewRawFile, RawFile, Redeploy}; use futures_util::TryStreamExt; use reqwest::header::{CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; -use super::upload::{content_type, ingest}; -use super::{Raw, RawError}; +use super::store::{content_type, ingest}; +use super::{FileError, Files}; /// The gate of one file's upstream requests, by `(repository id, path)`. type Gates = HashMap<(i64, String), Weak>>; @@ -55,23 +57,6 @@ fn gate(repository_id: i64, path: &str) -> Arc> { gate } -/// The upstream URL of `path`: `remote_url` (as a directory) with the -/// path's segments appended, each percent-encoded. -pub fn upstream_url(remote_url: &str, path: &str) -> Result { - let mut url = url::Url::parse(remote_url) - .map_err(|e| RawError::Internal(format!("proxy remote_url {remote_url:?}: {e}")))?; - { - let mut segments = url.path_segments_mut().map_err(|()| { - RawError::Internal(format!("proxy remote_url {remote_url:?} has no path")) - })?; - segments.pop_if_empty(); - for segment in path.split('/') { - segments.push(segment); - } - } - Ok(url.into()) -} - fn header_text( headers: &reqwest::header::HeaderMap, name: reqwest::header::HeaderName, @@ -84,7 +69,7 @@ fn header_text( .map(str::to_string) } -impl Raw<'_> { +impl Files<'_> { /// Keyed by the repository's `updated_at` too, so an administrator's /// cache invalidation (which bumps it) forgets the misses on every /// instance. @@ -97,9 +82,12 @@ impl Raw<'_> { let hit: Result = self .state .cache - .get_or_try_insert_local("raw:upstream-nf", &key, Duration::from_secs(1), || async { - Err(()) - }) + .get_or_try_insert_local( + "files:upstream-nf", + &key, + Duration::from_secs(1), + || async { Err(()) }, + ) .await; hit.unwrap_or(false) } @@ -118,7 +106,7 @@ impl Raw<'_> { .state .cache .get_or_try_insert_local( - "raw:upstream-nf", + "files:upstream-nf", &key, Duration::from_secs(ttl), || async { Ok(true) }, @@ -126,24 +114,43 @@ impl Raw<'_> { .await; } - async fn credentials(&self, repo: &Repository) -> Result, RawError> { + async fn credentials(&self, repo: &Repository) -> Result, FileError> { upstream_credentials(&self.state.pool, self.state.envelope.as_ref(), repo) .await - .map_err(|e| RawError::Internal(format!("proxy credentials: {e}"))) + .map_err(|e| FileError::Internal(format!("proxy credentials: {e}"))) + } + + /// The upstream URL of `path`, as the format's plugin names it; `None` + /// when the upstream cannot have it. + async fn upstream_url( + &self, + repo: &Repository, + path: &str, + ) -> Result, FileError> { + let format = self + .state + .formats + .for_repository(repo) + .map_err(|e| FileError::Unavailable(e.to_string()))?; + self.state + .proxy_repo(format) + .file_url(repo, path) + .await + .map_err(|e| { + tracing::warn!(repo = %repo.name, %path, error = %e, "file proxy: the upstream URL could not be resolved"); + FileError::BadGateway(format!("the upstream could not be reached: {e}")) + }) } - /// GET `path` from the upstream, conditionally on `cached`'s validators. + /// GET `url` (the upstream's `path`), conditionally on `cached`'s + /// validators. async fn upstream_get( &self, repo: &Repository, path: &str, + url: &str, cached: Option<&RawFile>, - ) -> Result { - let remote = repo - .remote_url - .as_deref() - .ok_or_else(|| RawError::Internal("proxy repository without remote_url".into()))?; - let url = upstream_url(remote, path)?; + ) -> Result { let credentials = self.credentials(repo).await?; let auth: Option> = credentials.as_ref().map(Credentials::auth); let mut headers = reqwest::header::HeaderMap::new(); @@ -163,15 +170,15 @@ impl Raw<'_> { .upstream .send( reqwest::Method::GET, - &url, + url, RequestClass::Download, headers, auth.as_ref(), ) .await .map_err(|e| { - tracing::warn!(repo = %repo.name, %path, error = %e, "raw proxy: upstream request failed"); - RawError::BadGateway(format!("the upstream could not be reached: {e}")) + tracing::warn!(repo = %repo.name, %path, error = %e, "file proxy: upstream request failed"); + FileError::BadGateway(format!("the upstream could not be reached: {e}")) }) } @@ -181,10 +188,10 @@ impl Raw<'_> { repo: &Repository, path: &str, response: reqwest::Response, - ) -> Result { + ) -> Result { let limit = self.state.upstream.config().max_package_bytes; if response.content_length().is_some_and(|len| len > limit) { - return Err(RawError::BadGateway(format!( + return Err(FileError::BadGateway(format!( "the upstream file is larger than proxy.max_package_bytes ({limit} bytes)" ))); } @@ -194,11 +201,11 @@ impl Raw<'_> { let declared = header_text(&headers, CONTENT_TYPE); let body = response.bytes_stream().map_err(std::io::Error::other); let ingested = ingest(self, body, limit).await.map_err(|e| match e { - RawError::TooLarge(limit) => RawError::BadGateway(format!( + FileError::TooLarge(limit) => FileError::BadGateway(format!( "the upstream file is larger than proxy.max_package_bytes ({limit} bytes)" )), - RawError::BadRequest(msg) => { - RawError::BadGateway(format!("the upstream download failed: {msg}")) + FileError::BadRequest(msg) => { + FileError::BadGateway(format!("the upstream download failed: {msg}")) } other => other, })?; @@ -206,7 +213,7 @@ impl Raw<'_> { let content_type = content_type(declared.as_deref(), path); let mut tx = self.state.pool.begin().await?; if !oci_dao::lock_proxy_upstream(&mut tx, repo.id, repo.remote_url.as_deref()).await? { - return Err(RawError::Unavailable( + return Err(FileError::Unavailable( "the proxy was pointed at another upstream while this was fetched; retry".into(), )); } @@ -216,7 +223,7 @@ impl Raw<'_> { &NewRawFile { path, sha256: &stored.sha256, - size: i64::try_from(stored.size).map_err(|_| RawError::TooLarge(stored.size))?, + size: i64::try_from(stored.size).map_err(|_| FileError::TooLarge(stored.size))?, content_type: &content_type, sha512_base64: &stored.sha512_base64, sha1: &ingested.sha1, @@ -233,7 +240,7 @@ impl Raw<'_> { tx.commit().await?; raw_dao::find(&self.state.pool, repo.id, path) .await? - .ok_or(RawError::NotFound) + .ok_or(FileError::NotFound) } /// The cached `file` of proxy `repo`, re-validated first when it is due. @@ -253,14 +260,21 @@ impl Raw<'_> { match self.revalidate(repo, &file).await { Ok(current) => current, Err(error) => { - tracing::warn!(repo = %repo.name, path = %file.path, error = %error.message(), "raw proxy: re-validation failed; serving the cached copy"); + tracing::warn!(repo = %repo.name, path = %file.path, error = %error.message(), "file proxy: re-validation failed; serving the cached copy"); file } } } - async fn revalidate(&self, repo: &Repository, file: &RawFile) -> Result { - let response = self.upstream_get(repo, &file.path, Some(file)).await?; + async fn revalidate(&self, repo: &Repository, file: &RawFile) -> Result { + let Some(url) = self.upstream_url(repo, &file.path).await? else { + // The upstream cannot have it (any more): the cached copy stands. + raw_dao::mark_checked(&self.state.pool, file.id, None, None).await?; + return Ok(file.clone()); + }; + let response = self + .upstream_get(repo, &file.path, &url, Some(file)) + .await?; let status = response.status(); if status == StatusCode::NOT_MODIFIED { let headers = response.headers(); @@ -277,18 +291,18 @@ impl Raw<'_> { return self.store_response(repo, &file.path, response).await; } if matches!(status, StatusCode::NOT_FOUND | StatusCode::GONE) { - tracing::warn!(repo = %repo.name, path = %file.path, "raw proxy: the upstream no longer has a cached file; serving the cached copy"); + tracing::warn!(repo = %repo.name, path = %file.path, "file proxy: the upstream no longer has a cached file; serving the cached copy"); raw_dao::mark_checked(&self.state.pool, file.id, None, None).await?; return Ok(file.clone()); } - Err(RawError::BadGateway(format!( + Err(FileError::BadGateway(format!( "the upstream answered {status}" ))) } /// Fetch `path` from the upstream of proxy `repo` and cache it. `None` /// when the upstream does not have it (or the proxy is offline). - pub async fn fetch(&self, repo: &Repository, path: &str) -> Result, RawError> { + pub async fn fetch(&self, repo: &Repository, path: &str) -> Result, FileError> { if !repo.online || self.negative_cached(repo, path).await { return Ok(None); } @@ -300,15 +314,19 @@ impl Raw<'_> { if self.negative_cached(repo, path).await { return Ok(None); } - let response = self.upstream_get(repo, path, None).await?; + let Some(url) = self.upstream_url(repo, path).await? else { + self.remember_not_found(repo, path).await; + return Ok(None); + }; + let response = self.upstream_get(repo, path, &url, None).await?; let status = response.status(); if matches!(status, StatusCode::NOT_FOUND | StatusCode::GONE) { self.remember_not_found(repo, path).await; return Ok(None); } if !status.is_success() { - tracing::warn!(repo = %repo.name, %path, %status, "raw proxy: upstream error"); - return Err(RawError::BadGateway(format!( + tracing::warn!(repo = %repo.name, %path, %status, "file proxy: upstream error"); + return Err(FileError::BadGateway(format!( "the upstream answered {status}" ))); } @@ -322,26 +340,3 @@ fn is_due(file: &RawFile, max_age: u64) -> bool { file.checked_at .is_none_or(|checked| chrono::Utc::now() - checked >= max_age) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn upstream_urls_append_encoded_segments() { - let base = "https://github.com/bblanchon/pdfium-binaries/releases/download/"; - assert_eq!( - upstream_url(base, "chromium/7350/pdfium-linux-x64.tgz").unwrap(), - "https://github.com/bblanchon/pdfium-binaries/releases/download/chromium/7350/pdfium-linux-x64.tgz" - ); - // Without a trailing slash the base is a directory all the same. - assert_eq!( - upstream_url("http://up.test/files", "a b/c?d#e.txt").unwrap(), - "http://up.test/files/a%20b/c%3Fd%23e.txt" - ); - assert_eq!( - upstream_url("http://up.test", "x.bin").unwrap(), - "http://up.test/x.bin" - ); - } -} diff --git a/crates/server/src/raw/serve.rs b/crates/server/src/files/serve.rs similarity index 72% rename from crates/server/src/raw/serve.rs rename to crates/server/src/files/serve.rs index 171ac596..f52c06e8 100644 --- a/crates/server/src/raw/serve.rs +++ b/crates/server/src/files/serve.rs @@ -1,70 +1,21 @@ -//! Answering for a stored file: its bytes (whole or one range), its -//! validators and conditional requests, and its checksum sidecars. +//! Serving a stored file (the `serve-file` action): its bytes (whole or one +//! range), its validators and conditional requests, its content type and +//! the repository's content disposition. use std::time::{Duration, SystemTime, UNIX_EPOCH}; use axum::body::Body; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; -use base64::Engine as _; use chrono::{DateTime, Utc}; use forge_db::model::Repository; use forge_db::raw_dao::{self, RawFile}; +use forge_plugin::api::files; use forge_storage::BlobRef; -use super::path; use crate::feed::{byte_range, ByteRange}; use crate::state::AppState; -/// A checksum sidecar: `.` answered from the file's stored -/// digests when no file of that name is stored. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Checksum { - Sha256, - Sha512, - Sha1, - Md5, -} - -impl Checksum { - const ALL: [Checksum; 4] = [ - Checksum::Sha256, - Checksum::Sha512, - Checksum::Sha1, - Checksum::Md5, - ]; - - fn extension(self) -> &'static str { - match self { - Checksum::Sha256 => ".sha256", - Checksum::Sha512 => ".sha512", - Checksum::Sha1 => ".sha1", - Checksum::Md5 => ".md5", - } - } - - /// The digest of `file` as lowercase hex. - pub fn hex(self, file: &RawFile) -> String { - match self { - Checksum::Sha256 => file.blob_sha256.clone(), - Checksum::Sha512 => base64::engine::general_purpose::STANDARD - .decode(&file.sha512_base64) - .map(hex::encode) - .unwrap_or_default(), - Checksum::Sha1 => file.sha1.clone(), - Checksum::Md5 => file.md5.clone(), - } - } -} - -/// The file a sidecar path is the checksum of, and which checksum. -pub fn sidecar_of(path: &str) -> Option<(&str, Checksum)> { - Checksum::ALL.into_iter().find_map(|checksum| { - let base = path.strip_suffix(checksum.extension())?; - (!base.is_empty() && !base.ends_with('/')).then_some((base, checksum)) - }) -} - /// The `ETag` of a file: its SHA-256, a strong validator. pub fn etag(file: &RawFile) -> String { format!("\"{}\"", file.blob_sha256) @@ -122,7 +73,7 @@ pub fn content_disposition(repo: &Repository, file_path: &str) -> HeaderValue { } else { "attachment" }; - let name = path::file_name(file_path); + let name = files::file_name(file_path); let plain = name .chars() .all(|c| c.is_ascii() && !c.is_ascii_control() && c != '"' && c != '\\'); @@ -200,7 +151,7 @@ pub async fn file( match state.blob_store.open(blob, span.clone()).await { Ok(stream) => Body::from_stream(stream), Err(error) if error.is_unavailable() => { - tracing::warn!(%error, path = %file.path, "raw: object store unavailable"); + tracing::warn!(%error, path = %file.path, "files: object store unavailable"); return ( StatusCode::SERVICE_UNAVAILABLE, [(header::RETRY_AFTER, "30")], @@ -209,7 +160,7 @@ pub async fn file( .into_response(); } Err(error) => { - tracing::error!(%error, path = %file.path, s3_key = %file.s3_key, "raw: blob read failed"); + tracing::error!(%error, path = %file.path, s3_key = %file.s3_key, "files: blob read failed"); return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(); } } @@ -240,56 +191,13 @@ pub async fn file( let id = file.id; tokio::spawn(async move { if let Err(error) = raw_dao::touch_downloaded(&pool, id).await { - tracing::debug!(%error, "raw: could not record a download"); + tracing::debug!(%error, "files: could not record a download"); } }); } response } -/// Answer `GET`/`HEAD` of the `checksum` sidecar of `base`: the digest and -/// the file name, as `sha256sum` and friends print them, so `sha256sum -c` -/// checks a download against it. -pub fn sidecar( - repo: &Repository, - sidecar_path: &str, - base: &RawFile, - checksum: Checksum, - request: &HeaderMap, - head: bool, -) -> Response { - let etag = format!( - "\"{}{}\"", - base.blob_sha256, - checksum.extension().replace('.', "-") - ); - let modified = http_date(base.updated_at); - if not_modified(request, &etag, base.updated_at) { - let mut response = StatusCode::NOT_MODIFIED.into_response(); - validators(response.headers_mut(), &etag, &modified); - return response; - } - let text = format!("{} {}\n", checksum.hex(base), path::file_name(&base.path)); - let length = text.len(); - let mut response = if head { - Response::new(Body::empty()) - } else { - Response::new(Body::from(text)) - }; - let out = response.headers_mut(); - out.insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - out.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); - out.insert( - header::CONTENT_DISPOSITION, - content_disposition(repo, sidecar_path), - ); - validators(out, &etag, &modified); - response -} - #[cfg(test)] mod tests { use super::*; @@ -306,8 +214,7 @@ mod tests { s3_key: "k".into(), size: 10, content_type: "application/gzip".into(), - // SHA-512 of nothing in particular: 64 zero bytes. - sha512_base64: base64::engine::general_purpose::STANDARD.encode([0u8; 64]), + sha512_base64: String::new(), sha1: "cd".repeat(20), md5: "ef".repeat(16), created_at: t, @@ -319,21 +226,6 @@ mod tests { } } - #[test] - fn sidecars_name_their_file() { - assert_eq!( - sidecar_of("a/x.tgz.sha256"), - Some(("a/x.tgz", Checksum::Sha256)) - ); - assert_eq!(sidecar_of("x.md5"), Some(("x", Checksum::Md5))); - assert_eq!(sidecar_of(".sha1"), None); - assert_eq!(sidecar_of("a/.sha1"), None); - assert_eq!(sidecar_of("a/x.tgz"), None); - let file = raw_file(); - assert_eq!(Checksum::Sha512.hex(&file), "00".repeat(64)); - assert_eq!(Checksum::Md5.hex(&file), "ef".repeat(16)); - } - #[test] fn preconditions() { let file = raw_file(); diff --git a/crates/server/src/raw/upload.rs b/crates/server/src/files/store.rs similarity index 53% rename from crates/server/src/raw/upload.rs rename to crates/server/src/files/store.rs index b143925e..6f57faa6 100644 --- a/crates/server/src/raw/upload.rs +++ b/crates/server/src/files/store.rs @@ -1,35 +1,36 @@ -//! Changing files of a hosted raw repository: `PUT` of one file, a -//! multipart form of several (the web UI's upload, and the form fields of -//! Nexus' component upload), and `DELETE`. +//! Changing the files of a hosted repository: receiving a multipart form +//! (the `form` query), storing files from the request body or the form's +//! parts (`store`), and deleting a file (`remove`), for the format's plugin +//! and for the web UI. //! //! An upload is streamed to the staging directory under the repository's //! upload limit, hashed on the way (SHA-256 and SHA-512 by the CAS, SHA-1 -//! and MD5 here, for the checksum sidecars), stored content-addressed, and +//! and MD5 here, for checksum sidecars), stored content-addressed, and //! recorded in one transaction with the deployment policy applied under the //! row lock: a path that exists is replaced only with `allow_redeploy` and //! the delete permission, and the same content again is accepted as is. -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use axum::body::Body; use axum::extract::Request; -use axum::http::{header, StatusCode}; -use axum::response::{IntoResponse, Json, Response}; +use axum::http::header; use bytes::Bytes; use forge_core::auth::principal::{AuthMethod, Principal}; use forge_core::authz::Action; use forge_core::repo::{PackageMutation, PublishAuthorizationContext, WriteError}; use forge_db::model::{DeploymentPolicy, RepoKind, Repository}; use forge_db::raw_dao::{self, NewRawFile, Redeploy, StoreOutcome}; +use forge_plugin::api::files; use forge_storage::{CasError, StoredBlob}; use futures_util::{Stream, TryStreamExt}; use md5::Md5; use serde::Serialize; use sha1::{Digest as _, Sha1}; -use super::{path, Raw, RawError}; +use super::{FileError, Files}; use crate::dispatch::principal_allows_protocol_mutations; /// Most files one multipart upload may carry. @@ -41,7 +42,7 @@ const MAX_TEXT_FIELD_BYTES: usize = 4096; /// Who may change files through a request. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Via { - /// The `/raw` protocol: an API token or a workload identity, as for + /// The format's protocol: an API token or a workload identity, as for /// every package client (never a browser session). Protocol, /// The web UI's endpoints: a signed-in session (whose CSRF token the @@ -63,8 +64,37 @@ struct Digests { md5: Md5, } +/// One part of a received form. +pub enum Part { + /// A file part: its declared name and content type, staged and stored. + File { + field: String, + name: String, + content_type: Option, + ingested: Box, + }, + /// Any other field. + Text { field: String, value: String }, +} + +/// A file to store, and where its content comes from. +pub struct Upload { + pub path: String, + /// The content type the client declared. + pub declared: Option, + pub content: Content, +} + +/// The content of an [`Upload`]. +pub enum Content { + /// The request body, still to be read. + Body(Request), + /// Received already (a form's file part). + Staged(Ingested), +} + /// Stage `body` (at most `limit` bytes) and store it content-addressed. -pub async fn ingest(raw: &Raw<'_>, body: S, limit: u64) -> Result +pub async fn ingest(files: &Files<'_>, body: S, limit: u64) -> Result where S: Stream> + Send + Unpin, E: Into>, @@ -78,14 +108,14 @@ where d.sha1.update(chunk); d.md5.update(chunk); }); - let staged = raw + let staged = files .state .cas .stage(tokio_util::io::StreamReader::new(body), limit) .await .map_err(cas_error)?; let digests = std::mem::take(&mut *digests.lock().unwrap_or_else(PoisonError::into_inner)); - let stored = raw.state.cas.commit(staged).await.map_err(cas_error)?; + let stored = files.state.cas.commit(staged).await.map_err(cas_error)?; Ok(Ingested { stored, sha1: hex::encode(digests.sha1.finalize()), @@ -94,15 +124,15 @@ where } /// An ingest failure as the answer to the request. -pub fn cas_error(error: CasError) -> RawError { +pub fn cas_error(error: CasError) -> FileError { match error { - CasError::TooLarge { limit } => RawError::TooLarge(limit), + CasError::TooLarge { limit } => FileError::TooLarge(limit), CasError::StagingFull { .. } => { - RawError::Unavailable("temporary staging space is exhausted; retry later".into()) + FileError::Unavailable("temporary staging space is exhausted; retry later".into()) } - e if e.is_unavailable() => RawError::Unavailable("object store unavailable".into()), - CasError::Io(e) => RawError::BadRequest(format!("the upload could not be read: {e}")), - e => RawError::Internal(format!("store upload: {e}")), + e if e.is_unavailable() => FileError::Unavailable("object store unavailable".into()), + CasError::Io(e) => FileError::BadRequest(format!("the upload could not be read: {e}")), + e => FileError::Internal(format!("store upload: {e}")), } } @@ -136,35 +166,47 @@ pub fn content_type(declared: Option<&str>, path: &str) -> String { } } -/// The answer to an upload or delete that needs `action` on `repo`, when it -/// may not happen; `None` when it may. -async fn refusal(raw: &Raw<'_>, repo: &Repository, action: Action, via: Via) -> Option { +/// A path the format named, checked against the store's rules. +fn checked_path(path: &str) -> Result<&str, FileError> { + files::validate_path(path) + .map(|()| path) + .map_err(|e| FileError::Internal(format!("the format named path {path:?}: {e}"))) +} + +/// Why an upload or delete that needs `action` on `repo` may not happen; +/// `None` when it may. +pub async fn refusal( + files: &Files<'_>, + repo: &Repository, + action: Action, + via: Via, +) -> Option { if repo.kind() != RepoKind::Hosted { - return Some(RawError::MethodNotAllowed( + return Some(FileError::MethodNotAllowed( "files are uploaded to and deleted from hosted repositories", )); } let credential = match via { - Via::Protocol => principal_allows_protocol_mutations(&raw.principal), + Via::Protocol => principal_allows_protocol_mutations(&files.principal), Via::Browser => { matches!( - raw.principal, + files.principal, Principal::User { method, .. } if method != AuthMethod::Password - ) || matches!(raw.principal, Principal::Workload { .. }) + ) || matches!(files.principal, Principal::Workload { .. }) } }; if !credential { - return Some(RawError::CredentialRequired); + return Some(FileError::CredentialRequired); } let allowed = match action { - Action::Delete => raw.state.authz.can_delete(&raw.principal, repo).await, - _ => raw.state.authz.can_write(&raw.principal, repo).await, + Action::Delete => files.state.authz.can_delete(&files.principal, repo).await, + _ => files.state.authz.can_write(&files.principal, repo).await, }; if !allowed { - return Some(RawError::Denied); + return Some(FileError::Denied); } if repo.deployment_policy() == DeploymentPolicy::ReadOnly { - return Some(RawError::MethodNotAllowed("the repository is read-only")); + return Some(FileError::MethodNotAllowed("the repository is read-only")); } None } @@ -172,18 +214,18 @@ async fn refusal(raw: &Raw<'_>, repo: &Repository, action: Action, via: Via) -> /// Record a workload's change in the trusted-publishing ledger, inside the /// transaction that makes it. async fn record( - raw: &Raw<'_>, + files: &Files<'_>, repo: &Repository, action: PackageMutation, path: &str, sha256: &str, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, -) -> Result<(), RawError> { - if !matches!(raw.principal, Principal::Workload { .. }) { +) -> Result<(), FileError> { + if !matches!(files.principal, Principal::Workload { .. }) { return Ok(()); } let context = PublishAuthorizationContext { - principal: raw.principal, + principal: files.principal, action, package_id: path.to_string(), package_version: sha256.to_string(), @@ -192,10 +234,10 @@ async fn record( .record_in_transaction(&context, tx) .await .map_err(|e| match e { - WriteError::ForbiddenPackage { .. } => RawError::Forbidden( + WriteError::ForbiddenPackage { .. } => FileError::Forbidden( "the workload identity may no longer change this repository".into(), ), - other => RawError::Internal(format!("trusted-publishing record: {other}")), + other => FileError::Internal(format!("trusted-publishing record: {other}")), }) } @@ -221,37 +263,37 @@ pub struct Stored { /// are locked in the order every other writer takes them in; the answer /// lists them in the order given. async fn record_all( - raw: &Raw<'_>, + files: &Files<'_>, repo: &Repository, - files: Vec, -) -> Result, RawError> { + pending: Vec, +) -> Result, FileError> { let policy = repo.deployment_policy(); - let replace = raw + let replace = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Delete) + .package_filter_for(&files.principal, repo, Action::Delete) .await; - let may_delete = raw.state.authz.can_delete(&raw.principal, repo).await; - let grace = raw.state.config.outbox.grace(); - let mut tx = raw.state.pool.begin().await?; - let mut order: Vec = (0..files.len()).collect(); + let may_delete = files.state.authz.can_delete(&files.principal, repo).await; + let grace = files.state.config.outbox.grace(); + let mut tx = files.state.pool.begin().await?; + let mut order: Vec = (0..pending.len()).collect(); order.sort_by(|&a, &b| { - files[a] + pending[a] .ingested .stored .sha256 - .cmp(&files[b].ingested.stored.sha256) + .cmp(&pending[b].ingested.stored.sha256) }); - let mut out: Vec> = (0..files.len()).map(|_| None).collect(); + let mut out: Vec> = (0..pending.len()).map(|_| None).collect(); for index in order { - let file = &files[index]; + let file = &pending[index]; let stored = &file.ingested.stored; let replaceable = policy == DeploymentPolicy::AllowRedeploy && may_delete && replace.allows(&file.path); let new = NewRawFile { path: &file.path, sha256: &stored.sha256, - size: i64::try_from(stored.size).map_err(|_| RawError::TooLarge(stored.size))?, + size: i64::try_from(stored.size).map_err(|_| FileError::TooLarge(stored.size))?, content_type: &file.content_type, sha512_base64: &stored.sha512_base64, sha1: &file.ingested.sha1, @@ -273,13 +315,13 @@ async fn record_all( StoreOutcome::Replaced(_) => "replaced", StoreOutcome::Unchanged(_) => "unchanged", StoreOutcome::Exists if policy == DeploymentPolicy::AllowRedeploy => { - return Err(RawError::Forbidden(format!( + return Err(FileError::Forbidden(format!( "{} exists; replacing a file needs the delete permission", file.path ))); } StoreOutcome::Exists => { - return Err(RawError::Conflict(format!( + return Err(FileError::Conflict(format!( "{} exists and the repository does not allow redeploying it", file.path ))); @@ -287,7 +329,7 @@ async fn record_all( }; if status != "unchanged" { record( - raw, + files, repo, PackageMutation::Publish, &file.path, @@ -307,100 +349,73 @@ async fn record_all( Ok(out.into_iter().flatten().collect()) } -/// `PUT /raw//`: store the request body as the file. -pub async fn put(raw: &Raw<'_>, repo: &Repository, path: &str, req: Request) -> Response { - match put_inner(raw, repo, path, req).await { - Ok(Stored { - status: "created", .. - }) => StatusCode::CREATED.into_response(), - Ok(_) => StatusCode::NO_CONTENT.into_response(), - Err(e) => e.into_response(), - } -} - -async fn put_inner( - raw: &Raw<'_>, +/// Store `uploads` in hosted `repo`, all of them or none, with the caller's +/// write permission (and selector) on every path. +pub async fn store( + files: &Files<'_>, repo: &Repository, - path: &str, - req: Request, -) -> Result { - if let Some(refused) = refusal(raw, repo, Action::Write, Via::Protocol).await { + uploads: Vec, + via: Via, +) -> Result, FileError> { + if let Some(refused) = refusal(files, repo, Action::Write, via).await { return Err(refused); } - let write = raw + let write = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Write) + .package_filter_for(&files.principal, repo, Action::Write) .await; - if !write.allows(path) { - return Err(RawError::Forbidden(format!("you may not upload to {path}"))); - } - let limit = raw.state.upload_limit(repo); - let declared_length = req - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - if declared_length.is_some_and(|len| len > limit) { - return Err(RawError::TooLarge(limit)); - } - let declared = req - .headers() - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .map(str::to_string); - let ingested = ingest(raw, req.into_body().into_data_stream(), limit).await?; - let pending = Pending { - path: path.to_string(), - content_type: content_type(declared.as_deref(), path), - ingested, - }; - let mut stored = record_all(raw, repo, vec![pending]).await?; - Ok(stored.remove(0)) -} - -/// `POST /raw///` (and the web UI's upload): a -/// `multipart/form-data` body whose file parts are stored under -/// `directory`, each at its part's file name. Nexus' field names work too: -/// `raw.directory` (or `directory`) names a subdirectory, and -/// `.filename` (`raw.asset1.filename`) the name of the file part -/// ``. Every file is recorded, or none. -pub async fn multipart( - raw: &Raw<'_>, - repo: &Repository, - directory: &str, - req: Request, - via: Via, -) -> Response { - match multipart_inner(raw, repo, directory, req, via).await { - Ok(files) => { - let status = if files.iter().any(|f| f.status == "created") { - StatusCode::CREATED - } else { - StatusCode::OK - }; - (status, Json(serde_json::json!({ "files": files }))).into_response() + let mut seen = HashSet::new(); + for upload in &uploads { + let path = checked_path(&upload.path)?; + if !seen.insert(path) { + return Err(FileError::BadRequest(format!( + "the upload holds {path} twice" + ))); + } + if !write.allows(path) { + return Err(FileError::Forbidden(format!( + "you may not upload to {path}" + ))); } - Err(e) => e.into_response(), } + let limit = files.state.upload_limit(repo); + let mut pending = Vec::with_capacity(uploads.len()); + for upload in uploads { + let ingested = match upload.content { + Content::Staged(ingested) => ingested, + Content::Body(req) => { + let declared_length = req + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + if declared_length.is_some_and(|len| len > limit) { + return Err(FileError::TooLarge(limit)); + } + ingest(files, req.into_body().into_data_stream(), limit).await? + } + }; + pending.push(Pending { + content_type: content_type(upload.declared.as_deref(), &upload.path), + path: upload.path, + ingested, + }); + } + record_all(files, repo, pending).await } -/// A file part, staged and stored, its name to be decided. -struct FilePart { - field: String, - file_name: String, - declared: Option, - ingested: Ingested, -} - -async fn multipart_inner( - raw: &Raw<'_>, +/// Receive the `multipart/form-data` body of `req` for hosted `repo`: every +/// file part staged and stored (at most [`MAX_FILES_PER_UPLOAD`], each +/// within the repository's upload limit), every other field read (at most +/// [`MAX_TEXT_FIELD_BYTES`]), in the order of the form. +pub async fn receive_form( + files: &Files<'_>, repo: &Repository, - directory: &str, req: Request, via: Via, -) -> Result, RawError> { - if let Some(refused) = refusal(raw, repo, Action::Write, via).await { +) -> Result, FileError> { + if let Some(refused) = refusal(files, repo, Action::Write, via).await { return Err(refused); } let Some(boundary) = req @@ -409,11 +424,11 @@ async fn multipart_inner( .and_then(|v| v.to_str().ok()) .and_then(crate::feed::multipart_boundary) else { - return Err(RawError::BadRequest( + return Err(FileError::BadRequest( "upload files as multipart/form-data (or PUT a single file to its path)".into(), )); }; - let limit = raw.state.upload_limit(repo); + let limit = files.state.upload_limit(repo); let overrun = Arc::new(AtomicBool::new(false)); let guarded = crate::feed::preamble_guard( req.into_body().into_data_stream(), @@ -424,31 +439,32 @@ async fn multipart_inner( .size_limit(multer::SizeLimit::new().per_field(limit.saturating_add(1))); let mut form = multer::Multipart::with_constraints(guarded, boundary, constraints); let form_error = |e: multer::Error| match e { - multer::Error::FieldSizeExceeded { .. } => RawError::TooLarge(limit), + multer::Error::FieldSizeExceeded { .. } => FileError::TooLarge(limit), _ if overrun.load(Ordering::Relaxed) => { - RawError::BadRequest("multipart body: no part within the first 64 KiB".into()) + FileError::BadRequest("multipart body: no part within the first 64 KiB".into()) } - e => RawError::BadRequest(format!("multipart body: {e}")), + e => FileError::BadRequest(format!("multipart body: {e}")), }; - let mut parts: Vec = Vec::new(); - let mut texts: BTreeMap = BTreeMap::new(); + let mut parts: Vec = Vec::new(); + let mut file_parts = 0; while let Some(mut part) = form.next_field().await.map_err(form_error)? { let field = part.name().unwrap_or("").to_string(); match part.file_name().map(str::to_string) { - Some(file_name) => { - if parts.len() == MAX_FILES_PER_UPLOAD { - return Err(RawError::BadRequest(format!( + Some(name) => { + if file_parts == MAX_FILES_PER_UPLOAD { + return Err(FileError::BadRequest(format!( "at most {MAX_FILES_PER_UPLOAD} files per upload" ))); } - let declared = part.content_type().map(|m| m.to_string()); - let ingested = ingest(raw, part, limit).await?; - parts.push(FilePart { + file_parts += 1; + let content_type = part.content_type().map(|m| m.to_string()); + let ingested = ingest(files, part, limit).await?; + parts.push(Part::File { field, - file_name, - declared, - ingested, + name, + content_type, + ingested: Box::new(ingested), }); } None => { @@ -456,110 +472,50 @@ async fn multipart_inner( while let Some(chunk) = part.chunk().await.map_err(form_error)? { text.extend_from_slice(&chunk); if text.len() > MAX_TEXT_FIELD_BYTES { - return Err(RawError::BadRequest(format!( + return Err(FileError::BadRequest(format!( "form field {field:?} is longer than {MAX_TEXT_FIELD_BYTES} bytes" ))); } } - let text = String::from_utf8(text).map_err(|_| { - RawError::BadRequest(format!("form field {field:?} is not UTF-8")) + let value = String::from_utf8(text).map_err(|_| { + FileError::BadRequest(format!("form field {field:?} is not UTF-8")) })?; - texts.insert(field, text); - } - } - } - if parts.is_empty() { - return Err(RawError::BadRequest("the form holds no file".into())); - } - - // The fields: a subdirectory, and the names of file parts. - let mut subdirectory = String::new(); - let fields: HashSet<&str> = parts.iter().map(|p| p.field.as_str()).collect(); - for (name, value) in &texts { - match name.as_str() { - "directory" | "raw.directory" => subdirectory = value.clone(), - other - if other - .strip_suffix(".filename") - .is_some_and(|field| fields.contains(field)) => {} - other => { - return Err(RawError::BadRequest(format!( - "unknown form field {other:?} (file parts, directory and .filename are accepted)" - ))) + parts.push(Part::Text { field, value }); } } } - let base = path::normalize_directory(&subdirectory) - .map_err(|e| RawError::BadRequest(format!("directory: {e}")))?; - let directory = format!("{directory}{base}"); - - let write = raw - .state - .authz - .package_filter_for(&raw.principal, repo, Action::Write) - .await; - let mut seen = HashSet::new(); - let mut pending = Vec::with_capacity(parts.len()); - for part in parts { - let name = texts - .get(&format!("{}.filename", part.field)) - .unwrap_or(&part.file_name); - let target = path::join(&directory, name) - .map_err(|e| RawError::BadRequest(format!("file name {name:?}: {e}")))?; - if !seen.insert(target.clone()) { - return Err(RawError::BadRequest(format!( - "the form holds {target} twice" - ))); - } - if !write.allows(&target) { - return Err(RawError::Forbidden(format!( - "you may not upload to {target}" - ))); - } - pending.push(Pending { - content_type: content_type(part.declared.as_deref(), &target), - path: target, - ingested: part.ingested, - }); - } - record_all(raw, repo, pending).await -} - -/// `DELETE /raw//` (and the web UI's delete). -pub async fn delete(raw: &Raw<'_>, repo: &Repository, path: &str, via: Via) -> Response { - match delete_inner(raw, repo, path, via).await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(e) => e.into_response(), - } + Ok(parts) } -async fn delete_inner( - raw: &Raw<'_>, +/// Delete the file at `path` of hosted `repo`: `false` when there is none +/// the caller may delete (one outside its selector is as unknown). +pub async fn remove( + files: &Files<'_>, repo: &Repository, path: &str, via: Via, -) -> Result<(), RawError> { - if let Some(refused) = refusal(raw, repo, Action::Delete, via).await { +) -> Result { + if let Some(refused) = refusal(files, repo, Action::Delete, via).await { return Err(refused); } - let filter = raw + let path = checked_path(path)?; + let filter = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Delete) + .package_filter_for(&files.principal, repo, Action::Delete) .await; - // Outside the caller's selector: as unknown. if !filter.allows(path) { - return Err(RawError::NotFound); + return Ok(false); } - let Some(file) = raw_dao::find(&raw.state.pool, repo.id, path).await? else { - return Err(RawError::NotFound); + let Some(file) = raw_dao::find(&files.state.pool, repo.id, path).await? else { + return Ok(false); }; - let mut tx = raw.state.pool.begin().await?; - if !raw_dao::delete(&mut tx, repo.id, path, raw.state.config.outbox.grace()).await? { - return Err(RawError::NotFound); + let mut tx = files.state.pool.begin().await?; + if !raw_dao::delete(&mut tx, repo.id, path, files.state.config.outbox.grace()).await? { + return Ok(false); } record( - raw, + files, repo, PackageMutation::Delete, path, @@ -568,7 +524,7 @@ async fn delete_inner( ) .await?; tx.commit().await?; - Ok(()) + Ok(true) } #[cfg(test)] diff --git a/crates/server/src/raw/web.rs b/crates/server/src/files/web.rs similarity index 64% rename from crates/server/src/raw/web.rs rename to crates/server/src/files/web.rs index c8369bb1..631adfb2 100644 --- a/crates/server/src/raw/web.rs +++ b/crates/server/src/files/web.rs @@ -1,12 +1,15 @@ -//! The raw endpoints of the web UI and the admin API. +//! The file store's endpoints of the web UI and the admin API, for the +//! repositories of a files format (raw). //! -//! The web UI reads listings from `/raw///` itself (as JSON, with -//! its session). Changing files through `/raw` takes an API token, like -//! every package client, so the UI changes them here, with its session -//! (whose CSRF token the principal middleware checks): +//! The web UI reads listings from its format's protocol itself +//! (`/raw///` as JSON, with its session). Changing files through +//! `/raw` takes an API token, like every package client, so the UI changes +//! them here, with its session (whose CSRF token the principal middleware +//! checks): //! //! * `POST /browse/raw/{repo}/files?directory=` — a multipart upload -//! into `` ([`upload::multipart`]); +//! into ``, each file part stored at its file name +//! ([`store::receive_form`], [`store::store`]); //! * `DELETE /browse/raw/{repo}/files?path=` — delete a file. //! //! Both take the same permissions as `/raw`: write (and, to replace a @@ -23,9 +26,11 @@ use axum::response::{IntoResponse, Json, Response}; use forge_core::auth::principal::Principal; use forge_db::model::{RepoKind, Repository}; use forge_db::{raw_dao, repository_dao}; +use forge_plugin::api::files; use serde::Deserialize; -use super::{path, upload, Raw, RawError}; +use super::store::{self, Content, Part, Upload, Via}; +use super::{FileError, Files}; use crate::middleware::principal_of; use crate::state::AppState; @@ -44,7 +49,7 @@ pub struct PathQuery { pub path: String, } -/// The raw repository `name` for a signed-in caller (`/browse` is for +/// The repository of files `name` for a signed-in caller (`/browse` is for /// signed-in users only). async fn browse_target( state: &AppState, @@ -55,9 +60,14 @@ async fn browse_target( return Err((StatusCode::UNAUTHORIZED, "authentication required").into_response()); } match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(repo)) if repo.is_raw() && state.authz.can_read(principal, &repo).await => Ok(repo), + Ok(Some(repo)) + if state.formats.stores_files(&repo.format) + && state.authz.can_read(principal, &repo).await => + { + Ok(repo) + } Ok(_) => Err((StatusCode::NOT_FOUND, "repository not found").into_response()), - Err(e) => Err(RawError::from(e).into_response()), + Err(e) => Err(FileError::from(e).into_response()), } } @@ -73,19 +83,65 @@ pub async fn browse_upload( Ok(repo) => repo, Err(response) => return response, }; - let directory = match path::normalize_directory(&query.directory) { + let directory = match files::normalize_directory(&query.directory) { Ok(dir) => dir, Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), }; - let raw = Raw { + let files = Files { state: &state, principal, }; - let response = upload::multipart(&raw, &repo, &directory, req, upload::Via::Browser).await; + let response = match upload(&files, &repo, &directory, req).await { + Ok(stored) => { + let status = if stored.iter().any(|f| f.status == "created") { + StatusCode::CREATED + } else { + StatusCode::OK + }; + (status, Json(serde_json::json!({ "files": stored }))).into_response() + } + Err(e) => e.into_response(), + }; audit(&repo, "POST", &directory, &principal, &response); response } +/// Store the file parts of the form `req` carries under `directory`, each +/// at its file name. +async fn upload( + files: &Files<'_>, + repo: &Repository, + directory: &str, + req: Request, +) -> Result, FileError> { + let parts = store::receive_form(files, repo, req, Via::Browser).await?; + let mut uploads = Vec::with_capacity(parts.len()); + for part in parts { + match part { + Part::File { + name, + content_type, + ingested, + .. + } => uploads.push(Upload { + path: files::join(directory, &name) + .map_err(|e| FileError::BadRequest(format!("file name {name:?}: {e}")))?, + declared: content_type, + content: Content::Staged(*ingested), + }), + Part::Text { field, .. } => { + return Err(FileError::BadRequest(format!( + "unknown form field {field:?} (only files are accepted)" + ))) + } + } + } + if uploads.is_empty() { + return Err(FileError::BadRequest("the form holds no file".into())); + } + store::store(files, repo, uploads, Via::Browser).await +} + /// `DELETE /browse/raw/{repo}/files?path=` pub async fn browse_delete( State(state): State, @@ -98,18 +154,22 @@ pub async fn browse_delete( Ok(repo) => repo, Err(response) => return response, }; - let file = match path::normalize(&query.path) { + let file = match files::normalize(&query.path) { Ok(file) if !file.is_empty() && !query.path.ends_with('/') => file, Ok(_) => { return (StatusCode::BAD_REQUEST, "path must name a file").into_response(); } Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), }; - let raw = Raw { + let files = Files { state: &state, principal, }; - let response = upload::delete(&raw, &repo, &file, upload::Via::Browser).await; + let response = match store::remove(&files, &repo, &file, Via::Browser).await { + Ok(true) => StatusCode::NO_CONTENT.into_response(), + Ok(false) => FileError::NotFound.into_response(), + Err(e) => e.into_response(), + }; audit(&repo, "DELETE", &file, &principal, &response); response } @@ -137,10 +197,10 @@ pub async fn admin_delete( Ok(None) => return (StatusCode::NOT_FOUND, "repository not found").into_response(), Err(e) => return crate::admin::db_error(e), }; - if !repo.is_raw() { + if !state.formats.stores_files(&repo.format) { return ( StatusCode::BAD_REQUEST, - "files are deleted from raw repositories", + "files are deleted from repositories of files (raw)", ) .into_response(); } @@ -153,9 +213,9 @@ pub async fn admin_delete( } let directory = query.path.is_empty() || query.path.ends_with('/'); let normalized = match if directory { - path::normalize_directory(&query.path) + files::normalize_directory(&query.path) } else { - path::normalize(&query.path) + files::normalize(&query.path) } { Ok(p) => p, Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 503d18de..a67418c8 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -23,6 +23,7 @@ pub mod crypto; pub mod dispatch; pub mod email; pub mod feed; +pub mod files; pub mod health; pub mod invitations; pub mod me; @@ -31,7 +32,6 @@ pub mod middleware; pub mod nexus_import; pub mod object_store; pub mod plugins; -pub mod raw; pub mod registry; pub mod serve; pub mod state; diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index 89314ea1..54c62195 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -12,21 +12,26 @@ //! * **Queries** the plugin asks for run through the same read ops as every //! other read ([`AppState::read_ops_for`]): package selectors, group //! members the caller cannot see, prefer-hosted and reserved prefixes all -//! apply, and an id the format does not accept is unknown. A request may -//! load at most [`MAX_QUERIES`] answers in at most [`MAX_ROUNDS`] rounds. +//! apply, and an id the format does not accept is unknown. A files +//! format's queries go to the file store ([`crate::files`]), with the same +//! selectors on paths. A request may load at most [`MAX_QUERIES`] answers +//! in at most [`MAX_ROUNDS`] rounds. //! * **Downloads** are streamed from the blob store and counted, exactly as -//! NuGet's, answering a single `Range` when the blob source serves ranges. +//! NuGet's, answering a single `Range` when the blob source serves ranges; +//! a file is served by the file store ([`crate::files::serve`]). //! * **Publishing, unlisting, relisting and deleting** go through the hosted //! write path (CAS staging, size cap, selector and redeploy policy, -//! trusted-publishing audit), and only when the request's method was +//! trusted-publishing audit), and receiving, storing and deleting files +//! through the file store's, only when the request's method was //! authorized for that action and carries an API token or workload //! identity. //! * **Responses** the plugin renders keep only the headers in -//! [`PASSED_RESPONSE_HEADERS`] (no cookies, no CORS, no redirects off this -//! repository) and get the feed's hardening headers. +//! [`PASSED_RESPONSE_HEADERS`] (no cookies, no CORS, no redirects or links +//! off this repository) and get the feed's hardening headers. +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use crate::feed::{byte_range, ByteRange, PackageBytes}; use axum::body::Body; @@ -42,10 +47,11 @@ use forge_core::repo::{ UploadReader, WriteError, }; use forge_db::model::Repository; +use forge_db::raw_dao::RawFile; use forge_db::repository_dao; use forge_plugin::api::{ self as protocol, Action, Answer, ChangeRequest, FormatPlugin, Loaded, PackageInfo, PackageRef, - PublishRequest, Query, Rendered, Upload, + PublishRequest, Query, Upload, }; use forge_plugin::Plugin; use futures_util::{StreamExt, TryStreamExt}; @@ -55,6 +61,8 @@ use crate::dispatch::{ action_for_method, authz_denied, offline_response, offline_serves_nothing, principal_allows_protocol_mutations, HostedWriteAdapter, }; +use crate::files::store::{Content, Ingested, Part, Via}; +use crate::files::{FileError, Files}; use crate::middleware::principal_of; use crate::state::AppState; @@ -82,7 +90,8 @@ pub const PASSED_REQUEST_HEADERS: &[&str] = &[ "user-agent", ]; -/// Response headers a plugin may set. `location` only within its repository. +/// Response headers a plugin may set. `location` and `link` only within its +/// repository. pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ "allow", "cache-control", @@ -91,18 +100,24 @@ pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ "content-type", "etag", "last-modified", + "link", "location", + "vary", ]; /// The format plugins built into this server: crates under `plugins//`, -/// linked in and always installed (NuGet). NuGet reads archives under +/// linked in and always installed (NuGet, raw). NuGet reads archives under /// `nuget_limits` (`[nuget]`). pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec> { - vec![Plugin::builtin( - forge_format_nuget::PLUGIN_TOML, - forge_format_nuget::Nuget::with_limits(nuget_limits), - ) - .expect("plugins/nuget/plugin.toml is valid")] + vec![ + Plugin::builtin( + forge_format_nuget::PLUGIN_TOML, + forge_format_nuget::Nuget::with_limits(nuget_limits), + ) + .expect("plugins/nuget/plugin.toml is valid"), + Plugin::builtin(forge_format_raw::PLUGIN_TOML, forge_format_raw::Raw) + .expect("plugins/raw/plugin.toml is valid"), + ] } /// Axum handler mounted at `/repository/{repo}` (and everything under it). @@ -113,13 +128,14 @@ pub async fn dispatch(State(state): State, req: Request) -> Resp } /// `(mount, repository name, rest of the path)` of a `{mount}/{repo}/...` -/// path, for one of `mounts`; the rest starts with `/`. +/// path, for one of `mounts`; the rest starts with `/`, or is empty when the +/// path ends with the repository's name. fn split_path<'p>(path: &'p str, mounts: &[&str]) -> Option<(&'p str, &'p str, &'p str)> { mounts.iter().find_map(|mount| { let rest = path.strip_prefix(mount)?.strip_prefix('/')?; let (name, rest) = match rest.find('/') { Some(i) => rest.split_at(i), - None => (rest, "/"), + None => (rest, ""), }; (!name.is_empty()).then(|| (&path[..mount.len()], name, rest)) }) @@ -182,13 +198,19 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { Ok(format) => format, Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(), }; + let stores_files = format.stores_files(); let package_filter = state .authz .package_filter_for(&principal, &repo, permission) .await; - let ops = match state.read_ops_for(&repo, principal).await { - Ok(ops) => ops, - Err(e) => return repo_error(e), + // A files format reads the file store, never the package read ops. + let ops = if stores_files { + None + } else { + match state.read_ops_for(&repo, principal).await { + Ok(ops) => Some(ops), + Err(e) => return repo_error(e), + } }; let target = protocol::Repository { @@ -199,6 +221,7 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { RepoKind::Group => protocol::RepositoryKind::Group, }, base_url: repository_url(&state, mount, &repo.name), + inline: stores_files && repo.inline_content(), }; let request = protocol::Request { method: method.as_str().to_string(), @@ -222,9 +245,13 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { headers: req.headers().clone(), plugin: plugin.implementation().clone(), format, + stores_files, ops, package_filter, target, + body: Mutex::new(Some(req)), + form: Mutex::new(None), + located: Mutex::new(HashMap::new()), }; let handled = { let (target, request) = (ctx.target.clone(), request.clone()); @@ -232,7 +259,7 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { .await }; let response = match handled { - Ok(action) => ctx.perform(action, request, req).await, + Ok(action) => ctx.perform(action, request).await, Err(response) => response, }; if permission != Permission::Read { @@ -264,10 +291,21 @@ struct Ctx<'a> { headers: axum::http::HeaderMap, plugin: Arc, format: Arc, - ops: Arc, + /// Whether the format's repositories hold files (its plugin asks for + /// files, not packages). + stores_files: bool, + /// The package read ops; `None` for a files format. + ops: Option>, /// The caller's package selector for [`Self::permission`]. package_filter: PackageFilter, target: protocol::Repository, + /// The request, until a publish or a file query reads its body. + body: Mutex>>, + /// The file parts of the form the plugin had received, by their index + /// among its parts, until a store takes them. + form: Mutex>>>, + /// The files queries found, by path, for serving one of them. + located: Mutex>, } impl Ctx<'_> { @@ -283,28 +321,61 @@ impl Ctx<'_> { .map_err(|e| plugin_failed(self.repo, &e)) } - async fn perform( - &self, - action: Action, - request: protocol::Request, - req: Request, - ) -> Response { + /// Do what the plugin decided, `handle` or the last round of `render`. + async fn perform(&self, action: Action, request: protocol::Request) -> Response { + if let Action::Load(queries) = action { + return self.render(queries, request).await; + } + self.act(action).await + } + + /// Do an action other than loading data. + async fn act(&self, action: Action) -> Response { + let files_action = matches!(action, Action::ServeFile(_)); + let package_action = !matches!(action, Action::Respond(_) | Action::ServeFile(_)); + if (files_action && !self.stores_files) || (package_action && self.stores_files) { + return self.plugin_misused("an action of the other storage"); + } match action { Action::Respond(response) => self.respond(response), - Action::Load(queries) => self.render(queries, request).await, + Action::Load(_) => self.plugin_misused("a load as an action"), Action::Download(target) => { if self.permission != Permission::Read { return refused("download"); } self.download(&target).await } - Action::Publish(publish) => self.publish(publish, req).await, + Action::Publish(publish) => match self.take_body() { + Some(req) => self.publish(publish, req).await, + None => self.plugin_misused("a publish after the body was read"), + }, Action::Unlist(change) => self.change(change, Change::Unlist).await, Action::Relist(change) => self.change(change, Change::Relist).await, Action::Delete(change) => self.change(change, Change::Delete).await, + Action::ServeFile(path) => { + if self.permission != Permission::Read { + return refused("serving a file"); + } + self.serve_file(&path).await + } } } + /// The request, for the one query or action that reads its body. + fn take_body(&self) -> Option> { + self.body + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + } + + /// The plugin asked for something the contract does not allow: a bug in + /// the plugin, answered with a server error. + fn plugin_misused(&self, what: &str) -> Response { + tracing::error!(repo = %self.repo.name, format = %self.repo.format, "format plugin asked for {what}"); + (StatusCode::INTERNAL_SERVER_ERROR, "format plugin failed").into_response() + } + /// The canonical id the format makes of `raw`, if it is a valid id. fn id(&self, raw: &str) -> Option { let id = self.format.lower_id(raw); @@ -325,6 +396,9 @@ impl Ctx<'_> { if data.len() + queries.len() > MAX_QUERIES { break; } + if let Err(response) = self.check_round(&queries) { + return response; + } let pending: Vec<_> = queries.iter().map(|query| self.answer(query)).collect(); let answers: Vec = match futures_util::stream::iter(pending) .buffered(CONCURRENT_QUERIES) @@ -332,7 +406,7 @@ impl Ctx<'_> { .await { Ok(answers) => answers, - Err(e) => return repo_error(e), + Err(response) => return response, }; data.extend( queries @@ -348,11 +422,11 @@ impl Ctx<'_> { }) .await; match rendered { - Ok((Rendered::Respond(response), _)) => return self.respond(response), - Ok((Rendered::Load(more), loaded)) => { + Ok((Action::Load(more), loaded)) => { data = loaded; queries = more; } + Ok((action, _)) => return self.act(action).await, Err(response) => return response, } } @@ -368,12 +442,226 @@ impl Ctx<'_> { .into_response() } - async fn answer(&self, query: &Query) -> Result { + /// Whether a round of queries keeps to the contract: queries of the + /// format's storage only, and a change (`form`, `store`, `remove`) alone + /// in its round. + fn check_round(&self, queries: &[Query]) -> Result<(), Response> { + if queries.iter().any(|q| q.is_files() != self.stores_files) { + return Err(self.plugin_misused("a query of the other storage")); + } + if queries.len() > 1 && queries.iter().any(Query::is_change) { + return Err(self.plugin_misused("a change with other queries in its round")); + } + Ok(()) + } + + /// The answer to one query, or the response that ends the request + /// instead. + async fn answer(&self, query: &Query) -> Result { + if query.is_files() { + return self.answer_files(query).await; + } + let Some(ops) = &self.ops else { + return Err(self.plugin_misused("packages of a files format")); + }; + self.answer_packages(ops.as_ref(), query) + .await + .map_err(repo_error) + } + + /// The answer to a files query, from the file store. + async fn answer_files(&self, query: &Query) -> Result { + let files = Files { + state: self.state, + principal: self.principal, + }; + let failed = |e: FileError| e.into_response(); + match query { + Query::File(path) | Query::FetchFile(path) => { + if forge_plugin::api::files::validate_path(path).is_err() { + return Err(self.plugin_misused("a file at an invalid path")); + } + let fetch = matches!(query, Query::FetchFile(_)); + let found = files + .locate(self.repo, path, fetch, Vec::new(), 0) + .await + .map_err(failed)?; + if let Some(file) = &found { + self.located + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(path.clone(), file.clone()); + } + Ok(Answer::File(found.as_ref().map(file_info))) + } + Query::Directory(wanted) => { + if forge_plugin::api::files::validate_directory(&wanted.directory).is_err() { + return Err(self.plugin_misused("an invalid directory")); + } + let page = files + .list( + self.repo, + &wanted.directory, + wanted.after.as_deref(), + wanted.limit as usize, + ) + .await + .map_err(failed)?; + Ok(Answer::Directory(page.map(directory_page))) + } + Query::Form => { + if self.permission != Permission::Write { + return Err(refused("receiving a form")); + } + let Some(req) = self.take_body() else { + return Err(self.plugin_misused("a form after the body was read")); + }; + let parts = + crate::files::store::receive_form(&files, self.repo, req, Via::Protocol) + .await + .map_err(failed)?; + let mut staged = Vec::with_capacity(parts.len()); + let mut answer = Vec::with_capacity(parts.len()); + for part in parts { + match part { + Part::File { + field, + name, + content_type, + ingested, + } => { + answer.push(protocol::FormPart { + field, + content: protocol::FormContent::File(protocol::FormFile { + name, + content_type, + size: ingested.stored.size, + }), + }); + staged.push(Some(*ingested)); + } + Part::Text { field, value } => { + answer.push(protocol::FormPart { + field, + content: protocol::FormContent::Text(value), + }); + staged.push(None); + } + } + } + *self.form.lock().unwrap_or_else(PoisonError::into_inner) = Some(staged); + Ok(Answer::Form(answer)) + } + Query::Store(wanted) => { + if self.permission != Permission::Write { + return Err(refused("storing files")); + } + let mut uploads = Vec::with_capacity(wanted.len()); + for file in wanted { + let content = match file.source { + protocol::FileSource::Body => match self.take_body() { + Some(req) => Content::Body(req), + None => { + return Err(self.plugin_misused("a store of a body already read")) + } + }, + protocol::FileSource::Part(index) => match self.take_part(index) { + Some(ingested) => Content::Staged(ingested), + None => { + return Err(self.plugin_misused( + "a store of a form part that is not a file it may store", + )) + } + }, + }; + uploads.push(crate::files::store::Upload { + path: file.path.clone(), + declared: file.content_type.clone(), + content, + }); + } + let stored = crate::files::store::store(&files, self.repo, uploads, Via::Protocol) + .await + .map_err(failed)?; + Ok(Answer::Stored( + stored + .into_iter() + .map(|stored| protocol::StoredFile { + path: stored.path, + size: stored.size, + sha256: stored.sha256, + status: match stored.status { + "created" => protocol::StoreStatus::Created, + "replaced" => protocol::StoreStatus::Replaced, + _ => protocol::StoreStatus::Unchanged, + }, + }) + .collect(), + )) + } + Query::Remove(path) => { + if self.permission != Permission::Delete { + return Err(refused("deleting a file")); + } + let removed = crate::files::store::remove(&files, self.repo, path, Via::Protocol) + .await + .map_err(failed)?; + Ok(Answer::Removed(removed)) + } + _ => Err(self.plugin_misused("a package query of a files format")), + } + } + + /// The staged file part `index` of the form the plugin had received, + /// taken (a part is stored once). + fn take_part(&self, index: u32) -> Option { + self.form + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_mut()? + .get_mut(usize::try_from(index).ok()?)? + .take() + } + + /// Serve the file at `path`, as a query found it or as `fetch-file` + /// finds it now. + async fn serve_file(&self, path: &str) -> Response { + let found = self + .located + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(path) + .cloned(); + let file = match found { + Some(file) => file, + None => { + if forge_plugin::api::files::validate_path(path).is_err() { + return self.plugin_misused("a file at an invalid path"); + } + let files = Files { + state: self.state, + principal: self.principal, + }; + match files.locate(self.repo, path, true, Vec::new(), 0).await { + Ok(Some(file)) => file, + Ok(None) => return FileError::NotFound.into_response(), + Err(e) => return e.into_response(), + } + } + }; + crate::files::serve::file(self.state, self.repo, &file, &self.headers, self.head).await + } + + /// The answer to a package query, through the package read ops. + async fn answer_packages( + &self, + ops: &dyn RepoReadOps, + query: &Query, + ) -> Result { let repo = &self.repo.name; Ok(match query { Query::Versions(id) => Answer::Versions(match self.id(id) { - Some(id) => self - .ops + Some(id) => ops .list_versions(repo, &id) .await? .into_iter() @@ -387,8 +675,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Packages(id) => Answer::Packages(match self.id(id) { - Some(id) => self - .ops + Some(id) => ops .registration(repo, &id) .await? .into_iter() @@ -397,8 +684,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Package(target) => Answer::Package(match self.package(target) { - Some((id, version)) => self - .ops + Some((id, version)) => ops .registration_entry(repo, &id, &version) .await? .map(|entry| self.package_info(entry)), @@ -415,7 +701,7 @@ impl Ctx<'_> { let mut entries = if versions.is_empty() { Vec::new() } else { - self.ops.registration_entries(repo, &id, &versions).await? + ops.registration_entries(repo, &id, &versions).await? }; entries.sort_by(|a, b| self.format.compare_versions(&a.version, &b.version)); entries @@ -426,7 +712,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Manifest(target) => Answer::Manifest(match self.package(target) { - Some((id, version)) => self.ops.resolve_nuspec(repo, &id, &version).await?, + Some((id, version)) => ops.resolve_nuspec(repo, &id, &version).await?, None => None, }), Query::Search(search) => { @@ -439,7 +725,7 @@ impl Ctx<'_> { include_semver2: search.semver2, include_upstream: search.upstream, }; - let result = self.ops.search(repo, ¶ms).await?; + let result = ops.search(repo, ¶ms).await?; Answer::Search(protocol::SearchResult { total: u32::try_from(result.total_hits).unwrap_or(u32::MAX), hits: result @@ -467,6 +753,17 @@ impl Ctx<'_> { .collect(), }) } + // Answered by `answer_files`. + Query::File(_) + | Query::FetchFile(_) + | Query::Directory(_) + | Query::Form + | Query::Store(_) + | Query::Remove(_) => { + return Err(RepoError::Storage( + "a files query of a package format".into(), + )) + } }) } @@ -475,13 +772,14 @@ impl Ctx<'_> { let Some((id, version)) = self.package(target) else { return (StatusCode::NOT_FOUND, "package not found").into_response(); }; + let Some(ops) = &self.ops else { + return self.plugin_misused("a package download of a files format"); + }; // A HEAD (a probe, a mirror tool) is not a download. let resolved = if self.head { - self.ops - .resolve_nupkg_head(&self.repo.name, &id, &version) - .await + ops.resolve_nupkg_head(&self.repo.name, &id, &version).await } else { - self.ops.resolve_nupkg(&self.repo.name, &id, &version).await + ops.resolve_nupkg(&self.repo.name, &id, &version).await }; let location = match resolved { Ok(Some(location)) => location, @@ -705,6 +1003,9 @@ impl Ctx<'_> { if name == "location" && !self.within_repository(value) { continue; } + if name == "link" && !self.links_within_repository(value) { + continue; + } let (Ok(name), Ok(value)) = ( header::HeaderName::from_bytes(name.as_bytes()), HeaderValue::from_str(value), @@ -713,10 +1014,13 @@ impl Ctx<'_> { }; out.headers_mut().append(name, value); } - out.headers_mut().insert( - header::CONTENT_LENGTH, - HeaderValue::from(response.body.len()), - ); + // A 204 or a 304 has no body, and says nothing of its length. + if !matches!(status, StatusCode::NO_CONTENT | StatusCode::NOT_MODIFIED) { + out.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from(response.body.len()), + ); + } if !self.head { *out.body_mut() = Body::from(response.body); } @@ -730,6 +1034,12 @@ impl Ctx<'_> { fn within_repository(&self, location: &str) -> bool { within(location, self.mount, &self.repo.name, &self.target.base_url) } + + /// Whether every target of a `Link` header (`; rel=...`, comma + /// separated) stays inside this repository. + fn links_within_repository(&self, value: &str) -> bool { + links_within(value, self.mount, &self.repo.name, &self.target.base_url) + } } #[derive(Clone, Copy)] @@ -821,6 +1131,39 @@ fn multipart_reader( Ok(Box::new(StreamReader::new(stream))) } +/// A stored file as the plugin sees it. +fn file_info(file: &RawFile) -> protocol::FileInfo { + let time = + |t: chrono::DateTime| t.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true); + protocol::FileInfo { + path: file.path.clone(), + size: u64::try_from(file.size).unwrap_or(0), + content_type: file.content_type.clone(), + sha256: file.blob_sha256.clone(), + sha512: file.sha512_base64.clone(), + sha1: file.sha1.clone(), + md5: file.md5.clone(), + modified: time(file.updated_at), + last_downloaded: file.last_downloaded_at.map(time), + } +} + +/// A page of a directory as the plugin sees it. +fn directory_page(page: crate::files::listing::Page) -> protocol::DirectoryPage { + use crate::files::listing::EntryKind; + protocol::DirectoryPage { + entries: page + .entries + .into_iter() + .map(|entry| match entry.kind { + EntryKind::File(file) => protocol::DirectoryEntry::File(file_info(&file)), + EntryKind::Directory(path) => protocol::DirectoryEntry::Directory(path), + }) + .collect(), + next: page.next, + } +} + /// The absolute URL of repository `name` served under `mount`: /// `{public_base_url}{mount}/{name}`, the base its plugin builds URLs on and /// the address its clients are given. @@ -831,12 +1174,20 @@ pub fn repository_url(state: &AppState, mount: &str, name: &str) -> String { ) } -/// See [`Ctx::within_repository`]. +/// See [`Ctx::within_repository`]. A path may also start with the path of +/// `base_url` (a public base URL with a path of its own). fn within(location: &str, mount: &str, repo: &str, base_url: &str) -> bool { let path_prefix = format!("{mount}/{repo}/"); let absolute_prefix = format!("{base_url}/"); + let base_path_prefix = url::Url::parse(base_url) + .ok() + .map(|url| format!("{}/", url.path().trim_end_matches('/'))); let lower = location.to_ascii_lowercase(); - (location.starts_with(&path_prefix) || location.starts_with(&absolute_prefix)) + (location.starts_with(&path_prefix) + || location.starts_with(&absolute_prefix) + || base_path_prefix + .as_deref() + .is_some_and(|prefix| prefix != "/" && location.starts_with(prefix))) && !location .split(['/', '?', '#']) .any(|seg| seg == ".." || seg == ".") @@ -844,6 +1195,27 @@ fn within(location: &str, mount: &str, repo: &str, base_url: &str) -> bool { && !location.contains('\\') } +/// See [`Ctx::links_within_repository`]: every `<…>` of `value` is +/// [`within`] the repository, and there is one at least. +fn links_within(value: &str, mount: &str, repo: &str, base_url: &str) -> bool { + let mut targets = 0; + for link in value.split(',') { + let Some(target) = link + .trim() + .strip_prefix('<') + .and_then(|rest| rest.split_once('>')) + .map(|(target, _)| target) + else { + return false; + }; + if !within(target, mount, repo, base_url) { + return false; + } + targets += 1; + } + targets > 0 +} + /// A header value of ASCII digits and punctuation. fn numeric_header(value: String) -> HeaderValue { HeaderValue::from_str(&value).expect("ASCII header value") @@ -928,7 +1300,7 @@ mod tests { ); assert_eq!( split("/repository/charts"), - Some(("/repository", "charts", "/")) + Some(("/repository", "charts", "")) ); assert_eq!( split("/repository/charts/"), @@ -949,6 +1321,28 @@ mod tests { assert_eq!(split("/v3/charts"), None); } + #[test] + fn links_stay_inside_the_repository() { + let base = "https://forge.example.com/raw/bin"; + let ok = "; rel=\"next\""; + assert!(links_within(ok, "/raw", "bin", base)); + assert!(links_within( + "; rel=\"next\", ; rel=\"up\"", + "/raw", + "bin", + base + )); + for bad in [ + "; rel=\"next\"", + "; rel=\"next\"", + "; rel=\"next\", ; rel=\"preload\"", + "rel=\"next\"", + "", + ] { + assert!(!links_within(bad, "/raw", "bin", base), "{bad}"); + } + } + #[test] fn redirects_stay_inside_the_repository() { let base = "https://forge.example.com/repository/charts"; @@ -958,6 +1352,13 @@ mod tests { ] { assert!(within(ok, "/repository", "charts", base), "{ok}"); } + // Behind a public base URL with a path, the repository's path. + assert!(within( + "/forge/repository/charts/index.yaml", + "/repository", + "charts", + "https://forge.example.com/forge/repository/charts" + )); for bad in [ "https://evil.example.com/", "//evil.example.com/repository/charts/x", diff --git a/crates/server/src/raw/listing.rs b/crates/server/src/raw/listing.rs deleted file mode 100644 index 63630ee8..00000000 --- a/crates/server/src/raw/listing.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Directory listings: `GET /raw///`, as HTML for browsers or as -//! JSON (`Accept: application/json`), a page at a time. -//! -//! A page holds at most `limit` entries (default 100, at most 1000) after -//! the entry named by `after`, in byte order of their names, a -//! subdirectory's name ending in `/`; `next` (and a `Link: rel="next"` -//! header) names where the following page starts. A listing walks the -//! index ([`raw_dao::walk_directory`]): a page costs `limit` index probes, -//! however many files are below the directory. -//! -//! Package selectors apply: a file outside the caller's selector is not -//! listed, and a subdirectory is listed only when it holds a file the -//! caller may see (looked for among its first files; a subdirectory none of -//! whose first [`SUBDIRECTORY_SCAN`] files is visible is left out). A -//! group lists the merged entries of the members the caller may read, the -//! highest-priority member's file first; a proxy lists what it has cached. - -use std::collections::BTreeMap; - -use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; -use chrono::{DateTime, Utc}; -use forge_core::authz::PackageFilter; -use forge_db::model::{RepoKind, Repository}; -use forge_db::raw_dao::{self, RawFile}; -use serde::Serialize; - -use super::{path, Raw, RawError}; - -/// Default page size. -pub const DEFAULT_LIMIT: usize = 100; - -/// Largest page size. -pub const MAX_LIMIT: usize = 1000; - -/// How many of a subdirectory's files a listing reads, at most, to find one -/// a selector-restricted caller may see. -pub const SUBDIRECTORY_SCAN: usize = 1000; - -/// One entry of a listing. -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct Entry { - /// The entry's name in its directory; a subdirectory's ends in `/`. - pub name: String, - /// The full path (a subdirectory's ends in `/`). - pub path: String, - /// `file` | `directory`. - #[serde(rename = "type")] - pub kind: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - pub size: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub content_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sha256: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_modified: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_downloaded: Option>, -} - -impl Entry { - fn file(file: &RawFile, name: &str) -> Self { - Entry { - name: name.to_string(), - path: file.path.clone(), - kind: "file", - size: Some(file.size), - content_type: Some(file.content_type.clone()), - sha256: Some(file.blob_sha256.clone()), - last_modified: Some(file.updated_at), - last_downloaded: file.last_downloaded_at, - } - } - - fn directory(prefix: &str, name: &str) -> Self { - Entry { - name: format!("{name}/"), - path: format!("{prefix}{name}/"), - kind: "directory", - size: None, - content_type: None, - sha256: None, - last_modified: None, - last_downloaded: None, - } - } -} - -/// A page of a listing. -#[derive(Debug, Clone, Serialize)] -pub struct Page { - pub repository: String, - /// The directory (`""` for the root). - pub path: String, - pub entries: Vec, - /// The `after` of the next page, if there is one. - pub next: Option, -} - -/// The query of a listing. -#[derive(Debug, Default, serde::Deserialize)] -pub struct ListQuery { - pub after: Option, - pub limit: Option, - /// `json` | `html`: overrides `Accept`. - pub format: Option, -} - -impl ListQuery { - /// Parse a request's query string; other parameters are ignored. - pub fn parse(query: Option<&str>) -> Result { - let mut out = ListQuery::default(); - for (key, value) in url::form_urlencoded::parse(query.unwrap_or("").as_bytes()) { - match key.as_ref() { - "after" => out.after = Some(value.into_owned()), - "limit" => { - out.limit = Some(value.parse().map_err(|_| { - RawError::BadRequest("limit must be a positive number".into()) - })?) - } - "format" => match value.as_ref() { - "json" | "html" => out.format = Some(value.into_owned()), - _ => return Err(RawError::BadRequest("format is json or html".into())), - }, - _ => {} - } - } - Ok(out) - } -} - -/// Where a page starting after the entry `after` (a name in directory -/// `prefix`, a subdirectory's ending in `/`) resumes: the least path past -/// that entry (past all of a subdirectory's files: `'0'` follows `'/'`; past -/// a file: no path holds a control character). -fn resume_at(prefix: &str, after: Option<&str>) -> Result { - let Some(after) = after.filter(|a| !a.is_empty()) else { - return Ok(prefix.to_string()); - }; - let (name, directory) = match after.strip_suffix('/') { - Some(name) => (name, true), - None => (after, false), - }; - path::validate_segment(name).map_err(|e| RawError::BadRequest(format!("after: {e}")))?; - Ok(if directory { - format!("{prefix}{name}0") - } else { - format!("{prefix}{name}\u{1}") - }) -} - -/// One repository's entries of a page: what it has after `from`, as far as -/// it read (`more`: it has entries past the last one returned, up to and -/// including the entry named `more`, which it has read). -struct Partial { - entries: Vec, - more: Option, -} - -impl Raw<'_> { - /// The entries of `prefix` in hosted or proxy `repo` from `from` on, - /// visible through every one of `filters`. - async fn list_local( - &self, - repo: &Repository, - prefix: &str, - from: &str, - limit: usize, - filters: &[PackageFilter], - ) -> Result { - let walked = raw_dao::walk_directory( - &self.state.pool, - repo.id, - prefix, - from, - i64::try_from(limit + 1).unwrap_or(i64::MAX), - ) - .await?; - let more = walked.len() > limit; - let mut entries = Vec::new(); - let mut last = None; - for walked in walked.into_iter().take(limit) { - let rest = &walked.path[prefix.len()..]; - let (entry, key) = match (rest.find('/'), &walked.file) { - (None, Some(file)) => { - let visible = filters.iter().all(|f| f.allows(&file.path)); - (visible.then(|| Entry::file(file, rest)), rest.to_string()) - } - (Some(i), _) => { - let name = &rest[..i]; - let dir = format!("{prefix}{name}/"); - let visible = self.directory_visible(repo, &dir, filters).await?; - ( - visible.then(|| Entry::directory(prefix, name)), - format!("{name}/"), - ) - } - // A file row vanished between the walk and the join. - (None, None) => (None, rest.to_string()), - }; - entries.extend(entry); - last = Some(key); - } - Ok(Partial { - entries, - more: if more { last } else { None }, - }) - } - - /// Whether directory `dir` of `repo` holds a file visible through every - /// one of `filters`. - async fn directory_visible( - &self, - repo: &Repository, - dir: &str, - filters: &[PackageFilter], - ) -> Result { - if filters.iter().all(|f| matches!(f, PackageFilter::AllowAll)) { - return Ok(true); - } - const BATCH: usize = 200; - let mut after = String::new(); - let mut read = 0; - while read < SUBDIRECTORY_SCAN { - let paths = - raw_dao::paths_under(&self.state.pool, repo.id, dir, &after, BATCH as i64).await?; - if paths.iter().any(|p| filters.iter().all(|f| f.allows(p))) { - return Ok(true); - } - if paths.len() < BATCH { - return Ok(false); - } - read += paths.len(); - after = paths.last().cloned().unwrap_or_default(); - } - Ok(false) - } - - /// The entries of `prefix` in `repo` (any kind) from `from` on. - fn list_in<'a>( - &'a self, - repo: &'a Repository, - prefix: &'a str, - from: &'a str, - limit: usize, - filters: Vec, - depth: u32, - ) -> futures_util::future::BoxFuture<'a, Result> { - Box::pin(async move { - let mut filters = filters; - filters.push(self.state.authz.package_filter(&self.principal, repo).await); - if repo.kind() != RepoKind::Group { - return self.list_local(repo, prefix, from, limit, &filters).await; - } - // A merge of the members' pages, complete up to the first point - // a member stopped reading at. - let mut merged: BTreeMap = BTreeMap::new(); - let mut cutoff: Option = None; - for member in self.serving_members(repo, depth).await? { - let part = self - .list_in(&member, prefix, from, limit, filters.clone(), depth + 1) - .await?; - for entry in part.entries { - merged.entry(entry.name.clone()).or_insert(entry); - } - if let Some(more) = part.more { - if cutoff.as_ref().is_none_or(|c| more < *c) { - cutoff = Some(more); - } - } - } - let mut entries: Vec = merged - .into_values() - .filter(|e| cutoff.as_ref().is_none_or(|c| e.name <= *c)) - .collect(); - let more = if entries.len() > limit { - entries.truncate(limit); - entries.last().map(|e| e.name.clone()) - } else { - cutoff - }; - Ok(Partial { entries, more }) - }) - } - - /// A page of directory `prefix` of `repo`, `None` when it does not - /// exist (a directory other than the root without visible entries). - pub async fn list( - &self, - repo: &Repository, - prefix: &str, - query: &ListQuery, - ) -> Result, RawError> { - let limit = query.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); - let from = resume_at(prefix, query.after.as_deref())?; - let partial = self - .list_in(repo, prefix, &from, limit, Vec::new(), 0) - .await?; - if partial.entries.is_empty() && partial.more.is_none() && !prefix.is_empty() { - // Past the end of an existing directory is an empty page, not a - // missing directory. - if query.after.is_none() { - return Ok(None); - } - } - Ok(Some(Page { - repository: repo.name.clone(), - path: prefix.to_string(), - entries: partial.entries, - next: partial.more, - })) - } -} - -/// Whether the client wants JSON: `format=json`, or an `Accept` preferring -/// `application/json` to HTML. -pub fn wants_json(headers: &HeaderMap, query: &ListQuery) -> bool { - match query.format.as_deref() { - Some("json") => return true, - Some("html") => return false, - _ => {} - } - let accept = headers - .get(header::ACCEPT) - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - let quality = |wanted: &str| { - accept - .split(',') - .filter_map(|item| { - let mut parts = item.split(';'); - let media = parts.next()?.trim().to_ascii_lowercase(); - (media == wanted).then(|| { - parts - .filter_map(|p| p.trim().strip_prefix("q=")) - .find_map(|q| q.parse::().ok()) - .unwrap_or(1.0) - }) - }) - .fold(0.0f32, f32::max) - }; - quality("application/json") > 0.0 && quality("application/json") >= quality("text/html") -} - -/// The URL of the page after `page`. -fn next_link(page: &Page, query: &ListQuery) -> Option { - let next = page.next.as_deref()?; - let mut params = url::form_urlencoded::Serializer::new(String::new()); - params.append_pair("after", next); - if let Some(limit) = query.limit { - params.append_pair("limit", &limit.to_string()); - } - if let Some(format) = &query.format { - params.append_pair("format", format); - } - Some(format!( - "/raw/{}/{}?{}", - page.repository, - path::encode(&page.path), - params.finish() - )) -} - -fn escape(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - for c in text.chars() { - match c { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - '\'' => out.push_str("'"), - c => out.push(c), - } - } - out -} - -/// The page as an HTML document: a table of the entries, linked. -pub fn html(page: &Page, query: &ListQuery) -> String { - let title = format!("Index of /raw/{}/{}", page.repository, page.path); - let mut out = format!( - "\n{t}\n\n

{t}

\n\n\n", - t = escape(&title) - ); - if !page.path.is_empty() { - out.push_str("\n"); - } - for entry in &page.entries { - let href = path::encode(&entry.name); - out.push_str(&format!( - "\n", - escape(&href), - escape(&entry.name), - entry.size.map(|s| s.to_string()).unwrap_or_default(), - entry - .last_modified - .map(|t| t.format("%Y-%m-%d %H:%M:%S UTC").to_string()) - .unwrap_or_default(), - escape(entry.sha256.as_deref().unwrap_or("")), - )); - } - out.push_str("
NameSizeLast modifiedSHA-256
../
{}{}{}{}
\n"); - if let Some(next) = next_link(page, query) { - out.push_str(&format!( - "

Next page

\n", - escape(&next) - )); - } - out.push_str("\n"); - out -} - -/// Answer a listing request. -pub fn respond(page: &Page, query: &ListQuery, json: bool, head: bool) -> Response { - let (content_type, body) = if json { - ( - "application/json", - serde_json::to_string(page).unwrap_or_default(), - ) - } else { - ("text/html; charset=utf-8", html(page, query)) - }; - let length = body.len(); - let mut response = if head { - StatusCode::OK.into_response() - } else { - (StatusCode::OK, body).into_response() - }; - let out = response.headers_mut(); - out.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); - out.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); - // Forge's own document, shown in the browser (and sandboxed). - out.insert( - header::CONTENT_DISPOSITION, - HeaderValue::from_static("inline"), - ); - out.insert(header::VARY, HeaderValue::from_static("Accept")); - if let Some(next) = next_link(page, query) { - if let Ok(value) = HeaderValue::from_str(&format!("<{next}>; rel=\"next\"")) { - out.insert(header::LINK, value); - } - } - response -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn pages_resume_past_the_named_entry() { - assert_eq!(resume_at("a/", None).unwrap(), "a/"); - assert_eq!(resume_at("a/", Some("")).unwrap(), "a/"); - assert_eq!(resume_at("a/", Some("x.bin")).unwrap(), "a/x.bin\u{1}"); - assert_eq!(resume_at("", Some("dir/")).unwrap(), "dir0"); - assert!(resume_at("a/", Some("../x")).is_err()); - assert!(resume_at("a/", Some("b/c")).is_err()); - } - - #[test] - fn json_is_asked_for() { - let query = ListQuery::default(); - let mut h = HeaderMap::new(); - assert!(!wants_json(&h, &query)); - h.insert(header::ACCEPT, HeaderValue::from_static("application/json")); - assert!(wants_json(&h, &query)); - h.insert( - header::ACCEPT, - HeaderValue::from_static( - "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", - ), - ); - assert!(!wants_json(&h, &query)); - let query = ListQuery { - format: Some("json".into()), - ..Default::default() - }; - assert!(wants_json(&h, &query)); - } - - #[test] - fn html_escapes_names_and_links_them_encoded() { - let page = Page { - repository: "bin".into(), - path: "a/".into(), - entries: vec![Entry { - name: " y.txt".into(), - path: "a/ y.txt".into(), - kind: "file", - size: Some(3), - content_type: Some("text/plain".into()), - sha256: Some("ab".into()), - last_modified: None, - last_downloaded: None, - }], - next: Some(" y.txt".into()), - }; - let html = html(&page, &ListQuery::default()); - assert!( - html.contains("href=\"%3Cx%3E%20y.txt\"><x> y.txt"), - "{html}" - ); - assert!(html.contains("href=\"../\"")); - assert!(html.contains("/raw/bin/a/?after=%3Cx%3E+y.txt"), "{html}"); - assert!(!html.contains("")); - } -} diff --git a/crates/server/src/raw/mod.rs b/crates/server/src/raw/mod.rs deleted file mode 100644 index b0918d1e..00000000 --- a/crates/server/src/raw/mod.rs +++ /dev/null @@ -1,436 +0,0 @@ -//! Raw (generic file) repositories: arbitrary files under arbitrary paths, -//! like Nexus' raw-hosted, raw-proxy and raw-group repositories, served at -//! `/raw//`. -//! -//! * `GET` / `HEAD` of a file: its bytes, with `ETag` (the SHA-256), -//! `Last-Modified`, `Content-Length` and the stored `Content-Type`; one -//! `Range`, `If-Range`, `If-None-Match` and `If-Modified-Since` ([`serve`]). -//! `.sha256`, `.sha512`, `.sha1` and `.md5` are answered from the -//! file's digests when no file of that name is stored. -//! * `GET` of a directory (`/`, or the root): a listing, HTML or JSON, -//! paginated ([`listing`]). -//! * `PUT` of a file, `POST` of a multipart form to a directory, `DELETE` of -//! a file, in hosted repositories ([`upload`]); directories are deleted -//! through the admin API. -//! * Proxies fetch from an HTTP(S) base URL and re-validate after a maximum -//! age ([`proxy`]); groups answer with the first member, by priority, that -//! has the path. -//! -//! Paths are normalized ([`path`]): `..`, `.`, empty segments, backslashes -//! and encoded slashes are refused, never resolved. Package selectors match -//! paths (`pdfium/**`). Files are CAS blobs, so the same content under -//! several paths or repositories is stored once. - -pub mod listing; -pub mod path; -pub mod proxy; -pub mod serve; -pub mod upload; -pub mod web; - -use std::collections::HashSet; - -use axum::body::Body; -use axum::extract::{Request, State}; -use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode}; -use axum::response::{IntoResponse, Response}; -use forge_core::auth::principal::Principal; -use forge_core::authz::{Action, PackageFilter}; -use forge_db::model::{RepoKind, Repository}; -use forge_db::{raw_dao, repository_dao}; - -use crate::dispatch::{action_for_method, authz_denied, offline_response, offline_serves_nothing}; -use crate::middleware::principal_of; -use crate::state::AppState; -use listing::ListQuery; -use path::Target; -use raw_dao::RawFile; -use serve::Checksum; - -/// Where raw repositories are served. -pub const MOUNT: &str = "/raw"; - -/// A request's view of raw repositories: the server's state and the caller. -pub struct Raw<'a> { - pub state: &'a AppState, - pub principal: Principal, -} - -/// Why a raw request failed, as its answer. -#[derive(Debug)] -pub enum RawError { - NotFound, - BadRequest(String), - /// A mutation needs an API token or a workload identity. - CredentialRequired, - /// The caller may not do this (repository-level). - Denied, - Forbidden(String), - Conflict(String), - MethodNotAllowed(&'static str), - TooLarge(u64), - Unavailable(String), - BadGateway(String), - Internal(String), -} - -impl RawError { - /// The error's text, for logs. - pub fn message(&self) -> String { - match self { - RawError::NotFound => "not found".into(), - RawError::CredentialRequired => "API token or workload identity required".into(), - RawError::Denied => "forbidden".into(), - RawError::MethodNotAllowed(m) => (*m).into(), - RawError::TooLarge(limit) => format!("larger than {limit} bytes"), - RawError::BadRequest(m) - | RawError::Forbidden(m) - | RawError::Conflict(m) - | RawError::Unavailable(m) - | RawError::BadGateway(m) - | RawError::Internal(m) => m.clone(), - } - } -} - -impl From for RawError { - fn from(e: sqlx::Error) -> Self { - match e { - sqlx::Error::PoolTimedOut => RawError::Unavailable("database busy".into()), - e => RawError::Internal(format!("database: {e}")), - } - } -} - -impl IntoResponse for RawError { - fn into_response(self) -> Response { - match self { - RawError::NotFound => (StatusCode::NOT_FOUND, "not found").into_response(), - RawError::BadRequest(m) => (StatusCode::BAD_REQUEST, m).into_response(), - RawError::CredentialRequired => ( - StatusCode::UNAUTHORIZED, - [(header::WWW_AUTHENTICATE, "Basic realm=\"Forge\"")], - "API token or workload identity required", - ) - .into_response(), - RawError::Denied => (StatusCode::FORBIDDEN, "forbidden").into_response(), - RawError::Forbidden(m) => (StatusCode::FORBIDDEN, m).into_response(), - RawError::Conflict(m) => (StatusCode::CONFLICT, m).into_response(), - RawError::MethodNotAllowed(m) => ( - StatusCode::METHOD_NOT_ALLOWED, - [(header::ALLOW, "GET, HEAD")], - m, - ) - .into_response(), - RawError::TooLarge(limit) => ( - StatusCode::PAYLOAD_TOO_LARGE, - format!("the upload exceeds the repository's limit of {limit} bytes"), - ) - .into_response(), - RawError::Unavailable(m) => ( - StatusCode::SERVICE_UNAVAILABLE, - [(header::RETRY_AFTER, "30")], - m, - ) - .into_response(), - RawError::BadGateway(m) => (StatusCode::BAD_GATEWAY, m).into_response(), - RawError::Internal(m) => { - tracing::error!(error = %m, "raw repository request failed"); - (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() - } - } - } -} - -/// A file a request found: stored at the path, or the file a checksum -/// sidecar is computed from. -pub enum Found { - File(RawFile), - Sidecar(RawFile, Checksum), -} - -/// The `/raw/...` handler. -pub async fn handle(State(state): State, req: Request) -> Response { - let mut response = handle_request(&state, req).await; - crate::feed::harden_response_headers(response.headers_mut()); - response -} - -/// `(repository name, rest)` of a `/raw/[/]` path; the rest is -/// `None` without the `/` after the name. -fn split(path: &str) -> Option<(&str, Option<&str>)> { - let rest = path.strip_prefix(MOUNT)?.strip_prefix('/')?; - let (name, rest) = match rest.find('/') { - Some(i) => (&rest[..i], Some(&rest[i + 1..])), - None => (rest, None), - }; - (!name.is_empty()).then_some((name, rest)) -} - -async fn handle_request(state: &AppState, req: Request) -> Response { - let principal = principal_of(&req); - let method = req.method().clone(); - let uri = req.uri().clone(); - let Some((name, rest)) = split(uri.path()) else { - return (StatusCode::NOT_FOUND, "repository not found").into_response(); - }; - let repo = match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(r)) => r, - // An unknown name answers like a repository the caller may not - // read, except to a global admin. - Ok(None) if state.authz.is_global_admin(&principal).await => { - return (StatusCode::NOT_FOUND, "repository not found").into_response(); - } - Ok(None) => return authz_denied(&principal), - Err(e) => return RawError::from(e).into_response(), - }; - let action = action_for_method(&method); - let allowed = match action { - Action::Read => state.authz.can_read(&principal, &repo).await, - Action::Write => state.authz.can_write(&principal, &repo).await, - Action::Delete => state.authz.can_delete(&principal, &repo).await, - Action::Admin => false, - }; - if !allowed { - return authz_denied(&principal); - } - if !repo.is_raw() { - return (StatusCode::NOT_FOUND, "repository is not served here").into_response(); - } - if offline_serves_nothing(&repo) { - return offline_response(); - } - let Some(rest) = rest else { - // `/raw/`: the root listing is `/raw//`. - if matches!(method, Method::GET | Method::HEAD) { - let location = format!("{MOUNT}/{}/", repo.name); - return ( - StatusCode::MOVED_PERMANENTLY, - [(header::LOCATION, location)], - ) - .into_response(); - } - return (StatusCode::NOT_FOUND, "not found").into_response(); - }; - let target = match path::parse_request(rest) { - Ok(target) => target, - Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), - }; - let raw = Raw { state, principal }; - let response = match (&method, &target) { - (&Method::GET | &Method::HEAD, Target::File(path)) => { - raw.get(&repo, path, req.headers(), method == Method::HEAD) - .await - } - (&Method::GET | &Method::HEAD, Target::Directory(dir)) => { - let query = match ListQuery::parse(uri.query()) { - Ok(query) => query, - Err(e) => return e.into_response(), - }; - match raw.list(&repo, dir, &query).await { - Ok(Some(page)) => listing::respond( - &page, - &query, - listing::wants_json(req.headers(), &query), - method == Method::HEAD, - ), - Ok(None) => RawError::NotFound.into_response(), - Err(e) => e.into_response(), - } - } - (&Method::PUT, Target::File(path)) => { - let path = path.clone(); - upload::put(&raw, &repo, &path, req).await - } - (&Method::PUT, Target::Directory(_)) => ( - StatusCode::BAD_REQUEST, - "PUT uploads a file: name it in the path", - ) - .into_response(), - (&Method::POST, Target::Directory(dir)) => { - let dir = dir.clone(); - upload::multipart(&raw, &repo, &dir, req, upload::Via::Protocol).await - } - (&Method::POST, Target::File(_)) => ( - StatusCode::BAD_REQUEST, - "a form upload goes to a directory: end the path with /", - ) - .into_response(), - (&Method::DELETE, Target::File(path)) => { - upload::delete(&raw, &repo, path, upload::Via::Protocol).await - } - (&Method::DELETE, Target::Directory(_)) => ( - StatusCode::BAD_REQUEST, - "directories are deleted through the admin API (DELETE /admin/repositories//files?path=/)", - ) - .into_response(), - _ => ( - StatusCode::METHOD_NOT_ALLOWED, - [(header::ALLOW, "GET, HEAD, PUT, POST, DELETE")], - "method not allowed", - ) - .into_response(), - }; - if action != Action::Read { - // The same audit trail as the package feeds' mutations. - tracing::info!( - target: "forge::audit", - %method, - repo = %repo.name, - path = %rest, - status = response.status().as_u16(), - principal = %principal.audit_label(), - "package mutation" - ); - } - response -} - -impl Raw<'_> { - /// `GET`/`HEAD` of `path` in `repo`. - async fn get( - &self, - repo: &Repository, - path: &str, - headers: &HeaderMap, - head: bool, - ) -> Response { - match self.locate(repo, path, Vec::new(), 0).await { - Ok(Some(Found::File(file))) => { - serve::file(self.state, repo, &file, headers, head).await - } - Ok(Some(Found::Sidecar(base, checksum))) => { - serve::sidecar(repo, path, &base, checksum, headers, head) - } - Ok(None) => { - // A directory named without its `/`. - let dir = format!("{path}/"); - match self.list(repo, &dir, &ListQuery::default()).await { - Ok(Some(_)) => { - let location = format!("{MOUNT}/{}/{}", repo.name, path::encode(&dir)); - let mut response = StatusCode::MOVED_PERMANENTLY.into_response(); - if let Ok(value) = HeaderValue::from_str(&location) { - response.headers_mut().insert(header::LOCATION, value); - } - response - } - Ok(None) => RawError::NotFound.into_response(), - Err(e) => e.into_response(), - } - } - Err(e) => e.into_response(), - } - } - - /// Find `path` in `repo` for the caller, through every one of `filters` - /// (a group's, for its members) and `repo`'s own package filter: the - /// stored file, else the checksum sidecar of a stored file, else (a - /// proxy) the upstream's file, else (a group) the first member's that - /// has one. - pub fn locate<'a>( - &'a self, - repo: &'a Repository, - path: &'a str, - filters: Vec, - depth: u32, - ) -> futures_util::future::BoxFuture<'a, Result, RawError>> { - Box::pin(async move { - let mut filters = filters; - filters.push(self.state.authz.package_filter(&self.principal, repo).await); - let visible = |p: &str| filters.iter().all(|f| f.allows(p)); - let sidecar = serve::sidecar_of(path).filter(|(base, _)| visible(base)); - match repo.kind() { - RepoKind::Hosted | RepoKind::Proxy => { - let proxy = repo.kind() == RepoKind::Proxy; - if visible(path) { - if let Some(file) = raw_dao::find(&self.state.pool, repo.id, path).await? { - let file = if proxy { - self.fresh(repo, file).await - } else { - file - }; - return Ok(Some(Found::File(file))); - } - } - if let Some((base, checksum)) = sidecar { - if let Some(file) = raw_dao::find(&self.state.pool, repo.id, base).await? { - let file = if proxy { - self.fresh(repo, file).await - } else { - file - }; - return Ok(Some(Found::Sidecar(file, checksum))); - } - } - if proxy && visible(path) { - return Ok(self.fetch(repo, path).await?.map(Found::File)); - } - Ok(None) - } - RepoKind::Group => { - if !visible(path) && sidecar.is_none() { - return Ok(None); - } - let mut failure = None; - for member in self.serving_members(repo, depth).await? { - match self.locate(&member, path, filters.clone(), depth + 1).await { - Ok(Some(found)) => return Ok(Some(found)), - Ok(None) => {} - Err(e) => { - tracing::warn!(group = %repo.name, member = %member.name, error = %e.message(), "raw group: member failed"); - failure = Some(e); - } - } - } - failure.map_or(Ok(None), Err) - } - } - }) - } - - /// The members of `group` that may serve the caller, highest priority - /// first: raw repositories the caller may read itself (reading a group - /// grants nothing on its members), without offline hosted or group - /// members, and without nested groups past the nesting limit or on a - /// cycle. - pub async fn serving_members( - &self, - group: &Repository, - depth: u32, - ) -> Result, RawError> { - let max_depth = self.state.config.groups.max_nesting_depth.max(1) as u32; - if depth >= max_depth { - return Ok(Vec::new()); - } - let mut seen = HashSet::from([group.id]); - let mut out = Vec::new(); - for (member, _priority) in repository_dao::group_members(&self.state.pool, group.id).await? - { - if !member.is_raw() || !seen.insert(member.id) || offline_serves_nothing(&member) { - continue; - } - if member.kind() == RepoKind::Group && depth + 1 >= max_depth { - continue; - } - if self.state.authz.can_read(&self.principal, &member).await { - out.push(member); - } - } - Ok(out) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn paths_split_into_repository_and_rest() { - assert_eq!(split("/raw/bin"), Some(("bin", None))); - assert_eq!(split("/raw/bin/"), Some(("bin", Some("")))); - assert_eq!(split("/raw/bin/a/b.txt"), Some(("bin", Some("a/b.txt")))); - assert_eq!(split("/raw/"), None); - assert_eq!(split("/raw"), None); - assert_eq!(split("/rawx/bin"), None); - } -} diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index 8f7ed1fa..aae89cd7 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -33,7 +33,7 @@ use forge_core::repo::{ }; use forge_core::retention::FormatResolver; use forge_core::session::SessionService; -use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT, RAW_FORMAT}; +use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT}; use forge_storage::{BlobRef, BlobStore, Cas, S3Error}; use sqlx::PgPool; @@ -139,15 +139,16 @@ impl Instance { } /// Format registry: maps a format id (the `repository.format` column) to its -/// [`Format`] and its plugin: the built-in format plugin -/// ([`crate::plugins::builtin`]: NuGet), Docker and raw. +/// [`Format`] and its plugin: the built-in format plugins +/// ([`crate::plugins::builtin`]: NuGet, raw) and Docker. /// -/// Docker ([`DOCKER_FORMAT`]) and raw ([`RAW_FORMAT`]) are installed but have -/// no [`Format`]: their repositories hold images (manifests and blobs) or -/// files under paths, not package versions, and are served by the registry -/// under `/v2` ([`crate::registry`]) and under `/raw` ([`crate::raw`]), so -/// none of the package seams (parsing, version order, the read ops) apply to -/// them. +/// Docker ([`DOCKER_FORMAT`]) is installed but has no [`Format`]: its +/// repositories hold images (manifests and blobs), not package versions, +/// and are served by the registry under `/v2` ([`crate::registry`]), so none +/// of the package seams (parsing, version order, the read ops) apply to +/// them. A files format (raw) has a [`Format`] whose +/// [`Format::stores_files`] says so: its repositories hold files in the +/// file store ([`crate::files`]). pub struct FormatRegistry { formats: BTreeMap>, /// Every format's plugin, by format id: its client protocol. @@ -212,16 +213,22 @@ impl FormatRegistry { } /// Whether format `id` is one served natively, without a [`Format`]: - /// Docker and raw. + /// Docker. pub fn is_native(id: &str) -> bool { - id == DOCKER_FORMAT || id == RAW_FORMAT + id == DOCKER_FORMAT } - /// Whether format `id` is installed: Docker, raw, or a format plugin. + /// Whether format `id` is installed: Docker, or a format plugin. pub fn is_installed(&self, id: &str) -> bool { Self::is_native(id) || self.formats.contains_key(id) } + /// Whether format `id`'s repositories hold files under paths (raw): it + /// is installed and its `plugin.toml` says `storage = "files"`. + pub fn stores_files(&self, id: &str) -> bool { + self.formats.get(id).is_some_and(|f| f.stores_files()) + } + /// The mounts plugin formats are served under, sorted and distinct /// (`/repository`, and any other a plugin names). pub fn mounts(&self) -> Vec<&str> { @@ -239,14 +246,13 @@ impl FormatRegistry { pub fn ids(&self) -> impl Iterator { let mut ids: Vec<&str> = self.formats.keys().map(String::as_str).collect(); ids.push(DOCKER_FORMAT); - ids.push(RAW_FORMAT); ids.sort_unstable(); ids.into_iter() } /// The repository kinds format `id` supports, `None` when it is not - /// installed. Docker and raw have every kind; a format plugin serves the - /// kinds its `plugin.toml` names (NuGet's all three). + /// installed. Docker has every kind; a format plugin serves the kinds its + /// `plugin.toml` names (NuGet's and raw's all three). pub fn kinds(&self, id: &str) -> Option> { if Self::is_native(id) { return Some(vec!["hosted", "proxy", "group"]); @@ -698,10 +704,13 @@ mod tests { registry.ids().collect::>(), ["docker", "nuget", "raw"] ); - assert!(registry.is_installed(RAW_FORMAT)); - assert!(registry.get(RAW_FORMAT).is_none()); + // Raw is a built-in plugin whose repositories hold files. + assert!(registry.get("raw").is_some()); + assert!(registry.stores_files("raw")); + assert!(!registry.stores_files("nuget")); + assert!(!registry.stores_files(DOCKER_FORMAT)); assert_eq!( - registry.kinds(RAW_FORMAT), + registry.kinds("raw"), Some(vec!["hosted", "proxy", "group"]) ); assert_eq!( @@ -718,8 +727,8 @@ mod tests { } #[test] - fn a_plugin_cannot_take_dockers_or_raws_id() { - for id in ["docker", "raw"] { + fn a_plugin_cannot_take_dockers_or_a_builtin_plugins_id() { + for id in ["docker", "nuget", "raw"] { let registry = FormatRegistry::new(Default::default()); let err = registry .with_format(Arc::new(Named(id))) diff --git a/crates/server/tests/plugin_contract.rs b/crates/server/tests/plugin_contract.rs index 25c25e06..9fff98fa 100644 --- a/crates/server/tests/plugin_contract.rs +++ b/crates/server/tests/plugin_contract.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use common::{start_with, unique, ServerOpts, TestServer}; use forge_plugin::api::{ Action, Answer, ChangeRequest, FormatError, FormatPlugin, Loaded, PackageInfo, PackageRef, - PackageSource, PackageVersions, ParsedPackage, PublishRequest, Query, Rendered, Repository, - Request, Response, SearchQuery, Upload, + PackageSource, PackageVersions, ParsedPackage, PublishRequest, Query, Repository, Request, + Response, SearchQuery, Upload, }; use forge_plugin::Plugin; use reqwest::StatusCode; @@ -25,7 +25,7 @@ const PROBE_TOML: &str = r#" id = "probe" name = "Probe" version = "0.0.1" -api = "0.3.0" +api = "0.4.0" mount = "/probe" kinds = ["hosted"] @@ -164,7 +164,7 @@ impl FormatPlugin for Probe { } } - fn render(&self, _repo: &Repository, _req: &Request, data: &[Loaded]) -> Rendered { + fn render(&self, _repo: &Repository, _req: &Request, data: &[Loaded]) -> Action { let answers: Vec = data .iter() .map(|loaded| match &loaded.answer { @@ -184,7 +184,7 @@ impl FormatPlugin for Probe { other => json!(format!("{other:?}")), }) .collect(); - Rendered::Respond(Response::new( + Action::Respond(Response::new( 200, "application/json", Value::Array(answers).to_string(), diff --git a/docs/changes/2.0.0/431.feature.md b/docs/changes/2.0.0/431.feature.md new file mode 100644 index 00000000..f96ec9fe --- /dev/null +++ b/docs/changes/2.0.0/431.feature.md @@ -0,0 +1,11 @@ +Raw file repositories are a format plugin, `plugins/raw`, like NuGet: the +plugin speaks the `/raw` protocol (paths, checksum sidecars, listings, +Nexus' form fields, a proxy's upstream URLs) and the server's new file store +keeps the files. `/raw` answers as before. The plugin contract, +`forge:format@0.4.0`, gains a second storage model for this: a plugin whose +`plugin.toml` says `storage = "files"` asks the host for files and +directories, stores and deletes files, and serves a file with `serve-file`; +`render` may now answer with any action `handle` can. `GET /admin/formats` +and the repositories API report what a repository holds (`storage`: +`packages`, `files` or `images`). See +[docs/format-plugins.md](docs/format-plugins.md#files-formats). diff --git a/docs/format-plugins.md b/docs/format-plugins.md index 278e8421..674e582b 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -11,17 +11,22 @@ streaming of package bytes. Every plugin is built in: a crate under `plugins//` linked into the server and called directly, like any other Rust code. Plugins need no -configuration. Forge ships one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). -Helm charts are served by the registry ([docker.md](docker.md#helm-charts)). +configuration. Forge ships two: **NuGet** ([`plugins/nuget`](../plugins/nuget)) +and **raw files** ([`plugins/raw`](../plugins/raw), +[Files formats](#files-formats)). Helm charts are served by the registry +([docker.md](docker.md#helm-charts)). -> **Status: preview.** The plugin contract is `forge:format@0.3.0`. It may +> **Status: preview.** The plugin contract is `forge:format@0.4.0`. It may > change before 1.x of the contract; a plugin declares the contract version it -> was built for, and the server refuses one it does not implement. +> was built for, and the server refuses one it does not implement. 0.4.0 added +> files formats ([Files formats](#files-formats)), and `render` returns an +> action, like `handle`. ## Contents - [Layout](#layout) - [How it fits together](#how-it-fits-together) +- [Files formats](#files-formats) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) - [Security model](#security-model) @@ -40,6 +45,15 @@ plugins/nuget/ ├── src/protocol.rs # the V3 protocol: handle and render ├── src/upstream.rs # proxies: reading another V3 feed (upstream) └── tests/ # the protocol against canned answers (forge-plugin-api `testing`) + +plugins/raw/ +├── Cargo.toml +├── plugin.toml # mount "/raw", storage "files", kinds hosted/proxy/group +├── src/lib.rs # impl FormatPlugin for Raw: paths, methods, upstream URLs +├── src/listing.rs # directory listings as HTML or JSON +├── src/sidecar.rs # .sha256/.sha512/.sha1/.md5 from the stored digests +├── src/form.rs # multipart uploads, Nexus' field names +└── tests/ # the protocol against canned answers ``` The pieces: @@ -49,7 +63,7 @@ The pieces: | [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, and, behind its `testing` feature, helpers that run a plugin's protocol against canned answers. Depends on nothing but `std`. | | [`plugins/`](../plugins) | One crate per plugin, implementing `FormatPlugin`. | | [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin: its `plugin.toml` and its implementation) and `PluginFormat`, which adapts it to Forge's `Format` seam. | -| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`) and the `{mount}/{repo}/...` dispatcher. | +| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`), the `{mount}/{repo}/...` dispatcher, and the file store files formats keep their files in ([`crates/server/src/files`](../crates/server/src/files)). | ## How it fits together @@ -63,6 +77,7 @@ The pieces: | Proxies | Sends every upstream request (credentials only to the upstream's origin, bounded, cached), downloads and verifies packages, keeps the package delay, the not-found markers and the offline mode | `upstream`: which documents an operation needs, and what they say ([Proxy repositories](#proxy-repositories)) | | Groups | Merges members by priority; the dependency-confusion guards apply | Nothing: a group of plugin repositories just works | | Retention | Evaluates policies, ordering versions with the plugin's `compare_versions` | Nothing more | +| Files (a files format) | Keeps files under paths, content-addressed with their digests: serving, ranges and validators, uploads under the deployment policy, listings, proxies and groups, selectors on paths, retention by path ([Files formats](#files-formats)) | Which path a request names, which file or directory it needs, what the response looks like | A request to a plugin repository is served under its format's mount, `{mount}/{repo}/...`: `/repository` unless the plugin's `plugin.toml` names @@ -82,9 +97,11 @@ in this order: 3. Act on the plugin's answer: - `respond`: send the response as given, keeping only the allowed headers. - `load(queries)`: run the queries, then call `render` with every query - and its answer. `render` either responds or asks for more data. One - request gets at most 4 rounds and 2,048 answers. + and its answer. `render` answers like `handle` does: a response, more + data to load, or one of the actions below. One request gets at most 4 + rounds and 2,048 answers. - `download`: stream the stored package. + - `serve-file`: stream a stored file (a files format). - `publish`, `unlist`, `relist`, `delete`: go through the hosted write path, but only when the request's method was authorized for that action (a plugin cannot turn a GET into a delete). The repository must be @@ -103,6 +120,9 @@ Its queries are targeted: | `manifest(id, version)` | The manifest stored at publish | | `search(text, skip, take, prerelease, semver2, upstream)` | Matching packages with their versions, the latest version's summary and the download total | +A files format asks for files instead ([Files formats](#files-formats)); a +query of the other kind fails the request. + Prerelease filtering is the host's, and its rule is that a normalized version is a prerelease when it contains a `-`; `semver2` adds SemVer 2.0.0 versions (dotted prerelease labels, build metadata). Publish times are RFC 3339 in UTC @@ -120,6 +140,57 @@ NuGet's). Every listing hands each version's summary to the plugin, so a summary should stay small. +## Files formats + +A format whose `plugin.toml` says `storage = "files"` holds files under +paths, not package versions: [raw](raw.md) is one. It keeps the defaults of +the identity rules and `parse_package` (which refuse every id, version and +package) and implements the protocol over the host's file store, which keeps +everything a file repository is: the files, content-addressed, with their +SHA-256, SHA-512, SHA-1 and MD5; uploads, under the repository's upload +limit and deployment policy (`allow_redeploy`, `disable_redeploy`, +`read_only`); serving with one `Range`, `If-Range`, `ETag`, `Last-Modified` +and conditional requests, under the repository's content disposition; +listings that walk an index; proxies that fetch on first use, follow +redirects and re-validate after `max_age_seconds`; groups; package selectors +on paths; retention by path prefix, age, downloads and the newest N per +directory; and the web UI's upload, delete and the admin API's directory +delete. + +The plugin decides what a request is, in the vocabulary of paths. A path is +segments joined by `/`, without a leading or trailing `/`, none of them +empty, `.` or `..`, nor holding a backslash or a control character +([`forge_plugin_api::files`](../crates/plugin-api/src/files.rs) parses +request paths by these rules); a directory is empty (the root) or a path +ending in `/`. The host refuses any other path a plugin names. + +| Query | Answer | +| --- | --- | +| `file(path)` | The stored file with its size, content type, digests, modification and last download time, or none. A proxy re-validates its copy once it is due; a group answers with its first member, by priority, that has the path | +| `fetch-file(path)` | `file`, and a proxy that has not cached the path asks its upstream (`upstream`'s `file` operation) | +| `directory(directory, after, limit)` | A page of a directory: files and subdirectories in byte order of their names, and the `after` of the next page; none for a directory that does not exist. A group merges its members' entries | +| `form` | Receives the request's `multipart/form-data` body: file parts are staged, other fields read | +| `store(files)` | Stores files, each from the request body or a form part, all or none, under the deployment policy; what became of each (`created`, `replaced`, `unchanged`) | +| `remove(path)` | Deletes a file; whether there was one | + +`form`, `store` and `remove` change the repository: each comes alone in its +round, needs the method's permission (write, or delete for `remove`), a +hosted repository and a package-client credential, and when one fails (403, +405, 409, 413) the host answers with its own status. The action +`serve-file(path)` serves a file as `fetch-file` finds it. The `repository` +a request is for says whether its files are shown inline (`inline`), for +responses the plugin renders itself. + +Raw maps its URLs onto these: `GET /raw//` is `file(path)` +(and `file` of the file a checksum sidecar names), then `serve-file` or +the sidecar rendered from the file's digests, else `fetch-file` in a proxy +or group, else a `directory` probe for a redirect; `GET …//` is a +`directory` page rendered as HTML or JSON; `PUT` is a `store` of the body; +`POST …//` is a `form`, then a `store` of its parts with Nexus' field +names applied; `DELETE` is a `remove`. [plugins/raw](../plugins/raw) is the +whole protocol, in a few hundred lines; the file store is +[`crates/server/src/files`](../crates/server/src/files). + ## Proxy repositories A plugin that lists `proxy` among its kinds implements the upstream side of @@ -140,6 +211,7 @@ what they say: | `manifest(id, version)` | The manifest, verified to be that version's, without downloading the package | | `metadata(versions)`, `dating(versions)` | Per version: listing state, publish time, a manifest summary for a version not downloaded yet, and (`dating`) when it was created upstream, which the package delay counts from | | `search(query)` | A page of the upstream's search results | +| `file(path)` | A files format: the file's URL, or none when the upstream cannot have it. The host downloads it (following redirects; credentials go to `remote_url`'s origin only), caches it, and re-validates it with a conditional GET once the repository's `max_age_seconds` has passed (raw: `/`) | Each operation is a conversation: `upstream` either answers or asks for GETs (`fetch`), and is called again with every fetch so far and how it went @@ -151,8 +223,8 @@ conversation gets at most 4 rounds and 256 fetches, 4 at a time. The host sends a fetch only where the upstream points: to the origin (scheme, host and port) of `remote_url`, or to an origin that an absolute `http` or `https` URL names in a document the host fetched for the same operation (in an -earlier round), or while resolving `index`. The URL `package` answers is held -to the same rule. Anything else is not sent: the fetch comes back as a +earlier round), or while resolving `index`. The URLs `package` and `file` +answer are held to the same rule. Anything else is not sent: the fetch comes back as a failure, a package URL fails the download like an upstream error, and the host logs a warning naming the format and the repository. What the plugin answers names no origin: its index is its own making (and the host keeps fields of its @@ -174,11 +246,12 @@ cached. ## Using a plugin `GET /admin/formats` lists every format the server can create repositories -of: the plugins (NuGet), Docker and raw files, each with the repository -kinds it supports and where its repositories are served. The web UI's *New -repository* form offers them. Plugins serve **hosted** and +of: the plugins (NuGet, raw files) and Docker, each with the repository +kinds it supports, what its repositories hold (`storage`: `packages`, +`files`, or Docker's `images`) and where its repositories are served. The +web UI's *New repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream -side (NuGet does). Docker has proxies of its own. +side (NuGet and raw do). Docker has proxies of its own. A repository of a plugin's format is created like any other, with the plugin's format id, and served under the plugin's mount (`/repository` @@ -241,10 +314,10 @@ The host enforces these for every plugin: plugin cannot have Forge read `http://localhost:8421/admin/…` or an internal search cluster that its upstream does not point at, and pass the answers on through the proxy's metadata. -- **Responses cannot attack the origin.** Only `cache-control`, +- **Responses cannot attack the origin.** Only `allow`, `cache-control`, `content-disposition`, `content-language`, `content-type`, `etag`, - `last-modified` and `location` survive; `location` survives only when it - stays inside the repository. Every response gets the feed's hardening + `last-modified`, `link`, `location` and `vary` survive; `location` and + `link` survive only when they stay inside the repository. Every response gets the feed's hardening headers (`Content-Security-Policy: sandbox`, `nosniff`, `Content-Disposition: attachment`). A plugin cannot set cookies, CORS headers, or redirect a client elsewhere. @@ -268,7 +341,7 @@ workspace, depending on `forge-plugin-api` and implementing its trait ([`plugins/nuget`](../plugins/nuget) is a complete one): ```rust -use forge_plugin_api::{Action, FormatPlugin, Loaded, Rendered, Repository, Request /* … */}; +use forge_plugin_api::{Action, FormatPlugin, Loaded, Repository, Request /* … */}; /// This crate's plugin.toml (id, name, version). pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); @@ -279,6 +352,7 @@ pub struct MyFormat; impl FormatPlugin for MyFormat { // identity: canonical_id, validate_id, normalize_version, compare_versions // packages: parse_package, manifest_summary + // (a files format keeps the defaults of these, which refuse everything) // protocol: handle, render // proxies (optional): upstream } @@ -290,10 +364,11 @@ impl FormatPlugin for MyFormat { id = "example" # the format id repositories are created with name = "Example packages" version = "0.1.0" -api = "0.3.0" # the forge:format contract +api = "0.4.0" # the forge:format contract mount = "/repository" # optional: served at {mount}/{repo}/… kinds = ["hosted", "group"] # optional: the repository kinds it serves # ("proxy" too when it implements `upstream`) +storage = "packages" # optional: or "files" (Files formats) [assets] # optional: how a version's files are stored package = "package" @@ -318,7 +393,8 @@ rules hold: the staged package. Read only what you need, and bound everything you decompress: a gzip bomb should fail fast. - **Ask for data, don't assume it.** `handle` returns `Action::Load(queries)`; - `render` gets each query with its answer and may load more. Build absolute + `render` gets each query with its answer and may load more, respond, or + hand the request to the host (a download, a publish, serving a file). Build absolute URLs from the repository's `base_url`. Groups and kinds are already resolved by the time data reaches you. - **Test the protocol without a server.** `forge-plugin-api`'s `testing` @@ -399,9 +475,11 @@ same way, a group holds members of its format only, and repositories, RBAC, package selectors (on image names), retention and the blob store are the same as for every other format. -Raw file repositories ([raw.md](raw.md)) are native for similar reasons: -they hold files under paths, not package versions, list directories, take -several files in one form, always answer ranges and conditional requests, -and their proxies follow redirects and re-validate with `If-None-Match`. They -live in the server (`crates/server/src/raw`, served under `/raw`) and are -registered like Docker, with selectors on paths. +Raw file repositories ([raw.md](raw.md)) were native for similar reasons +until contract 0.4.0: they hold files under paths, not package versions, +list directories, take several files in one form, always answer ranges and +conditional requests, and their proxies follow redirects and re-validate with +`If-None-Match`. 0.4.0 gave the contract a second storage model, files +([Files formats](#files-formats)), so raw is a plugin now +([`plugins/raw`](../plugins/raw)) like NuGet, and what was the raw server +module is the host's file store. diff --git a/docs/raw.md b/docs/raw.md index e60f98cf..02aa353f 100644 --- a/docs/raw.md +++ b/docs/raw.md @@ -6,9 +6,12 @@ have no package manager of their own (the [pdfium binaries](https://github.com/bblanchon/pdfium-binaries), for example), installers, firmware images, test fixtures, tool downloads. -Files live in repositories of `format: "raw"`, a format built into Forge -like Docker (`GET /admin/formats` lists it), served at -`https:///raw//`. They come in the usual three kinds: +Files live in repositories of `format: "raw"`, a format plugin built into +Forge like NuGet ([`plugins/raw`](../plugins/raw); `GET /admin/formats` +lists it), served at `https:///raw//`. The plugin +speaks the protocol below; the host's file store keeps the files +([format-plugins.md](format-plugins.md#files-formats)). They come in the +usual three kinds: - **hosted**: upload, download and delete your own files; - **proxy**: a cache of files under an HTTP(S) base URL, such as a GitHub diff --git a/plugins/README.md b/plugins/README.md index 482620bf..d0b2a277 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -8,6 +8,7 @@ the design, the contract and how to write one. | Plugin | Format id | What it serves | | --- | --- | --- | | [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | +| [`raw`](raw) | `raw` | Raw files at `/raw`: files under paths, uploads with `PUT` or a form, ranges, checksum sidecars, listings; hosted, proxy and group repositories ([docs/raw.md](../docs/raw.md)) | A plugin directory holds: @@ -15,7 +16,8 @@ A plugin directory holds: - `plugin.toml`: the format id (also the directory's name), display name, version, the `forge:format` contract version it targets (`api`), and optionally its mount, repository kinds (`proxy` when it implements - `upstream`) and `[assets]` names; + `upstream`), its `storage` (`packages`, or `files` for a files format such + as raw) and `[assets]` names; - `src/`: `impl FormatPlugin`, and `pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!();`. diff --git a/plugins/nuget/plugin.toml b/plugins/nuget/plugin.toml index f4165450..99cb0a5d 100644 --- a/plugins/nuget/plugin.toml +++ b/plugins/nuget/plugin.toml @@ -3,7 +3,7 @@ id = "nuget" name = "NuGet" version = "1.0.0" -api = "0.3.0" +api = "0.4.0" description = "NuGet V3 feeds: push, restore and search with dotnet, nuget.exe and Visual Studio; proxies of nuget.org and other V3 feeds." mount = "/v3" # Proxies mirror another V3 feed through the plugin's upstream side. diff --git a/plugins/nuget/src/lib.rs b/plugins/nuget/src/lib.rs index 0b6c3a1d..7c618432 100644 --- a/plugins/nuget/src/lib.rs +++ b/plugins/nuget/src/lib.rs @@ -29,7 +29,7 @@ use std::cmp::Ordering; use forge_plugin_api::{ Action, FetchResult, FormatError, FormatPlugin, Loaded, PackageSource, ParsedPackage, Remote, - Rendered, Repository, Request, UpstreamOp, UpstreamStep, + Repository, Request, UpstreamOp, UpstreamStep, }; pub use model::NupkgLimits; @@ -82,7 +82,7 @@ impl FormatPlugin for Nuget { protocol::handle(repo, req) } - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { protocol::render(repo, req, data) } diff --git a/plugins/nuget/src/protocol.rs b/plugins/nuget/src/protocol.rs index 4869c64f..dd821b19 100644 --- a/plugins/nuget/src/protocol.rs +++ b/plugins/nuget/src/protocol.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use forge_plugin_api::{ Action, Answer, ChangeRequest, Loaded, PackageInfo, PackageRef, PackageVersions, - PublishRequest, Query, Rendered, Repository, RepositoryKind, Request, Response, SearchQuery, + PublishRequest, Query, Repository, RepositoryKind, Request, Response, SearchQuery, SearchResult, Upload, VersionInfo, }; use serde_json::{json, Value}; @@ -263,9 +263,9 @@ pub(crate) fn handle(repo: &Repository, req: &Request) -> Action { } } -pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let respond = Rendered::Respond; - let not_found = |text: &str| Rendered::Respond(Response::text(404, text)); +pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + let respond = Action::Respond; + let not_found = |text: &str| Action::Respond(Response::text(404, text)); let Ok(route) = route(req, repo.kind) else { return not_found("not found"); }; @@ -319,7 +319,7 @@ pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rende base, &lower, &versions, ))) } else { - Rendered::Load(vec![Query::Packages(lower)]) + Action::Load(vec![Query::Packages(lower)]) } } Some([]) => not_found("registration not found"), @@ -340,12 +340,10 @@ pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rende let versions = version_strings(versions().unwrap_or_default()); match registration_page_versions(&versions, &lower, upper) { // Only the page's versions are loaded. - Some(in_range) => { - Rendered::Load(vec![Query::PackagesOf(PackageVersions { - id: lower_id, - versions: in_range.to_vec(), - })]) - } + Some(in_range) => Action::Load(vec![Query::PackagesOf(PackageVersions { + id: lower_id, + versions: in_range.to_vec(), + })]), None => not_found("registration page not found"), } } diff --git a/plugins/nuget/src/upstream.rs b/plugins/nuget/src/upstream.rs index f09df9c9..696c6e66 100644 --- a/plugins/nuget/src/upstream.rs +++ b/plugins/nuget/src/upstream.rs @@ -137,6 +137,8 @@ pub(crate) fn step( UpstreamOp::Metadata(wanted) => metadata(&resources, wanted, false, fetched), UpstreamOp::Dating(wanted) => metadata(&resources, wanted, true, fetched), UpstreamOp::Search(query) => search(&resources, query, fetched), + // A feed holds packages: it has no files to fetch. + UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), }) } diff --git a/plugins/nuget/tests/protocol.rs b/plugins/nuget/tests/protocol.rs index 39374941..1fca7409 100644 --- a/plugins/nuget/tests/protocol.rs +++ b/plugins/nuget/tests/protocol.rs @@ -23,6 +23,7 @@ fn repo(kind: RepositoryKind) -> Repository { name: "main".into(), kind, base_url: format!("{BASE}/v3/main"), + inline: false, } } @@ -100,6 +101,7 @@ impl MockRepo { .collect(), }) } + other => panic!("NuGet asked for files: {other:?}"), } } diff --git a/plugins/raw/Cargo.toml b/plugins/raw/Cargo.toml new file mode 100644 index 00000000..a8935ec1 --- /dev/null +++ b/plugins/raw/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "forge-format-raw" +# The plugin's own version: keep it equal to `version` in plugin.toml. +version = "1.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Raw file repositories for Forge, as a format plugin" + +[dependencies] +forge-plugin-api = { path = "../../crates/plugin-api" } +# The JSON listing. +serde = { workspace = true } +serde_json = { workspace = true } +# Last-Modified and If-Modified-Since of the checksum sidecars. +httpdate = "1" +# Upstream URLs of a proxy's files, listing queries and links. +url = { workspace = true } +percent-encoding = "2.3" +base64 = { workspace = true } + +[dev-dependencies] +forge-plugin-api = { path = "../../crates/plugin-api", features = ["testing"] } diff --git a/plugins/raw/plugin.toml b/plugins/raw/plugin.toml new file mode 100644 index 00000000..be343f08 --- /dev/null +++ b/plugins/raw/plugin.toml @@ -0,0 +1,13 @@ +# A Forge format plugin: raw file repositories (see plugins/README.md). +[plugin] +id = "raw" +name = "Raw files" +version = "1.0.0" +api = "0.4.0" +description = "Arbitrary files under arbitrary paths, like Nexus' raw repositories: uploads with PUT or a form, downloads with ranges and checksum sidecars, directory listings; proxies of any HTTP(S) base URL." +mount = "/raw" +# Proxies cache the files under an HTTP(S) base URL through the plugin's +# upstream side. +kinds = ["hosted", "proxy", "group"] +# Its repositories hold files under paths, in the host's file store. +storage = "files" diff --git a/plugins/raw/src/form.rs b/plugins/raw/src/form.rs new file mode 100644 index 00000000..85b75586 --- /dev/null +++ b/plugins/raw/src/form.rs @@ -0,0 +1,189 @@ +//! Uploads of several files: `POST /raw///` with a +//! `multipart/form-data` body, whose file parts are stored under +//! `directory`, each at its part's file name. Nexus' field names for its +//! component upload work too: `raw.directory` (or `directory`) names a +//! subdirectory, and `.filename` (`raw.asset1.filename`) the name of +//! the file part ``. The host receives the form +//! ([`Query::Form`]) and stores every file, or none ([`Query::Store`]). + +use std::collections::{BTreeMap, HashSet}; + +use forge_plugin_api::files; +use forge_plugin_api::{ + Action, Answer, FileSource, FormContent, FormPart, Loaded, NewFile, Query, Response, + StoreStatus, StoredFile, +}; + +/// The next step of a form upload into `directory`: store the form's files +/// once it is received, answer once they are stored. +pub fn render(directory: &str, data: &[Loaded]) -> Action { + match data.last().map(|loaded| &loaded.answer) { + Some(Answer::Form(parts)) => match place(directory, parts) { + Ok(files) => Action::Load(vec![Query::Store(files)]), + Err(message) => Action::Respond(Response::text(400, &message)), + }, + Some(Answer::Stored(files)) => Action::Respond(stored(files)), + _ => Action::Respond(Response::text(500, "raw: the form was not received")), + } +} + +/// Where each file of a form goes, or why the form is refused. +pub fn place(directory: &str, parts: &[FormPart]) -> Result, String> { + let mut texts: BTreeMap<&str, &str> = BTreeMap::new(); + let mut uploads = Vec::new(); + for (index, part) in parts.iter().enumerate() { + match &part.content { + FormContent::File(file) => uploads.push((index, part.field.as_str(), file)), + FormContent::Text(value) => { + texts.insert(part.field.as_str(), value.as_str()); + } + } + } + if uploads.is_empty() { + return Err("the form holds no file".into()); + } + + // The fields: a subdirectory, and the names of file parts. + let mut subdirectory = ""; + let fields: HashSet<&str> = uploads.iter().map(|(_, field, _)| *field).collect(); + for (name, value) in &texts { + match *name { + "directory" | "raw.directory" => subdirectory = value, + other + if other + .strip_suffix(".filename") + .is_some_and(|field| fields.contains(field)) => {} + other => { + return Err(format!( + "unknown form field {other:?} (file parts, directory and .filename are accepted)" + )) + } + } + } + let base = files::normalize_directory(subdirectory).map_err(|e| format!("directory: {e}"))?; + let directory = format!("{directory}{base}"); + + let mut seen = HashSet::new(); + let mut placed = Vec::with_capacity(uploads.len()); + for (index, field, file) in uploads { + let name = texts + .get(format!("{field}.filename").as_str()) + .copied() + .unwrap_or(&file.name); + let target = + files::join(&directory, name).map_err(|e| format!("file name {name:?}: {e}"))?; + if !seen.insert(target.clone()) { + return Err(format!("the form holds {target} twice")); + } + placed.push(NewFile { + path: target, + source: FileSource::Part(u32::try_from(index).unwrap_or(u32::MAX)), + content_type: file.content_type.clone(), + }); + } + Ok(placed) +} + +/// The answer to a stored form: what became of each file, 201 when one was +/// new. +fn stored(files: &[StoredFile]) -> Response { + let created = files.iter().any(|f| f.status == StoreStatus::Created); + let files: Vec = files + .iter() + .map(|f| { + serde_json::json!({ + "path": f.path, + "size": f.size, + "sha256": f.sha256, + "status": match f.status { + StoreStatus::Created => "created", + StoreStatus::Replaced => "replaced", + StoreStatus::Unchanged => "unchanged", + }, + }) + }) + .collect(); + Response::new( + if created { 201 } else { 200 }, + "application/json", + serde_json::json!({ "files": files }).to_string(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::FormFile; + + fn file(field: &str, name: &str) -> FormPart { + FormPart { + field: field.into(), + content: FormContent::File(FormFile { + name: name.into(), + content_type: Some("application/gzip".into()), + size: 3, + }), + } + } + + fn text(field: &str, value: &str) -> FormPart { + FormPart { + field: field.into(), + content: FormContent::Text(value.into()), + } + } + + #[test] + fn files_go_under_the_directory_by_their_names() { + let placed = place("pdfium/", &[file("file", "a.tgz"), file("file", "b.tgz")]).unwrap(); + let paths: Vec<_> = placed.iter().map(|f| f.path.as_str()).collect(); + assert_eq!(paths, ["pdfium/a.tgz", "pdfium/b.tgz"]); + assert_eq!(placed[1].source, FileSource::Part(1)); + assert_eq!(placed[0].content_type.as_deref(), Some("application/gzip")); + } + + #[test] + fn nexus_fields_name_the_directory_and_the_files() { + let parts = [ + text("raw.directory", "7350"), + file("raw.asset1", "upload.bin"), + text("raw.asset1.filename", "pdfium-linux-x64.tgz"), + file("raw.asset2", "pdfium-win-x64.tgz"), + ]; + let placed = place("pdfium/", &parts).unwrap(); + let placed: Vec<_> = placed + .iter() + .map(|f| (f.path.as_str(), &f.source)) + .collect(); + assert_eq!( + placed, + [ + ("pdfium/7350/pdfium-linux-x64.tgz", &FileSource::Part(1)), + ("pdfium/7350/pdfium-win-x64.tgz", &FileSource::Part(3)), + ] + ); + } + + #[test] + fn bad_forms_are_refused() { + assert_eq!( + place("", &[text("directory", "x")]).unwrap_err(), + "the form holds no file" + ); + assert!(place("", &[file("f", "a"), text("other", "x")]) + .unwrap_err() + .starts_with("unknown form field \"other\"")); + // A name for a part that is not there is unknown too. + assert!(place("", &[file("f", "a"), text("g.filename", "x")]).is_err()); + assert!(place("", &[file("f", "a"), text("directory", "../x")]) + .unwrap_err() + .starts_with("directory: ")); + assert!(place("", &[file("f", "../a")]) + .unwrap_err() + .starts_with("file name \"../a\": ")); + assert_eq!( + place("d/", &[file("f", "a"), file("g", "a")]).unwrap_err(), + "the form holds d/a twice" + ); + } +} diff --git a/plugins/raw/src/lib.rs b/plugins/raw/src/lib.rs new file mode 100644 index 00000000..7a2e0fc7 --- /dev/null +++ b/plugins/raw/src/lib.rs @@ -0,0 +1,274 @@ +//! Raw file repositories for Forge: files under paths, like Nexus' +//! raw-hosted, raw-proxy and raw-group repositories. +//! +//! Served under a Forge repository's base URL (`/raw/`): +//! +//! * `GET` / `HEAD /` — a file, which the host streams +//! ([`Action::ServeFile`]: one range, validators, conditional requests); +//! else the checksum sidecar `` of a stored file +//! (`.sha256`, `.sha512`, `.sha1`, `.md5`; [`sidecar`]); else, for a path +//! that is a directory, a redirect to the directory; +//! * `GET` / `HEAD //` — a listing, as HTML or JSON ([`listing`]); +//! * `PUT /` — store the request body as the file; +//! * `POST //` — store the files of a multipart form, Nexus' field names +//! included ([`form`]); +//! * `DELETE /` — delete a file. +//! +//! Paths follow [`forge_plugin_api::files`]: a path that needs normalizing +//! (`..`, `//`, an encoded `/`) is refused, never resolved. Everything else +//! a repository is, the host's file store keeps: the content-addressed +//! files and their digests, the deployment policy, package selectors on +//! paths, groups, and proxies, whose file `` is the upstream's +//! `/` ([`upstream_url`]). +//! +//! Forge links this plugin in ([`Raw`], [`PLUGIN_TOML`]), like NuGet. + +pub mod form; +pub mod listing; +pub mod sidecar; + +use forge_plugin_api::files::{self, Target}; +use forge_plugin_api::{ + Action, Answer, DirectoryQuery, FetchResult, FileSource, FormatError, FormatPlugin, Loaded, + NewFile, Query, Remote, Repository, RepositoryKind, Request, Response, StoreStatus, + UpstreamAnswer, UpstreamOp, UpstreamStep, +}; + +/// This plugin's `plugin.toml`: its format id, name and version. +pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); + +/// The raw format. +#[derive(Debug, Clone, Copy, Default)] +pub struct Raw; + +impl FormatPlugin for Raw { + fn handle(&self, repo: &Repository, req: &Request) -> Action { + let target = match target(req) { + Ok(Some(target)) => target, + // `/raw/`: the root listing is `/raw//`. + Ok(None) if is_read(req) => { + return redirect(&format!("{}/", listing::base_path(&repo.base_url))) + } + Ok(None) => return respond(Response::text(404, "not found")), + Err(e) => return respond(Response::text(400, &e.to_string())), + }; + match (req.method.as_str(), target) { + ("GET" | "HEAD", Target::File(path)) => { + let mut queries = vec![Query::File(path.clone())]; + if let Some((base, _)) = sidecar::of(&path) { + queries.push(Query::File(base.to_string())); + } + Action::Load(queries) + } + ("GET" | "HEAD", Target::Directory(directory)) => { + match listing::ListQuery::parse(&req.query) { + Ok(query) => Action::Load(vec![Query::Directory(DirectoryQuery { + directory, + limit: query.limit_for_host(), + after: query.after, + })]), + Err(message) => respond(Response::text(400, &message)), + } + } + ("PUT", Target::File(path)) => Action::Load(vec![Query::Store(vec![NewFile { + path, + source: FileSource::Body, + content_type: req.header("content-type").map(str::to_string), + }])]), + ("PUT", Target::Directory(_)) => respond(Response::text( + 400, + "PUT uploads a file: name it in the path", + )), + ("POST", Target::Directory(_)) => Action::Load(vec![Query::Form]), + ("POST", Target::File(_)) => respond(Response::text( + 400, + "a form upload goes to a directory: end the path with /", + )), + ("DELETE", Target::File(path)) => Action::Load(vec![Query::Remove(path)]), + ("DELETE", Target::Directory(_)) => respond(Response::text( + 400, + "directories are deleted through the admin API (DELETE /admin/repositories//files?path=/)", + )), + _ => respond( + Response::text(405, "method not allowed") + .with_header("allow", "GET, HEAD, PUT, POST, DELETE"), + ), + } + } + + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + let Ok(Some(target)) = target(req) else { + return unexpected("a request it did not load data for"); + }; + match (req.method.as_str(), target) { + ("GET" | "HEAD", Target::File(path)) => file(repo, req, &path, data), + ("GET" | "HEAD", Target::Directory(directory)) => { + listing::render(repo, req, &directory, data) + } + ("PUT", Target::File(_)) => match stored(data) { + Some([file]) if file.status == StoreStatus::Created => { + respond(Response::empty(201)) + } + Some([_]) => respond(Response::empty(204)), + _ => unexpected("a PUT without its stored file"), + }, + ("POST", Target::Directory(directory)) => form::render(&directory, data), + ("DELETE", Target::File(_)) => match data.last().map(|l| &l.answer) { + Some(Answer::Removed(true)) => respond(Response::empty(204)), + Some(Answer::Removed(false)) => respond(Response::text(404, "not found")), + _ => unexpected("a DELETE without its outcome"), + }, + _ => unexpected("a request it did not load data for"), + } + } + + fn upstream( + &self, + remote: &Remote, + op: &UpstreamOp, + _fetched: &[FetchResult], + ) -> Result { + Ok(UpstreamStep::Done(match op { + // Nothing to resolve: a file's URL is the remote's and its path. + UpstreamOp::Index => UpstreamAnswer::Index("{}".into()), + UpstreamOp::File(path) => UpstreamAnswer::File(upstream_url(&remote.url, path)), + _ => UpstreamAnswer::Failed("raw repositories hold files, not packages".into()), + })) + } +} + +/// What a request's path names: `None` for the repository's URL without its +/// `/`. +fn target(req: &Request) -> Result, files::PathError> { + match req.path.strip_prefix('/') { + Some(rest) => files::parse_request(rest).map(Some), + None if req.path.is_empty() => Ok(None), + None => files::parse_request(&req.path).map(Some), + } +} + +fn is_read(req: &Request) -> bool { + matches!(req.method.as_str(), "GET" | "HEAD") +} + +fn respond(response: Response) -> Action { + Action::Respond(response) +} + +/// A permanent redirect to `location`, a path within the repository: the +/// client stays on the host it asked. +fn redirect(location: &str) -> Action { + respond(Response::empty(301).with_header("location", location)) +} + +/// A response for a conversation the host should not have started: the +/// plugin and the host disagree, which is a bug. +fn unexpected(what: &str) -> Action { + respond(Response::text( + 500, + &format!("raw: the host rendered {what}"), + )) +} + +/// The answer of the store a request asked for. +fn stored(data: &[Loaded]) -> Option<&[forge_plugin_api::StoredFile]> { + data.iter().rev().find_map(|loaded| match &loaded.answer { + Answer::Stored(files) => Some(files.as_slice()), + _ => None, + }) +} + +/// The answer to `query`, if it was loaded. +fn answer<'a>(data: &'a [Loaded], query: &Query) -> Option<&'a Answer> { + data.iter() + .find(|loaded| loaded.query == *query) + .map(|loaded| &loaded.answer) +} + +/// The file a `file` or `fetch-file` query found: `None` when it was not +/// asked, `Some(None)` when there is no such file. +fn found<'a>(data: &'a [Loaded], query: &Query) -> Option> { + match answer(data, query)? { + Answer::File(file) => Some(file.as_ref()), + _ => None, + } +} + +/// `GET`/`HEAD` of a file path, round by round: the stored file, else the +/// checksum sidecar of a stored file, else (a proxy, or a group that may +/// have one) the upstream's file, else a redirect to the directory of that +/// name, else 404. +fn file(repo: &Repository, req: &Request, path: &str, data: &[Loaded]) -> Action { + if let Some(Some(_)) = found(data, &Query::File(path.to_string())) { + return Action::ServeFile(path.to_string()); + } + if let Some((base, checksum)) = sidecar::of(path) { + if let Some(Some(file)) = found(data, &Query::File(base.to_string())) { + return respond(sidecar::respond(repo, req, path, file, checksum)); + } + } + match found(data, &Query::FetchFile(path.to_string())) { + Some(Some(_)) => return Action::ServeFile(path.to_string()), + None if repo.kind != RepositoryKind::Hosted => { + return Action::Load(vec![Query::FetchFile(path.to_string())]) + } + _ => {} + } + let directory = format!("{path}/"); + if directory.len() > files::MAX_PATH_BYTES { + return respond(Response::text(404, "not found")); + } + let probe = Query::Directory(DirectoryQuery { + directory: directory.clone(), + after: None, + limit: 1, + }); + match answer(data, &probe) { + None => Action::Load(vec![probe]), + Some(Answer::Directory(Some(_))) => redirect(&format!( + "{}/{}", + listing::base_path(&repo.base_url), + files::encode(&directory) + )), + Some(_) => respond(Response::text(404, "not found")), + } +} + +/// The upstream URL of `path`: `remote_url` (as a directory) with the path's +/// segments appended, each percent-encoded. `None` when `remote_url` is not +/// a URL with a path. +pub fn upstream_url(remote_url: &str, path: &str) -> Option { + let mut url = url::Url::parse(remote_url).ok()?; + { + let mut segments = url.path_segments_mut().ok()?; + segments.pop_if_empty(); + for segment in path.split('/') { + segments.push(segment); + } + } + Some(url.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upstream_urls_append_encoded_segments() { + let base = "https://github.com/bblanchon/pdfium-binaries/releases/download/"; + assert_eq!( + upstream_url(base, "chromium/7350/pdfium-linux-x64.tgz").unwrap(), + "https://github.com/bblanchon/pdfium-binaries/releases/download/chromium/7350/pdfium-linux-x64.tgz" + ); + // Without a trailing slash the base is a directory all the same. + assert_eq!( + upstream_url("http://up.test/files", "a b/c?d#e.txt").unwrap(), + "http://up.test/files/a%20b/c%3Fd%23e.txt" + ); + assert_eq!( + upstream_url("http://up.test", "x.bin").unwrap(), + "http://up.test/x.bin" + ); + assert_eq!(upstream_url("not a url", "x.bin"), None); + } +} diff --git a/plugins/raw/src/listing.rs b/plugins/raw/src/listing.rs new file mode 100644 index 00000000..f5d22999 --- /dev/null +++ b/plugins/raw/src/listing.rs @@ -0,0 +1,473 @@ +//! Directory listings: `GET /raw///`, as HTML for browsers or as +//! JSON (`Accept: application/json`, or `?format=json`), a page at a time. +//! +//! A page holds at most `limit` entries (default 100, at most 1000) after +//! the entry named by `after`, in byte order of their names, a +//! subdirectory's name ending in `/`; `next` (and a `Link: rel="next"` +//! header) names where the following page starts. The host walks its file +//! store for the page ([`forge_plugin_api::Query::Directory`]): package +//! selectors, group members and a proxy's cache are its business. + +use forge_plugin_api::files; +use forge_plugin_api::{Action, Answer, DirectoryEntry, Loaded, Repository, Request, Response}; +use serde::Serialize; + +/// Largest page size. +pub const MAX_LIMIT: usize = 1000; + +/// One entry of a listing. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Entry { + /// The entry's name in its directory; a subdirectory's ends in `/`. + pub name: String, + /// The full path (a subdirectory's ends in `/`). + pub path: String, + /// `file` | `directory`. + #[serde(rename = "type")] + pub kind: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// RFC 3339. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_modified: Option, + /// RFC 3339. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_downloaded: Option, +} + +impl Entry { + fn of(directory: &str, entry: &DirectoryEntry) -> Self { + match entry { + DirectoryEntry::File(file) => Entry { + name: file + .path + .strip_prefix(directory) + .unwrap_or(&file.path) + .to_string(), + path: file.path.clone(), + kind: "file", + size: Some(file.size), + content_type: Some(file.content_type.clone()), + sha256: Some(file.sha256.clone()), + last_modified: Some(file.modified.clone()), + last_downloaded: file.last_downloaded.clone(), + }, + DirectoryEntry::Directory(path) => Entry { + name: path.strip_prefix(directory).unwrap_or(path).to_string(), + path: path.clone(), + kind: "directory", + size: None, + content_type: None, + sha256: None, + last_modified: None, + last_downloaded: None, + }, + } + } +} + +/// A page of a listing. +#[derive(Debug, Clone, Serialize)] +pub struct Page { + pub repository: String, + /// The directory (`""` for the root). + pub path: String, + pub entries: Vec, + /// The `after` of the next page, if there is one. + pub next: Option, +} + +/// The query of a listing. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ListQuery { + pub after: Option, + pub limit: Option, + /// `json` | `html`: overrides `Accept`. + pub format: Option, +} + +impl ListQuery { + /// Parse a request's query string; other parameters are ignored. + pub fn parse(query: &str) -> Result { + let mut out = ListQuery::default(); + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "after" => out.after = Some(value.into_owned()), + "limit" => { + out.limit = Some( + value + .parse() + .map_err(|_| "limit must be a positive number".to_string())?, + ) + } + "format" => match value.as_ref() { + "json" | "html" => out.format = Some(value.into_owned()), + _ => return Err("format is json or html".into()), + }, + _ => {} + } + } + if let Some(after) = out.after.as_deref().filter(|a| !a.is_empty()) { + let name = after.strip_suffix('/').unwrap_or(after); + files::validate_segment(name).map_err(|e| format!("after: {e}"))?; + } + Ok(out) + } + + /// The page size to ask the host for: 0 (its default) unless the client + /// named one. + pub fn limit_for_host(&self) -> u32 { + self.limit + .map_or(0, |limit| limit.clamp(1, MAX_LIMIT) as u32) + } +} + +/// Answer a listing request from the page the host loaded. +pub fn render(repo: &Repository, req: &Request, directory: &str, data: &[Loaded]) -> Action { + let page = data.iter().rev().find_map(|loaded| match &loaded.answer { + Answer::Directory(page) => Some(page), + _ => None, + }); + let Some(page) = page else { + return Action::Respond(Response::text(500, "raw: no listing was loaded")); + }; + let Some(page) = page else { + return Action::Respond(Response::text(404, "not found")); + }; + let query = ListQuery::parse(&req.query).unwrap_or_default(); + let page = Page { + repository: repo.name.clone(), + path: directory.to_string(), + entries: page + .entries + .iter() + .map(|entry| Entry::of(directory, entry)) + .collect(), + next: page.next.clone(), + }; + Action::Respond(respond(repo, &page, &query, wants_json(req, &query))) +} + +/// Whether the client wants JSON: `format=json`, or an `Accept` preferring +/// `application/json` to HTML. +pub fn wants_json(req: &Request, query: &ListQuery) -> bool { + match query.format.as_deref() { + Some("json") => return true, + Some("html") => return false, + _ => {} + } + let accept = req.header("accept").unwrap_or(""); + let quality = |wanted: &str| { + accept + .split(',') + .filter_map(|item| { + let mut parts = item.split(';'); + let media = parts.next()?.trim().to_ascii_lowercase(); + (media == wanted).then(|| { + parts + .filter_map(|p| p.trim().strip_prefix("q=")) + .find_map(|q| q.parse::().ok()) + .unwrap_or(1.0) + }) + }) + .fold(0.0f32, f32::max) + }; + quality("application/json") > 0.0 && quality("application/json") >= quality("text/html") +} + +/// The URL of the page after `page`. +fn next_link(repo: &Repository, page: &Page, query: &ListQuery) -> Option { + let next = page.next.as_deref()?; + let mut params = url::form_urlencoded::Serializer::new(String::new()); + params.append_pair("after", next); + if let Some(limit) = query.limit { + params.append_pair("limit", &limit.to_string()); + } + if let Some(format) = &query.format { + params.append_pair("format", format); + } + Some(format!( + "{}/{}?{}", + base_path(&repo.base_url), + files::encode(&page.path), + params.finish() + )) +} + +/// The path of the repository's URL (`/raw/`), for links and +/// redirects that keep the client on the host it asked. +pub fn base_path(base_url: &str) -> &str { + base_url + .split_once("://") + .and_then(|(_, rest)| rest.find('/').map(|i| &rest[i..])) + .unwrap_or("") +} + +fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c => out.push(c), + } + } + out +} + +/// An RFC 3339 time as `YYYY-MM-DD HH:MM:SS UTC` (the host's times are in +/// UTC). +fn display_time(rfc3339: &str) -> String { + match (rfc3339.get(..10), rfc3339.get(11..19)) { + (Some(date), Some(time)) => format!("{date} {time} UTC"), + _ => rfc3339.to_string(), + } +} + +/// The page as an HTML document: a table of the entries, linked. +pub fn html(repo: &Repository, page: &Page, query: &ListQuery) -> String { + let title = format!("Index of {}/{}", base_path(&repo.base_url), page.path); + let mut out = format!( + "\n{t}\n\n

{t}

\n\n\n", + t = escape(&title) + ); + if !page.path.is_empty() { + out.push_str("\n"); + } + for entry in &page.entries { + let href = files::encode(&entry.name); + out.push_str(&format!( + "\n", + escape(&href), + escape(&entry.name), + entry.size.map(|s| s.to_string()).unwrap_or_default(), + entry + .last_modified + .as_deref() + .map(display_time) + .unwrap_or_default(), + escape(entry.sha256.as_deref().unwrap_or("")), + )); + } + out.push_str("
NameSizeLast modifiedSHA-256
../
{}{}{}{}
\n"); + if let Some(next) = next_link(repo, page, query) { + out.push_str(&format!( + "

Next page

\n", + escape(&next) + )); + } + out.push_str("\n"); + out +} + +/// The listing response: the page, its `Link` to the next page. +pub fn respond(repo: &Repository, page: &Page, query: &ListQuery, json: bool) -> Response { + let mut response = if json { + Response::new( + 200, + "application/json", + serde_json::to_string(page).unwrap_or_default(), + ) + } else { + Response::new(200, "text/html; charset=utf-8", html(repo, page, query)) + }; + // Forge's own document, shown in the browser (and sandboxed). + response = response + .with_header("content-disposition", "inline") + .with_header("vary", "Accept"); + if let Some(next) = next_link(repo, page, query) { + response = response.with_header("link", &format!("<{next}>; rel=\"next\"")); + } + response +} + +/// Seconds since the epoch of an RFC 3339 time (`2026-10-01T12:00:00.250Z`, +/// or with a `+hh:mm` / `-hh:mm` offset), whole seconds. +pub fn epoch_seconds(rfc3339: &str) -> Option { + let number = |range: std::ops::Range| -> Option { + let text = rfc3339.get(range)?; + text.bytes() + .all(|b| b.is_ascii_digit()) + .then(|| text.parse().ok())? + }; + let (year, month, day) = (number(0..4)?, number(5..7)?, number(8..10)?); + let (hour, minute, second) = (number(11..13)?, number(14..16)?, number(17..19)?); + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + // Skip a fraction, then read the offset. + let rest = rfc3339.get(19..)?; + let rest = rest + .strip_prefix('.') + .map_or(rest, |r| r.trim_start_matches(|c: char| c.is_ascii_digit())); + let offset = match rest { + "Z" | "z" => 0, + _ => { + let sign = match rest.get(..1)? { + "+" => 1, + "-" => -1, + _ => return None, + }; + let hours: i64 = rest.get(1..3)?.parse().ok()?; + let minutes: i64 = rest.get(4..6)?.parse().ok()?; + sign * (hours * 3600 + minutes * 60) + } + }; + // Days since 1970-01-01 of a proleptic Gregorian date. + let (y, m) = if month <= 2 { + (year - 1, month + 9) + } else { + (year, month - 3) + }; + let era = y.div_euclid(400); + let year_of_era = y - era * 400; + let day_of_year = (153 * m + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + let days = era * 146_097 + day_of_era - 719_468; + let seconds = days * 86_400 + hour * 3600 + minute * 60 + second - offset; + u64::try_from(seconds).ok() +} + +/// Seconds since the epoch as an HTTP date (IMF-fixdate). +pub fn http_date(seconds: u64) -> String { + httpdate::fmt_http_date(std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds)) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::RepositoryKind; + + fn repo() -> Repository { + Repository { + name: "bin".into(), + kind: RepositoryKind::Hosted, + base_url: "https://forge.example.com/raw/bin".into(), + inline: false, + } + } + + #[test] + fn queries_are_checked() { + assert_eq!(ListQuery::parse("").unwrap(), ListQuery::default()); + let q = ListQuery::parse("after=x.bin&limit=5&format=json&other=1").unwrap(); + assert_eq!(q.after.as_deref(), Some("x.bin")); + assert_eq!(q.limit_for_host(), 5); + assert_eq!(ListQuery::parse("limit=0").unwrap().limit_for_host(), 1); + assert_eq!( + ListQuery::parse("limit=5000").unwrap().limit_for_host(), + 1000 + ); + assert_eq!(ListQuery::default().limit_for_host(), 0); + assert!(ListQuery::parse("limit=x").is_err()); + assert!(ListQuery::parse("format=xml").is_err()); + assert!(ListQuery::parse("after=..%2Fx").is_err()); + assert!(ListQuery::parse("after=b%2Fc").is_err()); + assert!(ListQuery::parse("after=dir%2F").is_ok()); + } + + #[test] + fn json_is_asked_for() { + let query = ListQuery::default(); + let req = Request::new("GET", "/"); + assert!(!wants_json(&req, &query)); + let req = Request::new("GET", "/").with_header("accept", "application/json"); + assert!(wants_json(&req, &query)); + let req = Request::new("GET", "/").with_header( + "accept", + "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", + ); + assert!(!wants_json(&req, &query)); + let query = ListQuery { + format: Some("json".into()), + ..Default::default() + }; + assert!(wants_json(&req, &query)); + } + + #[test] + fn html_escapes_names_and_links_them_encoded() { + let page = Page { + repository: "bin".into(), + path: "a/".into(), + entries: vec![Entry { + name: " y.txt".into(), + path: "a/ y.txt".into(), + kind: "file", + size: Some(3), + content_type: Some("text/plain".into()), + sha256: Some("ab".into()), + last_modified: Some("2026-10-21T10:58:44.123456Z".into()), + last_downloaded: None, + }], + next: Some(" y.txt".into()), + }; + let html = html(&repo(), &page, &ListQuery::default()); + assert!( + html.contains("href=\"%3Cx%3E%20y.txt\"><x> y.txt"), + "{html}" + ); + assert!( + html.contains("Index of /raw/bin/a/"), + "{html}" + ); + assert!(html.contains("href=\"../\"")); + assert!(html.contains("2026-10-21 10:58:44 UTC"), "{html}"); + assert!(html.contains("/raw/bin/a/?after=%3Cx%3E+y.txt"), "{html}"); + assert!(!html.contains("")); + } + + #[test] + fn json_keeps_the_listing_shape() { + let page = Page { + repository: "bin".into(), + path: String::new(), + entries: vec![ + Entry::of("", &DirectoryEntry::Directory("d/".into())), + Entry::of( + "", + &DirectoryEntry::File(forge_plugin_api::FileInfo { + path: "f.txt".into(), + size: 3, + content_type: "text/plain".into(), + sha256: "ab".into(), + sha512: String::new(), + sha1: String::new(), + md5: String::new(), + modified: "2026-10-21T10:58:44Z".into(), + last_downloaded: None, + }), + ), + ], + next: None, + }; + let response = respond(&repo(), &page, &ListQuery::default(), true); + assert_eq!( + String::from_utf8(response.body).unwrap(), + r#"{"repository":"bin","path":"","entries":[{"name":"d/","path":"d/","type":"directory"},{"name":"f.txt","path":"f.txt","type":"file","size":3,"content_type":"text/plain","sha256":"ab","last_modified":"2026-10-21T10:58:44Z"}],"next":null}"# + ); + } + + #[test] + fn times_convert() { + assert_eq!(epoch_seconds("1970-01-01T00:00:00Z"), Some(0)); + assert_eq!( + epoch_seconds("2026-10-01T12:00:00.250Z"), + Some(1_790_856_000) + ); + assert_eq!( + epoch_seconds("2026-10-01T14:00:00+02:00"), + Some(1_790_856_000) + ); + assert_eq!(epoch_seconds("2000-02-29T00:00:00Z"), Some(951_782_400)); + assert_eq!(epoch_seconds("not a time"), None); + assert_eq!(http_date(1_790_856_000), "Thu, 01 Oct 2026 12:00:00 GMT"); + } +} diff --git a/plugins/raw/src/sidecar.rs b/plugins/raw/src/sidecar.rs new file mode 100644 index 00000000..c1d380af --- /dev/null +++ b/plugins/raw/src/sidecar.rs @@ -0,0 +1,210 @@ +//! Checksum sidecars: `.sha256`, `.sha512`, `.sha1` and `.md5`, +//! answered from a stored file's digests when no file of that name is +//! stored, as Nexus does. The body is the digest and the file name, as +//! `sha256sum` and friends print them, so `sha256sum -c` checks a download +//! against it. + +use base64::Engine as _; +use forge_plugin_api::files; +use forge_plugin_api::{FileInfo, Repository, Request, Response}; + +use crate::listing::http_date; + +/// Which digest a sidecar holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Checksum { + Sha256, + Sha512, + Sha1, + Md5, +} + +impl Checksum { + const ALL: [Checksum; 4] = [ + Checksum::Sha256, + Checksum::Sha512, + Checksum::Sha1, + Checksum::Md5, + ]; + + fn extension(self) -> &'static str { + match self { + Checksum::Sha256 => ".sha256", + Checksum::Sha512 => ".sha512", + Checksum::Sha1 => ".sha1", + Checksum::Md5 => ".md5", + } + } + + /// The digest of `file` as lowercase hex. + pub fn hex(self, file: &FileInfo) -> String { + match self { + Checksum::Sha256 => file.sha256.clone(), + Checksum::Sha512 => base64::engine::general_purpose::STANDARD + .decode(&file.sha512) + .map(|bytes| bytes.iter().map(|b| format!("{b:02x}")).collect()) + .unwrap_or_default(), + Checksum::Sha1 => file.sha1.clone(), + Checksum::Md5 => file.md5.clone(), + } + } +} + +/// The file a sidecar path is the checksum of, and which checksum. +pub fn of(path: &str) -> Option<(&str, Checksum)> { + Checksum::ALL.into_iter().find_map(|checksum| { + let base = path.strip_suffix(checksum.extension())?; + (!base.is_empty() && !base.ends_with('/')).then_some((base, checksum)) + }) +} + +/// Whether the request's preconditions say the client's copy is current +/// (`If-None-Match`, else `If-Modified-Since`): answer 304. `modified` is in +/// seconds since the epoch. +pub fn not_modified(req: &Request, etag: &str, modified: Option) -> bool { + if let Some(value) = req.header("if-none-match") { + // Weak comparison: `W/"x"` matches `"x"`. + return value + .split(',') + .map(str::trim) + .any(|tag| tag == "*" || tag.strip_prefix("W/").unwrap_or(tag) == etag); + } + let since = req + .header("if-modified-since") + .and_then(|v| httpdate::parse_http_date(v).ok()); + match (since, modified) { + (Some(since), Some(modified)) => { + std::time::UNIX_EPOCH + std::time::Duration::from_secs(modified) <= since + } + _ => false, + } +} + +/// `Content-Disposition` of `file_path` in `repo`: a download +/// (`attachment`), or with the repository's `inline` option for display, +/// naming the file. +pub fn content_disposition(repo: &Repository, file_path: &str) -> String { + let kind = if repo.inline { "inline" } else { "attachment" }; + let name = files::file_name(file_path); + let plain = |c: char| c.is_ascii() && !c.is_ascii_control() && c != '"' && c != '\\'; + if name.chars().all(plain) { + format!("{kind}; filename=\"{name}\"") + } else { + // RFC 6266: an ASCII fallback, and the UTF-8 name. + let fallback: String = name + .chars() + .map(|c| if plain(c) { c } else { '_' }) + .collect(); + format!( + "{kind}; filename=\"{fallback}\"; filename*=UTF-8''{}", + percent_encoding::utf8_percent_encode(name, percent_encoding::NON_ALPHANUMERIC) + ) + } +} + +/// Answer `GET`/`HEAD` of `path`, the `checksum` sidecar of `base`. +pub fn respond( + repo: &Repository, + req: &Request, + path: &str, + base: &FileInfo, + checksum: Checksum, +) -> Response { + let etag = format!( + "\"{}{}\"", + base.sha256, + checksum.extension().replace('.', "-") + ); + let modified = crate::listing::epoch_seconds(&base.modified); + let validators = |response: Response| { + let response = response.with_header("etag", &etag); + match modified { + Some(seconds) => response.with_header("last-modified", &http_date(seconds)), + None => response, + } + }; + if not_modified(req, &etag, modified) { + return validators(Response::empty(304)); + } + let text = format!("{} {}\n", checksum.hex(base), files::file_name(&base.path)); + validators( + Response::text(200, &text) + .with_header("content-disposition", &content_disposition(repo, path)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::RepositoryKind; + + fn file() -> FileInfo { + FileInfo { + path: "dir/file.tgz".into(), + size: 10, + content_type: "application/gzip".into(), + sha256: "ab".repeat(32), + // SHA-512 of nothing in particular: 64 zero bytes. + sha512: base64::engine::general_purpose::STANDARD.encode([0u8; 64]), + sha1: "cd".repeat(20), + md5: "ef".repeat(16), + modified: "2026-10-01T12:00:00.250Z".into(), + last_downloaded: None, + } + } + + #[test] + fn sidecars_name_their_file() { + assert_eq!(of("a/x.tgz.sha256"), Some(("a/x.tgz", Checksum::Sha256))); + assert_eq!(of("x.md5"), Some(("x", Checksum::Md5))); + assert_eq!(of(".sha1"), None); + assert_eq!(of("a/.sha1"), None); + assert_eq!(of("a/x.tgz"), None); + let file = file(); + assert_eq!(Checksum::Sha512.hex(&file), "00".repeat(64)); + assert_eq!(Checksum::Md5.hex(&file), "ef".repeat(16)); + } + + #[test] + fn preconditions() { + let file = file(); + let modified = crate::listing::epoch_seconds(&file.modified); + let tag = "\"x-sha256\""; + let req = Request::new("GET", "/f.sha256"); + assert!(!not_modified(&req, tag, modified)); + let req = + Request::new("GET", "/f.sha256").with_header("if-none-match", "\"y\", W/\"x-sha256\""); + assert!(not_modified(&req, tag, modified)); + // An `If-None-Match` that fails decides; the date is not consulted. + let date = http_date(modified.unwrap()); + let req = Request::new("GET", "/f.sha256") + .with_header("if-none-match", "\"other\"") + .with_header("if-modified-since", &date); + assert!(!not_modified(&req, tag, modified)); + // Sub-second precision is not part of an HTTP date. + let req = Request::new("GET", "/f.sha256").with_header("if-modified-since", &date); + assert!(not_modified(&req, tag, modified)); + let earlier = http_date(modified.unwrap() - 5); + let req = Request::new("GET", "/f.sha256").with_header("if-modified-since", &earlier); + assert!(!not_modified(&req, tag, modified)); + } + + #[test] + fn dispositions_name_the_file() { + let mut repo = Repository { + name: "r".into(), + kind: RepositoryKind::Hosted, + base_url: "https://forge.example.com/raw/r".into(), + inline: false, + }; + assert_eq!( + content_disposition(&repo, "a/b.txt"), + "attachment; filename=\"b.txt\"" + ); + repo.inline = true; + assert_eq!( + content_disposition(&repo, "a/\"ä\".txt"), + "inline; filename=\"___.txt\"; filename*=UTF-8''%22%C3%A4%22%2Etxt" + ); + } +} diff --git a/plugins/raw/tests/protocol.rs b/plugins/raw/tests/protocol.rs new file mode 100644 index 00000000..44ca8872 --- /dev/null +++ b/plugins/raw/tests/protocol.rs @@ -0,0 +1,280 @@ +//! The raw protocol against canned answers: what each request asks of the +//! host, round by round, and what it answers. + +use forge_format_raw::Raw; +use forge_plugin_api::testing::{repository, serve}; +use forge_plugin_api::{ + Action, Answer, DirectoryEntry, DirectoryPage, DirectoryQuery, FileInfo, FileSource, + FormContent, FormFile, FormPart, NewFile, Query, RepositoryKind, Request, StoreStatus, + StoredFile, +}; + +fn file(path: &str) -> FileInfo { + FileInfo { + path: path.into(), + size: 3, + content_type: "application/gzip".into(), + sha256: "ab".repeat(32), + sha512: String::new(), + sha1: "cd".repeat(20), + md5: "ef".repeat(16), + modified: "2026-10-01T12:00:00.250Z".into(), + last_downloaded: None, + } +} + +fn hosted() -> forge_plugin_api::Repository { + repository(RepositoryKind::Hosted, "/raw") +} + +#[test] +fn a_stored_file_is_served_by_the_host() { + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz"), + |q| match q { + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.outcome, Action::ServeFile("a/x.tgz".into())); +} + +#[test] +fn a_sidecar_is_computed_from_its_file() { + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz.sha1"), + |q| match q { + Query::File(p) if p == "a/x.tgz.sha1" => Answer::File(None), + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.response().status, 200); + assert_eq!(served.text(), format!("{} x.tgz\n", "cd".repeat(20))); + assert_eq!( + served.header("etag").unwrap(), + format!("\"{}-sha1\"", "ab".repeat(32)) + ); + assert_eq!( + served.header("last-modified"), + Some("Thu, 01 Oct 2026 12:00:00 GMT") + ); + assert_eq!( + served.header("content-disposition"), + Some("attachment; filename=\"x.tgz.sha1\"") + ); + // A stored file of the sidecar's name is served as it is. + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz.sha1"), + |q| match q { + Query::File(p) => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.outcome, Action::ServeFile("a/x.tgz.sha1".into())); + // The client's copy is current. + let req = Request::new("GET", "/a/x.tgz.md5") + .with_header("if-none-match", &format!("\"{}-md5\"", "ab".repeat(32))); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + Query::File(_) => Answer::File(None), + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 304); + assert!(served.response().body.is_empty()); +} + +#[test] +fn a_proxy_fetches_what_it_has_not_cached() { + let proxy = repository(RepositoryKind::Proxy, "/raw"); + let served = serve(&Raw, &proxy, &Request::new("GET", "/x.bin"), |q| match q { + Query::File(_) => Answer::File(None), + Query::FetchFile(p) => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.outcome, Action::ServeFile("x.bin".into())); + assert_eq!( + served.queries, + [ + Query::File("x.bin".into()), + Query::FetchFile("x.bin".into()) + ] + ); +} + +#[test] +fn a_directory_named_without_its_slash_redirects() { + let directory = |answer: Option| { + move |q: &Query| match q { + Query::File(_) => Answer::File(None), + Query::Directory(d) => { + assert_eq!( + d, + &DirectoryQuery { + directory: "pdfium/".into(), + after: None, + limit: 1 + } + ); + Answer::Directory(answer.clone()) + } + other => panic!("unexpected {other:?}"), + } + }; + let page = DirectoryPage { + entries: vec![DirectoryEntry::Directory("pdfium/7350/".into())], + next: None, + }; + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/pdfium"), + directory(Some(page)), + ); + assert_eq!(served.response().status, 301); + assert_eq!(served.header("location"), Some("/raw/repo/pdfium/")); + // A hosted repository asks no upstream. + assert!(!served + .queries + .iter() + .any(|q| matches!(q, Query::FetchFile(_)))); + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/pdfium"), + directory(None), + ); + assert_eq!(served.response().status, 404); +} + +#[test] +fn the_repository_url_without_its_slash_redirects() { + let served = serve(&Raw, &hosted(), &Request::new("GET", ""), |q| { + panic!("unexpected {q:?}") + }); + assert_eq!(served.response().status, 301); + assert_eq!(served.header("location"), Some("/raw/repo/")); + let served = serve(&Raw, &hosted(), &Request::new("PUT", ""), |q| { + panic!("unexpected {q:?}") + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn listings_render_the_page_the_host_loaded() { + let req = Request::new("GET", "/a/?limit=1&format=json"); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::Directory(d) => { + assert_eq!(d.directory, "a/"); + assert_eq!(d.limit, 1); + Answer::Directory(Some(DirectoryPage { + entries: vec![DirectoryEntry::File(file("a/x.tgz"))], + next: Some("x.tgz".into()), + })) + } + other => panic!("unexpected {other:?}"), + }); + let response = served.response(); + assert_eq!(response.status, 200); + assert_eq!(served.header("content-type"), Some("application/json")); + assert_eq!( + served.header("link"), + Some("; rel=\"next\"") + ); + let page: serde_json::Value = serde_json::from_slice(&response.body).unwrap(); + assert_eq!(page["entries"][0]["name"], "x.tgz"); + assert_eq!(page["next"], "x.tgz"); + // A directory that does not exist. + let served = serve(&Raw, &hosted(), &Request::new("GET", "/b/"), |_| { + Answer::Directory(None) + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn uploads_and_deletes_go_through_the_host() { + let req = Request::new("PUT", "/a/x.tgz").with_header("content-type", "application/gzip"); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::Store(files) => { + assert_eq!( + files, + &[NewFile { + path: "a/x.tgz".into(), + source: FileSource::Body, + content_type: Some("application/gzip".into()), + }] + ); + Answer::Stored(vec![StoredFile { + path: "a/x.tgz".into(), + size: 3, + sha256: "ab".into(), + status: StoreStatus::Created, + }]) + } + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 201); + + let served = serve(&Raw, &hosted(), &Request::new("POST", "/a/"), |q| match q { + Query::Form => Answer::Form(vec![ + FormPart { + field: "raw.directory".into(), + content: FormContent::Text("7350".into()), + }, + FormPart { + field: "file".into(), + content: FormContent::File(FormFile { + name: "x.tgz".into(), + content_type: None, + size: 3, + }), + }, + ]), + Query::Store(files) => { + assert_eq!(files[0].path, "a/7350/x.tgz"); + assert_eq!(files[0].source, FileSource::Part(1)); + Answer::Stored(vec![StoredFile { + path: files[0].path.clone(), + size: 3, + sha256: "ab".into(), + status: StoreStatus::Unchanged, + }]) + } + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 200); + let body: serde_json::Value = serde_json::from_slice(&served.response().body).unwrap(); + assert_eq!(body["files"][0]["status"], "unchanged"); + + let served = serve(&Raw, &hosted(), &Request::new("DELETE", "/a/x.tgz"), |q| { + assert_eq!(q, &Query::Remove("a/x.tgz".into())); + Answer::Removed(false) + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn bad_paths_and_methods_are_refused() { + for (method, path, status) in [ + ("GET", "/a/../b", 400), + ("GET", "/a//b", 400), + ("GET", "/a%2fb", 400), + ("PUT", "/a/", 400), + ("POST", "/a", 400), + ("DELETE", "/a/", 400), + ("PATCH", "/a", 405), + ("GET", "/?limit=x", 400), + ("GET", "/?after=..", 400), + ] { + let served = serve(&Raw, &hosted(), &Request::new(method, path), |q| { + panic!("{method} {path}: unexpected {q:?}") + }); + assert_eq!(served.response().status, status, "{method} {path}"); + } +}