diff --git a/Cargo.lock b/Cargo.lock index 47c0d88a..7e4c4f22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1805,6 +1805,19 @@ dependencies = [ "zip", ] +[[package]] +name = "forge-format-raw" +version = "1.0.0" +dependencies = [ + "base64 0.23.1", + "forge-plugin-api", + "httpdate", + "percent-encoding", + "serde", + "serde_json", + "url", +] + [[package]] name = "forge-oci" version = "1.1.0" @@ -1873,6 +1886,7 @@ dependencies = [ "forge-crypto", "forge-db", "forge-format-nuget", + "forge-format-raw", "forge-oci", "forge-plugin", "forge-storage", diff --git a/Cargo.toml b/Cargo.toml index 585ee8ee..8a363f31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/server", "crates/e2e", "plugins/nuget", + "plugins/raw", "xtask", ] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fa289034..95a9c84e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -241,8 +241,9 @@ password: `docker login localhost:8421 -u admin@example.com`, then `docker push localhost:8421/docker-hosted/team/app:1`. See [docs/docker.md](docs/docker.md). -Raw repositories (`"format":"raw"`) hold files under arbitrary paths, served -under `/raw/`: `curl -u admin@example.com: --upload-file x.tgz +Raw repositories (`"format":"raw"`, the built-in plugin `plugins/raw`) hold +files under arbitrary paths, served under `/raw/`: `curl -u +admin@example.com: --upload-file x.tgz localhost:8421/raw/binaries/tools/x.tgz`, a proxy of any HTTP(S) base URL (`"remote_url":"https://github.com///releases/download/"`), and groups. See [docs/raw.md](docs/raw.md). diff --git a/README.md b/README.md index 12cbdb70..b2e02064 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,10 @@ A Cargo workspace of small crates plus the web SPA: | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. | | `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. | | `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. | -| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`), the Docker registry, raw file repositories (`/raw`), `/admin` + `/auth` REST, background workers, static SPA serving. | +| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. | | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). | | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. | -| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`. | +| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `raw`. | | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. | **Every format is a plugin in `plugins/`.** A package format is a crate in diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index 67df4ee1..cb736d2a 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -46,6 +46,13 @@ pub trait Format: Send + Sync { /// the `repository.format` column. fn id(&self) -> &'static str; + /// Whether the format's repositories hold files under paths (raw) rather + /// than package versions: their contents are the file store's + /// (`raw_file`), and none of the package seams below apply to them. + fn stores_files(&self) -> bool { + false + } + /// Parse an uploaded package into format-neutral [`ParsedUpload`] data. fn parse_upload(&self, file: &mut dyn ReadSeek) -> Result; diff --git a/crates/core/src/repo/proxy.rs b/crates/core/src/repo/proxy.rs index 1348207a..c15c3953 100644 --- a/crates/core/src/repo/proxy.rs +++ b/crates/core/src/repo/proxy.rs @@ -541,6 +541,16 @@ impl ProxyRepo { ); return Ok((api::UpstreamAnswer::Failed(reason), origins)); } + api::UpstreamStep::Done(api::UpstreamAnswer::File(Some(url))) + if !origins.permits(&url) => + { + tracing::warn!(format = %self.format.id(), repo = %repo.name, %url, "the format answered a file URL on an origin neither the remote_url nor a document fetched from the upstream names; not downloaded"); + let reason = format!( + "upstream file {url}: not downloaded, its origin is neither the \ + remote_url's nor named by a document fetched from the upstream" + ); + return Ok((api::UpstreamAnswer::Failed(reason), origins)); + } api::UpstreamStep::Done(api::UpstreamAnswer::Failed(reason)) if unavailable => { return Err(RepoError::Upstream(reason)); } @@ -669,6 +679,27 @@ impl ProxyRepo { } } + /// The upstream URL of the file at `path` of proxy `repo` + /// ([`api::UpstreamOp::File`], a files format), on an origin the + /// upstream points at; `None` when the upstream cannot have it. The + /// file store downloads and re-validates it. + pub async fn file_url( + &self, + repo: &Repository, + path: &str, + ) -> Result, RepoError> { + match self + .converse_resolved(repo, api::UpstreamOp::File(path.to_string())) + .await? + { + api::UpstreamAnswer::File(url) => Ok(url), + api::UpstreamAnswer::Failed(reason) => Err(RepoError::Storage(reason)), + _ => Err(RepoError::Storage( + "upstream protocol answered file with something else".into(), + )), + } + } + /// The upstream index, resolved, for `op` on `repo`. async fn converse_resolved( &self, diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index e35b86e1..a53428c8 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -231,7 +231,12 @@ impl RetentionWorker { if repo.format == forge_db::model::DOCKER_FORMAT { return self.process_images(repo, kind, run_id).await; } - if repo.format == forge_db::model::RAW_FORMAT { + // A files format's repositories (raw) hold files under paths. + if self + .formats + .resolve(&repo.format) + .is_some_and(|format| format.stores_files()) + { return self.process_files(repo, kind, run_id).await; } diff --git a/crates/db/src/model.rs b/crates/db/src/model.rs index 5f3e6a3b..31a8470a 100644 --- a/crates/db/src/model.rs +++ b/crates/db/src/model.rs @@ -282,9 +282,6 @@ impl Repository { } } -/// `repository.format` of raw (generic file) repositories. -pub const RAW_FORMAT: &str = "raw"; - /// `repository.config` key of [`Repository::deployment_policy`]. pub const DEPLOYMENT_POLICY_KEY: &str = "deployment_policy"; @@ -294,9 +291,9 @@ pub const CONTENT_DISPOSITION_KEY: &str = "content_disposition"; /// `repository.config` key of [`Repository::max_age_seconds`]. pub const MAX_AGE_SECONDS_KEY: &str = "max_age_seconds"; -/// How long a raw proxy serves a cached file before it asks its upstream -/// whether the file changed, when the repository sets no `max_age_seconds`: -/// a day. +/// How long a proxy of files (raw) serves a cached file before it asks its +/// upstream whether the file changed, when the repository sets no +/// `max_age_seconds`: a day. pub const DEFAULT_RAW_MAX_AGE_SECONDS: u64 = 24 * 60 * 60; /// What a hosted raw repository does with an upload to a path it has @@ -333,13 +330,11 @@ impl DeploymentPolicy { } } +/// The options of a repository of a files format (raw): which repositories +/// hold files is the format registry's to say (the server's +/// `FormatRegistry::stores_files`). impl Repository { - /// Whether this is a raw (generic file) repository. - pub fn is_raw(&self) -> bool { - self.format == RAW_FORMAT - } - - /// Hosted raw option (`config.deployment_policy`, default + /// Hosted files option (`config.deployment_policy`, default /// [`DeploymentPolicy::DisableRedeploy`]). pub fn deployment_policy(&self) -> DeploymentPolicy { self.config @@ -349,7 +344,7 @@ impl Repository { .unwrap_or_default() } - /// Raw option (`config.content_disposition = "inline"`, default + /// Files option (`config.content_disposition = "inline"`, default /// `"attachment"`): files are served for display in the browser rather /// than as downloads. pub fn inline_content(&self) -> bool { @@ -359,7 +354,7 @@ impl Repository { == Some("inline") } - /// Raw proxy option (`config.max_age_seconds`): how long a cached file + /// Files proxy option (`config.max_age_seconds`): how long a cached file /// is served before the upstream is asked whether it changed; `0` never /// asks (the upstream's files are immutable). Default /// [`DEFAULT_RAW_MAX_AGE_SECONDS`]. diff --git a/crates/server/src/raw/path.rs b/crates/plugin-api/src/files.rs similarity index 59% rename from crates/server/src/raw/path.rs rename to crates/plugin-api/src/files.rs index 31267441..73f7681c 100644 --- a/crates/server/src/raw/path.rs +++ b/crates/plugin-api/src/files.rs @@ -1,14 +1,20 @@ -//! Paths in raw repositories: what a request names, and how it is stored. +//! Paths in files repositories: what a request names, and how a file is +//! stored. //! //! A stored path is a list of segments joined by `/`, without a leading or -//! trailing `/`: `pdfium/7350/pdfium-linux-x64.tgz`. A request names it -//! percent-encoded after `/raw//`, and is refused when any decoded -//! segment is empty (`//`), `.` or `..` (also spelled `%2e%2e`), holds a `/` -//! (`%2F`), a backslash or a control character, or the path is too long. -//! Nothing is collapsed or resolved: a path that needs normalizing is -//! refused, so there is exactly one spelling of every stored path. +//! trailing `/`: `pdfium/7350/pdfium-linux-x64.tgz`. A directory is `""` (the +//! root) or a path ending in `/`. A request names a path percent-encoded, and +//! is refused when any decoded segment is empty (`//`), `.` or `..` (also +//! spelled `%2e%2e`), holds a `/` (`%2F`), a backslash or a control +//! character, or the path is too long. Nothing is collapsed or resolved: a +//! path that needs normalizing is refused, so there is exactly one spelling +//! of every stored path. +//! +//! The host refuses every path a plugin names that breaks these rules +//! ([`validate_path`], [`validate_directory`]); a plugin parses its requests +//! with [`parse_request`] to refuse them with the reason. -use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS}; +use std::fmt; /// Longest stored path, in bytes. pub const MAX_PATH_BYTES: usize = 1024; @@ -26,26 +32,37 @@ pub enum Target { } /// Why a path is refused. -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PathError { - #[error("invalid path: empty segment (`//`)")] EmptySegment, - #[error("invalid path: `.` and `..` segments are not allowed")] DotSegment, - #[error("invalid path: backslashes are not allowed")] Backslash, - #[error("invalid path: an encoded `/` (%2F) is not allowed in a segment")] EncodedSlash, - #[error("invalid path: control characters are not allowed")] Control, - #[error("invalid path: not valid percent-encoded UTF-8")] Encoding, - #[error("invalid path: longer than {MAX_PATH_BYTES} bytes")] TooLong, - #[error("invalid path: a segment is longer than {MAX_SEGMENT_BYTES} bytes")] SegmentTooLong, } +impl fmt::Display for PathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + PathError::EmptySegment => "invalid path: empty segment (`//`)", + PathError::DotSegment => "invalid path: `.` and `..` segments are not allowed", + PathError::Backslash => "invalid path: backslashes are not allowed", + PathError::EncodedSlash => { + "invalid path: an encoded `/` (%2F) is not allowed in a segment" + } + PathError::Control => "invalid path: control characters are not allowed", + PathError::Encoding => "invalid path: not valid percent-encoded UTF-8", + PathError::TooLong => "invalid path: longer than 1024 bytes", + PathError::SegmentTooLong => "invalid path: a segment is longer than 255 bytes", + }) + } +} + +impl std::error::Error for PathError {} + /// Check one decoded segment. pub fn validate_segment(segment: &str) -> Result<(), PathError> { if segment.is_empty() { @@ -69,9 +86,55 @@ pub fn validate_segment(segment: &str) -> Result<(), PathError> { Ok(()) } -/// Parse the percent-encoded rest of a request path (what follows -/// `/raw//`): empty or ending in `/` names a directory, anything else -/// a file. +/// Check a stored file path: segments joined by `/`, no leading or trailing +/// `/`. +pub fn validate_path(path: &str) -> Result<(), PathError> { + if path.len() > MAX_PATH_BYTES { + return Err(PathError::TooLong); + } + path.split('/').try_for_each(validate_segment) +} + +/// Check a directory: `""`, or a file path followed by `/`. +pub fn validate_directory(directory: &str) -> Result<(), PathError> { + if directory.is_empty() { + return Ok(()); + } + match directory.strip_suffix('/') { + Some(path) if directory.len() <= MAX_PATH_BYTES => validate_path(path), + Some(_) => Err(PathError::TooLong), + None => Err(PathError::EmptySegment), + } +} + +/// `%XX`-decode a segment: an escape of two hex digits is decoded, a `%` +/// before anything else stays as it is. +fn decode_segment(segment: &str) -> Result { + let bytes = segment.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let escaped = (bytes[i] == b'%') + .then(|| segment.get(i + 1..i + 3)) + .flatten() + .and_then(|hex| u8::from_str_radix(hex, 16).ok()); + match escaped { + Some(byte) => { + out.push(byte); + i += 3; + } + None => { + out.push(bytes[i]); + i += 1; + } + } + } + String::from_utf8(out).map_err(|_| PathError::Encoding) +} + +/// Parse a percent-encoded request path below a repository (without its +/// leading `/`): empty or ending in `/` names a directory, anything else a +/// file. pub fn parse_request(encoded: &str) -> Result { if encoded.is_empty() { return Ok(Target::Directory(String::new())); @@ -85,11 +148,9 @@ pub fn parse_request(encoded: &str) -> Result { if raw.contains('\\') { return Err(PathError::Backslash); } - let decoded = percent_decode_str(raw) - .decode_utf8() - .map_err(|_| PathError::Encoding)?; + let decoded = decode_segment(raw)?; validate_segment(&decoded)?; - segments.push(decoded.into_owned()); + segments.push(decoded); } let path = segments.join("/"); if path.len() + usize::from(directory) > MAX_PATH_BYTES { @@ -102,9 +163,9 @@ pub fn parse_request(encoded: &str) -> Result { }) } -/// Normalize a path given unencoded (an upload's file name, a `directory` -/// form field, an admin API parameter): segments separated by `/`, leading -/// and trailing `/` ignored. Empty yields `""`. +/// Normalize a path given unencoded (an upload's file name, a directory form +/// field, an admin API parameter): segments separated by `/`, leading and +/// trailing `/` ignored. Empty yields `""`. pub fn normalize(raw: &str) -> Result { let trimmed = raw.trim_matches('/'); if trimmed.is_empty() { @@ -144,34 +205,30 @@ pub fn join(directory: &str, name: &str) -> Result { Ok(path) } -/// Characters percent-encoded in a URL path segment: everything but the -/// unreserved characters and the sub-delimiters, `:` and `@` that a +/// Whether `byte` is percent-encoded in a URL path segment: everything but +/// the unreserved characters and the sub-delimiters, `:` and `@` that a /// segment may carry as they are. -const SEGMENT: &AsciiSet = &CONTROLS - .add(b' ') - .add(b'"') - .add(b'#') - .add(b'%') - .add(b'/') - .add(b'<') - .add(b'>') - .add(b'?') - .add(b'[') - .add(b'\\') - .add(b']') - .add(b'^') - .add(b'`') - .add(b'{') - .add(b'|') - .add(b'}'); +fn encoded_in_segment(byte: u8) -> bool { + !(0x20..0x7f).contains(&byte) || b" \"#%/<>?[\\]^`{|}".contains(&byte) +} /// `path` percent-encoded for a URL, segment by segment (a trailing `/` /// kept). pub fn encode(path: &str) -> String { - path.split('/') - .map(|segment| utf8_percent_encode(segment, SEGMENT).to_string()) - .collect::>() - .join("/") + let mut out = String::with_capacity(path.len()); + for (i, segment) in path.split('/').enumerate() { + if i > 0 { + out.push('/'); + } + for &byte in segment.as_bytes() { + if encoded_in_segment(byte) { + out.push_str(&format!("%{byte:02X}")); + } else { + out.push(char::from(byte)); + } + } + } + out } /// The last segment of a path (a directory's without its `/`). @@ -212,6 +269,9 @@ mod tests { assert_eq!(parse_request("%C3%A4/x"), Ok(Target::File("ä/x".into()))); assert_eq!(parse_request("..."), Ok(Target::File("...".into()))); assert_eq!(parse_request(".hidden"), Ok(Target::File(".hidden".into()))); + // A `%` without two hex digits is a `%`. + assert_eq!(parse_request("a%zzb"), Ok(Target::File("a%zzb".into()))); + assert_eq!(parse_request("a%2"), Ok(Target::File("a%2".into()))); } #[test] @@ -250,6 +310,21 @@ mod tests { assert_eq!(parse_request(&deep), Err(PathError::TooLong)); } + #[test] + fn stored_paths_and_directories_are_checked() { + assert_eq!(validate_path("a/b.txt"), Ok(())); + assert_eq!(validate_path(""), Err(PathError::EmptySegment)); + assert_eq!(validate_path("/a"), Err(PathError::EmptySegment)); + assert_eq!(validate_path("a/"), Err(PathError::EmptySegment)); + assert_eq!(validate_path("a/../b"), Err(PathError::DotSegment)); + assert_eq!(validate_path(&"a/".repeat(600)), Err(PathError::TooLong)); + assert_eq!(validate_directory(""), Ok(())); + assert_eq!(validate_directory("a/b/"), Ok(())); + assert_eq!(validate_directory("a/b"), Err(PathError::EmptySegment)); + assert_eq!(validate_directory("/"), Err(PathError::EmptySegment)); + assert_eq!(validate_directory("a//"), Err(PathError::EmptySegment)); + } + #[test] fn unencoded_names_normalize_their_slashes_only() { assert_eq!(normalize("/pdfium/7350/"), Ok("pdfium/7350".into())); @@ -286,6 +361,8 @@ mod tests { assert_eq!(parse_request(&encoded), Ok(expected), "{encoded}"); } assert_eq!(encode("a b/x?.txt"), "a%20b/x%3F.txt"); + assert_eq!(encode("ä"), "%C3%A4"); + assert_eq!(encode("a:b@c!$&'()*+,;=-._~"), "a:b@c!$&'()*+,;=-._~"); assert_eq!(file_name("a/b/c.txt"), "c.txt"); assert_eq!(file_name("a/b/"), "b"); assert_eq!(file_name("c.txt"), "c.txt"); diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 68c77a72..1e4921aa 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -7,10 +7,17 @@ //! package bytes — so a plugin is small and never sees a credential, a //! storage key, or a whole package in memory. //! +//! A format stores package versions, or files under paths (raw): its +//! `plugin.toml` says which (`storage`). A files format has no identity +//! rules and no packages to read: it keeps the defaults of those methods and +//! implements the protocol over the host's file store ([`Query::File`] … +//! [`Query::Remove`], [`Action::ServeFile`]). +//! //! A plugin is a crate under `plugins//` implementing [`FormatPlugin`], //! with a `plugin.toml` naming it, linked into the server and called -//! directly (NuGet). +//! directly (NuGet, raw). +pub mod files; mod types; #[cfg(feature = "testing")] @@ -24,7 +31,7 @@ pub use types::*; /// The `forge:format` contract version this crate implements: what a plugin /// built against it declares as `api` in its `plugin.toml`. -pub const API: &str = "0.3.0"; +pub const API: &str = "0.4.0"; /// The `plugin.toml` of the crate this is expanded in: how a built-in plugin /// hands the server its id and name (`pub const PLUGIN_TOML: &str = @@ -49,26 +56,48 @@ impl PackageSource for T {} /// they may take as long as they need; the rules ([`Self::canonical_id`] /// through [`Self::compare_versions`], [`Self::manifest_summary`]) run inline /// and must be quick. +/// +/// A package format implements the rules and [`Self::parse_package`]. A +/// files format has neither ids nor versions nor packages: it keeps their +/// defaults, which refuse every id, version and package. pub trait FormatPlugin: Send + Sync + 'static { /// The canonical form of a package id, used for lookups and uniqueness - /// (e.g. lowercased). - fn canonical_id(&self, id: &str) -> String; + /// (e.g. lowercased). The default keeps the id as it is. + fn canonical_id(&self, id: &str) -> String { + id.to_string() + } /// Whether `id` (canonical) is a well-formed id. The host treats a query /// or an action naming any other id like one naming an unknown package - /// (an empty answer, a 404). - fn validate_id(&self, id: &str) -> Result<(), FormatError>; + /// (an empty answer, a 404). The default accepts none. + fn validate_id(&self, id: &str) -> Result<(), FormatError> { + Err(FormatError::InvalidId(format!( + "{id:?}: this format has no packages" + ))) + } /// The normalized form of a version, used for lookups and uniqueness. - /// Prerelease versions must contain a `-` (the host filters on it). - fn normalize_version(&self, version: &str) -> Result; + /// Prerelease versions must contain a `-` (the host filters on it). The + /// default accepts none. + fn normalize_version(&self, version: &str) -> Result { + Err(FormatError::InvalidVersion(format!( + "{version:?}: this format has no versions" + ))) + } - /// The order of two normalized versions. - fn compare_versions(&self, a: &str, b: &str) -> Ordering; + /// The order of two normalized versions. The default is byte order. + fn compare_versions(&self, a: &str, b: &str) -> Ordering { + a.cmp(b) + } /// Read a package's identity, manifest and dependencies. `package` is - /// positioned at its start. - fn parse_package(&self, package: &mut dyn PackageSource) -> Result; + /// positioned at its start. The default reads none. + fn parse_package(&self, package: &mut dyn PackageSource) -> Result { + let _ = package; + Err(FormatError::InvalidPackage( + "this format stores files, not packages".into(), + )) + } /// [`Self::parse_package`] of a package the host staged at `path`. fn parse_package_file(&self, path: &Path) -> Result { @@ -80,8 +109,11 @@ pub trait FormatPlugin: Send + Sync + 'static { /// A JSON object summarizing `manifest`, stored with the version so that /// rendering never re-reads the manifest ([`PackageInfo::summary`]). /// `None` when there is nothing worth keeping. Every listing hands each - /// version's summary back, so keep it small. - fn manifest_summary(&self, manifest: &[u8]) -> Option; + /// version's summary back, so keep it small. The default keeps none. + fn manifest_summary(&self, manifest: &[u8]) -> Option { + let _ = manifest; + None + } /// What a request is: a response right away, data to load first (then /// [`Self::render`]), or an action only the host can perform. @@ -91,9 +123,11 @@ pub trait FormatPlugin: Send + Sync + 'static { /// write, DELETE delete) before it asks. fn handle(&self, repo: &Repository, req: &Request) -> Action; - /// The response to a request, from the data [`Self::handle`] (and earlier - /// rounds of `render`) asked for — or more data to load. - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered; + /// What to do once the data [`Self::handle`] (and earlier rounds of + /// `render`) asked for is loaded: `data` holds every query so far with + /// its answer. Anything `handle` may answer: a response, more data to + /// load, or an action of the host's. + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action; /// The upstream side of the format's proxy repositories: what to fetch /// from `remote` for `op`, and what came of it. `fetched` holds every diff --git a/crates/plugin-api/src/testing.rs b/crates/plugin-api/src/testing.rs index 2557d765..deaeb877 100644 --- a/crates/plugin-api/src/testing.rs +++ b/crates/plugin-api/src/testing.rs @@ -4,7 +4,8 @@ //! [`FormatPlugin::handle`], answers the queries the plugin asks for with //! `answer`, and calls [`FormatPlugin::render`] round by round, under the //! host's bounds. What only the host can do (a download, a publish, a -//! change) comes back as the action, for the test to inspect. +//! change, serving a file) comes back as the action, for the test to +//! inspect. //! //! [`converse`] does the same for the upstream side of a proxy //! ([`FormatPlugin::upstream`]): it answers every fetch the plugin asks for @@ -18,7 +19,7 @@ //! assert_eq!(served.response().status, 200); //! ``` -use crate::{Action, Answer, FormatPlugin, Loaded, Query, Rendered, Repository, RepositoryKind}; +use crate::{Action, Answer, FormatPlugin, Loaded, Query, Repository, RepositoryKind}; use crate::{Fetch, FetchResult, Fetched, FormatError, Remote, UpstreamAnswer, UpstreamOp}; use crate::{Request, Response, UpstreamStep}; @@ -35,6 +36,7 @@ pub fn repository(kind: RepositoryKind, mount: &str) -> Repository { name: "repo".into(), kind, base_url: format!("https://forge.example.com{mount}/repo"), + inline: false, } } @@ -97,13 +99,8 @@ pub fn serve( }); } match plugin.render(repo, req, &data) { - Rendered::Respond(response) => { - return Served { - outcome: Action::Respond(response), - queries, - } - } - Rendered::Load(more) => pending = more, + Action::Load(more) => pending = more, + outcome => return Served { outcome, queries }, } } Served { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index 30c8d5af..f1f7e023 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -87,6 +87,11 @@ pub struct Repository { /// The absolute URL the repository is served at, without a trailing /// slash (e.g. `https://forge.example.com/repository/charts`). pub base_url: String, + /// Files formats: whether the repository's files are displayed by a + /// browser (its `inline` content disposition) rather than downloaded, as + /// [`Action::ServeFile`] serves them. Always `false` for a package + /// format. + pub inline: bool, } /// An HTTP request from a package client. @@ -95,7 +100,8 @@ pub struct Request { /// The method, upper case. pub method: String, /// The path below `base_url`, starting with `/`, still percent-encoded - /// as the client sent it. + /// as the client sent it; empty when the URL ends with the repository's + /// name. pub path: String, /// The query string without its `?`, empty when there is none. pub query: String, @@ -193,8 +199,8 @@ fn percent_decode(s: &str, plus_is_space: bool) -> Option { } /// An HTTP response. The host keeps only the headers it allows (content -/// type, caching and validators, content-disposition, location) and adds its -/// own security headers. +/// type, caching and validators, content-disposition, vary, and location and +/// link within the repository) and adds its own security headers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { pub status: u16, @@ -208,6 +214,15 @@ impl Response { Response::new(status, "text/plain; charset=utf-8", body) } + /// A response without a body or headers (a 204, a redirect). + pub fn empty(status: u16) -> Self { + Response { + status, + headers: Vec::new(), + body: Vec::new(), + } + } + /// A response with a body of `content_type`. pub fn new(status: u16, content_type: &str, body: impl Into>) -> Self { Response { @@ -256,9 +271,145 @@ pub struct SearchQuery { pub upstream: bool, } -/// Data a plugin can ask for. Package ids are canonical, versions -/// normalized. Everything the caller may not read (package selectors, group -/// members it cannot see) is left out, exactly as for an unknown package. +/// A page of a directory of a files repository ([`Query::Directory`]). +/// +/// A file path is a list of segments joined by `/`, without a leading or +/// trailing `/` (see [`crate::files`]). A directory is `""` (the root) or a +/// path ending in `/`. The host refuses any other path a plugin names. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryQuery { + pub directory: String, + /// Start after this entry, a name in the directory (a subdirectory's ends + /// in `/`); `None` starts at the first. + pub after: Option, + /// The page size: 0 means the default (100), more than 1000 is 1000. + pub limit: u32, +} + +/// A file of a files repository. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileInfo { + pub path: String, + pub size: u64, + /// What the upload declared, else what its name suggests. + pub content_type: String, + /// Lowercase hex. + pub sha256: String, + /// Base64. + pub sha512: String, + /// Lowercase hex. + pub sha1: String, + /// Lowercase hex. + pub md5: String, + /// When its content was last stored (uploaded, or fetched by a proxy): + /// RFC 3339 in UTC. + pub modified: String, + /// When it was last downloaded, to the hour: RFC 3339 in UTC. + pub last_downloaded: Option, +} + +/// One entry of a [`DirectoryPage`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DirectoryEntry { + File(FileInfo), + /// A subdirectory: its path, ending in `/`. + Directory(String), +} + +/// A page of a directory ([`Answer::Directory`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryPage { + /// In byte order of their names, a subdirectory's ending in `/`. + pub entries: Vec, + /// The `after` of the next page; `None` on the last. + pub next: Option, +} + +/// One part of a `multipart/form-data` body ([`Query::Form`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormPart { + /// The part's field name. + pub field: String, + pub content: FormContent, +} + +/// What a [`FormPart`] holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FormContent { + /// A file: received and staged by the host, stored with + /// [`FileSource::Part`] and its index among the form's parts. + File(FormFile), + /// Any other field: its value (UTF-8, at most 4 KiB). + Text(String), +} + +/// A file part of a form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormFile { + /// The file name the part declares. + pub name: String, + /// The content type the part declares. + pub content_type: Option, + pub size: u64, +} + +/// Where a file to store comes from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileSource { + /// The request body. + Body, + /// A file part of the request's form, by its index ([`Query::Form`]). + Part(u32), +} + +/// A file to store ([`Query::Store`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewFile { + pub path: String, + pub source: FileSource, + /// The content type the client declared. The host keeps it unless it + /// says no more than "bytes" (none, `application/octet-stream`, a form + /// encoding); then it takes the one the path's extension suggests. + pub content_type: Option, +} + +/// What storing a file did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoreStatus { + /// The path was new. + Created, + /// The path held other content, which this replaced. + Replaced, + /// The path held this content already. + Unchanged, +} + +/// A file stored ([`Answer::Stored`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredFile { + pub path: String, + pub size: u64, + /// Lowercase hex. + pub sha256: String, + pub status: StoreStatus, +} + +/// What a plugin can ask of the host: data, and in a files repository +/// changes to its files. Package ids are canonical, versions normalized. +/// Everything the caller may not read (package selectors, group members it +/// cannot see) is left out, exactly as for an unknown package or file. +/// +/// A package format asks for packages ([`Query::Versions`] … +/// [`Query::Search`]), a files format for files ([`Query::File`] … +/// [`Query::Remove`]): a query of the other kind fails the request. +/// [`Query::Form`], [`Query::Store`] and [`Query::Remove`] change the +/// repository: each is asked alone in its round, at most once per request +/// for `Form` and `Store`, and only by a request authorized for them (write +/// for `Form` and `Store`, delete for `Remove`) in a hosted repository, with +/// an API token or a workload identity. When one fails (403 not allowed, 405 +/// not hosted or read-only, 409 the path exists and the repository does not +/// allow redeploying it, 413 too large) the host answers with its own +/// status. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Query { /// Every version of a package, ascending. @@ -274,6 +425,49 @@ pub enum Query { Manifest(PackageRef), /// The packages matching a search. Search(SearchQuery), + /// The file stored at a path. A proxy answers with its cached copy, + /// re-validated with its upstream once it is due; a group with its first + /// member, by priority, that has the path. + File(String), + /// [`Query::File`], and a proxy (a group's proxy members too) that has + /// not cached the path asks its upstream for it ([`UpstreamOp::File`]) + /// and caches what it gets. + FetchFile(String), + /// A page of a directory. A proxy lists the files it has cached, a group + /// the merged entries of its members (the first member's file of a + /// name). A subdirectory is listed when the caller may see a file in it. + Directory(DirectoryQuery), + /// Receive the request's `multipart/form-data` body: its file parts are + /// staged (each within the repository's upload limit, at most 100), its + /// other fields read. + Form, + /// Store files, all of them or none. A path that holds a file is replaced + /// only when the repository allows redeploying and the caller may delete; + /// the same content again is accepted as it is. + Store(Vec), + /// Delete the file at a path. + Remove(String), +} + +impl Query { + /// Whether the query changes the repository (it is asked alone in its + /// round). + pub fn is_change(&self) -> bool { + matches!(self, Query::Form | Query::Store(_) | Query::Remove(_)) + } + + /// Whether the query is about files rather than packages. + pub fn is_files(&self) -> bool { + matches!( + self, + Query::File(_) + | Query::FetchFile(_) + | Query::Directory(_) + | Query::Form + | Query::Store(_) + | Query::Remove(_) + ) + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -334,6 +528,16 @@ pub enum Answer { Package(Option), Manifest(Option>), Search(SearchResult), + /// The answer to [`Query::File`] and to [`Query::FetchFile`]. + File(Option), + /// `None`: the directory does not exist (a directory other than the root + /// without an entry the caller may see, asked for without `after`). + Directory(Option), + Form(Vec), + /// What became of each file, in the order of [`Query::Store`]. + Stored(Vec), + /// Whether there was a file to delete. + Removed(bool), } /// A query with its answer. @@ -372,14 +576,17 @@ pub struct ChangeRequest { pub done: Response, } -/// What a request is ([`FormatPlugin::handle`]). +/// What a request is ([`FormatPlugin::handle`]), or what to do once the data +/// it asked for is loaded ([`FormatPlugin::render`]). /// /// [`FormatPlugin::handle`]: crate::FormatPlugin::handle +/// [`FormatPlugin::render`]: crate::FormatPlugin::render #[derive(Debug, Clone, PartialEq, Eq)] pub enum Action { /// Answer with this response. Respond(Response), - /// Load this data, then call `render` with it. + /// Load this data, then call `render` with everything loaded so far (the + /// host bounds the rounds and the number of queries). Load(Vec), /// Serve the stored package of this version, or 404 when the caller /// cannot read it. The host streams it and counts the download. @@ -393,18 +600,12 @@ pub enum Action { Relist(ChangeRequest), /// Remove a version and its bytes (delete permission). Delete(ChangeRequest), -} - -/// What a round of [`FormatPlugin::render`] produced. -/// -/// [`FormatPlugin::render`]: crate::FormatPlugin::render -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Rendered { - /// Answer with this response. - Respond(Response), - /// Load more data, then call `render` again with everything loaded so - /// far (the host bounds the rounds and the number of queries). - Load(Vec), + /// Serve the file at this path (read requests), found as + /// [`Query::FetchFile`] finds it, or 404. The host streams it with its + /// validators (`ETag`, its SHA-256; `Last-Modified`), answers conditional + /// requests and one range, sets its content type and the repository's + /// content disposition, and counts the download. + ServeFile(String), } /// The upstream a proxy repository mirrors ([`FormatPlugin::upstream`]). @@ -446,6 +647,11 @@ pub enum UpstreamOp { Dating(PackageVersions), /// A page of the upstream's search results. Search(SearchQuery), + /// The URL of the file at a path (files formats). The host downloads it, + /// following redirects (its credentials go to the remote's origin only), + /// caches it as the file, and re-validates it with a conditional request + /// once the repository's maximum age has passed. + File(String), } /// How long the host may reuse a fetched document. @@ -568,6 +774,9 @@ pub enum UpstreamAnswer { Metadata(PackageMetadata), /// `None`: the upstream offers no search. Search(Option), + /// The file's URL, on an origin a [`Fetch`] may reach; `None`: the + /// upstream cannot have the path. + File(Option), /// The upstream answered something unusable: the host serves what it /// has cached. Failed(String), diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index f4483cc2..60178b6a 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -8,7 +8,7 @@ use forge_core::domain::{Dependency, DependencyGroup, FormatError, ParsedUpload} use forge_core::format::{Format, ReadSeek}; use forge_plugin_api::{self as api, FormatPlugin}; -use crate::manifest::Assets; +use crate::manifest::{Assets, Storage}; use crate::Plugin; /// The longest version the host takes from a format plugin, in bytes: a @@ -35,6 +35,7 @@ pub struct PluginFormat { id: &'static str, plugin: Arc, assets: Assets, + storage: Storage, } impl PluginFormat { @@ -44,6 +45,7 @@ impl PluginFormat { id: plugin.id(), plugin: plugin.implementation().clone(), assets: plugin.manifest().assets.clone(), + storage: plugin.manifest().storage, } } } @@ -128,6 +130,10 @@ impl Format for PluginFormat { self.id } + fn stores_files(&self) -> bool { + self.storage == Storage::Files + } + fn package_asset_path(&self) -> &str { &self.assets.package } @@ -263,8 +269,8 @@ mod tests { _repo: &api::Repository, _req: &api::Request, _data: &[api::Loaded], - ) -> api::Rendered { - api::Rendered::Respond(api::Response::text(404, "not found")) + ) -> api::Action { + api::Action::Respond(api::Response::text(404, "not found")) } } @@ -284,7 +290,7 @@ mod tests { /// [`Unlimited`] as the host sees it. fn unlimited(normalized: Option<&str>, parsed: api::ParsedPackage) -> PluginFormat { let plugin = Plugin::builtin( - "[plugin]\nid = \"unlimited\"\nname = \"Unlimited\"\nversion = \"0.1.0\"\napi = \"0.3.0\"\n", + "[plugin]\nid = \"unlimited\"\nname = \"Unlimited\"\nversion = \"0.1.0\"\napi = \"0.4.0\"\n", Unlimited { normalized: normalized.map(str::to_string), parsed, diff --git a/crates/plugin/src/manifest.rs b/crates/plugin/src/manifest.rs index b77d8217..1c3a0997 100644 --- a/crates/plugin/src/manifest.rs +++ b/crates/plugin/src/manifest.rs @@ -6,12 +6,14 @@ //! id = "nuget" # the format id repositories are created with //! name = "NuGet" //! version = "1.0.0" # the plugin's own version (informational) -//! api = "0.3.0" # the forge:format contract it is built against +//! api = "0.4.0" # the forge:format contract it is built against //! mount = "/v3" # optional: served at {mount}/{repository}/… //! kinds = ["hosted", "group"] # optional: the repository kinds it supports //! # ("proxy" too, when it implements `upstream`) +//! storage = "packages" # optional: what its repositories hold, +//! # package versions or "files" under paths //! -//! [assets] # optional: how its files are stored +//! [assets] # optional: how a version's files are stored //! package = "package" //! package_type = "application/octet-stream" //! manifest = "manifest" @@ -28,10 +30,9 @@ pub const SUPPORTED_APIS: &[&str] = &[forge_plugin_api::API]; /// Format ids a plugin can never take: the built-in Docker registry's, and /// names the server routes or reserves. (A built-in plugin's id, such as -/// NuGet's, is taken by that plugin: see the server's registry.) +/// NuGet's or raw's, is taken by that plugin: see the server's registry.) const RESERVED_IDS: &[&str] = &[ "docker", - "raw", "v2", "v3", "admin", @@ -58,7 +59,6 @@ const RESERVED_MOUNTS: &[&str] = &[ "login", "manage", "packages", - "raw", "readyz", "realms", "repos", @@ -87,10 +87,33 @@ pub struct Manifest { /// The repository kinds the format supports. `proxy` means the plugin /// implements [`forge_plugin_api::FormatPlugin::upstream`]. pub kinds: Vec, + /// What the format's repositories hold. + pub storage: Storage, /// How a version's files are stored. pub assets: Assets, } +/// What a format's repositories hold (`storage`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Storage { + /// Package versions: an id, a version, a package file and its manifest + /// (NuGet, Helm). + #[default] + Packages, + /// Files under paths, with their digests (raw). + Files, +} + +impl Storage { + /// The name `plugin.toml` and the admin API use. + pub fn as_str(self) -> &'static str { + match self { + Storage::Packages => "packages", + Storage::Files => "files", + } + } +} + /// The `[assets]` table: the names (within a version) and content types of /// a version's package and manifest. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -131,6 +154,7 @@ struct RawPlugin { description: Option, mount: Option, kinds: Option>, + storage: Option, } impl Manifest { @@ -155,6 +179,15 @@ impl Manifest { None => vec![RepositoryKind::Hosted, RepositoryKind::Group], Some(kinds) => parse_kinds(&kinds)?, }; + let storage = match p.storage.as_deref() { + None | Some("packages") => Storage::Packages, + Some("files") => Storage::Files, + Some(other) => { + return Err(PluginError::Manifest(format!( + "unknown storage {other:?} (packages, files)" + ))) + } + }; validate_assets(&raw.assets)?; Ok(Manifest { id: p.id, @@ -164,6 +197,7 @@ impl Manifest { description: p.description, mount, kinds, + storage, assets: raw.assets, }) } @@ -287,7 +321,7 @@ mod tests { id = "helm" name = "Helm charts" version = "0.1.0" - api = "0.3.0" + api = "0.4.0" "#; #[test] @@ -327,8 +361,8 @@ mod tests { let with = |plugin_keys: &str, tables: &str| { let text = GOOD.replace( - "api = \"0.3.0\"", - &format!("api = \"0.3.0\"\n{plugin_keys}"), + "api = \"0.4.0\"", + &format!("api = \"0.4.0\"\n{plugin_keys}"), ); Manifest::parse(&format!("{text}\n{tables}")) }; @@ -354,6 +388,10 @@ mod tests { // A plugin with an upstream side serves proxies. let m = with("kinds = [\"hosted\", \"proxy\"]", "").unwrap(); assert_eq!(m.kinds, [RepositoryKind::Hosted, RepositoryKind::Proxy]); + assert_eq!(m.storage, Storage::Packages); + let m = with("storage = \"files\"", "").unwrap(); + assert_eq!(m.storage, Storage::Files); + assert!(with("storage = \"images\"", "").is_err()); // `[proxy]` (0.2's host-side upstream protocols) is gone. assert!(with("", "[proxy]\nupstream = \"nuget-v3\"\n").is_err()); for kinds in ["[]", "[\"mirror\"]", "[\"hosted\", \"hosted\"]"] { @@ -401,9 +439,12 @@ mod tests { #[test] fn refuses_unknown_apis_and_keys() { - assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"9.0.0\"")).is_err()); - // 0.1.0 was a preview that no release shipped. - assert!(Manifest::parse(&GOOD.replace("api = \"0.3.0\"", "api = \"0.2.0\"")).is_err()); + assert!(Manifest::parse(&GOOD.replace("api = \"0.4.0\"", "api = \"9.0.0\"")).is_err()); + // Contracts before 0.4.0 had no files. + for old in ["0.2.0", "0.3.0"] { + let text = GOOD.replace("api = \"0.4.0\"", &format!("api = {old:?}")); + assert!(Manifest::parse(&text).is_err(), "{old} accepted"); + } assert!(Manifest::parse(&format!("{GOOD}\nunknown = 1\n")).is_err()); // What a WebAssembly component named is gone with it. assert!(Manifest::parse(&format!("{GOOD}component = \"helm.wasm\"\n")).is_err()); diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 1e6a5a7e..a156baf2 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -22,8 +22,9 @@ forge-core = { path = "../core" } forge-crypto = { path = "../crypto" } forge-plugin = { path = "../plugin" } forge-oci = { path = "../format-oci" } -# The built-in format plugin (plugins/nuget), linked in. +# The built-in format plugins (plugins/nuget, plugins/raw), linked in. forge-format-nuget = { path = "../../plugins/nuget" } +forge-format-raw = { path = "../../plugins/raw" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs", "io-util", "time", "signal"] } axum = { workspace = true, features = ["multipart"] } @@ -62,9 +63,9 @@ argon2 = { workspace = true, features = ["password-hash"] } tokio-util = { workspace = true } # Format-plugin uploads: the named file part of a multipart body, streamed. multer = "3" -# Raw repositories: the MD5 of uploads (the `.md5` sidecar), file paths in -# URLs, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content types -# guessed from file names. +# The file store: the MD5 of uploads (the `.md5` sidecar), file names in +# headers, HTTP dates (`Last-Modified`, `If-Modified-Since`) and content +# types guessed from file names. md-5 = "0.11" percent-encoding = "2.3" httpdate = "1" diff --git a/crates/server/src/admin/formats.rs b/crates/server/src/admin/formats.rs index 2f26119a..49dc1d30 100644 --- a/crates/server/src/admin/formats.rs +++ b/crates/server/src/admin/formats.rs @@ -1,9 +1,9 @@ //! `GET /admin/formats` — the package formats this server can create -//! repositories of: the format plugins (NuGet), Docker and raw files. +//! repositories of: the format plugins (NuGet, raw files) and Docker. use axum::extract::State; use axum::response::{IntoResponse, Json, Response}; -use forge_db::model::{DOCKER_FORMAT, RAW_FORMAT}; +use forge_db::model::DOCKER_FORMAT; use serde::Serialize; use crate::state::AppState; @@ -17,6 +17,9 @@ pub struct FormatView { pub name: String, /// The repository kinds it supports. pub kinds: Vec<&'static str>, + /// What its repositories hold: `packages` (versions of packages), + /// `files` (files under paths: raw) or `images` (Docker). + pub storage: &'static str, /// Where its repositories are served: `{mount}/{repository}` (for /// Docker, `/v2/{repository}/{image}`, or a repository's own host). pub mount: String, @@ -35,7 +38,7 @@ pub struct FormatView { } /// `GET /admin/formats`: NuGet first, then Docker, then the other formats -/// (the other plugins and raw files) by id. +/// (the other plugins, raw files among them) by id. pub async fn list(State(state): State) -> Response { let kinds = |id: &str| state.formats.kinds(id).unwrap_or_default(); let ceiling = state.config.max_repository_upload_bytes(); @@ -45,6 +48,7 @@ pub async fn list(State(state): State) -> Response { id: plugin.id().to_string(), name: manifest.name.clone(), kinds: kinds(plugin.id()), + storage: manifest.storage.as_str(), mount: manifest.mount.clone(), version: Some(manifest.version.clone()), description: manifest.description.clone(), @@ -60,6 +64,7 @@ pub async fn list(State(state): State) -> Response { id: DOCKER_FORMAT.to_string(), name: "Docker / OCI".into(), kinds: kinds(DOCKER_FORMAT), + storage: "images", mount: "/v2".into(), version: None, description: None, @@ -67,16 +72,6 @@ pub async fn list(State(state): State) -> Response { max_upload_bytes_limit: ceiling, }); let mut others: Vec = others.into_iter().map(|p| view(p)).collect(); - others.push(FormatView { - id: RAW_FORMAT.to_string(), - name: "Raw files".into(), - kinds: kinds(RAW_FORMAT), - mount: crate::raw::MOUNT.into(), - version: None, - description: Some("Arbitrary files under arbitrary paths".into()), - default_max_upload_bytes: state.config.default_upload_limit(RAW_FORMAT), - max_upload_bytes_limit: ceiling, - }); others.sort_by(|a, b| a.id.cmp(&b.id)); formats.extend(others); Json(formats).into_response() diff --git a/crates/server/src/admin/members.rs b/crates/server/src/admin/members.rs index a919b853..1e2be7ba 100644 --- a/crates/server/src/admin/members.rs +++ b/crates/server/src/admin/members.rs @@ -66,14 +66,14 @@ pub struct UpdateMember { /// Refuse to flag a proxy as an owner: it would never own anything, and a /// group whose only flagged member is a proxy would serve its reserved ids /// from nowhere. -fn owner_kind_error(member: &forge_db::model::Repository) -> Option { - if member.is_raw() { - // Owners guard package ids; a raw group serves a path from its first - // member that has it. +fn owner_kind_error(state: &AppState, member: &forge_db::model::Repository) -> Option { + if state.formats.stores_files(&member.format) { + // Owners guard package ids; a group of files (raw) serves a path + // from its first member that has it. return Some( ( StatusCode::BAD_REQUEST, - "raw groups have no owner members: a path is served by the first member that has it", + "groups of files (raw) have no owner members: a path is served by the first member that has it", ) .into_response(), ); @@ -161,7 +161,7 @@ pub async fn add( .into_response(); } if body.owner { - if let Some(resp) = owner_kind_error(&member) { + if let Some(resp) = owner_kind_error(&state, &member) { return resp; } } @@ -215,7 +215,7 @@ pub async fn update( Err(e) => return db_error(e), }; if body.owner { - if let Some(resp) = owner_kind_error(&member) { + if let Some(resp) = owner_kind_error(&state, &member) { return resp; } } diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index 1364f502..3809cfa7 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -192,7 +192,7 @@ pub fn admin_router(state: AppState) -> Router { ) .route( "/repositories/{name}/files", - axum::routing::delete(crate::raw::web::admin_delete), + axum::routing::delete(crate::files::web::admin_delete), ) .route( "/repositories/{name}/proxy-cache/invalidate", diff --git a/crates/server/src/admin/proxy_cache.rs b/crates/server/src/admin/proxy_cache.rs index a109e2b5..dd043152 100644 --- a/crates/server/src/admin/proxy_cache.rs +++ b/crates/server/src/admin/proxy_cache.rs @@ -81,16 +81,17 @@ pub async fn invalidate( return bad_request("only a proxy repository caches what an upstream answered"); } let docker = repo.is_docker(); + let stores_files = state.formats.stores_files(&repo.format); // The package as the proxy stores it: a docker image under Docker Hub's // `library/` spelling, any other id lowercased. let package = match body.and_then(|b| b.0.package) { None => None, - Some(p) if repo.is_raw() => { + Some(p) if stores_files => { let normalized = if p.ends_with('/') { - crate::raw::path::normalize_directory(&p) + forge_plugin::api::files::normalize_directory(&p) } else { - crate::raw::path::normalize(&p) + forge_plugin::api::files::normalize(&p) }; match normalized { Ok(path) => Some(path), @@ -116,7 +117,7 @@ pub async fn invalidate( } }; - match forget(&state, &repo, docker, package.as_deref()).await { + match forget(&state, &repo, docker, stores_files, package.as_deref()).await { Ok(Some(invalidated)) => { tracing::info!( repository = %repo.name, @@ -140,6 +141,7 @@ async fn forget( state: &AppState, repo: &forge_db::model::Repository, docker: bool, + stores_files: bool, package: Option<&str>, ) -> sqlx::Result> { let mut tx = state.pool.begin().await?; @@ -148,7 +150,7 @@ async fn forget( tags_expired: 0, files_expired: None, }; - if repo.is_raw() { + if stores_files { // A file is the range of its own path; a directory, of its files. let prefix = package.unwrap_or(""); invalidated.files_expired = Some(if prefix.is_empty() || prefix.ends_with('/') { diff --git a/crates/server/src/admin/repositories.rs b/crates/server/src/admin/repositories.rs index bcc42398..abe23813 100644 --- a/crates/server/src/admin/repositories.rs +++ b/crates/server/src/admin/repositories.rs @@ -23,7 +23,7 @@ use forge_core::net::ssrf::validate_url; use forge_db::model::{ DeploymentPolicy, RepoKind, Repository, CONTENT_DISPOSITION_KEY, DEPLOYMENT_POLICY_KEY, DOCKER_FORMAT, DOCKER_HOST_KEY, IMMUTABLE_TAGS_KEY, LATEST_MUTABLE_KEY, MAX_AGE_SECONDS_KEY, - MAX_UPLOAD_BYTES_KEY, NUGET_FORMAT, PREFER_HOSTED_KEY, RAW_FORMAT, RESERVED_PREFIXES_KEY, + MAX_UPLOAD_BYTES_KEY, NUGET_FORMAT, PREFER_HOSTED_KEY, RESERVED_PREFIXES_KEY, WRITABLE_MEMBER_ID_KEY, }; use forge_db::repository_dao::{self, NewRepository, RepositoryUpdate}; @@ -70,19 +70,24 @@ pub struct RepoView { /// Hosted: the largest push it accepts (for docker, the largest blob), /// `None` for the default of its format (`GET /admin/formats`). pub max_upload_bytes: Option, - /// Hosted raw: `allow_redeploy` | `disable_redeploy` | `read_only`. + /// Hosted, of a files format (raw): `allow_redeploy` | + /// `disable_redeploy` | `read_only`. pub deployment_policy: Option<&'static str>, - /// Raw: `attachment` (files download) | `inline` (files display). + /// A files format (raw): `attachment` (files download) | `inline` + /// (files display). pub content_disposition: Option<&'static str>, - /// Raw proxy: seconds a cached file is served before the upstream is - /// asked whether it changed (`0`: never). + /// Proxy of a files format (raw): seconds a cached file is served + /// before the upstream is asked whether it changed (`0`: never). pub max_age_seconds: Option, + /// What the repository holds: `packages`, `files` (a files format: raw) + /// or `images` (Docker). + pub storage: &'static str, } impl RepoView { /// The view of `r`, naming its writable member as `names` (repository - /// names by id) does. - fn new(r: &Repository, names: &HashMap) -> Self { + /// names by id) does; `stores_files` when its format holds files. + fn new(r: &Repository, names: &HashMap, stores_files: bool) -> Self { RepoView { id: r.id, name: r.name.clone(), @@ -107,30 +112,41 @@ impl RepoView { .writable_member_id() .and_then(|id| names.get(&id).cloned()), max_upload_bytes: r.max_upload_bytes(), - deployment_policy: (r.is_raw() && r.kind() == RepoKind::Hosted) + deployment_policy: (stores_files && r.kind() == RepoKind::Hosted) .then(|| r.deployment_policy().as_str()), - content_disposition: r.is_raw().then(|| { + content_disposition: stores_files.then(|| { if r.inline_content() { "inline" } else { "attachment" } }), - max_age_seconds: (r.is_raw() && r.kind() == RepoKind::Proxy) + max_age_seconds: (stores_files && r.kind() == RepoKind::Proxy) .then(|| r.max_age_seconds()), + storage: if r.is_docker() { + "images" + } else if stores_files { + "files" + } else { + "packages" + }, } } } /// The view of `r`, its writable member's current name looked up. -async fn view(pool: &sqlx::PgPool, r: &Repository) -> sqlx::Result { +async fn view(state: &AppState, r: &Repository) -> sqlx::Result { let mut names = HashMap::new(); if let Some(id) = r.writable_member_id() { - if let Some(member) = repository_dao::find_repository_by_id(pool, id).await? { + if let Some(member) = repository_dao::find_repository_by_id(&state.pool, id).await? { names.insert(id, member.name); } } - Ok(RepoView::new(r, &names)) + Ok(RepoView::new( + r, + &names, + state.formats.stores_files(&r.format), + )) } /// Create-repository request body. Unknown fields are rejected: a misspelled @@ -521,7 +537,10 @@ pub async fn list(State(state): State) -> Response { Ok(repos) => { let names: HashMap = repos.iter().map(|r| (r.id, r.name.clone())).collect(); - let views: Vec = repos.iter().map(|r| RepoView::new(r, &names)).collect(); + let views: Vec = repos + .iter() + .map(|r| RepoView::new(r, &names, state.formats.stores_files(&r.format))) + .collect(); Json(views).into_response() } Err(e) => db_error(e), @@ -531,7 +550,7 @@ pub async fn list(State(state): State) -> Response { /// `GET /admin/repositories/{name}` pub async fn get(State(state): State, Path(name): Path) -> Response { match repository_dao::find_repository_by_name(&state.pool, &name).await { - Ok(Some(r)) => match view(&state.pool, &r).await { + Ok(Some(r)) => match view(&state, &r).await { Ok(view) => Json(view).into_response(), Err(e) => db_error(e), }, @@ -595,9 +614,9 @@ pub async fn create(State(state): State, Json(body): Json) "allow_redeploy is a NuGet option; docker tags move unless immutable_tags is set", ); } - let raw = body.format == RAW_FORMAT; - if raw { - if let Err(resp) = refuse_for_raw( + let files = state.formats.stores_files(&body.format); + if files { + if let Err(resp) = refuse_for_files( Some(body.package_delay_seconds), Some(body.allow_redeploy), Some(body.metadata_ttl_seconds), @@ -606,10 +625,10 @@ pub async fn create(State(state): State, Json(body): Json) return resp; } } - let docker_config = match raw_options( - raw, + let docker_config = match file_options( + files, parsed_kind, - RawOptions { + FileOptions { deployment_policy: body.deployment_policy.clone(), content_disposition: body.content_disposition.clone(), max_age_seconds: body.max_age_seconds.map(Some), @@ -628,8 +647,8 @@ pub async fn create(State(state): State, Json(body): Json) if let Err(resp) = validate_remote_url(url) { return resp; } - if raw { - if let Err(resp) = validate_raw_remote_url(url) { + if files { + if let Err(resp) = validate_files_remote_url(url) { return resp; } } @@ -740,7 +759,7 @@ pub async fn create(State(state): State, Json(body): Json) } else { repo }; - match view(&state.pool, &repo).await { + match view(&state, &repo).await { Ok(view) => (StatusCode::CREATED, Json(view)).into_response(), Err(e) => db_error(e), } @@ -833,15 +852,15 @@ async fn docker_options( Ok(config) } -/// The raw options of a create or update request (`None`: not sent; for -/// `max_age_seconds`, `Some(None)`: cleared). -struct RawOptions { +/// The options of a files format (raw) in a create or update request +/// (`None`: not sent; for `max_age_seconds`, `Some(None)`: cleared). +struct FileOptions { deployment_policy: Option, content_disposition: Option, max_age_seconds: Option>, } -impl RawOptions { +impl FileOptions { fn any(&self) -> bool { self.deployment_policy.is_some() || self.content_disposition.is_some() @@ -850,21 +869,21 @@ impl RawOptions { } /// Longest accepted `max_age_seconds`: ten years. -const MAX_RAW_MAX_AGE_SECONDS: u64 = 10 * 366 * 24 * 60 * 60; +const MAX_FILE_MAX_AGE_SECONDS: u64 = 10 * 366 * 24 * 60 * 60; -/// Validate the raw options against the repository's format and kind and +/// Validate the files options against the repository's format and kind and /// merge them into `config` (the config object to store). #[allow(clippy::result_large_err)] -fn raw_options( - raw: bool, +fn file_options( + files: bool, kind: RepoKind, - options: RawOptions, + options: FileOptions, mut config: serde_json::Map, ) -> Result, Response> { - if !raw { + if !files { if options.any() { return Err(bad_request( - "deployment_policy, content_disposition and max_age_seconds are only valid for raw repositories", + "deployment_policy, content_disposition and max_age_seconds are only valid for repositories of files (raw)", )); } return Ok(config); @@ -897,9 +916,9 @@ fn raw_options( )); } match max_age { - Some(seconds) if seconds > MAX_RAW_MAX_AGE_SECONDS => { + Some(seconds) if seconds > MAX_FILE_MAX_AGE_SECONDS => { return Err(bad_request(format!( - "max_age_seconds must be at most {MAX_RAW_MAX_AGE_SECONDS} (0: never re-validate)" + "max_age_seconds must be at most {MAX_FILE_MAX_AGE_SECONDS} (0: never re-validate)" ))); } Some(seconds) => { @@ -913,9 +932,9 @@ fn raw_options( Ok(config) } -/// The options of other formats a raw repository refuses. +/// The options of other formats a repository of files (raw) refuses. #[allow(clippy::result_large_err)] -fn refuse_for_raw( +fn refuse_for_files( package_delay_seconds: Option, allow_redeploy: Option, metadata_ttl_seconds: Option>, @@ -923,34 +942,34 @@ fn refuse_for_raw( ) -> Result<(), Response> { if package_delay_seconds.is_some_and(|d| d != 0) { return Err(bad_request( - "package_delay_seconds is not supported for raw repositories", + "package_delay_seconds is not supported for repositories of files (raw)", )); } if allow_redeploy == Some(true) { return Err(bad_request( - "allow_redeploy is a NuGet option; raw repositories have deployment_policy", + "allow_redeploy is a NuGet option; repositories of files (raw) have deployment_policy", )); } if matches!(metadata_ttl_seconds, Some(Some(_))) { return Err(bad_request( - "metadata_ttl_seconds is not used by raw proxies; set max_age_seconds", + "metadata_ttl_seconds is not used by proxies of files (raw); set max_age_seconds", )); } if group_guard { return Err(bad_request( - "a raw group serves a path from its first member that has it; prefer_hosted and reserved_prefixes do not apply", + "a group of files (raw) serves a path from its first member that has it; prefer_hosted and reserved_prefixes do not apply", )); } Ok(()) } -/// A raw proxy's `remote_url` is the base its paths are appended to: no +/// A files proxy's `remote_url` is the base its paths are appended to: no /// query or fragment. #[allow(clippy::result_large_err)] -fn validate_raw_remote_url(url: &str) -> Result<(), Response> { +fn validate_files_remote_url(url: &str) -> Result<(), Response> { match url::Url::parse(url) { Ok(parsed) if parsed.query().is_some() || parsed.fragment().is_some() => Err(bad_request( - "a raw proxy's remote_url is the base URL files are under: no query or fragment", + "a proxy of files' remote_url is the base URL the files are under: no query or fragment", )), _ => Ok(()), } @@ -1016,16 +1035,16 @@ pub async fn update( if let Err(resp) = validate_remote_url(url) { return resp; } - if existing.is_raw() { - if let Err(resp) = validate_raw_remote_url(url) { + if state.formats.stores_files(&existing.format) { + if let Err(resp) = validate_files_remote_url(url) { return resp; } } } } } - if existing.is_raw() { - if let Err(resp) = refuse_for_raw( + if state.formats.stores_files(&existing.format) { + if let Err(resp) = refuse_for_files( body.package_delay_seconds, body.allow_redeploy, body.metadata_ttl_seconds, @@ -1064,10 +1083,10 @@ pub async fn update( Ok(config) => config, Err(resp) => return resp, }; - let mut config = match raw_options( - existing.is_raw(), + let mut config = match file_options( + state.formats.stores_files(&existing.format), existing.kind(), - RawOptions { + FileOptions { deployment_policy: body.deployment_policy.clone(), content_disposition: body.content_disposition.clone(), max_age_seconds: body.max_age_seconds, @@ -1225,8 +1244,8 @@ pub async fn update( return db_error(e); } } - // A raw proxy's files are the previous upstream's: likewise. - if existing.is_raw() { + // A files proxy's files are the previous upstream's: likewise. + if state.formats.stores_files(&existing.format) { match repository_dao::lock_repository(&mut *tx, updated.id).await { Ok(_) => {} Err(e) => return db_error(e), @@ -1293,7 +1312,7 @@ pub async fn update( } else { updated }; - match view(&state.pool, &updated).await { + match view(&state, &updated).await { Ok(repo) => Json(UpdatedRepoView { repo, upstream_credentials_dropped: credentials_dropped, @@ -1347,7 +1366,7 @@ pub async fn delete(State(state): State, Path(name): Path) -> "repository still has images; delete their manifests first", ) .into_response(), - Ok(RepoDelete::HasPackages) if existing.is_raw() => ( + Ok(RepoDelete::HasPackages) if state.formats.stores_files(&existing.format) => ( StatusCode::CONFLICT, "repository still has files; delete them first (DELETE /admin/repositories//files?path=)", ) @@ -1485,7 +1504,7 @@ pub async fn hard_delete_package( ) .into_response(); } - if repo.is_raw() { + if state.formats.stores_files(&repo.format) { return ( StatusCode::BAD_REQUEST, "delete files through DELETE /admin/repositories//files?path=", diff --git a/crates/server/src/admin/retention.rs b/crates/server/src/admin/retention.rs index 6b98de72..96ccdaa2 100644 --- a/crates/server/src/admin/retention.rs +++ b/crates/server/src/admin/retention.rs @@ -154,7 +154,7 @@ pub async fn put( }; // Parsed with its repository's model and stored in its canonical form. - let normalized = if repo.is_raw() { + let normalized = if state.formats.stores_files(&repo.format) { let policy: FileRetentionPolicy = match serde_json::from_value(body.policy) { Ok(p) => p, Err(e) => return unprocessable(e), @@ -331,7 +331,7 @@ pub async fn preview( decisions: Vec::new(), proxy_decisions: Vec::new(), image_decisions: repo.is_docker().then(ImageDecisions::default), - file_decisions: repo.is_raw().then(Vec::new), + file_decisions: state.formats.stores_files(&repo.format).then(Vec::new), counts: PreviewCounts::default(), }) .into_response() @@ -342,7 +342,7 @@ pub async fn preview( if repo.is_docker() { return preview_images(&state, &repo, policy_json, source, at).await; } - if repo.is_raw() { + if state.formats.stores_files(&repo.format) { return preview_files(&state, &repo, policy_json, source, at).await; } let mut policy: RetentionPolicy = match source.parse(policy_json) { diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index 7c0bf839..3220d3a6 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -129,14 +129,9 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { .into_iter() .map(str::to_string) .collect(); - // Raw repositories are feeds too, for the response policy. - let feeds: std::sync::Arc> = std::sync::Arc::new( - mounts - .iter() - .cloned() - .chain([crate::raw::MOUNT.to_string()]) - .collect(), - ); + // Every format's mount is a feed, for the response policy (raw's + // `/raw` among them). + let feeds: std::sync::Arc> = std::sync::Arc::new(mounts.clone()); let mut app = Router::new() // Liveness / readiness probes (unauthenticated, dependency-light). .merge(crate::health::health_router()) @@ -162,17 +157,16 @@ pub fn build_router(state: AppState, layer: PrincipalLayer) -> Router { // the host) itself. .route("/v2", any(crate::registry::registry)) .route("/v2/", any(crate::registry::registry)) - .route("/v2/{*rest}", any(crate::registry::registry)) - // Raw (generic file) repositories: `/raw//`. - .route("/raw/{repo}", any(crate::raw::handle)) - .route("/raw/{repo}/", any(crate::raw::handle)) - .route("/raw/{repo}/{*rest}", any(crate::raw::handle)); + .route("/v2/{*rest}", any(crate::registry::registry)); // Repositories, each served by its format's plugin under the format's - // mount (`crate::plugins`; NuGet at `/v3`). `any` covers + // mount (`crate::plugins`; NuGet at `/v3`, raw files at `/raw`). `any` covers // GET/HEAD/PUT/POST/DELETE; the dispatcher authorizes per method. for mount in mounts.iter() { app = app .route(&format!("{mount}/{{repo}}"), any(crate::plugins::dispatch)) + // A wildcard matches no empty rest: the root of a repository + // (a files format's root listing) needs its own route. + .route(&format!("{mount}/{{repo}}/"), any(crate::plugins::dispatch)) .route( &format!("{mount}/{{repo}}/{{*rest}}"), any(crate::plugins::dispatch), diff --git a/crates/server/src/browse.rs b/crates/server/src/browse.rs index e2685d31..98142186 100644 --- a/crates/server/src/browse.rs +++ b/crates/server/src/browse.rs @@ -100,6 +100,9 @@ pub struct BrowseRepo { /// `/browse/images` names, and for a format not installed here. #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, + /// What the repository holds: `packages`, `files` (a files format: raw, + /// browsed through its listings) or `images` (docker). + pub storage: &'static str, } /// Query parameters for `GET /browse/repos`. @@ -143,16 +146,15 @@ pub async fn list_repos( kind: repo.kind.clone(), online: repo.online, format: repo.format.clone(), - url: if repo.is_raw() { - Some(crate::plugins::repository_url( - &state, - crate::raw::MOUNT, - &repo.name, - )) + url: state.formats.plugin(&repo.format).map(|plugin| { + crate::plugins::repository_url(&state, &plugin.manifest().mount, &repo.name) + }), + storage: if repo.is_docker() { + "images" + } else if state.formats.stores_files(&repo.format) { + "files" } else { - state.formats.plugin(&repo.format).map(|plugin| { - crate::plugins::repository_url(&state, &plugin.manifest().mount, &repo.name) - }) + "packages" }, }); } @@ -251,8 +253,8 @@ pub async fn search_packages( } } // Package search covers the package formats; images are listed by - // /browse/images, raw files by their repository's listings. - if repo.is_docker() || repo.is_raw() { + // /browse/images, files (raw) by their repository's listings. + if repo.is_docker() || state.formats.stores_files(&repo.format) { continue; } if state.authz.can_read(&principal, repo).await { @@ -413,9 +415,9 @@ pub async fn package_detail( } let repo = match repository_dao::find_repository_by_name(&state.pool, &repo_name).await { - // A docker repository holds images (`/browse/images`) and a raw one - // files (its listings), not packages. - Ok(Some(r)) if !r.is_docker() && !r.is_raw() => r, + // A docker repository holds images (`/browse/images`) and a files + // (raw) one files (its listings), not packages. + Ok(Some(r)) if !r.is_docker() && !state.formats.stores_files(&r.format) => r, // An unknown name answers like an unreadable repository (no name // enumeration), except to a global admin. Ok(_) if state.authz.is_global_admin(&principal).await => { @@ -666,7 +668,7 @@ pub fn browse_router() -> Router { ) .route( "/browse/raw/{repo}/files", - axum::routing::post(crate::raw::web::browse_upload) - .delete(crate::raw::web::browse_delete), + axum::routing::post(crate::files::web::browse_upload) + .delete(crate::files::web::browse_delete), ) } diff --git a/crates/server/src/files/listing.rs b/crates/server/src/files/listing.rs new file mode 100644 index 00000000..b9bd76fd --- /dev/null +++ b/crates/server/src/files/listing.rs @@ -0,0 +1,268 @@ +//! Directory pages of the file store (the `directory` query): what a +//! directory holds, a page at a time, for the format's plugin to render. +//! +//! A page holds at most `limit` entries (default 100, at most 1000) after +//! the entry named by `after`, in byte order of their names, a +//! subdirectory's name ending in `/`; `next` names where the following page +//! starts. A listing walks the index ([`raw_dao::walk_directory`]): a page +//! costs `limit` index probes, however many files are below the directory. +//! +//! Package selectors apply: a file outside the caller's selector is not +//! listed, and a subdirectory is listed only when it holds a file the +//! caller may see (looked for among its first files; a subdirectory none of +//! whose first [`SUBDIRECTORY_SCAN`] files is visible is left out). A +//! group lists the merged entries of the members the caller may read, the +//! highest-priority member's file first; a proxy lists what it has cached. + +use std::collections::BTreeMap; + +use forge_core::authz::PackageFilter; +use forge_db::model::{RepoKind, Repository}; +use forge_db::raw_dao::{self, RawFile}; +use forge_plugin::api::files; + +use super::{FileError, Files}; + +/// Default page size. +pub const DEFAULT_LIMIT: usize = 100; + +/// Largest page size. +pub const MAX_LIMIT: usize = 1000; + +/// How many of a subdirectory's files a listing reads, at most, to find one +/// a selector-restricted caller may see. +pub const SUBDIRECTORY_SCAN: usize = 1000; + +/// One entry of a page. +#[derive(Debug, Clone)] +pub struct Entry { + /// The entry's name in its directory; a subdirectory's ends in `/`. + pub name: String, + pub kind: EntryKind, +} + +#[derive(Debug, Clone)] +pub enum EntryKind { + File(Box), + /// A subdirectory: its path, ending in `/`. + Directory(String), +} + +/// A page of a directory. +#[derive(Debug, Clone)] +pub struct Page { + pub entries: Vec, + /// The `after` of the next page, if there is one. + pub next: Option, +} + +/// Where a page starting after the entry `after` (a name in directory +/// `prefix`, a subdirectory's ending in `/`) resumes: the least path past +/// that entry (past all of a subdirectory's files: `'0'` follows `'/'`; past +/// a file: no path holds a control character). +fn resume_at(prefix: &str, after: Option<&str>) -> Result { + let Some(after) = after.filter(|a| !a.is_empty()) else { + return Ok(prefix.to_string()); + }; + let (name, directory) = match after.strip_suffix('/') { + Some(name) => (name, true), + None => (after, false), + }; + files::validate_segment(name).map_err(|e| FileError::BadRequest(format!("after: {e}")))?; + Ok(if directory { + format!("{prefix}{name}0") + } else { + format!("{prefix}{name}\u{1}") + }) +} + +/// One repository's entries of a page: what it has after `from`, as far as +/// it read (`more`: it has entries past the last one returned, up to and +/// including the entry named `more`, which it has read). +struct Partial { + entries: Vec, + more: Option, +} + +impl Files<'_> { + /// The entries of `prefix` in hosted or proxy `repo` from `from` on, + /// visible through every one of `filters`. + async fn list_local( + &self, + repo: &Repository, + prefix: &str, + from: &str, + limit: usize, + filters: &[PackageFilter], + ) -> Result { + let walked = raw_dao::walk_directory( + &self.state.pool, + repo.id, + prefix, + from, + i64::try_from(limit + 1).unwrap_or(i64::MAX), + ) + .await?; + let more = walked.len() > limit; + let mut entries = Vec::new(); + let mut last = None; + for walked in walked.into_iter().take(limit) { + let rest = &walked.path[prefix.len()..]; + let (entry, key) = match (rest.find('/'), &walked.file) { + (None, Some(file)) => { + let visible = filters.iter().all(|f| f.allows(&file.path)); + let entry = Entry { + name: rest.to_string(), + kind: EntryKind::File(Box::new(file.clone())), + }; + (visible.then_some(entry), rest.to_string()) + } + (Some(i), _) => { + let name = &rest[..i]; + let dir = format!("{prefix}{name}/"); + let visible = self.directory_visible(repo, &dir, filters).await?; + let entry = Entry { + name: format!("{name}/"), + kind: EntryKind::Directory(dir), + }; + (visible.then_some(entry), format!("{name}/")) + } + // A file row vanished between the walk and the join. + (None, None) => (None, rest.to_string()), + }; + entries.extend(entry); + last = Some(key); + } + Ok(Partial { + entries, + more: if more { last } else { None }, + }) + } + + /// Whether directory `dir` of `repo` holds a file visible through every + /// one of `filters`. + async fn directory_visible( + &self, + repo: &Repository, + dir: &str, + filters: &[PackageFilter], + ) -> Result { + if filters.iter().all(|f| matches!(f, PackageFilter::AllowAll)) { + return Ok(true); + } + const BATCH: usize = 200; + let mut after = String::new(); + let mut read = 0; + while read < SUBDIRECTORY_SCAN { + let paths = + raw_dao::paths_under(&self.state.pool, repo.id, dir, &after, BATCH as i64).await?; + if paths.iter().any(|p| filters.iter().all(|f| f.allows(p))) { + return Ok(true); + } + if paths.len() < BATCH { + return Ok(false); + } + read += paths.len(); + after = paths.last().cloned().unwrap_or_default(); + } + Ok(false) + } + + /// The entries of `prefix` in `repo` (any kind) from `from` on. + fn list_in<'a>( + &'a self, + repo: &'a Repository, + prefix: &'a str, + from: &'a str, + limit: usize, + filters: Vec, + depth: u32, + ) -> futures_util::future::BoxFuture<'a, Result> { + Box::pin(async move { + let mut filters = filters; + filters.push(self.state.authz.package_filter(&self.principal, repo).await); + if repo.kind() != RepoKind::Group { + return self.list_local(repo, prefix, from, limit, &filters).await; + } + // A merge of the members' pages, complete up to the first point + // a member stopped reading at. + let mut merged: BTreeMap = BTreeMap::new(); + let mut cutoff: Option = None; + for member in self.serving_members(repo, depth).await? { + let part = self + .list_in(&member, prefix, from, limit, filters.clone(), depth + 1) + .await?; + for entry in part.entries { + merged.entry(entry.name.clone()).or_insert(entry); + } + if let Some(more) = part.more { + if cutoff.as_ref().is_none_or(|c| more < *c) { + cutoff = Some(more); + } + } + } + let mut entries: Vec = merged + .into_values() + .filter(|e| cutoff.as_ref().is_none_or(|c| e.name <= *c)) + .collect(); + let more = if entries.len() > limit { + entries.truncate(limit); + entries.last().map(|e| e.name.clone()) + } else { + cutoff + }; + Ok(Partial { entries, more }) + }) + } + + /// A page of directory `prefix` of `repo` after the entry `after`, of + /// `limit` entries (0: the default), `None` when it does not exist (a + /// directory other than the root without visible entries, asked for + /// without `after`). + pub async fn list( + &self, + repo: &Repository, + prefix: &str, + after: Option<&str>, + limit: usize, + ) -> Result, FileError> { + files::validate_directory(prefix).map_err(|e| { + FileError::Internal(format!("the format named directory {prefix:?}: {e}")) + })?; + let limit = if limit == 0 { + DEFAULT_LIMIT + } else { + limit.min(MAX_LIMIT) + }; + let from = resume_at(prefix, after)?; + let partial = self + .list_in(repo, prefix, &from, limit, Vec::new(), 0) + .await?; + if partial.entries.is_empty() && partial.more.is_none() && !prefix.is_empty() { + // Past the end of an existing directory is an empty page, not a + // missing directory. + if after.is_none() { + return Ok(None); + } + } + Ok(Some(Page { + entries: partial.entries, + next: partial.more, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pages_resume_past_the_named_entry() { + assert_eq!(resume_at("a/", None).unwrap(), "a/"); + assert_eq!(resume_at("a/", Some("")).unwrap(), "a/"); + assert_eq!(resume_at("a/", Some("x.bin")).unwrap(), "a/x.bin\u{1}"); + assert_eq!(resume_at("", Some("dir/")).unwrap(), "dir0"); + assert!(resume_at("a/", Some("../x")).is_err()); + assert!(resume_at("a/", Some("b/c")).is_err()); + } +} diff --git a/crates/server/src/files/mod.rs b/crates/server/src/files/mod.rs new file mode 100644 index 00000000..9debdcc2 --- /dev/null +++ b/crates/server/src/files/mod.rs @@ -0,0 +1,226 @@ +//! The file store: the repositories of a format whose `plugin.toml` says +//! `storage = "files"` (raw) hold files under paths, like Nexus' raw-hosted, +//! raw-proxy and raw-group repositories. The format's plugin owns the +//! protocol (`plugins/raw`, served under its mount by [`crate::plugins`]); +//! this module answers the file queries and actions it asks for: +//! +//! * `file` and `fetch-file`: [`Files::locate`] — the stored file; a proxy's +//! copy, re-validated with its upstream once it is due, or fetched from it +//! ([`proxy`]); a group's first member, by priority, that has the path; +//! * `directory`: a page of a directory ([`listing`]); +//! * `form`, `store` and `remove`: receiving a multipart form, storing files +//! under the deployment policy, deleting a file ([`store`]); +//! * `serve-file`: its bytes, one `Range`, `If-Range`, `If-None-Match` and +//! `If-Modified-Since` ([`serve`]). +//! +//! The web UI's upload and delete and the admin API's directory delete are +//! here too ([`web`]). +//! +//! Paths follow [`forge_plugin::api::files`]: `..`, `.`, empty segments, +//! backslashes and encoded slashes are refused, never resolved, and so is any +//! other path a plugin names. Package selectors match paths (`pdfium/**`). +//! Files are CAS blobs, so the same content under several paths or +//! repositories is stored once (one `raw_file` row, one reference each). + +pub mod listing; +pub mod proxy; +pub mod serve; +pub mod store; +pub mod web; + +use std::collections::HashSet; + +use axum::http::{header, StatusCode}; +use axum::response::{IntoResponse, Response}; +use forge_core::auth::principal::Principal; +use forge_core::authz::PackageFilter; +use forge_db::model::{RepoKind, Repository}; +use forge_db::{raw_dao, repository_dao}; + +use crate::dispatch::offline_serves_nothing; +use crate::state::AppState; +use raw_dao::RawFile; + +/// A request's view of the file store: the server's state and the caller. +pub struct Files<'a> { + pub state: &'a AppState, + pub principal: Principal, +} + +/// Why a file request failed, as its answer. +#[derive(Debug)] +pub enum FileError { + NotFound, + BadRequest(String), + /// A mutation needs an API token or a workload identity. + CredentialRequired, + /// The caller may not do this (repository-level). + Denied, + Forbidden(String), + Conflict(String), + MethodNotAllowed(&'static str), + TooLarge(u64), + Unavailable(String), + BadGateway(String), + Internal(String), +} + +impl FileError { + /// The error's text, for logs. + pub fn message(&self) -> String { + match self { + FileError::NotFound => "not found".into(), + FileError::CredentialRequired => "API token or workload identity required".into(), + FileError::Denied => "forbidden".into(), + FileError::MethodNotAllowed(m) => (*m).into(), + FileError::TooLarge(limit) => format!("larger than {limit} bytes"), + FileError::BadRequest(m) + | FileError::Forbidden(m) + | FileError::Conflict(m) + | FileError::Unavailable(m) + | FileError::BadGateway(m) + | FileError::Internal(m) => m.clone(), + } + } +} + +impl From for FileError { + fn from(e: sqlx::Error) -> Self { + match e { + sqlx::Error::PoolTimedOut => FileError::Unavailable("database busy".into()), + e => FileError::Internal(format!("database: {e}")), + } + } +} + +impl IntoResponse for FileError { + fn into_response(self) -> Response { + match self { + FileError::NotFound => (StatusCode::NOT_FOUND, "not found").into_response(), + FileError::BadRequest(m) => (StatusCode::BAD_REQUEST, m).into_response(), + FileError::CredentialRequired => ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Basic realm=\"Forge\"")], + "API token or workload identity required", + ) + .into_response(), + FileError::Denied => (StatusCode::FORBIDDEN, "forbidden").into_response(), + FileError::Forbidden(m) => (StatusCode::FORBIDDEN, m).into_response(), + FileError::Conflict(m) => (StatusCode::CONFLICT, m).into_response(), + FileError::MethodNotAllowed(m) => ( + StatusCode::METHOD_NOT_ALLOWED, + [(header::ALLOW, "GET, HEAD")], + m, + ) + .into_response(), + FileError::TooLarge(limit) => ( + StatusCode::PAYLOAD_TOO_LARGE, + format!("the upload exceeds the repository's limit of {limit} bytes"), + ) + .into_response(), + FileError::Unavailable(m) => ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "30")], + m, + ) + .into_response(), + FileError::BadGateway(m) => (StatusCode::BAD_GATEWAY, m).into_response(), + FileError::Internal(m) => { + tracing::error!(error = %m, "file store request failed"); + (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() + } + } + } +} + +impl Files<'_> { + /// Find `path` in `repo` for the caller, through every one of `filters` + /// (a group's, for its members) and `repo`'s own package filter: the + /// stored file (a proxy's re-validated once it is due), else with + /// `fetch` a proxy's upstream file, else (a group) the first member's + /// that has one. A member that fails is skipped while another has the + /// path. + pub fn locate<'a>( + &'a self, + repo: &'a Repository, + path: &'a str, + fetch: bool, + filters: Vec, + depth: u32, + ) -> futures_util::future::BoxFuture<'a, Result, FileError>> { + Box::pin(async move { + let mut filters = filters; + filters.push(self.state.authz.package_filter(&self.principal, repo).await); + if !filters.iter().all(|f| f.allows(path)) { + return Ok(None); + } + match repo.kind() { + RepoKind::Hosted | RepoKind::Proxy => { + let proxy = repo.kind() == RepoKind::Proxy; + if let Some(file) = raw_dao::find(&self.state.pool, repo.id, path).await? { + return Ok(Some(if proxy { + self.fresh(repo, file).await + } else { + file + })); + } + if proxy && fetch { + return self.fetch(repo, path).await; + } + Ok(None) + } + RepoKind::Group => { + let mut failure = None; + for member in self.serving_members(repo, depth).await? { + match self + .locate(&member, path, fetch, filters.clone(), depth + 1) + .await + { + Ok(Some(found)) => return Ok(Some(found)), + Ok(None) => {} + Err(e) => { + tracing::warn!(group = %repo.name, member = %member.name, error = %e.message(), "file group: member failed"); + failure = Some(e); + } + } + } + failure.map_or(Ok(None), Err) + } + } + }) + } + + /// The members of `group` that may serve the caller, highest priority + /// first: repositories of the group's format the caller may read itself + /// (reading a group grants nothing on its members), without offline hosted or group + /// members, and without nested groups past the nesting limit or on a + /// cycle. + pub async fn serving_members( + &self, + group: &Repository, + depth: u32, + ) -> Result, FileError> { + let max_depth = self.state.config.groups.max_nesting_depth.max(1) as u32; + if depth >= max_depth { + return Ok(Vec::new()); + } + let mut seen = HashSet::from([group.id]); + let mut out = Vec::new(); + for (member, _priority) in repository_dao::group_members(&self.state.pool, group.id).await? + { + if member.format != group.format + || !seen.insert(member.id) + || offline_serves_nothing(&member) + { + continue; + } + if member.kind() == RepoKind::Group && depth + 1 >= max_depth { + continue; + } + if self.state.authz.can_read(&self.principal, &member).await { + out.push(member); + } + } + Ok(out) + } +} diff --git a/crates/server/src/raw/proxy.rs b/crates/server/src/files/proxy.rs similarity index 72% rename from crates/server/src/raw/proxy.rs rename to crates/server/src/files/proxy.rs index d6be2ac6..c882064a 100644 --- a/crates/server/src/raw/proxy.rs +++ b/crates/server/src/files/proxy.rs @@ -1,12 +1,14 @@ -//! Raw proxy repositories: files fetched from an HTTP(S) base URL on first -//! request, cached content-addressed, and re-validated against the upstream -//! once they are older than the repository's `max_age_seconds`. +//! Proxy repositories of the file store: files fetched from the upstream on +//! first request, cached content-addressed, and re-validated against the +//! upstream once they are older than the repository's `max_age_seconds`. //! -//! * A file `` is `/`, each segment -//! percent-encoded. Redirects are followed (GitHub release assets -//! redirect to `objects.githubusercontent.com`), every hop through the -//! upstream client's cloud-metadata guard; the upstream credentials go to -//! the origin of `remote_url` only. +//! * The format's plugin names the upstream URL of a path (its upstream +//! side's `file` operation; raw's is `/`, each segment +//! percent-encoded), on an origin the upstream points at. Redirects are +//! followed (GitHub release assets redirect to +//! `objects.githubusercontent.com`), every hop through the upstream +//! client's cloud-metadata guard; the upstream credentials go to the +//! origin of `remote_url` only. //! * Re-validation is a conditional GET (`If-None-Match` with the upstream's //! `ETag`, `If-Modified-Since` with its `Last-Modified`): a `304` keeps //! the copy, new content replaces it. A `max_age_seconds` of `0` never @@ -31,8 +33,8 @@ use forge_db::raw_dao::{self, NewRawFile, RawFile, Redeploy}; use futures_util::TryStreamExt; use reqwest::header::{CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED}; -use super::upload::{content_type, ingest}; -use super::{Raw, RawError}; +use super::store::{content_type, ingest}; +use super::{FileError, Files}; /// The gate of one file's upstream requests, by `(repository id, path)`. type Gates = HashMap<(i64, String), Weak>>; @@ -55,23 +57,6 @@ fn gate(repository_id: i64, path: &str) -> Arc> { gate } -/// The upstream URL of `path`: `remote_url` (as a directory) with the -/// path's segments appended, each percent-encoded. -pub fn upstream_url(remote_url: &str, path: &str) -> Result { - let mut url = url::Url::parse(remote_url) - .map_err(|e| RawError::Internal(format!("proxy remote_url {remote_url:?}: {e}")))?; - { - let mut segments = url.path_segments_mut().map_err(|()| { - RawError::Internal(format!("proxy remote_url {remote_url:?} has no path")) - })?; - segments.pop_if_empty(); - for segment in path.split('/') { - segments.push(segment); - } - } - Ok(url.into()) -} - fn header_text( headers: &reqwest::header::HeaderMap, name: reqwest::header::HeaderName, @@ -84,7 +69,7 @@ fn header_text( .map(str::to_string) } -impl Raw<'_> { +impl Files<'_> { /// Keyed by the repository's `updated_at` too, so an administrator's /// cache invalidation (which bumps it) forgets the misses on every /// instance. @@ -97,9 +82,12 @@ impl Raw<'_> { let hit: Result = self .state .cache - .get_or_try_insert_local("raw:upstream-nf", &key, Duration::from_secs(1), || async { - Err(()) - }) + .get_or_try_insert_local( + "files:upstream-nf", + &key, + Duration::from_secs(1), + || async { Err(()) }, + ) .await; hit.unwrap_or(false) } @@ -118,7 +106,7 @@ impl Raw<'_> { .state .cache .get_or_try_insert_local( - "raw:upstream-nf", + "files:upstream-nf", &key, Duration::from_secs(ttl), || async { Ok(true) }, @@ -126,24 +114,43 @@ impl Raw<'_> { .await; } - async fn credentials(&self, repo: &Repository) -> Result, RawError> { + async fn credentials(&self, repo: &Repository) -> Result, FileError> { upstream_credentials(&self.state.pool, self.state.envelope.as_ref(), repo) .await - .map_err(|e| RawError::Internal(format!("proxy credentials: {e}"))) + .map_err(|e| FileError::Internal(format!("proxy credentials: {e}"))) + } + + /// The upstream URL of `path`, as the format's plugin names it; `None` + /// when the upstream cannot have it. + async fn upstream_url( + &self, + repo: &Repository, + path: &str, + ) -> Result, FileError> { + let format = self + .state + .formats + .for_repository(repo) + .map_err(|e| FileError::Unavailable(e.to_string()))?; + self.state + .proxy_repo(format) + .file_url(repo, path) + .await + .map_err(|e| { + tracing::warn!(repo = %repo.name, %path, error = %e, "file proxy: the upstream URL could not be resolved"); + FileError::BadGateway(format!("the upstream could not be reached: {e}")) + }) } - /// GET `path` from the upstream, conditionally on `cached`'s validators. + /// GET `url` (the upstream's `path`), conditionally on `cached`'s + /// validators. async fn upstream_get( &self, repo: &Repository, path: &str, + url: &str, cached: Option<&RawFile>, - ) -> Result { - let remote = repo - .remote_url - .as_deref() - .ok_or_else(|| RawError::Internal("proxy repository without remote_url".into()))?; - let url = upstream_url(remote, path)?; + ) -> Result { let credentials = self.credentials(repo).await?; let auth: Option> = credentials.as_ref().map(Credentials::auth); let mut headers = reqwest::header::HeaderMap::new(); @@ -163,15 +170,15 @@ impl Raw<'_> { .upstream .send( reqwest::Method::GET, - &url, + url, RequestClass::Download, headers, auth.as_ref(), ) .await .map_err(|e| { - tracing::warn!(repo = %repo.name, %path, error = %e, "raw proxy: upstream request failed"); - RawError::BadGateway(format!("the upstream could not be reached: {e}")) + tracing::warn!(repo = %repo.name, %path, error = %e, "file proxy: upstream request failed"); + FileError::BadGateway(format!("the upstream could not be reached: {e}")) }) } @@ -181,10 +188,10 @@ impl Raw<'_> { repo: &Repository, path: &str, response: reqwest::Response, - ) -> Result { + ) -> Result { let limit = self.state.upstream.config().max_package_bytes; if response.content_length().is_some_and(|len| len > limit) { - return Err(RawError::BadGateway(format!( + return Err(FileError::BadGateway(format!( "the upstream file is larger than proxy.max_package_bytes ({limit} bytes)" ))); } @@ -194,11 +201,11 @@ impl Raw<'_> { let declared = header_text(&headers, CONTENT_TYPE); let body = response.bytes_stream().map_err(std::io::Error::other); let ingested = ingest(self, body, limit).await.map_err(|e| match e { - RawError::TooLarge(limit) => RawError::BadGateway(format!( + FileError::TooLarge(limit) => FileError::BadGateway(format!( "the upstream file is larger than proxy.max_package_bytes ({limit} bytes)" )), - RawError::BadRequest(msg) => { - RawError::BadGateway(format!("the upstream download failed: {msg}")) + FileError::BadRequest(msg) => { + FileError::BadGateway(format!("the upstream download failed: {msg}")) } other => other, })?; @@ -206,7 +213,7 @@ impl Raw<'_> { let content_type = content_type(declared.as_deref(), path); let mut tx = self.state.pool.begin().await?; if !oci_dao::lock_proxy_upstream(&mut tx, repo.id, repo.remote_url.as_deref()).await? { - return Err(RawError::Unavailable( + return Err(FileError::Unavailable( "the proxy was pointed at another upstream while this was fetched; retry".into(), )); } @@ -216,7 +223,7 @@ impl Raw<'_> { &NewRawFile { path, sha256: &stored.sha256, - size: i64::try_from(stored.size).map_err(|_| RawError::TooLarge(stored.size))?, + size: i64::try_from(stored.size).map_err(|_| FileError::TooLarge(stored.size))?, content_type: &content_type, sha512_base64: &stored.sha512_base64, sha1: &ingested.sha1, @@ -233,7 +240,7 @@ impl Raw<'_> { tx.commit().await?; raw_dao::find(&self.state.pool, repo.id, path) .await? - .ok_or(RawError::NotFound) + .ok_or(FileError::NotFound) } /// The cached `file` of proxy `repo`, re-validated first when it is due. @@ -253,14 +260,21 @@ impl Raw<'_> { match self.revalidate(repo, &file).await { Ok(current) => current, Err(error) => { - tracing::warn!(repo = %repo.name, path = %file.path, error = %error.message(), "raw proxy: re-validation failed; serving the cached copy"); + tracing::warn!(repo = %repo.name, path = %file.path, error = %error.message(), "file proxy: re-validation failed; serving the cached copy"); file } } } - async fn revalidate(&self, repo: &Repository, file: &RawFile) -> Result { - let response = self.upstream_get(repo, &file.path, Some(file)).await?; + async fn revalidate(&self, repo: &Repository, file: &RawFile) -> Result { + let Some(url) = self.upstream_url(repo, &file.path).await? else { + // The upstream cannot have it (any more): the cached copy stands. + raw_dao::mark_checked(&self.state.pool, file.id, None, None).await?; + return Ok(file.clone()); + }; + let response = self + .upstream_get(repo, &file.path, &url, Some(file)) + .await?; let status = response.status(); if status == StatusCode::NOT_MODIFIED { let headers = response.headers(); @@ -277,18 +291,18 @@ impl Raw<'_> { return self.store_response(repo, &file.path, response).await; } if matches!(status, StatusCode::NOT_FOUND | StatusCode::GONE) { - tracing::warn!(repo = %repo.name, path = %file.path, "raw proxy: the upstream no longer has a cached file; serving the cached copy"); + tracing::warn!(repo = %repo.name, path = %file.path, "file proxy: the upstream no longer has a cached file; serving the cached copy"); raw_dao::mark_checked(&self.state.pool, file.id, None, None).await?; return Ok(file.clone()); } - Err(RawError::BadGateway(format!( + Err(FileError::BadGateway(format!( "the upstream answered {status}" ))) } /// Fetch `path` from the upstream of proxy `repo` and cache it. `None` /// when the upstream does not have it (or the proxy is offline). - pub async fn fetch(&self, repo: &Repository, path: &str) -> Result, RawError> { + pub async fn fetch(&self, repo: &Repository, path: &str) -> Result, FileError> { if !repo.online || self.negative_cached(repo, path).await { return Ok(None); } @@ -300,15 +314,19 @@ impl Raw<'_> { if self.negative_cached(repo, path).await { return Ok(None); } - let response = self.upstream_get(repo, path, None).await?; + let Some(url) = self.upstream_url(repo, path).await? else { + self.remember_not_found(repo, path).await; + return Ok(None); + }; + let response = self.upstream_get(repo, path, &url, None).await?; let status = response.status(); if matches!(status, StatusCode::NOT_FOUND | StatusCode::GONE) { self.remember_not_found(repo, path).await; return Ok(None); } if !status.is_success() { - tracing::warn!(repo = %repo.name, %path, %status, "raw proxy: upstream error"); - return Err(RawError::BadGateway(format!( + tracing::warn!(repo = %repo.name, %path, %status, "file proxy: upstream error"); + return Err(FileError::BadGateway(format!( "the upstream answered {status}" ))); } @@ -322,26 +340,3 @@ fn is_due(file: &RawFile, max_age: u64) -> bool { file.checked_at .is_none_or(|checked| chrono::Utc::now() - checked >= max_age) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn upstream_urls_append_encoded_segments() { - let base = "https://github.com/bblanchon/pdfium-binaries/releases/download/"; - assert_eq!( - upstream_url(base, "chromium/7350/pdfium-linux-x64.tgz").unwrap(), - "https://github.com/bblanchon/pdfium-binaries/releases/download/chromium/7350/pdfium-linux-x64.tgz" - ); - // Without a trailing slash the base is a directory all the same. - assert_eq!( - upstream_url("http://up.test/files", "a b/c?d#e.txt").unwrap(), - "http://up.test/files/a%20b/c%3Fd%23e.txt" - ); - assert_eq!( - upstream_url("http://up.test", "x.bin").unwrap(), - "http://up.test/x.bin" - ); - } -} diff --git a/crates/server/src/raw/serve.rs b/crates/server/src/files/serve.rs similarity index 72% rename from crates/server/src/raw/serve.rs rename to crates/server/src/files/serve.rs index 171ac596..f52c06e8 100644 --- a/crates/server/src/raw/serve.rs +++ b/crates/server/src/files/serve.rs @@ -1,70 +1,21 @@ -//! Answering for a stored file: its bytes (whole or one range), its -//! validators and conditional requests, and its checksum sidecars. +//! Serving a stored file (the `serve-file` action): its bytes (whole or one +//! range), its validators and conditional requests, its content type and +//! the repository's content disposition. use std::time::{Duration, SystemTime, UNIX_EPOCH}; use axum::body::Body; use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; -use base64::Engine as _; use chrono::{DateTime, Utc}; use forge_db::model::Repository; use forge_db::raw_dao::{self, RawFile}; +use forge_plugin::api::files; use forge_storage::BlobRef; -use super::path; use crate::feed::{byte_range, ByteRange}; use crate::state::AppState; -/// A checksum sidecar: `.` answered from the file's stored -/// digests when no file of that name is stored. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Checksum { - Sha256, - Sha512, - Sha1, - Md5, -} - -impl Checksum { - const ALL: [Checksum; 4] = [ - Checksum::Sha256, - Checksum::Sha512, - Checksum::Sha1, - Checksum::Md5, - ]; - - fn extension(self) -> &'static str { - match self { - Checksum::Sha256 => ".sha256", - Checksum::Sha512 => ".sha512", - Checksum::Sha1 => ".sha1", - Checksum::Md5 => ".md5", - } - } - - /// The digest of `file` as lowercase hex. - pub fn hex(self, file: &RawFile) -> String { - match self { - Checksum::Sha256 => file.blob_sha256.clone(), - Checksum::Sha512 => base64::engine::general_purpose::STANDARD - .decode(&file.sha512_base64) - .map(hex::encode) - .unwrap_or_default(), - Checksum::Sha1 => file.sha1.clone(), - Checksum::Md5 => file.md5.clone(), - } - } -} - -/// The file a sidecar path is the checksum of, and which checksum. -pub fn sidecar_of(path: &str) -> Option<(&str, Checksum)> { - Checksum::ALL.into_iter().find_map(|checksum| { - let base = path.strip_suffix(checksum.extension())?; - (!base.is_empty() && !base.ends_with('/')).then_some((base, checksum)) - }) -} - /// The `ETag` of a file: its SHA-256, a strong validator. pub fn etag(file: &RawFile) -> String { format!("\"{}\"", file.blob_sha256) @@ -122,7 +73,7 @@ pub fn content_disposition(repo: &Repository, file_path: &str) -> HeaderValue { } else { "attachment" }; - let name = path::file_name(file_path); + let name = files::file_name(file_path); let plain = name .chars() .all(|c| c.is_ascii() && !c.is_ascii_control() && c != '"' && c != '\\'); @@ -200,7 +151,7 @@ pub async fn file( match state.blob_store.open(blob, span.clone()).await { Ok(stream) => Body::from_stream(stream), Err(error) if error.is_unavailable() => { - tracing::warn!(%error, path = %file.path, "raw: object store unavailable"); + tracing::warn!(%error, path = %file.path, "files: object store unavailable"); return ( StatusCode::SERVICE_UNAVAILABLE, [(header::RETRY_AFTER, "30")], @@ -209,7 +160,7 @@ pub async fn file( .into_response(); } Err(error) => { - tracing::error!(%error, path = %file.path, s3_key = %file.s3_key, "raw: blob read failed"); + tracing::error!(%error, path = %file.path, s3_key = %file.s3_key, "files: blob read failed"); return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(); } } @@ -240,56 +191,13 @@ pub async fn file( let id = file.id; tokio::spawn(async move { if let Err(error) = raw_dao::touch_downloaded(&pool, id).await { - tracing::debug!(%error, "raw: could not record a download"); + tracing::debug!(%error, "files: could not record a download"); } }); } response } -/// Answer `GET`/`HEAD` of the `checksum` sidecar of `base`: the digest and -/// the file name, as `sha256sum` and friends print them, so `sha256sum -c` -/// checks a download against it. -pub fn sidecar( - repo: &Repository, - sidecar_path: &str, - base: &RawFile, - checksum: Checksum, - request: &HeaderMap, - head: bool, -) -> Response { - let etag = format!( - "\"{}{}\"", - base.blob_sha256, - checksum.extension().replace('.', "-") - ); - let modified = http_date(base.updated_at); - if not_modified(request, &etag, base.updated_at) { - let mut response = StatusCode::NOT_MODIFIED.into_response(); - validators(response.headers_mut(), &etag, &modified); - return response; - } - let text = format!("{} {}\n", checksum.hex(base), path::file_name(&base.path)); - let length = text.len(); - let mut response = if head { - Response::new(Body::empty()) - } else { - Response::new(Body::from(text)) - }; - let out = response.headers_mut(); - out.insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - out.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); - out.insert( - header::CONTENT_DISPOSITION, - content_disposition(repo, sidecar_path), - ); - validators(out, &etag, &modified); - response -} - #[cfg(test)] mod tests { use super::*; @@ -306,8 +214,7 @@ mod tests { s3_key: "k".into(), size: 10, content_type: "application/gzip".into(), - // SHA-512 of nothing in particular: 64 zero bytes. - sha512_base64: base64::engine::general_purpose::STANDARD.encode([0u8; 64]), + sha512_base64: String::new(), sha1: "cd".repeat(20), md5: "ef".repeat(16), created_at: t, @@ -319,21 +226,6 @@ mod tests { } } - #[test] - fn sidecars_name_their_file() { - assert_eq!( - sidecar_of("a/x.tgz.sha256"), - Some(("a/x.tgz", Checksum::Sha256)) - ); - assert_eq!(sidecar_of("x.md5"), Some(("x", Checksum::Md5))); - assert_eq!(sidecar_of(".sha1"), None); - assert_eq!(sidecar_of("a/.sha1"), None); - assert_eq!(sidecar_of("a/x.tgz"), None); - let file = raw_file(); - assert_eq!(Checksum::Sha512.hex(&file), "00".repeat(64)); - assert_eq!(Checksum::Md5.hex(&file), "ef".repeat(16)); - } - #[test] fn preconditions() { let file = raw_file(); diff --git a/crates/server/src/raw/upload.rs b/crates/server/src/files/store.rs similarity index 53% rename from crates/server/src/raw/upload.rs rename to crates/server/src/files/store.rs index b143925e..6f57faa6 100644 --- a/crates/server/src/raw/upload.rs +++ b/crates/server/src/files/store.rs @@ -1,35 +1,36 @@ -//! Changing files of a hosted raw repository: `PUT` of one file, a -//! multipart form of several (the web UI's upload, and the form fields of -//! Nexus' component upload), and `DELETE`. +//! Changing the files of a hosted repository: receiving a multipart form +//! (the `form` query), storing files from the request body or the form's +//! parts (`store`), and deleting a file (`remove`), for the format's plugin +//! and for the web UI. //! //! An upload is streamed to the staging directory under the repository's //! upload limit, hashed on the way (SHA-256 and SHA-512 by the CAS, SHA-1 -//! and MD5 here, for the checksum sidecars), stored content-addressed, and +//! and MD5 here, for checksum sidecars), stored content-addressed, and //! recorded in one transaction with the deployment policy applied under the //! row lock: a path that exists is replaced only with `allow_redeploy` and //! the delete permission, and the same content again is accepted as is. -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use axum::body::Body; use axum::extract::Request; -use axum::http::{header, StatusCode}; -use axum::response::{IntoResponse, Json, Response}; +use axum::http::header; use bytes::Bytes; use forge_core::auth::principal::{AuthMethod, Principal}; use forge_core::authz::Action; use forge_core::repo::{PackageMutation, PublishAuthorizationContext, WriteError}; use forge_db::model::{DeploymentPolicy, RepoKind, Repository}; use forge_db::raw_dao::{self, NewRawFile, Redeploy, StoreOutcome}; +use forge_plugin::api::files; use forge_storage::{CasError, StoredBlob}; use futures_util::{Stream, TryStreamExt}; use md5::Md5; use serde::Serialize; use sha1::{Digest as _, Sha1}; -use super::{path, Raw, RawError}; +use super::{FileError, Files}; use crate::dispatch::principal_allows_protocol_mutations; /// Most files one multipart upload may carry. @@ -41,7 +42,7 @@ const MAX_TEXT_FIELD_BYTES: usize = 4096; /// Who may change files through a request. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Via { - /// The `/raw` protocol: an API token or a workload identity, as for + /// The format's protocol: an API token or a workload identity, as for /// every package client (never a browser session). Protocol, /// The web UI's endpoints: a signed-in session (whose CSRF token the @@ -63,8 +64,37 @@ struct Digests { md5: Md5, } +/// One part of a received form. +pub enum Part { + /// A file part: its declared name and content type, staged and stored. + File { + field: String, + name: String, + content_type: Option, + ingested: Box, + }, + /// Any other field. + Text { field: String, value: String }, +} + +/// A file to store, and where its content comes from. +pub struct Upload { + pub path: String, + /// The content type the client declared. + pub declared: Option, + pub content: Content, +} + +/// The content of an [`Upload`]. +pub enum Content { + /// The request body, still to be read. + Body(Request), + /// Received already (a form's file part). + Staged(Ingested), +} + /// Stage `body` (at most `limit` bytes) and store it content-addressed. -pub async fn ingest(raw: &Raw<'_>, body: S, limit: u64) -> Result +pub async fn ingest(files: &Files<'_>, body: S, limit: u64) -> Result where S: Stream> + Send + Unpin, E: Into>, @@ -78,14 +108,14 @@ where d.sha1.update(chunk); d.md5.update(chunk); }); - let staged = raw + let staged = files .state .cas .stage(tokio_util::io::StreamReader::new(body), limit) .await .map_err(cas_error)?; let digests = std::mem::take(&mut *digests.lock().unwrap_or_else(PoisonError::into_inner)); - let stored = raw.state.cas.commit(staged).await.map_err(cas_error)?; + let stored = files.state.cas.commit(staged).await.map_err(cas_error)?; Ok(Ingested { stored, sha1: hex::encode(digests.sha1.finalize()), @@ -94,15 +124,15 @@ where } /// An ingest failure as the answer to the request. -pub fn cas_error(error: CasError) -> RawError { +pub fn cas_error(error: CasError) -> FileError { match error { - CasError::TooLarge { limit } => RawError::TooLarge(limit), + CasError::TooLarge { limit } => FileError::TooLarge(limit), CasError::StagingFull { .. } => { - RawError::Unavailable("temporary staging space is exhausted; retry later".into()) + FileError::Unavailable("temporary staging space is exhausted; retry later".into()) } - e if e.is_unavailable() => RawError::Unavailable("object store unavailable".into()), - CasError::Io(e) => RawError::BadRequest(format!("the upload could not be read: {e}")), - e => RawError::Internal(format!("store upload: {e}")), + e if e.is_unavailable() => FileError::Unavailable("object store unavailable".into()), + CasError::Io(e) => FileError::BadRequest(format!("the upload could not be read: {e}")), + e => FileError::Internal(format!("store upload: {e}")), } } @@ -136,35 +166,47 @@ pub fn content_type(declared: Option<&str>, path: &str) -> String { } } -/// The answer to an upload or delete that needs `action` on `repo`, when it -/// may not happen; `None` when it may. -async fn refusal(raw: &Raw<'_>, repo: &Repository, action: Action, via: Via) -> Option { +/// A path the format named, checked against the store's rules. +fn checked_path(path: &str) -> Result<&str, FileError> { + files::validate_path(path) + .map(|()| path) + .map_err(|e| FileError::Internal(format!("the format named path {path:?}: {e}"))) +} + +/// Why an upload or delete that needs `action` on `repo` may not happen; +/// `None` when it may. +pub async fn refusal( + files: &Files<'_>, + repo: &Repository, + action: Action, + via: Via, +) -> Option { if repo.kind() != RepoKind::Hosted { - return Some(RawError::MethodNotAllowed( + return Some(FileError::MethodNotAllowed( "files are uploaded to and deleted from hosted repositories", )); } let credential = match via { - Via::Protocol => principal_allows_protocol_mutations(&raw.principal), + Via::Protocol => principal_allows_protocol_mutations(&files.principal), Via::Browser => { matches!( - raw.principal, + files.principal, Principal::User { method, .. } if method != AuthMethod::Password - ) || matches!(raw.principal, Principal::Workload { .. }) + ) || matches!(files.principal, Principal::Workload { .. }) } }; if !credential { - return Some(RawError::CredentialRequired); + return Some(FileError::CredentialRequired); } let allowed = match action { - Action::Delete => raw.state.authz.can_delete(&raw.principal, repo).await, - _ => raw.state.authz.can_write(&raw.principal, repo).await, + Action::Delete => files.state.authz.can_delete(&files.principal, repo).await, + _ => files.state.authz.can_write(&files.principal, repo).await, }; if !allowed { - return Some(RawError::Denied); + return Some(FileError::Denied); } if repo.deployment_policy() == DeploymentPolicy::ReadOnly { - return Some(RawError::MethodNotAllowed("the repository is read-only")); + return Some(FileError::MethodNotAllowed("the repository is read-only")); } None } @@ -172,18 +214,18 @@ async fn refusal(raw: &Raw<'_>, repo: &Repository, action: Action, via: Via) -> /// Record a workload's change in the trusted-publishing ledger, inside the /// transaction that makes it. async fn record( - raw: &Raw<'_>, + files: &Files<'_>, repo: &Repository, action: PackageMutation, path: &str, sha256: &str, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, -) -> Result<(), RawError> { - if !matches!(raw.principal, Principal::Workload { .. }) { +) -> Result<(), FileError> { + if !matches!(files.principal, Principal::Workload { .. }) { return Ok(()); } let context = PublishAuthorizationContext { - principal: raw.principal, + principal: files.principal, action, package_id: path.to_string(), package_version: sha256.to_string(), @@ -192,10 +234,10 @@ async fn record( .record_in_transaction(&context, tx) .await .map_err(|e| match e { - WriteError::ForbiddenPackage { .. } => RawError::Forbidden( + WriteError::ForbiddenPackage { .. } => FileError::Forbidden( "the workload identity may no longer change this repository".into(), ), - other => RawError::Internal(format!("trusted-publishing record: {other}")), + other => FileError::Internal(format!("trusted-publishing record: {other}")), }) } @@ -221,37 +263,37 @@ pub struct Stored { /// are locked in the order every other writer takes them in; the answer /// lists them in the order given. async fn record_all( - raw: &Raw<'_>, + files: &Files<'_>, repo: &Repository, - files: Vec, -) -> Result, RawError> { + pending: Vec, +) -> Result, FileError> { let policy = repo.deployment_policy(); - let replace = raw + let replace = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Delete) + .package_filter_for(&files.principal, repo, Action::Delete) .await; - let may_delete = raw.state.authz.can_delete(&raw.principal, repo).await; - let grace = raw.state.config.outbox.grace(); - let mut tx = raw.state.pool.begin().await?; - let mut order: Vec = (0..files.len()).collect(); + let may_delete = files.state.authz.can_delete(&files.principal, repo).await; + let grace = files.state.config.outbox.grace(); + let mut tx = files.state.pool.begin().await?; + let mut order: Vec = (0..pending.len()).collect(); order.sort_by(|&a, &b| { - files[a] + pending[a] .ingested .stored .sha256 - .cmp(&files[b].ingested.stored.sha256) + .cmp(&pending[b].ingested.stored.sha256) }); - let mut out: Vec> = (0..files.len()).map(|_| None).collect(); + let mut out: Vec> = (0..pending.len()).map(|_| None).collect(); for index in order { - let file = &files[index]; + let file = &pending[index]; let stored = &file.ingested.stored; let replaceable = policy == DeploymentPolicy::AllowRedeploy && may_delete && replace.allows(&file.path); let new = NewRawFile { path: &file.path, sha256: &stored.sha256, - size: i64::try_from(stored.size).map_err(|_| RawError::TooLarge(stored.size))?, + size: i64::try_from(stored.size).map_err(|_| FileError::TooLarge(stored.size))?, content_type: &file.content_type, sha512_base64: &stored.sha512_base64, sha1: &file.ingested.sha1, @@ -273,13 +315,13 @@ async fn record_all( StoreOutcome::Replaced(_) => "replaced", StoreOutcome::Unchanged(_) => "unchanged", StoreOutcome::Exists if policy == DeploymentPolicy::AllowRedeploy => { - return Err(RawError::Forbidden(format!( + return Err(FileError::Forbidden(format!( "{} exists; replacing a file needs the delete permission", file.path ))); } StoreOutcome::Exists => { - return Err(RawError::Conflict(format!( + return Err(FileError::Conflict(format!( "{} exists and the repository does not allow redeploying it", file.path ))); @@ -287,7 +329,7 @@ async fn record_all( }; if status != "unchanged" { record( - raw, + files, repo, PackageMutation::Publish, &file.path, @@ -307,100 +349,73 @@ async fn record_all( Ok(out.into_iter().flatten().collect()) } -/// `PUT /raw//`: store the request body as the file. -pub async fn put(raw: &Raw<'_>, repo: &Repository, path: &str, req: Request) -> Response { - match put_inner(raw, repo, path, req).await { - Ok(Stored { - status: "created", .. - }) => StatusCode::CREATED.into_response(), - Ok(_) => StatusCode::NO_CONTENT.into_response(), - Err(e) => e.into_response(), - } -} - -async fn put_inner( - raw: &Raw<'_>, +/// Store `uploads` in hosted `repo`, all of them or none, with the caller's +/// write permission (and selector) on every path. +pub async fn store( + files: &Files<'_>, repo: &Repository, - path: &str, - req: Request, -) -> Result { - if let Some(refused) = refusal(raw, repo, Action::Write, Via::Protocol).await { + uploads: Vec, + via: Via, +) -> Result, FileError> { + if let Some(refused) = refusal(files, repo, Action::Write, via).await { return Err(refused); } - let write = raw + let write = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Write) + .package_filter_for(&files.principal, repo, Action::Write) .await; - if !write.allows(path) { - return Err(RawError::Forbidden(format!("you may not upload to {path}"))); - } - let limit = raw.state.upload_limit(repo); - let declared_length = req - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - if declared_length.is_some_and(|len| len > limit) { - return Err(RawError::TooLarge(limit)); - } - let declared = req - .headers() - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .map(str::to_string); - let ingested = ingest(raw, req.into_body().into_data_stream(), limit).await?; - let pending = Pending { - path: path.to_string(), - content_type: content_type(declared.as_deref(), path), - ingested, - }; - let mut stored = record_all(raw, repo, vec![pending]).await?; - Ok(stored.remove(0)) -} - -/// `POST /raw///` (and the web UI's upload): a -/// `multipart/form-data` body whose file parts are stored under -/// `directory`, each at its part's file name. Nexus' field names work too: -/// `raw.directory` (or `directory`) names a subdirectory, and -/// `.filename` (`raw.asset1.filename`) the name of the file part -/// ``. Every file is recorded, or none. -pub async fn multipart( - raw: &Raw<'_>, - repo: &Repository, - directory: &str, - req: Request, - via: Via, -) -> Response { - match multipart_inner(raw, repo, directory, req, via).await { - Ok(files) => { - let status = if files.iter().any(|f| f.status == "created") { - StatusCode::CREATED - } else { - StatusCode::OK - }; - (status, Json(serde_json::json!({ "files": files }))).into_response() + let mut seen = HashSet::new(); + for upload in &uploads { + let path = checked_path(&upload.path)?; + if !seen.insert(path) { + return Err(FileError::BadRequest(format!( + "the upload holds {path} twice" + ))); + } + if !write.allows(path) { + return Err(FileError::Forbidden(format!( + "you may not upload to {path}" + ))); } - Err(e) => e.into_response(), } + let limit = files.state.upload_limit(repo); + let mut pending = Vec::with_capacity(uploads.len()); + for upload in uploads { + let ingested = match upload.content { + Content::Staged(ingested) => ingested, + Content::Body(req) => { + let declared_length = req + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + if declared_length.is_some_and(|len| len > limit) { + return Err(FileError::TooLarge(limit)); + } + ingest(files, req.into_body().into_data_stream(), limit).await? + } + }; + pending.push(Pending { + content_type: content_type(upload.declared.as_deref(), &upload.path), + path: upload.path, + ingested, + }); + } + record_all(files, repo, pending).await } -/// A file part, staged and stored, its name to be decided. -struct FilePart { - field: String, - file_name: String, - declared: Option, - ingested: Ingested, -} - -async fn multipart_inner( - raw: &Raw<'_>, +/// Receive the `multipart/form-data` body of `req` for hosted `repo`: every +/// file part staged and stored (at most [`MAX_FILES_PER_UPLOAD`], each +/// within the repository's upload limit), every other field read (at most +/// [`MAX_TEXT_FIELD_BYTES`]), in the order of the form. +pub async fn receive_form( + files: &Files<'_>, repo: &Repository, - directory: &str, req: Request, via: Via, -) -> Result, RawError> { - if let Some(refused) = refusal(raw, repo, Action::Write, via).await { +) -> Result, FileError> { + if let Some(refused) = refusal(files, repo, Action::Write, via).await { return Err(refused); } let Some(boundary) = req @@ -409,11 +424,11 @@ async fn multipart_inner( .and_then(|v| v.to_str().ok()) .and_then(crate::feed::multipart_boundary) else { - return Err(RawError::BadRequest( + return Err(FileError::BadRequest( "upload files as multipart/form-data (or PUT a single file to its path)".into(), )); }; - let limit = raw.state.upload_limit(repo); + let limit = files.state.upload_limit(repo); let overrun = Arc::new(AtomicBool::new(false)); let guarded = crate::feed::preamble_guard( req.into_body().into_data_stream(), @@ -424,31 +439,32 @@ async fn multipart_inner( .size_limit(multer::SizeLimit::new().per_field(limit.saturating_add(1))); let mut form = multer::Multipart::with_constraints(guarded, boundary, constraints); let form_error = |e: multer::Error| match e { - multer::Error::FieldSizeExceeded { .. } => RawError::TooLarge(limit), + multer::Error::FieldSizeExceeded { .. } => FileError::TooLarge(limit), _ if overrun.load(Ordering::Relaxed) => { - RawError::BadRequest("multipart body: no part within the first 64 KiB".into()) + FileError::BadRequest("multipart body: no part within the first 64 KiB".into()) } - e => RawError::BadRequest(format!("multipart body: {e}")), + e => FileError::BadRequest(format!("multipart body: {e}")), }; - let mut parts: Vec = Vec::new(); - let mut texts: BTreeMap = BTreeMap::new(); + let mut parts: Vec = Vec::new(); + let mut file_parts = 0; while let Some(mut part) = form.next_field().await.map_err(form_error)? { let field = part.name().unwrap_or("").to_string(); match part.file_name().map(str::to_string) { - Some(file_name) => { - if parts.len() == MAX_FILES_PER_UPLOAD { - return Err(RawError::BadRequest(format!( + Some(name) => { + if file_parts == MAX_FILES_PER_UPLOAD { + return Err(FileError::BadRequest(format!( "at most {MAX_FILES_PER_UPLOAD} files per upload" ))); } - let declared = part.content_type().map(|m| m.to_string()); - let ingested = ingest(raw, part, limit).await?; - parts.push(FilePart { + file_parts += 1; + let content_type = part.content_type().map(|m| m.to_string()); + let ingested = ingest(files, part, limit).await?; + parts.push(Part::File { field, - file_name, - declared, - ingested, + name, + content_type, + ingested: Box::new(ingested), }); } None => { @@ -456,110 +472,50 @@ async fn multipart_inner( while let Some(chunk) = part.chunk().await.map_err(form_error)? { text.extend_from_slice(&chunk); if text.len() > MAX_TEXT_FIELD_BYTES { - return Err(RawError::BadRequest(format!( + return Err(FileError::BadRequest(format!( "form field {field:?} is longer than {MAX_TEXT_FIELD_BYTES} bytes" ))); } } - let text = String::from_utf8(text).map_err(|_| { - RawError::BadRequest(format!("form field {field:?} is not UTF-8")) + let value = String::from_utf8(text).map_err(|_| { + FileError::BadRequest(format!("form field {field:?} is not UTF-8")) })?; - texts.insert(field, text); - } - } - } - if parts.is_empty() { - return Err(RawError::BadRequest("the form holds no file".into())); - } - - // The fields: a subdirectory, and the names of file parts. - let mut subdirectory = String::new(); - let fields: HashSet<&str> = parts.iter().map(|p| p.field.as_str()).collect(); - for (name, value) in &texts { - match name.as_str() { - "directory" | "raw.directory" => subdirectory = value.clone(), - other - if other - .strip_suffix(".filename") - .is_some_and(|field| fields.contains(field)) => {} - other => { - return Err(RawError::BadRequest(format!( - "unknown form field {other:?} (file parts, directory and .filename are accepted)" - ))) + parts.push(Part::Text { field, value }); } } } - let base = path::normalize_directory(&subdirectory) - .map_err(|e| RawError::BadRequest(format!("directory: {e}")))?; - let directory = format!("{directory}{base}"); - - let write = raw - .state - .authz - .package_filter_for(&raw.principal, repo, Action::Write) - .await; - let mut seen = HashSet::new(); - let mut pending = Vec::with_capacity(parts.len()); - for part in parts { - let name = texts - .get(&format!("{}.filename", part.field)) - .unwrap_or(&part.file_name); - let target = path::join(&directory, name) - .map_err(|e| RawError::BadRequest(format!("file name {name:?}: {e}")))?; - if !seen.insert(target.clone()) { - return Err(RawError::BadRequest(format!( - "the form holds {target} twice" - ))); - } - if !write.allows(&target) { - return Err(RawError::Forbidden(format!( - "you may not upload to {target}" - ))); - } - pending.push(Pending { - content_type: content_type(part.declared.as_deref(), &target), - path: target, - ingested: part.ingested, - }); - } - record_all(raw, repo, pending).await -} - -/// `DELETE /raw//` (and the web UI's delete). -pub async fn delete(raw: &Raw<'_>, repo: &Repository, path: &str, via: Via) -> Response { - match delete_inner(raw, repo, path, via).await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), - Err(e) => e.into_response(), - } + Ok(parts) } -async fn delete_inner( - raw: &Raw<'_>, +/// Delete the file at `path` of hosted `repo`: `false` when there is none +/// the caller may delete (one outside its selector is as unknown). +pub async fn remove( + files: &Files<'_>, repo: &Repository, path: &str, via: Via, -) -> Result<(), RawError> { - if let Some(refused) = refusal(raw, repo, Action::Delete, via).await { +) -> Result { + if let Some(refused) = refusal(files, repo, Action::Delete, via).await { return Err(refused); } - let filter = raw + let path = checked_path(path)?; + let filter = files .state .authz - .package_filter_for(&raw.principal, repo, Action::Delete) + .package_filter_for(&files.principal, repo, Action::Delete) .await; - // Outside the caller's selector: as unknown. if !filter.allows(path) { - return Err(RawError::NotFound); + return Ok(false); } - let Some(file) = raw_dao::find(&raw.state.pool, repo.id, path).await? else { - return Err(RawError::NotFound); + let Some(file) = raw_dao::find(&files.state.pool, repo.id, path).await? else { + return Ok(false); }; - let mut tx = raw.state.pool.begin().await?; - if !raw_dao::delete(&mut tx, repo.id, path, raw.state.config.outbox.grace()).await? { - return Err(RawError::NotFound); + let mut tx = files.state.pool.begin().await?; + if !raw_dao::delete(&mut tx, repo.id, path, files.state.config.outbox.grace()).await? { + return Ok(false); } record( - raw, + files, repo, PackageMutation::Delete, path, @@ -568,7 +524,7 @@ async fn delete_inner( ) .await?; tx.commit().await?; - Ok(()) + Ok(true) } #[cfg(test)] diff --git a/crates/server/src/raw/web.rs b/crates/server/src/files/web.rs similarity index 64% rename from crates/server/src/raw/web.rs rename to crates/server/src/files/web.rs index c8369bb1..631adfb2 100644 --- a/crates/server/src/raw/web.rs +++ b/crates/server/src/files/web.rs @@ -1,12 +1,15 @@ -//! The raw endpoints of the web UI and the admin API. +//! The file store's endpoints of the web UI and the admin API, for the +//! repositories of a files format (raw). //! -//! The web UI reads listings from `/raw///` itself (as JSON, with -//! its session). Changing files through `/raw` takes an API token, like -//! every package client, so the UI changes them here, with its session -//! (whose CSRF token the principal middleware checks): +//! The web UI reads listings from its format's protocol itself +//! (`/raw///` as JSON, with its session). Changing files through +//! `/raw` takes an API token, like every package client, so the UI changes +//! them here, with its session (whose CSRF token the principal middleware +//! checks): //! //! * `POST /browse/raw/{repo}/files?directory=` — a multipart upload -//! into `` ([`upload::multipart`]); +//! into ``, each file part stored at its file name +//! ([`store::receive_form`], [`store::store`]); //! * `DELETE /browse/raw/{repo}/files?path=` — delete a file. //! //! Both take the same permissions as `/raw`: write (and, to replace a @@ -23,9 +26,11 @@ use axum::response::{IntoResponse, Json, Response}; use forge_core::auth::principal::Principal; use forge_db::model::{RepoKind, Repository}; use forge_db::{raw_dao, repository_dao}; +use forge_plugin::api::files; use serde::Deserialize; -use super::{path, upload, Raw, RawError}; +use super::store::{self, Content, Part, Upload, Via}; +use super::{FileError, Files}; use crate::middleware::principal_of; use crate::state::AppState; @@ -44,7 +49,7 @@ pub struct PathQuery { pub path: String, } -/// The raw repository `name` for a signed-in caller (`/browse` is for +/// The repository of files `name` for a signed-in caller (`/browse` is for /// signed-in users only). async fn browse_target( state: &AppState, @@ -55,9 +60,14 @@ async fn browse_target( return Err((StatusCode::UNAUTHORIZED, "authentication required").into_response()); } match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(repo)) if repo.is_raw() && state.authz.can_read(principal, &repo).await => Ok(repo), + Ok(Some(repo)) + if state.formats.stores_files(&repo.format) + && state.authz.can_read(principal, &repo).await => + { + Ok(repo) + } Ok(_) => Err((StatusCode::NOT_FOUND, "repository not found").into_response()), - Err(e) => Err(RawError::from(e).into_response()), + Err(e) => Err(FileError::from(e).into_response()), } } @@ -73,19 +83,65 @@ pub async fn browse_upload( Ok(repo) => repo, Err(response) => return response, }; - let directory = match path::normalize_directory(&query.directory) { + let directory = match files::normalize_directory(&query.directory) { Ok(dir) => dir, Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), }; - let raw = Raw { + let files = Files { state: &state, principal, }; - let response = upload::multipart(&raw, &repo, &directory, req, upload::Via::Browser).await; + let response = match upload(&files, &repo, &directory, req).await { + Ok(stored) => { + let status = if stored.iter().any(|f| f.status == "created") { + StatusCode::CREATED + } else { + StatusCode::OK + }; + (status, Json(serde_json::json!({ "files": stored }))).into_response() + } + Err(e) => e.into_response(), + }; audit(&repo, "POST", &directory, &principal, &response); response } +/// Store the file parts of the form `req` carries under `directory`, each +/// at its file name. +async fn upload( + files: &Files<'_>, + repo: &Repository, + directory: &str, + req: Request, +) -> Result, FileError> { + let parts = store::receive_form(files, repo, req, Via::Browser).await?; + let mut uploads = Vec::with_capacity(parts.len()); + for part in parts { + match part { + Part::File { + name, + content_type, + ingested, + .. + } => uploads.push(Upload { + path: files::join(directory, &name) + .map_err(|e| FileError::BadRequest(format!("file name {name:?}: {e}")))?, + declared: content_type, + content: Content::Staged(*ingested), + }), + Part::Text { field, .. } => { + return Err(FileError::BadRequest(format!( + "unknown form field {field:?} (only files are accepted)" + ))) + } + } + } + if uploads.is_empty() { + return Err(FileError::BadRequest("the form holds no file".into())); + } + store::store(files, repo, uploads, Via::Browser).await +} + /// `DELETE /browse/raw/{repo}/files?path=` pub async fn browse_delete( State(state): State, @@ -98,18 +154,22 @@ pub async fn browse_delete( Ok(repo) => repo, Err(response) => return response, }; - let file = match path::normalize(&query.path) { + let file = match files::normalize(&query.path) { Ok(file) if !file.is_empty() && !query.path.ends_with('/') => file, Ok(_) => { return (StatusCode::BAD_REQUEST, "path must name a file").into_response(); } Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), }; - let raw = Raw { + let files = Files { state: &state, principal, }; - let response = upload::delete(&raw, &repo, &file, upload::Via::Browser).await; + let response = match store::remove(&files, &repo, &file, Via::Browser).await { + Ok(true) => StatusCode::NO_CONTENT.into_response(), + Ok(false) => FileError::NotFound.into_response(), + Err(e) => e.into_response(), + }; audit(&repo, "DELETE", &file, &principal, &response); response } @@ -137,10 +197,10 @@ pub async fn admin_delete( Ok(None) => return (StatusCode::NOT_FOUND, "repository not found").into_response(), Err(e) => return crate::admin::db_error(e), }; - if !repo.is_raw() { + if !state.formats.stores_files(&repo.format) { return ( StatusCode::BAD_REQUEST, - "files are deleted from raw repositories", + "files are deleted from repositories of files (raw)", ) .into_response(); } @@ -153,9 +213,9 @@ pub async fn admin_delete( } let directory = query.path.is_empty() || query.path.ends_with('/'); let normalized = match if directory { - path::normalize_directory(&query.path) + files::normalize_directory(&query.path) } else { - path::normalize(&query.path) + files::normalize(&query.path) } { Ok(p) => p, Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 503d18de..a67418c8 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -23,6 +23,7 @@ pub mod crypto; pub mod dispatch; pub mod email; pub mod feed; +pub mod files; pub mod health; pub mod invitations; pub mod me; @@ -31,7 +32,6 @@ pub mod middleware; pub mod nexus_import; pub mod object_store; pub mod plugins; -pub mod raw; pub mod registry; pub mod serve; pub mod state; diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index b9e7adb9..abff7ba0 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -12,21 +12,26 @@ //! * **Queries** the plugin asks for run through the same read ops as every //! other read ([`AppState::read_ops_for`]): package selectors, group //! members the caller cannot see, prefer-hosted and reserved prefixes all -//! apply, and an id the format does not accept is unknown. A request may -//! load at most [`MAX_QUERIES`] answers in at most [`MAX_ROUNDS`] rounds. +//! apply, and an id the format does not accept is unknown. A files +//! format's queries go to the file store ([`crate::files`]), with the same +//! selectors on paths. A request may load at most [`MAX_QUERIES`] answers +//! in at most [`MAX_ROUNDS`] rounds. //! * **Downloads** are streamed from the blob store and counted, exactly as -//! NuGet's, answering a single `Range` when the blob source serves ranges. +//! NuGet's, answering a single `Range` when the blob source serves ranges; +//! a file is served by the file store ([`crate::files::serve`]). //! * **Publishing, unlisting, relisting and deleting** go through the hosted //! write path (CAS staging, size cap, selector and redeploy policy, -//! trusted-publishing audit), and only when the request's method was +//! trusted-publishing audit), and receiving, storing and deleting files +//! through the file store's, only when the request's method was //! authorized for that action and carries an API token or workload //! identity. //! * **Responses** the plugin renders keep only the headers in -//! [`PASSED_RESPONSE_HEADERS`] (no cookies, no CORS, no redirects off this -//! repository) and get the feed's hardening headers. +//! [`PASSED_RESPONSE_HEADERS`] (no cookies, no CORS, no redirects or links +//! off this repository) and get the feed's hardening headers. +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use crate::feed::{byte_range, ByteRange, PackageBytes}; use axum::body::Body; @@ -42,10 +47,11 @@ use forge_core::repo::{ UploadReader, WriteError, }; use forge_db::model::Repository; +use forge_db::raw_dao::RawFile; use forge_db::repository_dao; use forge_plugin::api::{ self as protocol, Action, Answer, ChangeRequest, FormatPlugin, Loaded, PackageInfo, PackageRef, - PublishRequest, Query, Rendered, Upload, + PublishRequest, Query, Upload, }; use forge_plugin::Plugin; use futures_util::{StreamExt, TryStreamExt}; @@ -55,6 +61,8 @@ use crate::dispatch::{ action_for_method, authz_denied, offline_response, offline_serves_nothing, principal_allows_protocol_mutations, HostedWriteAdapter, }; +use crate::files::store::{Content, Ingested, Part, Via}; +use crate::files::{FileError, Files}; use crate::middleware::principal_of; use crate::state::AppState; @@ -82,7 +90,8 @@ pub const PASSED_REQUEST_HEADERS: &[&str] = &[ "user-agent", ]; -/// Response headers a plugin may set. `location` only within its repository. +/// Response headers a plugin may set. `location` and `link` only within its +/// repository. pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ "allow", "cache-control", @@ -91,18 +100,24 @@ pub const PASSED_RESPONSE_HEADERS: &[&str] = &[ "content-type", "etag", "last-modified", + "link", "location", + "vary", ]; /// The format plugins built into this server: crates under `plugins//`, -/// linked in and always installed (NuGet). NuGet reads archives under +/// linked in and always installed (NuGet, raw). NuGet reads archives under /// `nuget_limits` (`[nuget]`). pub fn builtin(nuget_limits: forge_format_nuget::NupkgLimits) -> Vec> { - vec![Plugin::builtin( - forge_format_nuget::PLUGIN_TOML, - forge_format_nuget::Nuget::with_limits(nuget_limits), - ) - .expect("plugins/nuget/plugin.toml is valid")] + vec![ + Plugin::builtin( + forge_format_nuget::PLUGIN_TOML, + forge_format_nuget::Nuget::with_limits(nuget_limits), + ) + .expect("plugins/nuget/plugin.toml is valid"), + Plugin::builtin(forge_format_raw::PLUGIN_TOML, forge_format_raw::Raw) + .expect("plugins/raw/plugin.toml is valid"), + ] } /// Axum handler mounted at `/repository/{repo}` (and everything under it). @@ -113,13 +128,14 @@ pub async fn dispatch(State(state): State, req: Request) -> Resp } /// `(mount, repository name, rest of the path)` of a `{mount}/{repo}/...` -/// path, for one of `mounts`; the rest starts with `/`. +/// path, for one of `mounts`; the rest starts with `/`, or is empty when the +/// path ends with the repository's name. fn split_path<'p>(path: &'p str, mounts: &[&str]) -> Option<(&'p str, &'p str, &'p str)> { mounts.iter().find_map(|mount| { let rest = path.strip_prefix(mount)?.strip_prefix('/')?; let (name, rest) = match rest.find('/') { Some(i) => rest.split_at(i), - None => (rest, "/"), + None => (rest, ""), }; (!name.is_empty()).then(|| (&path[..mount.len()], name, rest)) }) @@ -182,13 +198,19 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { Ok(format) => format, Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response(), }; + let stores_files = format.stores_files(); let package_filter = state .authz .package_filter_for(&principal, &repo, permission) .await; - let ops = match state.read_ops_for(&repo, principal).await { - Ok(ops) => ops, - Err(e) => return repo_error(e), + // A files format reads the file store, never the package read ops. + let ops = if stores_files { + None + } else { + match state.read_ops_for(&repo, principal).await { + Ok(ops) => Some(ops), + Err(e) => return repo_error(e), + } }; let target = protocol::Repository { @@ -199,6 +221,7 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { RepoKind::Group => protocol::RepositoryKind::Group, }, base_url: repository_url(&state, mount, &repo.name), + inline: stores_files && repo.inline_content(), }; let request = protocol::Request { method: method.as_str().to_string(), @@ -222,9 +245,13 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { headers: req.headers().clone(), plugin: plugin.implementation().clone(), format, + stores_files, ops, package_filter, target, + body: Mutex::new(Some(req)), + form: Mutex::new(None), + located: Mutex::new(HashMap::new()), }; let handled = { let (target, request) = (ctx.target.clone(), request.clone()); @@ -232,7 +259,7 @@ async fn dispatch_request(state: AppState, req: Request) -> Response { .await }; let response = match handled { - Ok(action) => ctx.perform(action, request, req).await, + Ok(action) => ctx.perform(action, request).await, Err(response) => response, }; if permission != Permission::Read { @@ -264,10 +291,21 @@ struct Ctx<'a> { headers: axum::http::HeaderMap, plugin: Arc, format: Arc, - ops: Arc, + /// Whether the format's repositories hold files (its plugin asks for + /// files, not packages). + stores_files: bool, + /// The package read ops; `None` for a files format. + ops: Option>, /// The caller's package selector for [`Self::permission`]. package_filter: PackageFilter, target: protocol::Repository, + /// The request, until a publish or a file query reads its body. + body: Mutex>>, + /// The file parts of the form the plugin had received, by their index + /// among its parts, until a store takes them. + form: Mutex>>>, + /// The files queries found, by path, for serving one of them. + located: Mutex>, } impl Ctx<'_> { @@ -283,28 +321,61 @@ impl Ctx<'_> { .map_err(|e| plugin_failed(self.repo, &e)) } - async fn perform( - &self, - action: Action, - request: protocol::Request, - req: Request, - ) -> Response { + /// Do what the plugin decided, `handle` or the last round of `render`. + async fn perform(&self, action: Action, request: protocol::Request) -> Response { + if let Action::Load(queries) = action { + return self.render(queries, request).await; + } + self.act(action).await + } + + /// Do an action other than loading data. + async fn act(&self, action: Action) -> Response { + let files_action = matches!(action, Action::ServeFile(_)); + let package_action = !matches!(action, Action::Respond(_) | Action::ServeFile(_)); + if (files_action && !self.stores_files) || (package_action && self.stores_files) { + return self.plugin_misused("an action of the other storage"); + } match action { Action::Respond(response) => self.respond(response), - Action::Load(queries) => self.render(queries, request).await, + Action::Load(_) => self.plugin_misused("a load as an action"), Action::Download(target) => { if self.permission != Permission::Read { return refused("download"); } self.download(&target).await } - Action::Publish(publish) => self.publish(publish, req).await, + Action::Publish(publish) => match self.take_body() { + Some(req) => self.publish(publish, req).await, + None => self.plugin_misused("a publish after the body was read"), + }, Action::Unlist(change) => self.change(change, Change::Unlist).await, Action::Relist(change) => self.change(change, Change::Relist).await, Action::Delete(change) => self.change(change, Change::Delete).await, + Action::ServeFile(path) => { + if self.permission != Permission::Read { + return refused("serving a file"); + } + self.serve_file(&path).await + } } } + /// The request, for the one query or action that reads its body. + fn take_body(&self) -> Option> { + self.body + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + } + + /// The plugin asked for something the contract does not allow: a bug in + /// the plugin, answered with a server error. + fn plugin_misused(&self, what: &str) -> Response { + tracing::error!(repo = %self.repo.name, format = %self.repo.format, "format plugin asked for {what}"); + (StatusCode::INTERNAL_SERVER_ERROR, "format plugin failed").into_response() + } + /// The canonical id the format makes of `raw`, if it is a valid id. fn id(&self, raw: &str) -> Option { let id = self.format.lower_id(raw); @@ -325,6 +396,9 @@ impl Ctx<'_> { if data.len() + queries.len() > MAX_QUERIES { break; } + if let Err(response) = self.check_round(&queries) { + return response; + } let pending: Vec<_> = queries.iter().map(|query| self.answer(query)).collect(); let answers: Vec = match futures_util::stream::iter(pending) .buffered(CONCURRENT_QUERIES) @@ -332,7 +406,7 @@ impl Ctx<'_> { .await { Ok(answers) => answers, - Err(e) => return repo_error(e), + Err(response) => return response, }; data.extend( queries @@ -348,11 +422,11 @@ impl Ctx<'_> { }) .await; match rendered { - Ok((Rendered::Respond(response), _)) => return self.respond(response), - Ok((Rendered::Load(more), loaded)) => { + Ok((Action::Load(more), loaded)) => { data = loaded; queries = more; } + Ok((action, _)) => return self.act(action).await, Err(response) => return response, } } @@ -368,12 +442,226 @@ impl Ctx<'_> { .into_response() } - async fn answer(&self, query: &Query) -> Result { + /// Whether a round of queries keeps to the contract: queries of the + /// format's storage only, and a change (`form`, `store`, `remove`) alone + /// in its round. + fn check_round(&self, queries: &[Query]) -> Result<(), Response> { + if queries.iter().any(|q| q.is_files() != self.stores_files) { + return Err(self.plugin_misused("a query of the other storage")); + } + if queries.len() > 1 && queries.iter().any(Query::is_change) { + return Err(self.plugin_misused("a change with other queries in its round")); + } + Ok(()) + } + + /// The answer to one query, or the response that ends the request + /// instead. + async fn answer(&self, query: &Query) -> Result { + if query.is_files() { + return self.answer_files(query).await; + } + let Some(ops) = &self.ops else { + return Err(self.plugin_misused("packages of a files format")); + }; + self.answer_packages(ops.as_ref(), query) + .await + .map_err(repo_error) + } + + /// The answer to a files query, from the file store. + async fn answer_files(&self, query: &Query) -> Result { + let files = Files { + state: self.state, + principal: self.principal, + }; + let failed = |e: FileError| e.into_response(); + match query { + Query::File(path) | Query::FetchFile(path) => { + if forge_plugin::api::files::validate_path(path).is_err() { + return Err(self.plugin_misused("a file at an invalid path")); + } + let fetch = matches!(query, Query::FetchFile(_)); + let found = files + .locate(self.repo, path, fetch, Vec::new(), 0) + .await + .map_err(failed)?; + if let Some(file) = &found { + self.located + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(path.clone(), file.clone()); + } + Ok(Answer::File(found.as_ref().map(file_info))) + } + Query::Directory(wanted) => { + if forge_plugin::api::files::validate_directory(&wanted.directory).is_err() { + return Err(self.plugin_misused("an invalid directory")); + } + let page = files + .list( + self.repo, + &wanted.directory, + wanted.after.as_deref(), + wanted.limit as usize, + ) + .await + .map_err(failed)?; + Ok(Answer::Directory(page.map(directory_page))) + } + Query::Form => { + if self.permission != Permission::Write { + return Err(refused("receiving a form")); + } + let Some(req) = self.take_body() else { + return Err(self.plugin_misused("a form after the body was read")); + }; + let parts = + crate::files::store::receive_form(&files, self.repo, req, Via::Protocol) + .await + .map_err(failed)?; + let mut staged = Vec::with_capacity(parts.len()); + let mut answer = Vec::with_capacity(parts.len()); + for part in parts { + match part { + Part::File { + field, + name, + content_type, + ingested, + } => { + answer.push(protocol::FormPart { + field, + content: protocol::FormContent::File(protocol::FormFile { + name, + content_type, + size: ingested.stored.size, + }), + }); + staged.push(Some(*ingested)); + } + Part::Text { field, value } => { + answer.push(protocol::FormPart { + field, + content: protocol::FormContent::Text(value), + }); + staged.push(None); + } + } + } + *self.form.lock().unwrap_or_else(PoisonError::into_inner) = Some(staged); + Ok(Answer::Form(answer)) + } + Query::Store(wanted) => { + if self.permission != Permission::Write { + return Err(refused("storing files")); + } + let mut uploads = Vec::with_capacity(wanted.len()); + for file in wanted { + let content = match file.source { + protocol::FileSource::Body => match self.take_body() { + Some(req) => Content::Body(req), + None => { + return Err(self.plugin_misused("a store of a body already read")) + } + }, + protocol::FileSource::Part(index) => match self.take_part(index) { + Some(ingested) => Content::Staged(ingested), + None => { + return Err(self.plugin_misused( + "a store of a form part that is not a file it may store", + )) + } + }, + }; + uploads.push(crate::files::store::Upload { + path: file.path.clone(), + declared: file.content_type.clone(), + content, + }); + } + let stored = crate::files::store::store(&files, self.repo, uploads, Via::Protocol) + .await + .map_err(failed)?; + Ok(Answer::Stored( + stored + .into_iter() + .map(|stored| protocol::StoredFile { + path: stored.path, + size: stored.size, + sha256: stored.sha256, + status: match stored.status { + "created" => protocol::StoreStatus::Created, + "replaced" => protocol::StoreStatus::Replaced, + _ => protocol::StoreStatus::Unchanged, + }, + }) + .collect(), + )) + } + Query::Remove(path) => { + if self.permission != Permission::Delete { + return Err(refused("deleting a file")); + } + let removed = crate::files::store::remove(&files, self.repo, path, Via::Protocol) + .await + .map_err(failed)?; + Ok(Answer::Removed(removed)) + } + _ => Err(self.plugin_misused("a package query of a files format")), + } + } + + /// The staged file part `index` of the form the plugin had received, + /// taken (a part is stored once). + fn take_part(&self, index: u32) -> Option { + self.form + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_mut()? + .get_mut(usize::try_from(index).ok()?)? + .take() + } + + /// Serve the file at `path`, as a query found it or as `fetch-file` + /// finds it now. + async fn serve_file(&self, path: &str) -> Response { + let found = self + .located + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(path) + .cloned(); + let file = match found { + Some(file) => file, + None => { + if forge_plugin::api::files::validate_path(path).is_err() { + return self.plugin_misused("a file at an invalid path"); + } + let files = Files { + state: self.state, + principal: self.principal, + }; + match files.locate(self.repo, path, true, Vec::new(), 0).await { + Ok(Some(file)) => file, + Ok(None) => return FileError::NotFound.into_response(), + Err(e) => return e.into_response(), + } + } + }; + crate::files::serve::file(self.state, self.repo, &file, &self.headers, self.head).await + } + + /// The answer to a package query, through the package read ops. + async fn answer_packages( + &self, + ops: &dyn RepoReadOps, + query: &Query, + ) -> Result { let repo = &self.repo.name; Ok(match query { Query::Versions(id) => Answer::Versions(match self.id(id) { - Some(id) => self - .ops + Some(id) => ops .list_versions(repo, &id) .await? .into_iter() @@ -387,8 +675,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Packages(id) => Answer::Packages(match self.id(id) { - Some(id) => self - .ops + Some(id) => ops .registration(repo, &id) .await? .into_iter() @@ -397,8 +684,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Package(target) => Answer::Package(match self.package(target) { - Some((id, version)) => self - .ops + Some((id, version)) => ops .registration_entry(repo, &id, &version) .await? .map(|entry| self.package_info(entry)), @@ -415,7 +701,7 @@ impl Ctx<'_> { let mut entries = if versions.is_empty() { Vec::new() } else { - self.ops.registration_entries(repo, &id, &versions).await? + ops.registration_entries(repo, &id, &versions).await? }; entries.sort_by(|a, b| self.format.compare_versions(&a.version, &b.version)); entries @@ -426,7 +712,7 @@ impl Ctx<'_> { None => Vec::new(), }), Query::Manifest(target) => Answer::Manifest(match self.package(target) { - Some((id, version)) => self.ops.resolve_nuspec(repo, &id, &version).await?, + Some((id, version)) => ops.resolve_nuspec(repo, &id, &version).await?, None => None, }), Query::Search(search) => { @@ -439,7 +725,7 @@ impl Ctx<'_> { include_semver2: search.semver2, include_upstream: search.upstream, }; - let result = self.ops.search(repo, ¶ms).await?; + let result = ops.search(repo, ¶ms).await?; Answer::Search(protocol::SearchResult { total: u32::try_from(result.total_hits).unwrap_or(u32::MAX), hits: result @@ -467,6 +753,17 @@ impl Ctx<'_> { .collect(), }) } + // Answered by `answer_files`. + Query::File(_) + | Query::FetchFile(_) + | Query::Directory(_) + | Query::Form + | Query::Store(_) + | Query::Remove(_) => { + return Err(RepoError::Storage( + "a files query of a package format".into(), + )) + } }) } @@ -475,13 +772,14 @@ impl Ctx<'_> { let Some((id, version)) = self.package(target) else { return (StatusCode::NOT_FOUND, "package not found").into_response(); }; + let Some(ops) = &self.ops else { + return self.plugin_misused("a package download of a files format"); + }; // A HEAD (a probe, a mirror tool) is not a download. let resolved = if self.head { - self.ops - .resolve_nupkg_head(&self.repo.name, &id, &version) - .await + ops.resolve_nupkg_head(&self.repo.name, &id, &version).await } else { - self.ops.resolve_nupkg(&self.repo.name, &id, &version).await + ops.resolve_nupkg(&self.repo.name, &id, &version).await }; let location = match resolved { Ok(Some(location)) => location, @@ -705,6 +1003,9 @@ impl Ctx<'_> { if name == "location" && !self.within_repository(value) { continue; } + if name == "link" && !self.links_within_repository(value) { + continue; + } let (Ok(name), Ok(value)) = ( header::HeaderName::from_bytes(name.as_bytes()), HeaderValue::from_str(value), @@ -713,10 +1014,13 @@ impl Ctx<'_> { }; out.headers_mut().append(name, value); } - out.headers_mut().insert( - header::CONTENT_LENGTH, - HeaderValue::from(response.body.len()), - ); + // A 204 or a 304 has no body, and says nothing of its length. + if !matches!(status, StatusCode::NO_CONTENT | StatusCode::NOT_MODIFIED) { + out.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from(response.body.len()), + ); + } if !self.head { *out.body_mut() = Body::from(response.body); } @@ -730,6 +1034,12 @@ impl Ctx<'_> { fn within_repository(&self, location: &str) -> bool { within(location, self.mount, &self.repo.name, &self.target.base_url) } + + /// Whether every target of a `Link` header (`; rel=...`, comma + /// separated) stays inside this repository. + fn links_within_repository(&self, value: &str) -> bool { + links_within(value, self.mount, &self.repo.name, &self.target.base_url) + } } #[derive(Clone, Copy)] @@ -821,6 +1131,39 @@ fn multipart_reader( Ok(Box::new(StreamReader::new(stream))) } +/// A stored file as the plugin sees it. +fn file_info(file: &RawFile) -> protocol::FileInfo { + let time = + |t: chrono::DateTime| t.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true); + protocol::FileInfo { + path: file.path.clone(), + size: u64::try_from(file.size).unwrap_or(0), + content_type: file.content_type.clone(), + sha256: file.blob_sha256.clone(), + sha512: file.sha512_base64.clone(), + sha1: file.sha1.clone(), + md5: file.md5.clone(), + modified: time(file.updated_at), + last_downloaded: file.last_downloaded_at.map(time), + } +} + +/// A page of a directory as the plugin sees it. +fn directory_page(page: crate::files::listing::Page) -> protocol::DirectoryPage { + use crate::files::listing::EntryKind; + protocol::DirectoryPage { + entries: page + .entries + .into_iter() + .map(|entry| match entry.kind { + EntryKind::File(file) => protocol::DirectoryEntry::File(file_info(&file)), + EntryKind::Directory(path) => protocol::DirectoryEntry::Directory(path), + }) + .collect(), + next: page.next, + } +} + /// The absolute URL of repository `name` served under `mount`: /// `{public_base_url}{mount}/{name}`, the base its plugin builds URLs on and /// the address its clients are given. @@ -831,12 +1174,20 @@ pub fn repository_url(state: &AppState, mount: &str, name: &str) -> String { ) } -/// See [`Ctx::within_repository`]. +/// See [`Ctx::within_repository`]. A path may also start with the path of +/// `base_url` (a public base URL with a path of its own). fn within(location: &str, mount: &str, repo: &str, base_url: &str) -> bool { let path_prefix = format!("{mount}/{repo}/"); let absolute_prefix = format!("{base_url}/"); + let base_path_prefix = url::Url::parse(base_url) + .ok() + .map(|url| format!("{}/", url.path().trim_end_matches('/'))); let lower = location.to_ascii_lowercase(); - (location.starts_with(&path_prefix) || location.starts_with(&absolute_prefix)) + (location.starts_with(&path_prefix) + || location.starts_with(&absolute_prefix) + || base_path_prefix + .as_deref() + .is_some_and(|prefix| prefix != "/" && location.starts_with(prefix))) && !location .split(['/', '?', '#']) .any(|seg| seg == ".." || seg == ".") @@ -844,6 +1195,27 @@ fn within(location: &str, mount: &str, repo: &str, base_url: &str) -> bool { && !location.contains('\\') } +/// See [`Ctx::links_within_repository`]: every `<…>` of `value` is +/// [`within`] the repository, and there is one at least. +fn links_within(value: &str, mount: &str, repo: &str, base_url: &str) -> bool { + let mut targets = 0; + for link in value.split(',') { + let Some(target) = link + .trim() + .strip_prefix('<') + .and_then(|rest| rest.split_once('>')) + .map(|(target, _)| target) + else { + return false; + }; + if !within(target, mount, repo, base_url) { + return false; + } + targets += 1; + } + targets > 0 +} + /// A header value of ASCII digits and punctuation. fn numeric_header(value: String) -> HeaderValue { HeaderValue::from_str(&value).expect("ASCII header value") @@ -939,7 +1311,7 @@ mod tests { ); assert_eq!( split("/repository/charts"), - Some(("/repository", "charts", "/")) + Some(("/repository", "charts", "")) ); assert_eq!( split("/repository/charts/"), @@ -960,6 +1332,28 @@ mod tests { assert_eq!(split("/v3/charts"), None); } + #[test] + fn links_stay_inside_the_repository() { + let base = "https://forge.example.com/raw/bin"; + let ok = "; rel=\"next\""; + assert!(links_within(ok, "/raw", "bin", base)); + assert!(links_within( + "; rel=\"next\", ; rel=\"up\"", + "/raw", + "bin", + base + )); + for bad in [ + "; rel=\"next\"", + "; rel=\"next\"", + "; rel=\"next\", ; rel=\"preload\"", + "rel=\"next\"", + "", + ] { + assert!(!links_within(bad, "/raw", "bin", base), "{bad}"); + } + } + #[test] fn redirects_stay_inside_the_repository() { let base = "https://forge.example.com/repository/charts"; @@ -969,6 +1363,13 @@ mod tests { ] { assert!(within(ok, "/repository", "charts", base), "{ok}"); } + // Behind a public base URL with a path, the repository's path. + assert!(within( + "/forge/repository/charts/index.yaml", + "/repository", + "charts", + "https://forge.example.com/forge/repository/charts" + )); for bad in [ "https://evil.example.com/", "//evil.example.com/repository/charts/x", diff --git a/crates/server/src/raw/listing.rs b/crates/server/src/raw/listing.rs deleted file mode 100644 index 63630ee8..00000000 --- a/crates/server/src/raw/listing.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Directory listings: `GET /raw///`, as HTML for browsers or as -//! JSON (`Accept: application/json`), a page at a time. -//! -//! A page holds at most `limit` entries (default 100, at most 1000) after -//! the entry named by `after`, in byte order of their names, a -//! subdirectory's name ending in `/`; `next` (and a `Link: rel="next"` -//! header) names where the following page starts. A listing walks the -//! index ([`raw_dao::walk_directory`]): a page costs `limit` index probes, -//! however many files are below the directory. -//! -//! Package selectors apply: a file outside the caller's selector is not -//! listed, and a subdirectory is listed only when it holds a file the -//! caller may see (looked for among its first files; a subdirectory none of -//! whose first [`SUBDIRECTORY_SCAN`] files is visible is left out). A -//! group lists the merged entries of the members the caller may read, the -//! highest-priority member's file first; a proxy lists what it has cached. - -use std::collections::BTreeMap; - -use axum::http::{header, HeaderMap, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; -use chrono::{DateTime, Utc}; -use forge_core::authz::PackageFilter; -use forge_db::model::{RepoKind, Repository}; -use forge_db::raw_dao::{self, RawFile}; -use serde::Serialize; - -use super::{path, Raw, RawError}; - -/// Default page size. -pub const DEFAULT_LIMIT: usize = 100; - -/// Largest page size. -pub const MAX_LIMIT: usize = 1000; - -/// How many of a subdirectory's files a listing reads, at most, to find one -/// a selector-restricted caller may see. -pub const SUBDIRECTORY_SCAN: usize = 1000; - -/// One entry of a listing. -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub struct Entry { - /// The entry's name in its directory; a subdirectory's ends in `/`. - pub name: String, - /// The full path (a subdirectory's ends in `/`). - pub path: String, - /// `file` | `directory`. - #[serde(rename = "type")] - pub kind: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - pub size: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub content_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sha256: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_modified: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_downloaded: Option>, -} - -impl Entry { - fn file(file: &RawFile, name: &str) -> Self { - Entry { - name: name.to_string(), - path: file.path.clone(), - kind: "file", - size: Some(file.size), - content_type: Some(file.content_type.clone()), - sha256: Some(file.blob_sha256.clone()), - last_modified: Some(file.updated_at), - last_downloaded: file.last_downloaded_at, - } - } - - fn directory(prefix: &str, name: &str) -> Self { - Entry { - name: format!("{name}/"), - path: format!("{prefix}{name}/"), - kind: "directory", - size: None, - content_type: None, - sha256: None, - last_modified: None, - last_downloaded: None, - } - } -} - -/// A page of a listing. -#[derive(Debug, Clone, Serialize)] -pub struct Page { - pub repository: String, - /// The directory (`""` for the root). - pub path: String, - pub entries: Vec, - /// The `after` of the next page, if there is one. - pub next: Option, -} - -/// The query of a listing. -#[derive(Debug, Default, serde::Deserialize)] -pub struct ListQuery { - pub after: Option, - pub limit: Option, - /// `json` | `html`: overrides `Accept`. - pub format: Option, -} - -impl ListQuery { - /// Parse a request's query string; other parameters are ignored. - pub fn parse(query: Option<&str>) -> Result { - let mut out = ListQuery::default(); - for (key, value) in url::form_urlencoded::parse(query.unwrap_or("").as_bytes()) { - match key.as_ref() { - "after" => out.after = Some(value.into_owned()), - "limit" => { - out.limit = Some(value.parse().map_err(|_| { - RawError::BadRequest("limit must be a positive number".into()) - })?) - } - "format" => match value.as_ref() { - "json" | "html" => out.format = Some(value.into_owned()), - _ => return Err(RawError::BadRequest("format is json or html".into())), - }, - _ => {} - } - } - Ok(out) - } -} - -/// Where a page starting after the entry `after` (a name in directory -/// `prefix`, a subdirectory's ending in `/`) resumes: the least path past -/// that entry (past all of a subdirectory's files: `'0'` follows `'/'`; past -/// a file: no path holds a control character). -fn resume_at(prefix: &str, after: Option<&str>) -> Result { - let Some(after) = after.filter(|a| !a.is_empty()) else { - return Ok(prefix.to_string()); - }; - let (name, directory) = match after.strip_suffix('/') { - Some(name) => (name, true), - None => (after, false), - }; - path::validate_segment(name).map_err(|e| RawError::BadRequest(format!("after: {e}")))?; - Ok(if directory { - format!("{prefix}{name}0") - } else { - format!("{prefix}{name}\u{1}") - }) -} - -/// One repository's entries of a page: what it has after `from`, as far as -/// it read (`more`: it has entries past the last one returned, up to and -/// including the entry named `more`, which it has read). -struct Partial { - entries: Vec, - more: Option, -} - -impl Raw<'_> { - /// The entries of `prefix` in hosted or proxy `repo` from `from` on, - /// visible through every one of `filters`. - async fn list_local( - &self, - repo: &Repository, - prefix: &str, - from: &str, - limit: usize, - filters: &[PackageFilter], - ) -> Result { - let walked = raw_dao::walk_directory( - &self.state.pool, - repo.id, - prefix, - from, - i64::try_from(limit + 1).unwrap_or(i64::MAX), - ) - .await?; - let more = walked.len() > limit; - let mut entries = Vec::new(); - let mut last = None; - for walked in walked.into_iter().take(limit) { - let rest = &walked.path[prefix.len()..]; - let (entry, key) = match (rest.find('/'), &walked.file) { - (None, Some(file)) => { - let visible = filters.iter().all(|f| f.allows(&file.path)); - (visible.then(|| Entry::file(file, rest)), rest.to_string()) - } - (Some(i), _) => { - let name = &rest[..i]; - let dir = format!("{prefix}{name}/"); - let visible = self.directory_visible(repo, &dir, filters).await?; - ( - visible.then(|| Entry::directory(prefix, name)), - format!("{name}/"), - ) - } - // A file row vanished between the walk and the join. - (None, None) => (None, rest.to_string()), - }; - entries.extend(entry); - last = Some(key); - } - Ok(Partial { - entries, - more: if more { last } else { None }, - }) - } - - /// Whether directory `dir` of `repo` holds a file visible through every - /// one of `filters`. - async fn directory_visible( - &self, - repo: &Repository, - dir: &str, - filters: &[PackageFilter], - ) -> Result { - if filters.iter().all(|f| matches!(f, PackageFilter::AllowAll)) { - return Ok(true); - } - const BATCH: usize = 200; - let mut after = String::new(); - let mut read = 0; - while read < SUBDIRECTORY_SCAN { - let paths = - raw_dao::paths_under(&self.state.pool, repo.id, dir, &after, BATCH as i64).await?; - if paths.iter().any(|p| filters.iter().all(|f| f.allows(p))) { - return Ok(true); - } - if paths.len() < BATCH { - return Ok(false); - } - read += paths.len(); - after = paths.last().cloned().unwrap_or_default(); - } - Ok(false) - } - - /// The entries of `prefix` in `repo` (any kind) from `from` on. - fn list_in<'a>( - &'a self, - repo: &'a Repository, - prefix: &'a str, - from: &'a str, - limit: usize, - filters: Vec, - depth: u32, - ) -> futures_util::future::BoxFuture<'a, Result> { - Box::pin(async move { - let mut filters = filters; - filters.push(self.state.authz.package_filter(&self.principal, repo).await); - if repo.kind() != RepoKind::Group { - return self.list_local(repo, prefix, from, limit, &filters).await; - } - // A merge of the members' pages, complete up to the first point - // a member stopped reading at. - let mut merged: BTreeMap = BTreeMap::new(); - let mut cutoff: Option = None; - for member in self.serving_members(repo, depth).await? { - let part = self - .list_in(&member, prefix, from, limit, filters.clone(), depth + 1) - .await?; - for entry in part.entries { - merged.entry(entry.name.clone()).or_insert(entry); - } - if let Some(more) = part.more { - if cutoff.as_ref().is_none_or(|c| more < *c) { - cutoff = Some(more); - } - } - } - let mut entries: Vec = merged - .into_values() - .filter(|e| cutoff.as_ref().is_none_or(|c| e.name <= *c)) - .collect(); - let more = if entries.len() > limit { - entries.truncate(limit); - entries.last().map(|e| e.name.clone()) - } else { - cutoff - }; - Ok(Partial { entries, more }) - }) - } - - /// A page of directory `prefix` of `repo`, `None` when it does not - /// exist (a directory other than the root without visible entries). - pub async fn list( - &self, - repo: &Repository, - prefix: &str, - query: &ListQuery, - ) -> Result, RawError> { - let limit = query.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); - let from = resume_at(prefix, query.after.as_deref())?; - let partial = self - .list_in(repo, prefix, &from, limit, Vec::new(), 0) - .await?; - if partial.entries.is_empty() && partial.more.is_none() && !prefix.is_empty() { - // Past the end of an existing directory is an empty page, not a - // missing directory. - if query.after.is_none() { - return Ok(None); - } - } - Ok(Some(Page { - repository: repo.name.clone(), - path: prefix.to_string(), - entries: partial.entries, - next: partial.more, - })) - } -} - -/// Whether the client wants JSON: `format=json`, or an `Accept` preferring -/// `application/json` to HTML. -pub fn wants_json(headers: &HeaderMap, query: &ListQuery) -> bool { - match query.format.as_deref() { - Some("json") => return true, - Some("html") => return false, - _ => {} - } - let accept = headers - .get(header::ACCEPT) - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - let quality = |wanted: &str| { - accept - .split(',') - .filter_map(|item| { - let mut parts = item.split(';'); - let media = parts.next()?.trim().to_ascii_lowercase(); - (media == wanted).then(|| { - parts - .filter_map(|p| p.trim().strip_prefix("q=")) - .find_map(|q| q.parse::().ok()) - .unwrap_or(1.0) - }) - }) - .fold(0.0f32, f32::max) - }; - quality("application/json") > 0.0 && quality("application/json") >= quality("text/html") -} - -/// The URL of the page after `page`. -fn next_link(page: &Page, query: &ListQuery) -> Option { - let next = page.next.as_deref()?; - let mut params = url::form_urlencoded::Serializer::new(String::new()); - params.append_pair("after", next); - if let Some(limit) = query.limit { - params.append_pair("limit", &limit.to_string()); - } - if let Some(format) = &query.format { - params.append_pair("format", format); - } - Some(format!( - "/raw/{}/{}?{}", - page.repository, - path::encode(&page.path), - params.finish() - )) -} - -fn escape(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - for c in text.chars() { - match c { - '&' => out.push_str("&"), - '<' => out.push_str("<"), - '>' => out.push_str(">"), - '"' => out.push_str("""), - '\'' => out.push_str("'"), - c => out.push(c), - } - } - out -} - -/// The page as an HTML document: a table of the entries, linked. -pub fn html(page: &Page, query: &ListQuery) -> String { - let title = format!("Index of /raw/{}/{}", page.repository, page.path); - let mut out = format!( - "\n{t}\n\n

{t}

\n\n\n", - t = escape(&title) - ); - if !page.path.is_empty() { - out.push_str("\n"); - } - for entry in &page.entries { - let href = path::encode(&entry.name); - out.push_str(&format!( - "\n", - escape(&href), - escape(&entry.name), - entry.size.map(|s| s.to_string()).unwrap_or_default(), - entry - .last_modified - .map(|t| t.format("%Y-%m-%d %H:%M:%S UTC").to_string()) - .unwrap_or_default(), - escape(entry.sha256.as_deref().unwrap_or("")), - )); - } - out.push_str("
NameSizeLast modifiedSHA-256
../
{}{}{}{}
\n"); - if let Some(next) = next_link(page, query) { - out.push_str(&format!( - "

Next page

\n", - escape(&next) - )); - } - out.push_str("\n"); - out -} - -/// Answer a listing request. -pub fn respond(page: &Page, query: &ListQuery, json: bool, head: bool) -> Response { - let (content_type, body) = if json { - ( - "application/json", - serde_json::to_string(page).unwrap_or_default(), - ) - } else { - ("text/html; charset=utf-8", html(page, query)) - }; - let length = body.len(); - let mut response = if head { - StatusCode::OK.into_response() - } else { - (StatusCode::OK, body).into_response() - }; - let out = response.headers_mut(); - out.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); - out.insert(header::CONTENT_LENGTH, HeaderValue::from(length)); - // Forge's own document, shown in the browser (and sandboxed). - out.insert( - header::CONTENT_DISPOSITION, - HeaderValue::from_static("inline"), - ); - out.insert(header::VARY, HeaderValue::from_static("Accept")); - if let Some(next) = next_link(page, query) { - if let Ok(value) = HeaderValue::from_str(&format!("<{next}>; rel=\"next\"")) { - out.insert(header::LINK, value); - } - } - response -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn pages_resume_past_the_named_entry() { - assert_eq!(resume_at("a/", None).unwrap(), "a/"); - assert_eq!(resume_at("a/", Some("")).unwrap(), "a/"); - assert_eq!(resume_at("a/", Some("x.bin")).unwrap(), "a/x.bin\u{1}"); - assert_eq!(resume_at("", Some("dir/")).unwrap(), "dir0"); - assert!(resume_at("a/", Some("../x")).is_err()); - assert!(resume_at("a/", Some("b/c")).is_err()); - } - - #[test] - fn json_is_asked_for() { - let query = ListQuery::default(); - let mut h = HeaderMap::new(); - assert!(!wants_json(&h, &query)); - h.insert(header::ACCEPT, HeaderValue::from_static("application/json")); - assert!(wants_json(&h, &query)); - h.insert( - header::ACCEPT, - HeaderValue::from_static( - "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", - ), - ); - assert!(!wants_json(&h, &query)); - let query = ListQuery { - format: Some("json".into()), - ..Default::default() - }; - assert!(wants_json(&h, &query)); - } - - #[test] - fn html_escapes_names_and_links_them_encoded() { - let page = Page { - repository: "bin".into(), - path: "a/".into(), - entries: vec![Entry { - name: " y.txt".into(), - path: "a/ y.txt".into(), - kind: "file", - size: Some(3), - content_type: Some("text/plain".into()), - sha256: Some("ab".into()), - last_modified: None, - last_downloaded: None, - }], - next: Some(" y.txt".into()), - }; - let html = html(&page, &ListQuery::default()); - assert!( - html.contains("href=\"%3Cx%3E%20y.txt\"><x> y.txt"), - "{html}" - ); - assert!(html.contains("href=\"../\"")); - assert!(html.contains("/raw/bin/a/?after=%3Cx%3E+y.txt"), "{html}"); - assert!(!html.contains("")); - } -} diff --git a/crates/server/src/raw/mod.rs b/crates/server/src/raw/mod.rs deleted file mode 100644 index b0918d1e..00000000 --- a/crates/server/src/raw/mod.rs +++ /dev/null @@ -1,436 +0,0 @@ -//! Raw (generic file) repositories: arbitrary files under arbitrary paths, -//! like Nexus' raw-hosted, raw-proxy and raw-group repositories, served at -//! `/raw//`. -//! -//! * `GET` / `HEAD` of a file: its bytes, with `ETag` (the SHA-256), -//! `Last-Modified`, `Content-Length` and the stored `Content-Type`; one -//! `Range`, `If-Range`, `If-None-Match` and `If-Modified-Since` ([`serve`]). -//! `.sha256`, `.sha512`, `.sha1` and `.md5` are answered from the -//! file's digests when no file of that name is stored. -//! * `GET` of a directory (`/`, or the root): a listing, HTML or JSON, -//! paginated ([`listing`]). -//! * `PUT` of a file, `POST` of a multipart form to a directory, `DELETE` of -//! a file, in hosted repositories ([`upload`]); directories are deleted -//! through the admin API. -//! * Proxies fetch from an HTTP(S) base URL and re-validate after a maximum -//! age ([`proxy`]); groups answer with the first member, by priority, that -//! has the path. -//! -//! Paths are normalized ([`path`]): `..`, `.`, empty segments, backslashes -//! and encoded slashes are refused, never resolved. Package selectors match -//! paths (`pdfium/**`). Files are CAS blobs, so the same content under -//! several paths or repositories is stored once. - -pub mod listing; -pub mod path; -pub mod proxy; -pub mod serve; -pub mod upload; -pub mod web; - -use std::collections::HashSet; - -use axum::body::Body; -use axum::extract::{Request, State}; -use axum::http::{header, HeaderMap, HeaderValue, Method, StatusCode}; -use axum::response::{IntoResponse, Response}; -use forge_core::auth::principal::Principal; -use forge_core::authz::{Action, PackageFilter}; -use forge_db::model::{RepoKind, Repository}; -use forge_db::{raw_dao, repository_dao}; - -use crate::dispatch::{action_for_method, authz_denied, offline_response, offline_serves_nothing}; -use crate::middleware::principal_of; -use crate::state::AppState; -use listing::ListQuery; -use path::Target; -use raw_dao::RawFile; -use serve::Checksum; - -/// Where raw repositories are served. -pub const MOUNT: &str = "/raw"; - -/// A request's view of raw repositories: the server's state and the caller. -pub struct Raw<'a> { - pub state: &'a AppState, - pub principal: Principal, -} - -/// Why a raw request failed, as its answer. -#[derive(Debug)] -pub enum RawError { - NotFound, - BadRequest(String), - /// A mutation needs an API token or a workload identity. - CredentialRequired, - /// The caller may not do this (repository-level). - Denied, - Forbidden(String), - Conflict(String), - MethodNotAllowed(&'static str), - TooLarge(u64), - Unavailable(String), - BadGateway(String), - Internal(String), -} - -impl RawError { - /// The error's text, for logs. - pub fn message(&self) -> String { - match self { - RawError::NotFound => "not found".into(), - RawError::CredentialRequired => "API token or workload identity required".into(), - RawError::Denied => "forbidden".into(), - RawError::MethodNotAllowed(m) => (*m).into(), - RawError::TooLarge(limit) => format!("larger than {limit} bytes"), - RawError::BadRequest(m) - | RawError::Forbidden(m) - | RawError::Conflict(m) - | RawError::Unavailable(m) - | RawError::BadGateway(m) - | RawError::Internal(m) => m.clone(), - } - } -} - -impl From for RawError { - fn from(e: sqlx::Error) -> Self { - match e { - sqlx::Error::PoolTimedOut => RawError::Unavailable("database busy".into()), - e => RawError::Internal(format!("database: {e}")), - } - } -} - -impl IntoResponse for RawError { - fn into_response(self) -> Response { - match self { - RawError::NotFound => (StatusCode::NOT_FOUND, "not found").into_response(), - RawError::BadRequest(m) => (StatusCode::BAD_REQUEST, m).into_response(), - RawError::CredentialRequired => ( - StatusCode::UNAUTHORIZED, - [(header::WWW_AUTHENTICATE, "Basic realm=\"Forge\"")], - "API token or workload identity required", - ) - .into_response(), - RawError::Denied => (StatusCode::FORBIDDEN, "forbidden").into_response(), - RawError::Forbidden(m) => (StatusCode::FORBIDDEN, m).into_response(), - RawError::Conflict(m) => (StatusCode::CONFLICT, m).into_response(), - RawError::MethodNotAllowed(m) => ( - StatusCode::METHOD_NOT_ALLOWED, - [(header::ALLOW, "GET, HEAD")], - m, - ) - .into_response(), - RawError::TooLarge(limit) => ( - StatusCode::PAYLOAD_TOO_LARGE, - format!("the upload exceeds the repository's limit of {limit} bytes"), - ) - .into_response(), - RawError::Unavailable(m) => ( - StatusCode::SERVICE_UNAVAILABLE, - [(header::RETRY_AFTER, "30")], - m, - ) - .into_response(), - RawError::BadGateway(m) => (StatusCode::BAD_GATEWAY, m).into_response(), - RawError::Internal(m) => { - tracing::error!(error = %m, "raw repository request failed"); - (StatusCode::INTERNAL_SERVER_ERROR, "internal error").into_response() - } - } - } -} - -/// A file a request found: stored at the path, or the file a checksum -/// sidecar is computed from. -pub enum Found { - File(RawFile), - Sidecar(RawFile, Checksum), -} - -/// The `/raw/...` handler. -pub async fn handle(State(state): State, req: Request) -> Response { - let mut response = handle_request(&state, req).await; - crate::feed::harden_response_headers(response.headers_mut()); - response -} - -/// `(repository name, rest)` of a `/raw/[/]` path; the rest is -/// `None` without the `/` after the name. -fn split(path: &str) -> Option<(&str, Option<&str>)> { - let rest = path.strip_prefix(MOUNT)?.strip_prefix('/')?; - let (name, rest) = match rest.find('/') { - Some(i) => (&rest[..i], Some(&rest[i + 1..])), - None => (rest, None), - }; - (!name.is_empty()).then_some((name, rest)) -} - -async fn handle_request(state: &AppState, req: Request) -> Response { - let principal = principal_of(&req); - let method = req.method().clone(); - let uri = req.uri().clone(); - let Some((name, rest)) = split(uri.path()) else { - return (StatusCode::NOT_FOUND, "repository not found").into_response(); - }; - let repo = match repository_dao::find_repository_by_name(&state.pool, name).await { - Ok(Some(r)) => r, - // An unknown name answers like a repository the caller may not - // read, except to a global admin. - Ok(None) if state.authz.is_global_admin(&principal).await => { - return (StatusCode::NOT_FOUND, "repository not found").into_response(); - } - Ok(None) => return authz_denied(&principal), - Err(e) => return RawError::from(e).into_response(), - }; - let action = action_for_method(&method); - let allowed = match action { - Action::Read => state.authz.can_read(&principal, &repo).await, - Action::Write => state.authz.can_write(&principal, &repo).await, - Action::Delete => state.authz.can_delete(&principal, &repo).await, - Action::Admin => false, - }; - if !allowed { - return authz_denied(&principal); - } - if !repo.is_raw() { - return (StatusCode::NOT_FOUND, "repository is not served here").into_response(); - } - if offline_serves_nothing(&repo) { - return offline_response(); - } - let Some(rest) = rest else { - // `/raw/`: the root listing is `/raw//`. - if matches!(method, Method::GET | Method::HEAD) { - let location = format!("{MOUNT}/{}/", repo.name); - return ( - StatusCode::MOVED_PERMANENTLY, - [(header::LOCATION, location)], - ) - .into_response(); - } - return (StatusCode::NOT_FOUND, "not found").into_response(); - }; - let target = match path::parse_request(rest) { - Ok(target) => target, - Err(e) => return (StatusCode::BAD_REQUEST, e.to_string()).into_response(), - }; - let raw = Raw { state, principal }; - let response = match (&method, &target) { - (&Method::GET | &Method::HEAD, Target::File(path)) => { - raw.get(&repo, path, req.headers(), method == Method::HEAD) - .await - } - (&Method::GET | &Method::HEAD, Target::Directory(dir)) => { - let query = match ListQuery::parse(uri.query()) { - Ok(query) => query, - Err(e) => return e.into_response(), - }; - match raw.list(&repo, dir, &query).await { - Ok(Some(page)) => listing::respond( - &page, - &query, - listing::wants_json(req.headers(), &query), - method == Method::HEAD, - ), - Ok(None) => RawError::NotFound.into_response(), - Err(e) => e.into_response(), - } - } - (&Method::PUT, Target::File(path)) => { - let path = path.clone(); - upload::put(&raw, &repo, &path, req).await - } - (&Method::PUT, Target::Directory(_)) => ( - StatusCode::BAD_REQUEST, - "PUT uploads a file: name it in the path", - ) - .into_response(), - (&Method::POST, Target::Directory(dir)) => { - let dir = dir.clone(); - upload::multipart(&raw, &repo, &dir, req, upload::Via::Protocol).await - } - (&Method::POST, Target::File(_)) => ( - StatusCode::BAD_REQUEST, - "a form upload goes to a directory: end the path with /", - ) - .into_response(), - (&Method::DELETE, Target::File(path)) => { - upload::delete(&raw, &repo, path, upload::Via::Protocol).await - } - (&Method::DELETE, Target::Directory(_)) => ( - StatusCode::BAD_REQUEST, - "directories are deleted through the admin API (DELETE /admin/repositories//files?path=/)", - ) - .into_response(), - _ => ( - StatusCode::METHOD_NOT_ALLOWED, - [(header::ALLOW, "GET, HEAD, PUT, POST, DELETE")], - "method not allowed", - ) - .into_response(), - }; - if action != Action::Read { - // The same audit trail as the package feeds' mutations. - tracing::info!( - target: "forge::audit", - %method, - repo = %repo.name, - path = %rest, - status = response.status().as_u16(), - principal = %principal.audit_label(), - "package mutation" - ); - } - response -} - -impl Raw<'_> { - /// `GET`/`HEAD` of `path` in `repo`. - async fn get( - &self, - repo: &Repository, - path: &str, - headers: &HeaderMap, - head: bool, - ) -> Response { - match self.locate(repo, path, Vec::new(), 0).await { - Ok(Some(Found::File(file))) => { - serve::file(self.state, repo, &file, headers, head).await - } - Ok(Some(Found::Sidecar(base, checksum))) => { - serve::sidecar(repo, path, &base, checksum, headers, head) - } - Ok(None) => { - // A directory named without its `/`. - let dir = format!("{path}/"); - match self.list(repo, &dir, &ListQuery::default()).await { - Ok(Some(_)) => { - let location = format!("{MOUNT}/{}/{}", repo.name, path::encode(&dir)); - let mut response = StatusCode::MOVED_PERMANENTLY.into_response(); - if let Ok(value) = HeaderValue::from_str(&location) { - response.headers_mut().insert(header::LOCATION, value); - } - response - } - Ok(None) => RawError::NotFound.into_response(), - Err(e) => e.into_response(), - } - } - Err(e) => e.into_response(), - } - } - - /// Find `path` in `repo` for the caller, through every one of `filters` - /// (a group's, for its members) and `repo`'s own package filter: the - /// stored file, else the checksum sidecar of a stored file, else (a - /// proxy) the upstream's file, else (a group) the first member's that - /// has one. - pub fn locate<'a>( - &'a self, - repo: &'a Repository, - path: &'a str, - filters: Vec, - depth: u32, - ) -> futures_util::future::BoxFuture<'a, Result, RawError>> { - Box::pin(async move { - let mut filters = filters; - filters.push(self.state.authz.package_filter(&self.principal, repo).await); - let visible = |p: &str| filters.iter().all(|f| f.allows(p)); - let sidecar = serve::sidecar_of(path).filter(|(base, _)| visible(base)); - match repo.kind() { - RepoKind::Hosted | RepoKind::Proxy => { - let proxy = repo.kind() == RepoKind::Proxy; - if visible(path) { - if let Some(file) = raw_dao::find(&self.state.pool, repo.id, path).await? { - let file = if proxy { - self.fresh(repo, file).await - } else { - file - }; - return Ok(Some(Found::File(file))); - } - } - if let Some((base, checksum)) = sidecar { - if let Some(file) = raw_dao::find(&self.state.pool, repo.id, base).await? { - let file = if proxy { - self.fresh(repo, file).await - } else { - file - }; - return Ok(Some(Found::Sidecar(file, checksum))); - } - } - if proxy && visible(path) { - return Ok(self.fetch(repo, path).await?.map(Found::File)); - } - Ok(None) - } - RepoKind::Group => { - if !visible(path) && sidecar.is_none() { - return Ok(None); - } - let mut failure = None; - for member in self.serving_members(repo, depth).await? { - match self.locate(&member, path, filters.clone(), depth + 1).await { - Ok(Some(found)) => return Ok(Some(found)), - Ok(None) => {} - Err(e) => { - tracing::warn!(group = %repo.name, member = %member.name, error = %e.message(), "raw group: member failed"); - failure = Some(e); - } - } - } - failure.map_or(Ok(None), Err) - } - } - }) - } - - /// The members of `group` that may serve the caller, highest priority - /// first: raw repositories the caller may read itself (reading a group - /// grants nothing on its members), without offline hosted or group - /// members, and without nested groups past the nesting limit or on a - /// cycle. - pub async fn serving_members( - &self, - group: &Repository, - depth: u32, - ) -> Result, RawError> { - let max_depth = self.state.config.groups.max_nesting_depth.max(1) as u32; - if depth >= max_depth { - return Ok(Vec::new()); - } - let mut seen = HashSet::from([group.id]); - let mut out = Vec::new(); - for (member, _priority) in repository_dao::group_members(&self.state.pool, group.id).await? - { - if !member.is_raw() || !seen.insert(member.id) || offline_serves_nothing(&member) { - continue; - } - if member.kind() == RepoKind::Group && depth + 1 >= max_depth { - continue; - } - if self.state.authz.can_read(&self.principal, &member).await { - out.push(member); - } - } - Ok(out) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn paths_split_into_repository_and_rest() { - assert_eq!(split("/raw/bin"), Some(("bin", None))); - assert_eq!(split("/raw/bin/"), Some(("bin", Some("")))); - assert_eq!(split("/raw/bin/a/b.txt"), Some(("bin", Some("a/b.txt")))); - assert_eq!(split("/raw/"), None); - assert_eq!(split("/raw"), None); - assert_eq!(split("/rawx/bin"), None); - } -} diff --git a/crates/server/src/state.rs b/crates/server/src/state.rs index 8f7ed1fa..aae89cd7 100644 --- a/crates/server/src/state.rs +++ b/crates/server/src/state.rs @@ -33,7 +33,7 @@ use forge_core::repo::{ }; use forge_core::retention::FormatResolver; use forge_core::session::SessionService; -use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT, RAW_FORMAT}; +use forge_db::model::{RepoKind, Repository, DOCKER_FORMAT}; use forge_storage::{BlobRef, BlobStore, Cas, S3Error}; use sqlx::PgPool; @@ -139,15 +139,16 @@ impl Instance { } /// Format registry: maps a format id (the `repository.format` column) to its -/// [`Format`] and its plugin: the built-in format plugin -/// ([`crate::plugins::builtin`]: NuGet), Docker and raw. +/// [`Format`] and its plugin: the built-in format plugins +/// ([`crate::plugins::builtin`]: NuGet, raw) and Docker. /// -/// Docker ([`DOCKER_FORMAT`]) and raw ([`RAW_FORMAT`]) are installed but have -/// no [`Format`]: their repositories hold images (manifests and blobs) or -/// files under paths, not package versions, and are served by the registry -/// under `/v2` ([`crate::registry`]) and under `/raw` ([`crate::raw`]), so -/// none of the package seams (parsing, version order, the read ops) apply to -/// them. +/// Docker ([`DOCKER_FORMAT`]) is installed but has no [`Format`]: its +/// repositories hold images (manifests and blobs), not package versions, +/// and are served by the registry under `/v2` ([`crate::registry`]), so none +/// of the package seams (parsing, version order, the read ops) apply to +/// them. A files format (raw) has a [`Format`] whose +/// [`Format::stores_files`] says so: its repositories hold files in the +/// file store ([`crate::files`]). pub struct FormatRegistry { formats: BTreeMap>, /// Every format's plugin, by format id: its client protocol. @@ -212,16 +213,22 @@ impl FormatRegistry { } /// Whether format `id` is one served natively, without a [`Format`]: - /// Docker and raw. + /// Docker. pub fn is_native(id: &str) -> bool { - id == DOCKER_FORMAT || id == RAW_FORMAT + id == DOCKER_FORMAT } - /// Whether format `id` is installed: Docker, raw, or a format plugin. + /// Whether format `id` is installed: Docker, or a format plugin. pub fn is_installed(&self, id: &str) -> bool { Self::is_native(id) || self.formats.contains_key(id) } + /// Whether format `id`'s repositories hold files under paths (raw): it + /// is installed and its `plugin.toml` says `storage = "files"`. + pub fn stores_files(&self, id: &str) -> bool { + self.formats.get(id).is_some_and(|f| f.stores_files()) + } + /// The mounts plugin formats are served under, sorted and distinct /// (`/repository`, and any other a plugin names). pub fn mounts(&self) -> Vec<&str> { @@ -239,14 +246,13 @@ impl FormatRegistry { pub fn ids(&self) -> impl Iterator { let mut ids: Vec<&str> = self.formats.keys().map(String::as_str).collect(); ids.push(DOCKER_FORMAT); - ids.push(RAW_FORMAT); ids.sort_unstable(); ids.into_iter() } /// The repository kinds format `id` supports, `None` when it is not - /// installed. Docker and raw have every kind; a format plugin serves the - /// kinds its `plugin.toml` names (NuGet's all three). + /// installed. Docker has every kind; a format plugin serves the kinds its + /// `plugin.toml` names (NuGet's and raw's all three). pub fn kinds(&self, id: &str) -> Option> { if Self::is_native(id) { return Some(vec!["hosted", "proxy", "group"]); @@ -698,10 +704,13 @@ mod tests { registry.ids().collect::>(), ["docker", "nuget", "raw"] ); - assert!(registry.is_installed(RAW_FORMAT)); - assert!(registry.get(RAW_FORMAT).is_none()); + // Raw is a built-in plugin whose repositories hold files. + assert!(registry.get("raw").is_some()); + assert!(registry.stores_files("raw")); + assert!(!registry.stores_files("nuget")); + assert!(!registry.stores_files(DOCKER_FORMAT)); assert_eq!( - registry.kinds(RAW_FORMAT), + registry.kinds("raw"), Some(vec!["hosted", "proxy", "group"]) ); assert_eq!( @@ -718,8 +727,8 @@ mod tests { } #[test] - fn a_plugin_cannot_take_dockers_or_raws_id() { - for id in ["docker", "raw"] { + fn a_plugin_cannot_take_dockers_or_a_builtin_plugins_id() { + for id in ["docker", "nuget", "raw"] { let registry = FormatRegistry::new(Default::default()); let err = registry .with_format(Arc::new(Named(id))) diff --git a/crates/server/tests/plugin_contract.rs b/crates/server/tests/plugin_contract.rs index 25c25e06..9fff98fa 100644 --- a/crates/server/tests/plugin_contract.rs +++ b/crates/server/tests/plugin_contract.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use common::{start_with, unique, ServerOpts, TestServer}; use forge_plugin::api::{ Action, Answer, ChangeRequest, FormatError, FormatPlugin, Loaded, PackageInfo, PackageRef, - PackageSource, PackageVersions, ParsedPackage, PublishRequest, Query, Rendered, Repository, - Request, Response, SearchQuery, Upload, + PackageSource, PackageVersions, ParsedPackage, PublishRequest, Query, Repository, Request, + Response, SearchQuery, Upload, }; use forge_plugin::Plugin; use reqwest::StatusCode; @@ -25,7 +25,7 @@ const PROBE_TOML: &str = r#" id = "probe" name = "Probe" version = "0.0.1" -api = "0.3.0" +api = "0.4.0" mount = "/probe" kinds = ["hosted"] @@ -164,7 +164,7 @@ impl FormatPlugin for Probe { } } - fn render(&self, _repo: &Repository, _req: &Request, data: &[Loaded]) -> Rendered { + fn render(&self, _repo: &Repository, _req: &Request, data: &[Loaded]) -> Action { let answers: Vec = data .iter() .map(|loaded| match &loaded.answer { @@ -184,7 +184,7 @@ impl FormatPlugin for Probe { other => json!(format!("{other:?}")), }) .collect(); - Rendered::Respond(Response::new( + Action::Respond(Response::new( 200, "application/json", Value::Array(answers).to_string(), diff --git a/docs/changes/2.0.0/431.feature.md b/docs/changes/2.0.0/431.feature.md new file mode 100644 index 00000000..f96ec9fe --- /dev/null +++ b/docs/changes/2.0.0/431.feature.md @@ -0,0 +1,11 @@ +Raw file repositories are a format plugin, `plugins/raw`, like NuGet: the +plugin speaks the `/raw` protocol (paths, checksum sidecars, listings, +Nexus' form fields, a proxy's upstream URLs) and the server's new file store +keeps the files. `/raw` answers as before. The plugin contract, +`forge:format@0.4.0`, gains a second storage model for this: a plugin whose +`plugin.toml` says `storage = "files"` asks the host for files and +directories, stores and deletes files, and serves a file with `serve-file`; +`render` may now answer with any action `handle` can. `GET /admin/formats` +and the repositories API report what a repository holds (`storage`: +`packages`, `files` or `images`). See +[docs/format-plugins.md](docs/format-plugins.md#files-formats). diff --git a/docs/format-plugins.md b/docs/format-plugins.md index 95d13212..2ffe6166 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -11,17 +11,22 @@ streaming of package bytes. Every plugin is built in: a crate under `plugins//` linked into the server and called directly, like any other Rust code. Plugins need no -configuration. Forge ships one: **NuGet** ([`plugins/nuget`](../plugins/nuget)). -Helm charts are served by the registry ([docker.md](docker.md#helm-charts)). +configuration. Forge ships two: **NuGet** ([`plugins/nuget`](../plugins/nuget)) +and **raw files** ([`plugins/raw`](../plugins/raw), +[Files formats](#files-formats)). Helm charts are served by the registry +([docker.md](docker.md#helm-charts)). -> **Status: preview.** The plugin contract is `forge:format@0.3.0`. It may +> **Status: preview.** The plugin contract is `forge:format@0.4.0`. It may > change before 1.x of the contract; a plugin declares the contract version it -> was built for, and the server refuses one it does not implement. +> was built for, and the server refuses one it does not implement. 0.4.0 added +> files formats ([Files formats](#files-formats)), and `render` returns an +> action, like `handle`. ## Contents - [Layout](#layout) - [How it fits together](#how-it-fits-together) +- [Files formats](#files-formats) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) - [Security model](#security-model) @@ -40,6 +45,15 @@ plugins/nuget/ ├── src/protocol.rs # the V3 protocol: handle and render ├── src/upstream.rs # proxies: reading another V3 feed (upstream) └── tests/ # the protocol against canned answers (forge-plugin-api `testing`) + +plugins/raw/ +├── Cargo.toml +├── plugin.toml # mount "/raw", storage "files", kinds hosted/proxy/group +├── src/lib.rs # impl FormatPlugin for Raw: paths, methods, upstream URLs +├── src/listing.rs # directory listings as HTML or JSON +├── src/sidecar.rs # .sha256/.sha512/.sha1/.md5 from the stored digests +├── src/form.rs # multipart uploads, Nexus' field names +└── tests/ # the protocol against canned answers ``` The pieces: @@ -49,7 +63,7 @@ The pieces: | [`forge-plugin-api`](../crates/plugin-api) | The contract: the `FormatPlugin` trait and its types, and, behind its `testing` feature, helpers that run a plugin's protocol against canned answers. Depends on nothing but `std`. | | [`plugins/`](../plugins) | One crate per plugin, implementing `FormatPlugin`. | | [`forge-plugin`](../crates/plugin) | The host side: `Plugin` (an installed plugin: its `plugin.toml` and its implementation) and `PluginFormat`, which adapts it to Forge's `Format` seam. | -| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`) and the `{mount}/{repo}/...` dispatcher. | +| [`forge-server`](../crates/server/src/plugins.rs) | The plugins it links in (`plugins::builtin`), the `{mount}/{repo}/...` dispatcher, and the file store files formats keep their files in ([`crates/server/src/files`](../crates/server/src/files)). | ## How it fits together @@ -63,6 +77,7 @@ The pieces: | Proxies | Sends every upstream request (credentials only to the upstream's origin, bounded, cached), downloads and verifies packages, keeps the package delay, the not-found markers and the offline mode | `upstream`: which documents an operation needs, and what they say ([Proxy repositories](#proxy-repositories)) | | Groups | Merges members by priority; the dependency-confusion guards apply | Nothing: a group of plugin repositories just works | | Retention | Evaluates policies, ordering versions with the plugin's `compare_versions` | Nothing more | +| Files (a files format) | Keeps files under paths, content-addressed with their digests: serving, ranges and validators, uploads under the deployment policy, listings, proxies and groups, selectors on paths, retention by path ([Files formats](#files-formats)) | Which path a request names, which file or directory it needs, what the response looks like | A request to a plugin repository is served under its format's mount, `{mount}/{repo}/...`: `/repository` unless the plugin's `plugin.toml` names @@ -82,9 +97,11 @@ in this order: 3. Act on the plugin's answer: - `respond`: send the response as given, keeping only the allowed headers. - `load(queries)`: run the queries, then call `render` with every query - and its answer. `render` either responds or asks for more data. One - request gets at most 4 rounds and 2,048 answers. + and its answer. `render` answers like `handle` does: a response, more + data to load, or one of the actions below. One request gets at most 4 + rounds and 2,048 answers. - `download`: stream the stored package. + - `serve-file`: stream a stored file (a files format). - `publish`, `unlist`, `relist`, `delete`: go through the hosted write path, but only when the request's method was authorized for that action (a plugin cannot turn a GET into a delete). The repository must be @@ -103,6 +120,9 @@ Its queries are targeted: | `manifest(id, version)` | The manifest stored at publish | | `search(text, skip, take, prerelease, semver2, upstream)` | Matching packages with their versions, the latest version's summary and the download total | +A files format asks for files instead ([Files formats](#files-formats)); a +query of the other kind fails the request. + Prerelease filtering is the host's, and its rule is that a normalized version is a prerelease when it contains a `-`; `semver2` adds SemVer 2.0.0 versions (dotted prerelease labels, build metadata). Publish times are RFC 3339 in UTC @@ -120,6 +140,57 @@ NuGet's). Every listing hands each version's summary to the plugin, so a summary should stay small. +## Files formats + +A format whose `plugin.toml` says `storage = "files"` holds files under +paths, not package versions: [raw](raw.md) is one. It keeps the defaults of +the identity rules and `parse_package` (which refuse every id, version and +package) and implements the protocol over the host's file store, which keeps +everything a file repository is: the files, content-addressed, with their +SHA-256, SHA-512, SHA-1 and MD5; uploads, under the repository's upload +limit and deployment policy (`allow_redeploy`, `disable_redeploy`, +`read_only`); serving with one `Range`, `If-Range`, `ETag`, `Last-Modified` +and conditional requests, under the repository's content disposition; +listings that walk an index; proxies that fetch on first use, follow +redirects and re-validate after `max_age_seconds`; groups; package selectors +on paths; retention by path prefix, age, downloads and the newest N per +directory; and the web UI's upload, delete and the admin API's directory +delete. + +The plugin decides what a request is, in the vocabulary of paths. A path is +segments joined by `/`, without a leading or trailing `/`, none of them +empty, `.` or `..`, nor holding a backslash or a control character +([`forge_plugin_api::files`](../crates/plugin-api/src/files.rs) parses +request paths by these rules); a directory is empty (the root) or a path +ending in `/`. The host refuses any other path a plugin names. + +| Query | Answer | +| --- | --- | +| `file(path)` | The stored file with its size, content type, digests, modification and last download time, or none. A proxy re-validates its copy once it is due; a group answers with its first member, by priority, that has the path | +| `fetch-file(path)` | `file`, and a proxy that has not cached the path asks its upstream (`upstream`'s `file` operation) | +| `directory(directory, after, limit)` | A page of a directory: files and subdirectories in byte order of their names, and the `after` of the next page; none for a directory that does not exist. A group merges its members' entries | +| `form` | Receives the request's `multipart/form-data` body: file parts are staged, other fields read | +| `store(files)` | Stores files, each from the request body or a form part, all or none, under the deployment policy; what became of each (`created`, `replaced`, `unchanged`) | +| `remove(path)` | Deletes a file; whether there was one | + +`form`, `store` and `remove` change the repository: each comes alone in its +round, needs the method's permission (write, or delete for `remove`), a +hosted repository and a package-client credential, and when one fails (403, +405, 409, 413) the host answers with its own status. The action +`serve-file(path)` serves a file as `fetch-file` finds it. The `repository` +a request is for says whether its files are shown inline (`inline`), for +responses the plugin renders itself. + +Raw maps its URLs onto these: `GET /raw//` is `file(path)` +(and `file` of the file a checksum sidecar names), then `serve-file` or +the sidecar rendered from the file's digests, else `fetch-file` in a proxy +or group, else a `directory` probe for a redirect; `GET …//` is a +`directory` page rendered as HTML or JSON; `PUT` is a `store` of the body; +`POST …//` is a `form`, then a `store` of its parts with Nexus' field +names applied; `DELETE` is a `remove`. [plugins/raw](../plugins/raw) is the +whole protocol, in a few hundred lines; the file store is +[`crates/server/src/files`](../crates/server/src/files). + ## Proxy repositories A plugin that lists `proxy` among its kinds implements the upstream side of @@ -140,6 +211,7 @@ what they say: | `manifest(id, version)` | The manifest, verified to be that version's, without downloading the package | | `metadata(versions)`, `dating(versions)` | Per version: listing state, publish time, a manifest summary for a version not downloaded yet, and (`dating`) when it was created upstream, which the package delay counts from | | `search(query)` | A page of the upstream's search results | +| `file(path)` | A files format: the file's URL, or none when the upstream cannot have it. The host downloads it (following redirects; credentials go to `remote_url`'s origin only), caches it, and re-validates it with a conditional GET once the repository's `max_age_seconds` has passed (raw: `/`) | Each operation is a conversation: `upstream` either answers or asks for GETs (`fetch`), and is called again with every fetch so far and how it went @@ -160,8 +232,8 @@ size bound. The host sends a fetch only where the upstream points: to the origin (scheme, host and port) of `remote_url`, or to an origin that an absolute `http` or `https` URL names in a document the host fetched for the same operation (in an -earlier round), or while resolving `index`. The URL `package` answers is held -to the same rule. Anything else is not sent: the fetch comes back as a +earlier round), or while resolving `index`. The URLs `package` and `file` +answer are held to the same rule. Anything else is not sent: the fetch comes back as a failure, a package URL fails the download like an upstream error, and the host logs a warning naming the format and the repository. What the plugin answers names no origin: its index is its own making (and the host keeps fields of its @@ -183,11 +255,12 @@ cached. ## Using a plugin `GET /admin/formats` lists every format the server can create repositories -of: the plugins (NuGet), Docker and raw files, each with the repository -kinds it supports and where its repositories are served. The web UI's *New -repository* form offers them. Plugins serve **hosted** and +of: the plugins (NuGet, raw files) and Docker, each with the repository +kinds it supports, what its repositories hold (`storage`: `packages`, +`files`, or Docker's `images`) and where its repositories are served. The +web UI's *New repository* form offers them. Plugins serve **hosted** and **group** repositories, and **proxies** when they implement the upstream -side (NuGet does). Docker has proxies of its own. +side (NuGet and raw do). Docker has proxies of its own. A repository of a plugin's format is created like any other, with the plugin's format id, and served under the plugin's mount (`/repository` @@ -250,10 +323,10 @@ The host enforces these for every plugin: plugin cannot have Forge read `http://localhost:8421/admin/…` or an internal search cluster that its upstream does not point at, and pass the answers on through the proxy's metadata. -- **Responses cannot attack the origin.** Only `cache-control`, +- **Responses cannot attack the origin.** Only `allow`, `cache-control`, `content-disposition`, `content-language`, `content-type`, `etag`, - `last-modified` and `location` survive; `location` survives only when it - stays inside the repository. Every response gets the feed's hardening + `last-modified`, `link`, `location` and `vary` survive; `location` and + `link` survive only when they stay inside the repository. Every response gets the feed's hardening headers (`Content-Security-Policy: sandbox`, `nosniff`, `Content-Disposition: attachment`). A plugin cannot set cookies, CORS headers, or redirect a client elsewhere. @@ -277,7 +350,7 @@ workspace, depending on `forge-plugin-api` and implementing its trait ([`plugins/nuget`](../plugins/nuget) is a complete one): ```rust -use forge_plugin_api::{Action, FormatPlugin, Loaded, Rendered, Repository, Request /* … */}; +use forge_plugin_api::{Action, FormatPlugin, Loaded, Repository, Request /* … */}; /// This crate's plugin.toml (id, name, version). pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); @@ -288,6 +361,7 @@ pub struct MyFormat; impl FormatPlugin for MyFormat { // identity: canonical_id, validate_id, normalize_version, compare_versions // packages: parse_package, manifest_summary + // (a files format keeps the defaults of these, which refuse everything) // protocol: handle, render // proxies (optional): upstream } @@ -299,10 +373,11 @@ impl FormatPlugin for MyFormat { id = "example" # the format id repositories are created with name = "Example packages" version = "0.1.0" -api = "0.3.0" # the forge:format contract +api = "0.4.0" # the forge:format contract mount = "/repository" # optional: served at {mount}/{repo}/… kinds = ["hosted", "group"] # optional: the repository kinds it serves # ("proxy" too when it implements `upstream`) +storage = "packages" # optional: or "files" (Files formats) [assets] # optional: how a version's files are stored package = "package" @@ -327,7 +402,8 @@ rules hold: the staged package. Read only what you need, and bound everything you decompress: a gzip bomb should fail fast. - **Ask for data, don't assume it.** `handle` returns `Action::Load(queries)`; - `render` gets each query with its answer and may load more. Build absolute + `render` gets each query with its answer and may load more, respond, or + hand the request to the host (a download, a publish, serving a file). Build absolute URLs from the repository's `base_url`. Groups and kinds are already resolved by the time data reaches you. - **Test the protocol without a server.** `forge-plugin-api`'s `testing` @@ -408,9 +484,11 @@ same way, a group holds members of its format only, and repositories, RBAC, package selectors (on image names), retention and the blob store are the same as for every other format. -Raw file repositories ([raw.md](raw.md)) are native for similar reasons: -they hold files under paths, not package versions, list directories, take -several files in one form, always answer ranges and conditional requests, -and their proxies follow redirects and re-validate with `If-None-Match`. They -live in the server (`crates/server/src/raw`, served under `/raw`) and are -registered like Docker, with selectors on paths. +Raw file repositories ([raw.md](raw.md)) were native for similar reasons +until contract 0.4.0: they hold files under paths, not package versions, +list directories, take several files in one form, always answer ranges and +conditional requests, and their proxies follow redirects and re-validate with +`If-None-Match`. 0.4.0 gave the contract a second storage model, files +([Files formats](#files-formats)), so raw is a plugin now +([`plugins/raw`](../plugins/raw)) like NuGet, and what was the raw server +module is the host's file store. diff --git a/docs/raw.md b/docs/raw.md index fba034d4..e9fc880b 100644 --- a/docs/raw.md +++ b/docs/raw.md @@ -6,9 +6,12 @@ have no package manager of their own (the [pdfium binaries](https://github.com/bblanchon/pdfium-binaries), for example), installers, firmware images, test fixtures, tool downloads. -Files live in repositories of `format: "raw"`, a format built into Forge -like Docker (`GET /admin/formats` lists it), served at -`https:///raw//`. They come in the usual three kinds: +Files live in repositories of `format: "raw"`, a format plugin built into +Forge like NuGet ([`plugins/raw`](../plugins/raw); `GET /admin/formats` +lists it), served at `https:///raw//`. The plugin +speaks the protocol below; the host's file store keeps the files +([format-plugins.md](format-plugins.md#files-formats)). They come in the +usual three kinds: - **hosted**: upload, download and delete your own files; - **proxy**: a cache of files under an HTTP(S) base URL, such as a GitHub diff --git a/plugins/README.md b/plugins/README.md index 482620bf..d0b2a277 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -8,6 +8,7 @@ the design, the contract and how to write one. | Plugin | Format id | What it serves | | --- | --- | --- | | [`nuget`](nuget) | `nuget` | NuGet V3 feeds at `/v3`: push, restore, search, unlist and relist; hosted, proxy and group repositories | +| [`raw`](raw) | `raw` | Raw files at `/raw`: files under paths, uploads with `PUT` or a form, ranges, checksum sidecars, listings; hosted, proxy and group repositories ([docs/raw.md](../docs/raw.md)) | A plugin directory holds: @@ -15,7 +16,8 @@ A plugin directory holds: - `plugin.toml`: the format id (also the directory's name), display name, version, the `forge:format` contract version it targets (`api`), and optionally its mount, repository kinds (`proxy` when it implements - `upstream`) and `[assets]` names; + `upstream`), its `storage` (`packages`, or `files` for a files format such + as raw) and `[assets]` names; - `src/`: `impl FormatPlugin`, and `pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!();`. diff --git a/plugins/nuget/plugin.toml b/plugins/nuget/plugin.toml index f4165450..99cb0a5d 100644 --- a/plugins/nuget/plugin.toml +++ b/plugins/nuget/plugin.toml @@ -3,7 +3,7 @@ id = "nuget" name = "NuGet" version = "1.0.0" -api = "0.3.0" +api = "0.4.0" description = "NuGet V3 feeds: push, restore and search with dotnet, nuget.exe and Visual Studio; proxies of nuget.org and other V3 feeds." mount = "/v3" # Proxies mirror another V3 feed through the plugin's upstream side. diff --git a/plugins/nuget/src/lib.rs b/plugins/nuget/src/lib.rs index 0b6c3a1d..7c618432 100644 --- a/plugins/nuget/src/lib.rs +++ b/plugins/nuget/src/lib.rs @@ -29,7 +29,7 @@ use std::cmp::Ordering; use forge_plugin_api::{ Action, FetchResult, FormatError, FormatPlugin, Loaded, PackageSource, ParsedPackage, Remote, - Rendered, Repository, Request, UpstreamOp, UpstreamStep, + Repository, Request, UpstreamOp, UpstreamStep, }; pub use model::NupkgLimits; @@ -82,7 +82,7 @@ impl FormatPlugin for Nuget { protocol::handle(repo, req) } - fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { protocol::render(repo, req, data) } diff --git a/plugins/nuget/src/protocol.rs b/plugins/nuget/src/protocol.rs index 4869c64f..dd821b19 100644 --- a/plugins/nuget/src/protocol.rs +++ b/plugins/nuget/src/protocol.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use forge_plugin_api::{ Action, Answer, ChangeRequest, Loaded, PackageInfo, PackageRef, PackageVersions, - PublishRequest, Query, Rendered, Repository, RepositoryKind, Request, Response, SearchQuery, + PublishRequest, Query, Repository, RepositoryKind, Request, Response, SearchQuery, SearchResult, Upload, VersionInfo, }; use serde_json::{json, Value}; @@ -263,9 +263,9 @@ pub(crate) fn handle(repo: &Repository, req: &Request) -> Action { } } -pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rendered { - let respond = Rendered::Respond; - let not_found = |text: &str| Rendered::Respond(Response::text(404, text)); +pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + let respond = Action::Respond; + let not_found = |text: &str| Action::Respond(Response::text(404, text)); let Ok(route) = route(req, repo.kind) else { return not_found("not found"); }; @@ -319,7 +319,7 @@ pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rende base, &lower, &versions, ))) } else { - Rendered::Load(vec![Query::Packages(lower)]) + Action::Load(vec![Query::Packages(lower)]) } } Some([]) => not_found("registration not found"), @@ -340,12 +340,10 @@ pub(crate) fn render(repo: &Repository, req: &Request, data: &[Loaded]) -> Rende let versions = version_strings(versions().unwrap_or_default()); match registration_page_versions(&versions, &lower, upper) { // Only the page's versions are loaded. - Some(in_range) => { - Rendered::Load(vec![Query::PackagesOf(PackageVersions { - id: lower_id, - versions: in_range.to_vec(), - })]) - } + Some(in_range) => Action::Load(vec![Query::PackagesOf(PackageVersions { + id: lower_id, + versions: in_range.to_vec(), + })]), None => not_found("registration page not found"), } } diff --git a/plugins/nuget/src/upstream.rs b/plugins/nuget/src/upstream.rs index f09df9c9..696c6e66 100644 --- a/plugins/nuget/src/upstream.rs +++ b/plugins/nuget/src/upstream.rs @@ -137,6 +137,8 @@ pub(crate) fn step( UpstreamOp::Metadata(wanted) => metadata(&resources, wanted, false, fetched), UpstreamOp::Dating(wanted) => metadata(&resources, wanted, true, fetched), UpstreamOp::Search(query) => search(&resources, query, fetched), + // A feed holds packages: it has no files to fetch. + UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), }) } diff --git a/plugins/nuget/tests/protocol.rs b/plugins/nuget/tests/protocol.rs index 39374941..1fca7409 100644 --- a/plugins/nuget/tests/protocol.rs +++ b/plugins/nuget/tests/protocol.rs @@ -23,6 +23,7 @@ fn repo(kind: RepositoryKind) -> Repository { name: "main".into(), kind, base_url: format!("{BASE}/v3/main"), + inline: false, } } @@ -100,6 +101,7 @@ impl MockRepo { .collect(), }) } + other => panic!("NuGet asked for files: {other:?}"), } } diff --git a/plugins/raw/Cargo.toml b/plugins/raw/Cargo.toml new file mode 100644 index 00000000..a8935ec1 --- /dev/null +++ b/plugins/raw/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "forge-format-raw" +# The plugin's own version: keep it equal to `version` in plugin.toml. +version = "1.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Raw file repositories for Forge, as a format plugin" + +[dependencies] +forge-plugin-api = { path = "../../crates/plugin-api" } +# The JSON listing. +serde = { workspace = true } +serde_json = { workspace = true } +# Last-Modified and If-Modified-Since of the checksum sidecars. +httpdate = "1" +# Upstream URLs of a proxy's files, listing queries and links. +url = { workspace = true } +percent-encoding = "2.3" +base64 = { workspace = true } + +[dev-dependencies] +forge-plugin-api = { path = "../../crates/plugin-api", features = ["testing"] } diff --git a/plugins/raw/plugin.toml b/plugins/raw/plugin.toml new file mode 100644 index 00000000..be343f08 --- /dev/null +++ b/plugins/raw/plugin.toml @@ -0,0 +1,13 @@ +# A Forge format plugin: raw file repositories (see plugins/README.md). +[plugin] +id = "raw" +name = "Raw files" +version = "1.0.0" +api = "0.4.0" +description = "Arbitrary files under arbitrary paths, like Nexus' raw repositories: uploads with PUT or a form, downloads with ranges and checksum sidecars, directory listings; proxies of any HTTP(S) base URL." +mount = "/raw" +# Proxies cache the files under an HTTP(S) base URL through the plugin's +# upstream side. +kinds = ["hosted", "proxy", "group"] +# Its repositories hold files under paths, in the host's file store. +storage = "files" diff --git a/plugins/raw/src/form.rs b/plugins/raw/src/form.rs new file mode 100644 index 00000000..85b75586 --- /dev/null +++ b/plugins/raw/src/form.rs @@ -0,0 +1,189 @@ +//! Uploads of several files: `POST /raw///` with a +//! `multipart/form-data` body, whose file parts are stored under +//! `directory`, each at its part's file name. Nexus' field names for its +//! component upload work too: `raw.directory` (or `directory`) names a +//! subdirectory, and `.filename` (`raw.asset1.filename`) the name of +//! the file part ``. The host receives the form +//! ([`Query::Form`]) and stores every file, or none ([`Query::Store`]). + +use std::collections::{BTreeMap, HashSet}; + +use forge_plugin_api::files; +use forge_plugin_api::{ + Action, Answer, FileSource, FormContent, FormPart, Loaded, NewFile, Query, Response, + StoreStatus, StoredFile, +}; + +/// The next step of a form upload into `directory`: store the form's files +/// once it is received, answer once they are stored. +pub fn render(directory: &str, data: &[Loaded]) -> Action { + match data.last().map(|loaded| &loaded.answer) { + Some(Answer::Form(parts)) => match place(directory, parts) { + Ok(files) => Action::Load(vec![Query::Store(files)]), + Err(message) => Action::Respond(Response::text(400, &message)), + }, + Some(Answer::Stored(files)) => Action::Respond(stored(files)), + _ => Action::Respond(Response::text(500, "raw: the form was not received")), + } +} + +/// Where each file of a form goes, or why the form is refused. +pub fn place(directory: &str, parts: &[FormPart]) -> Result, String> { + let mut texts: BTreeMap<&str, &str> = BTreeMap::new(); + let mut uploads = Vec::new(); + for (index, part) in parts.iter().enumerate() { + match &part.content { + FormContent::File(file) => uploads.push((index, part.field.as_str(), file)), + FormContent::Text(value) => { + texts.insert(part.field.as_str(), value.as_str()); + } + } + } + if uploads.is_empty() { + return Err("the form holds no file".into()); + } + + // The fields: a subdirectory, and the names of file parts. + let mut subdirectory = ""; + let fields: HashSet<&str> = uploads.iter().map(|(_, field, _)| *field).collect(); + for (name, value) in &texts { + match *name { + "directory" | "raw.directory" => subdirectory = value, + other + if other + .strip_suffix(".filename") + .is_some_and(|field| fields.contains(field)) => {} + other => { + return Err(format!( + "unknown form field {other:?} (file parts, directory and .filename are accepted)" + )) + } + } + } + let base = files::normalize_directory(subdirectory).map_err(|e| format!("directory: {e}"))?; + let directory = format!("{directory}{base}"); + + let mut seen = HashSet::new(); + let mut placed = Vec::with_capacity(uploads.len()); + for (index, field, file) in uploads { + let name = texts + .get(format!("{field}.filename").as_str()) + .copied() + .unwrap_or(&file.name); + let target = + files::join(&directory, name).map_err(|e| format!("file name {name:?}: {e}"))?; + if !seen.insert(target.clone()) { + return Err(format!("the form holds {target} twice")); + } + placed.push(NewFile { + path: target, + source: FileSource::Part(u32::try_from(index).unwrap_or(u32::MAX)), + content_type: file.content_type.clone(), + }); + } + Ok(placed) +} + +/// The answer to a stored form: what became of each file, 201 when one was +/// new. +fn stored(files: &[StoredFile]) -> Response { + let created = files.iter().any(|f| f.status == StoreStatus::Created); + let files: Vec = files + .iter() + .map(|f| { + serde_json::json!({ + "path": f.path, + "size": f.size, + "sha256": f.sha256, + "status": match f.status { + StoreStatus::Created => "created", + StoreStatus::Replaced => "replaced", + StoreStatus::Unchanged => "unchanged", + }, + }) + }) + .collect(); + Response::new( + if created { 201 } else { 200 }, + "application/json", + serde_json::json!({ "files": files }).to_string(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::FormFile; + + fn file(field: &str, name: &str) -> FormPart { + FormPart { + field: field.into(), + content: FormContent::File(FormFile { + name: name.into(), + content_type: Some("application/gzip".into()), + size: 3, + }), + } + } + + fn text(field: &str, value: &str) -> FormPart { + FormPart { + field: field.into(), + content: FormContent::Text(value.into()), + } + } + + #[test] + fn files_go_under_the_directory_by_their_names() { + let placed = place("pdfium/", &[file("file", "a.tgz"), file("file", "b.tgz")]).unwrap(); + let paths: Vec<_> = placed.iter().map(|f| f.path.as_str()).collect(); + assert_eq!(paths, ["pdfium/a.tgz", "pdfium/b.tgz"]); + assert_eq!(placed[1].source, FileSource::Part(1)); + assert_eq!(placed[0].content_type.as_deref(), Some("application/gzip")); + } + + #[test] + fn nexus_fields_name_the_directory_and_the_files() { + let parts = [ + text("raw.directory", "7350"), + file("raw.asset1", "upload.bin"), + text("raw.asset1.filename", "pdfium-linux-x64.tgz"), + file("raw.asset2", "pdfium-win-x64.tgz"), + ]; + let placed = place("pdfium/", &parts).unwrap(); + let placed: Vec<_> = placed + .iter() + .map(|f| (f.path.as_str(), &f.source)) + .collect(); + assert_eq!( + placed, + [ + ("pdfium/7350/pdfium-linux-x64.tgz", &FileSource::Part(1)), + ("pdfium/7350/pdfium-win-x64.tgz", &FileSource::Part(3)), + ] + ); + } + + #[test] + fn bad_forms_are_refused() { + assert_eq!( + place("", &[text("directory", "x")]).unwrap_err(), + "the form holds no file" + ); + assert!(place("", &[file("f", "a"), text("other", "x")]) + .unwrap_err() + .starts_with("unknown form field \"other\"")); + // A name for a part that is not there is unknown too. + assert!(place("", &[file("f", "a"), text("g.filename", "x")]).is_err()); + assert!(place("", &[file("f", "a"), text("directory", "../x")]) + .unwrap_err() + .starts_with("directory: ")); + assert!(place("", &[file("f", "../a")]) + .unwrap_err() + .starts_with("file name \"../a\": ")); + assert_eq!( + place("d/", &[file("f", "a"), file("g", "a")]).unwrap_err(), + "the form holds d/a twice" + ); + } +} diff --git a/plugins/raw/src/lib.rs b/plugins/raw/src/lib.rs new file mode 100644 index 00000000..7a2e0fc7 --- /dev/null +++ b/plugins/raw/src/lib.rs @@ -0,0 +1,274 @@ +//! Raw file repositories for Forge: files under paths, like Nexus' +//! raw-hosted, raw-proxy and raw-group repositories. +//! +//! Served under a Forge repository's base URL (`/raw/`): +//! +//! * `GET` / `HEAD /` — a file, which the host streams +//! ([`Action::ServeFile`]: one range, validators, conditional requests); +//! else the checksum sidecar `` of a stored file +//! (`.sha256`, `.sha512`, `.sha1`, `.md5`; [`sidecar`]); else, for a path +//! that is a directory, a redirect to the directory; +//! * `GET` / `HEAD //` — a listing, as HTML or JSON ([`listing`]); +//! * `PUT /` — store the request body as the file; +//! * `POST //` — store the files of a multipart form, Nexus' field names +//! included ([`form`]); +//! * `DELETE /` — delete a file. +//! +//! Paths follow [`forge_plugin_api::files`]: a path that needs normalizing +//! (`..`, `//`, an encoded `/`) is refused, never resolved. Everything else +//! a repository is, the host's file store keeps: the content-addressed +//! files and their digests, the deployment policy, package selectors on +//! paths, groups, and proxies, whose file `` is the upstream's +//! `/` ([`upstream_url`]). +//! +//! Forge links this plugin in ([`Raw`], [`PLUGIN_TOML`]), like NuGet. + +pub mod form; +pub mod listing; +pub mod sidecar; + +use forge_plugin_api::files::{self, Target}; +use forge_plugin_api::{ + Action, Answer, DirectoryQuery, FetchResult, FileSource, FormatError, FormatPlugin, Loaded, + NewFile, Query, Remote, Repository, RepositoryKind, Request, Response, StoreStatus, + UpstreamAnswer, UpstreamOp, UpstreamStep, +}; + +/// This plugin's `plugin.toml`: its format id, name and version. +pub const PLUGIN_TOML: &str = forge_plugin_api::plugin_toml!(); + +/// The raw format. +#[derive(Debug, Clone, Copy, Default)] +pub struct Raw; + +impl FormatPlugin for Raw { + fn handle(&self, repo: &Repository, req: &Request) -> Action { + let target = match target(req) { + Ok(Some(target)) => target, + // `/raw/`: the root listing is `/raw//`. + Ok(None) if is_read(req) => { + return redirect(&format!("{}/", listing::base_path(&repo.base_url))) + } + Ok(None) => return respond(Response::text(404, "not found")), + Err(e) => return respond(Response::text(400, &e.to_string())), + }; + match (req.method.as_str(), target) { + ("GET" | "HEAD", Target::File(path)) => { + let mut queries = vec![Query::File(path.clone())]; + if let Some((base, _)) = sidecar::of(&path) { + queries.push(Query::File(base.to_string())); + } + Action::Load(queries) + } + ("GET" | "HEAD", Target::Directory(directory)) => { + match listing::ListQuery::parse(&req.query) { + Ok(query) => Action::Load(vec![Query::Directory(DirectoryQuery { + directory, + limit: query.limit_for_host(), + after: query.after, + })]), + Err(message) => respond(Response::text(400, &message)), + } + } + ("PUT", Target::File(path)) => Action::Load(vec![Query::Store(vec![NewFile { + path, + source: FileSource::Body, + content_type: req.header("content-type").map(str::to_string), + }])]), + ("PUT", Target::Directory(_)) => respond(Response::text( + 400, + "PUT uploads a file: name it in the path", + )), + ("POST", Target::Directory(_)) => Action::Load(vec![Query::Form]), + ("POST", Target::File(_)) => respond(Response::text( + 400, + "a form upload goes to a directory: end the path with /", + )), + ("DELETE", Target::File(path)) => Action::Load(vec![Query::Remove(path)]), + ("DELETE", Target::Directory(_)) => respond(Response::text( + 400, + "directories are deleted through the admin API (DELETE /admin/repositories//files?path=/)", + )), + _ => respond( + Response::text(405, "method not allowed") + .with_header("allow", "GET, HEAD, PUT, POST, DELETE"), + ), + } + } + + fn render(&self, repo: &Repository, req: &Request, data: &[Loaded]) -> Action { + let Ok(Some(target)) = target(req) else { + return unexpected("a request it did not load data for"); + }; + match (req.method.as_str(), target) { + ("GET" | "HEAD", Target::File(path)) => file(repo, req, &path, data), + ("GET" | "HEAD", Target::Directory(directory)) => { + listing::render(repo, req, &directory, data) + } + ("PUT", Target::File(_)) => match stored(data) { + Some([file]) if file.status == StoreStatus::Created => { + respond(Response::empty(201)) + } + Some([_]) => respond(Response::empty(204)), + _ => unexpected("a PUT without its stored file"), + }, + ("POST", Target::Directory(directory)) => form::render(&directory, data), + ("DELETE", Target::File(_)) => match data.last().map(|l| &l.answer) { + Some(Answer::Removed(true)) => respond(Response::empty(204)), + Some(Answer::Removed(false)) => respond(Response::text(404, "not found")), + _ => unexpected("a DELETE without its outcome"), + }, + _ => unexpected("a request it did not load data for"), + } + } + + fn upstream( + &self, + remote: &Remote, + op: &UpstreamOp, + _fetched: &[FetchResult], + ) -> Result { + Ok(UpstreamStep::Done(match op { + // Nothing to resolve: a file's URL is the remote's and its path. + UpstreamOp::Index => UpstreamAnswer::Index("{}".into()), + UpstreamOp::File(path) => UpstreamAnswer::File(upstream_url(&remote.url, path)), + _ => UpstreamAnswer::Failed("raw repositories hold files, not packages".into()), + })) + } +} + +/// What a request's path names: `None` for the repository's URL without its +/// `/`. +fn target(req: &Request) -> Result, files::PathError> { + match req.path.strip_prefix('/') { + Some(rest) => files::parse_request(rest).map(Some), + None if req.path.is_empty() => Ok(None), + None => files::parse_request(&req.path).map(Some), + } +} + +fn is_read(req: &Request) -> bool { + matches!(req.method.as_str(), "GET" | "HEAD") +} + +fn respond(response: Response) -> Action { + Action::Respond(response) +} + +/// A permanent redirect to `location`, a path within the repository: the +/// client stays on the host it asked. +fn redirect(location: &str) -> Action { + respond(Response::empty(301).with_header("location", location)) +} + +/// A response for a conversation the host should not have started: the +/// plugin and the host disagree, which is a bug. +fn unexpected(what: &str) -> Action { + respond(Response::text( + 500, + &format!("raw: the host rendered {what}"), + )) +} + +/// The answer of the store a request asked for. +fn stored(data: &[Loaded]) -> Option<&[forge_plugin_api::StoredFile]> { + data.iter().rev().find_map(|loaded| match &loaded.answer { + Answer::Stored(files) => Some(files.as_slice()), + _ => None, + }) +} + +/// The answer to `query`, if it was loaded. +fn answer<'a>(data: &'a [Loaded], query: &Query) -> Option<&'a Answer> { + data.iter() + .find(|loaded| loaded.query == *query) + .map(|loaded| &loaded.answer) +} + +/// The file a `file` or `fetch-file` query found: `None` when it was not +/// asked, `Some(None)` when there is no such file. +fn found<'a>(data: &'a [Loaded], query: &Query) -> Option> { + match answer(data, query)? { + Answer::File(file) => Some(file.as_ref()), + _ => None, + } +} + +/// `GET`/`HEAD` of a file path, round by round: the stored file, else the +/// checksum sidecar of a stored file, else (a proxy, or a group that may +/// have one) the upstream's file, else a redirect to the directory of that +/// name, else 404. +fn file(repo: &Repository, req: &Request, path: &str, data: &[Loaded]) -> Action { + if let Some(Some(_)) = found(data, &Query::File(path.to_string())) { + return Action::ServeFile(path.to_string()); + } + if let Some((base, checksum)) = sidecar::of(path) { + if let Some(Some(file)) = found(data, &Query::File(base.to_string())) { + return respond(sidecar::respond(repo, req, path, file, checksum)); + } + } + match found(data, &Query::FetchFile(path.to_string())) { + Some(Some(_)) => return Action::ServeFile(path.to_string()), + None if repo.kind != RepositoryKind::Hosted => { + return Action::Load(vec![Query::FetchFile(path.to_string())]) + } + _ => {} + } + let directory = format!("{path}/"); + if directory.len() > files::MAX_PATH_BYTES { + return respond(Response::text(404, "not found")); + } + let probe = Query::Directory(DirectoryQuery { + directory: directory.clone(), + after: None, + limit: 1, + }); + match answer(data, &probe) { + None => Action::Load(vec![probe]), + Some(Answer::Directory(Some(_))) => redirect(&format!( + "{}/{}", + listing::base_path(&repo.base_url), + files::encode(&directory) + )), + Some(_) => respond(Response::text(404, "not found")), + } +} + +/// The upstream URL of `path`: `remote_url` (as a directory) with the path's +/// segments appended, each percent-encoded. `None` when `remote_url` is not +/// a URL with a path. +pub fn upstream_url(remote_url: &str, path: &str) -> Option { + let mut url = url::Url::parse(remote_url).ok()?; + { + let mut segments = url.path_segments_mut().ok()?; + segments.pop_if_empty(); + for segment in path.split('/') { + segments.push(segment); + } + } + Some(url.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upstream_urls_append_encoded_segments() { + let base = "https://github.com/bblanchon/pdfium-binaries/releases/download/"; + assert_eq!( + upstream_url(base, "chromium/7350/pdfium-linux-x64.tgz").unwrap(), + "https://github.com/bblanchon/pdfium-binaries/releases/download/chromium/7350/pdfium-linux-x64.tgz" + ); + // Without a trailing slash the base is a directory all the same. + assert_eq!( + upstream_url("http://up.test/files", "a b/c?d#e.txt").unwrap(), + "http://up.test/files/a%20b/c%3Fd%23e.txt" + ); + assert_eq!( + upstream_url("http://up.test", "x.bin").unwrap(), + "http://up.test/x.bin" + ); + assert_eq!(upstream_url("not a url", "x.bin"), None); + } +} diff --git a/plugins/raw/src/listing.rs b/plugins/raw/src/listing.rs new file mode 100644 index 00000000..f5d22999 --- /dev/null +++ b/plugins/raw/src/listing.rs @@ -0,0 +1,473 @@ +//! Directory listings: `GET /raw///`, as HTML for browsers or as +//! JSON (`Accept: application/json`, or `?format=json`), a page at a time. +//! +//! A page holds at most `limit` entries (default 100, at most 1000) after +//! the entry named by `after`, in byte order of their names, a +//! subdirectory's name ending in `/`; `next` (and a `Link: rel="next"` +//! header) names where the following page starts. The host walks its file +//! store for the page ([`forge_plugin_api::Query::Directory`]): package +//! selectors, group members and a proxy's cache are its business. + +use forge_plugin_api::files; +use forge_plugin_api::{Action, Answer, DirectoryEntry, Loaded, Repository, Request, Response}; +use serde::Serialize; + +/// Largest page size. +pub const MAX_LIMIT: usize = 1000; + +/// One entry of a listing. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Entry { + /// The entry's name in its directory; a subdirectory's ends in `/`. + pub name: String, + /// The full path (a subdirectory's ends in `/`). + pub path: String, + /// `file` | `directory`. + #[serde(rename = "type")] + pub kind: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// RFC 3339. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_modified: Option, + /// RFC 3339. + #[serde(skip_serializing_if = "Option::is_none")] + pub last_downloaded: Option, +} + +impl Entry { + fn of(directory: &str, entry: &DirectoryEntry) -> Self { + match entry { + DirectoryEntry::File(file) => Entry { + name: file + .path + .strip_prefix(directory) + .unwrap_or(&file.path) + .to_string(), + path: file.path.clone(), + kind: "file", + size: Some(file.size), + content_type: Some(file.content_type.clone()), + sha256: Some(file.sha256.clone()), + last_modified: Some(file.modified.clone()), + last_downloaded: file.last_downloaded.clone(), + }, + DirectoryEntry::Directory(path) => Entry { + name: path.strip_prefix(directory).unwrap_or(path).to_string(), + path: path.clone(), + kind: "directory", + size: None, + content_type: None, + sha256: None, + last_modified: None, + last_downloaded: None, + }, + } + } +} + +/// A page of a listing. +#[derive(Debug, Clone, Serialize)] +pub struct Page { + pub repository: String, + /// The directory (`""` for the root). + pub path: String, + pub entries: Vec, + /// The `after` of the next page, if there is one. + pub next: Option, +} + +/// The query of a listing. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ListQuery { + pub after: Option, + pub limit: Option, + /// `json` | `html`: overrides `Accept`. + pub format: Option, +} + +impl ListQuery { + /// Parse a request's query string; other parameters are ignored. + pub fn parse(query: &str) -> Result { + let mut out = ListQuery::default(); + for (key, value) in url::form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "after" => out.after = Some(value.into_owned()), + "limit" => { + out.limit = Some( + value + .parse() + .map_err(|_| "limit must be a positive number".to_string())?, + ) + } + "format" => match value.as_ref() { + "json" | "html" => out.format = Some(value.into_owned()), + _ => return Err("format is json or html".into()), + }, + _ => {} + } + } + if let Some(after) = out.after.as_deref().filter(|a| !a.is_empty()) { + let name = after.strip_suffix('/').unwrap_or(after); + files::validate_segment(name).map_err(|e| format!("after: {e}"))?; + } + Ok(out) + } + + /// The page size to ask the host for: 0 (its default) unless the client + /// named one. + pub fn limit_for_host(&self) -> u32 { + self.limit + .map_or(0, |limit| limit.clamp(1, MAX_LIMIT) as u32) + } +} + +/// Answer a listing request from the page the host loaded. +pub fn render(repo: &Repository, req: &Request, directory: &str, data: &[Loaded]) -> Action { + let page = data.iter().rev().find_map(|loaded| match &loaded.answer { + Answer::Directory(page) => Some(page), + _ => None, + }); + let Some(page) = page else { + return Action::Respond(Response::text(500, "raw: no listing was loaded")); + }; + let Some(page) = page else { + return Action::Respond(Response::text(404, "not found")); + }; + let query = ListQuery::parse(&req.query).unwrap_or_default(); + let page = Page { + repository: repo.name.clone(), + path: directory.to_string(), + entries: page + .entries + .iter() + .map(|entry| Entry::of(directory, entry)) + .collect(), + next: page.next.clone(), + }; + Action::Respond(respond(repo, &page, &query, wants_json(req, &query))) +} + +/// Whether the client wants JSON: `format=json`, or an `Accept` preferring +/// `application/json` to HTML. +pub fn wants_json(req: &Request, query: &ListQuery) -> bool { + match query.format.as_deref() { + Some("json") => return true, + Some("html") => return false, + _ => {} + } + let accept = req.header("accept").unwrap_or(""); + let quality = |wanted: &str| { + accept + .split(',') + .filter_map(|item| { + let mut parts = item.split(';'); + let media = parts.next()?.trim().to_ascii_lowercase(); + (media == wanted).then(|| { + parts + .filter_map(|p| p.trim().strip_prefix("q=")) + .find_map(|q| q.parse::().ok()) + .unwrap_or(1.0) + }) + }) + .fold(0.0f32, f32::max) + }; + quality("application/json") > 0.0 && quality("application/json") >= quality("text/html") +} + +/// The URL of the page after `page`. +fn next_link(repo: &Repository, page: &Page, query: &ListQuery) -> Option { + let next = page.next.as_deref()?; + let mut params = url::form_urlencoded::Serializer::new(String::new()); + params.append_pair("after", next); + if let Some(limit) = query.limit { + params.append_pair("limit", &limit.to_string()); + } + if let Some(format) = &query.format { + params.append_pair("format", format); + } + Some(format!( + "{}/{}?{}", + base_path(&repo.base_url), + files::encode(&page.path), + params.finish() + )) +} + +/// The path of the repository's URL (`/raw/`), for links and +/// redirects that keep the client on the host it asked. +pub fn base_path(base_url: &str) -> &str { + base_url + .split_once("://") + .and_then(|(_, rest)| rest.find('/').map(|i| &rest[i..])) + .unwrap_or("") +} + +fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c => out.push(c), + } + } + out +} + +/// An RFC 3339 time as `YYYY-MM-DD HH:MM:SS UTC` (the host's times are in +/// UTC). +fn display_time(rfc3339: &str) -> String { + match (rfc3339.get(..10), rfc3339.get(11..19)) { + (Some(date), Some(time)) => format!("{date} {time} UTC"), + _ => rfc3339.to_string(), + } +} + +/// The page as an HTML document: a table of the entries, linked. +pub fn html(repo: &Repository, page: &Page, query: &ListQuery) -> String { + let title = format!("Index of {}/{}", base_path(&repo.base_url), page.path); + let mut out = format!( + "\n{t}\n\n

{t}

\n\n\n", + t = escape(&title) + ); + if !page.path.is_empty() { + out.push_str("\n"); + } + for entry in &page.entries { + let href = files::encode(&entry.name); + out.push_str(&format!( + "\n", + escape(&href), + escape(&entry.name), + entry.size.map(|s| s.to_string()).unwrap_or_default(), + entry + .last_modified + .as_deref() + .map(display_time) + .unwrap_or_default(), + escape(entry.sha256.as_deref().unwrap_or("")), + )); + } + out.push_str("
NameSizeLast modifiedSHA-256
../
{}{}{}{}
\n"); + if let Some(next) = next_link(repo, page, query) { + out.push_str(&format!( + "

Next page

\n", + escape(&next) + )); + } + out.push_str("\n"); + out +} + +/// The listing response: the page, its `Link` to the next page. +pub fn respond(repo: &Repository, page: &Page, query: &ListQuery, json: bool) -> Response { + let mut response = if json { + Response::new( + 200, + "application/json", + serde_json::to_string(page).unwrap_or_default(), + ) + } else { + Response::new(200, "text/html; charset=utf-8", html(repo, page, query)) + }; + // Forge's own document, shown in the browser (and sandboxed). + response = response + .with_header("content-disposition", "inline") + .with_header("vary", "Accept"); + if let Some(next) = next_link(repo, page, query) { + response = response.with_header("link", &format!("<{next}>; rel=\"next\"")); + } + response +} + +/// Seconds since the epoch of an RFC 3339 time (`2026-10-01T12:00:00.250Z`, +/// or with a `+hh:mm` / `-hh:mm` offset), whole seconds. +pub fn epoch_seconds(rfc3339: &str) -> Option { + let number = |range: std::ops::Range| -> Option { + let text = rfc3339.get(range)?; + text.bytes() + .all(|b| b.is_ascii_digit()) + .then(|| text.parse().ok())? + }; + let (year, month, day) = (number(0..4)?, number(5..7)?, number(8..10)?); + let (hour, minute, second) = (number(11..13)?, number(14..16)?, number(17..19)?); + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + // Skip a fraction, then read the offset. + let rest = rfc3339.get(19..)?; + let rest = rest + .strip_prefix('.') + .map_or(rest, |r| r.trim_start_matches(|c: char| c.is_ascii_digit())); + let offset = match rest { + "Z" | "z" => 0, + _ => { + let sign = match rest.get(..1)? { + "+" => 1, + "-" => -1, + _ => return None, + }; + let hours: i64 = rest.get(1..3)?.parse().ok()?; + let minutes: i64 = rest.get(4..6)?.parse().ok()?; + sign * (hours * 3600 + minutes * 60) + } + }; + // Days since 1970-01-01 of a proleptic Gregorian date. + let (y, m) = if month <= 2 { + (year - 1, month + 9) + } else { + (year, month - 3) + }; + let era = y.div_euclid(400); + let year_of_era = y - era * 400; + let day_of_year = (153 * m + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + let days = era * 146_097 + day_of_era - 719_468; + let seconds = days * 86_400 + hour * 3600 + minute * 60 + second - offset; + u64::try_from(seconds).ok() +} + +/// Seconds since the epoch as an HTTP date (IMF-fixdate). +pub fn http_date(seconds: u64) -> String { + httpdate::fmt_http_date(std::time::UNIX_EPOCH + std::time::Duration::from_secs(seconds)) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::RepositoryKind; + + fn repo() -> Repository { + Repository { + name: "bin".into(), + kind: RepositoryKind::Hosted, + base_url: "https://forge.example.com/raw/bin".into(), + inline: false, + } + } + + #[test] + fn queries_are_checked() { + assert_eq!(ListQuery::parse("").unwrap(), ListQuery::default()); + let q = ListQuery::parse("after=x.bin&limit=5&format=json&other=1").unwrap(); + assert_eq!(q.after.as_deref(), Some("x.bin")); + assert_eq!(q.limit_for_host(), 5); + assert_eq!(ListQuery::parse("limit=0").unwrap().limit_for_host(), 1); + assert_eq!( + ListQuery::parse("limit=5000").unwrap().limit_for_host(), + 1000 + ); + assert_eq!(ListQuery::default().limit_for_host(), 0); + assert!(ListQuery::parse("limit=x").is_err()); + assert!(ListQuery::parse("format=xml").is_err()); + assert!(ListQuery::parse("after=..%2Fx").is_err()); + assert!(ListQuery::parse("after=b%2Fc").is_err()); + assert!(ListQuery::parse("after=dir%2F").is_ok()); + } + + #[test] + fn json_is_asked_for() { + let query = ListQuery::default(); + let req = Request::new("GET", "/"); + assert!(!wants_json(&req, &query)); + let req = Request::new("GET", "/").with_header("accept", "application/json"); + assert!(wants_json(&req, &query)); + let req = Request::new("GET", "/").with_header( + "accept", + "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", + ); + assert!(!wants_json(&req, &query)); + let query = ListQuery { + format: Some("json".into()), + ..Default::default() + }; + assert!(wants_json(&req, &query)); + } + + #[test] + fn html_escapes_names_and_links_them_encoded() { + let page = Page { + repository: "bin".into(), + path: "a/".into(), + entries: vec![Entry { + name: " y.txt".into(), + path: "a/ y.txt".into(), + kind: "file", + size: Some(3), + content_type: Some("text/plain".into()), + sha256: Some("ab".into()), + last_modified: Some("2026-10-21T10:58:44.123456Z".into()), + last_downloaded: None, + }], + next: Some(" y.txt".into()), + }; + let html = html(&repo(), &page, &ListQuery::default()); + assert!( + html.contains("href=\"%3Cx%3E%20y.txt\"><x> y.txt"), + "{html}" + ); + assert!( + html.contains("Index of /raw/bin/a/"), + "{html}" + ); + assert!(html.contains("href=\"../\"")); + assert!(html.contains("2026-10-21 10:58:44 UTC"), "{html}"); + assert!(html.contains("/raw/bin/a/?after=%3Cx%3E+y.txt"), "{html}"); + assert!(!html.contains("")); + } + + #[test] + fn json_keeps_the_listing_shape() { + let page = Page { + repository: "bin".into(), + path: String::new(), + entries: vec![ + Entry::of("", &DirectoryEntry::Directory("d/".into())), + Entry::of( + "", + &DirectoryEntry::File(forge_plugin_api::FileInfo { + path: "f.txt".into(), + size: 3, + content_type: "text/plain".into(), + sha256: "ab".into(), + sha512: String::new(), + sha1: String::new(), + md5: String::new(), + modified: "2026-10-21T10:58:44Z".into(), + last_downloaded: None, + }), + ), + ], + next: None, + }; + let response = respond(&repo(), &page, &ListQuery::default(), true); + assert_eq!( + String::from_utf8(response.body).unwrap(), + r#"{"repository":"bin","path":"","entries":[{"name":"d/","path":"d/","type":"directory"},{"name":"f.txt","path":"f.txt","type":"file","size":3,"content_type":"text/plain","sha256":"ab","last_modified":"2026-10-21T10:58:44Z"}],"next":null}"# + ); + } + + #[test] + fn times_convert() { + assert_eq!(epoch_seconds("1970-01-01T00:00:00Z"), Some(0)); + assert_eq!( + epoch_seconds("2026-10-01T12:00:00.250Z"), + Some(1_790_856_000) + ); + assert_eq!( + epoch_seconds("2026-10-01T14:00:00+02:00"), + Some(1_790_856_000) + ); + assert_eq!(epoch_seconds("2000-02-29T00:00:00Z"), Some(951_782_400)); + assert_eq!(epoch_seconds("not a time"), None); + assert_eq!(http_date(1_790_856_000), "Thu, 01 Oct 2026 12:00:00 GMT"); + } +} diff --git a/plugins/raw/src/sidecar.rs b/plugins/raw/src/sidecar.rs new file mode 100644 index 00000000..c1d380af --- /dev/null +++ b/plugins/raw/src/sidecar.rs @@ -0,0 +1,210 @@ +//! Checksum sidecars: `.sha256`, `.sha512`, `.sha1` and `.md5`, +//! answered from a stored file's digests when no file of that name is +//! stored, as Nexus does. The body is the digest and the file name, as +//! `sha256sum` and friends print them, so `sha256sum -c` checks a download +//! against it. + +use base64::Engine as _; +use forge_plugin_api::files; +use forge_plugin_api::{FileInfo, Repository, Request, Response}; + +use crate::listing::http_date; + +/// Which digest a sidecar holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Checksum { + Sha256, + Sha512, + Sha1, + Md5, +} + +impl Checksum { + const ALL: [Checksum; 4] = [ + Checksum::Sha256, + Checksum::Sha512, + Checksum::Sha1, + Checksum::Md5, + ]; + + fn extension(self) -> &'static str { + match self { + Checksum::Sha256 => ".sha256", + Checksum::Sha512 => ".sha512", + Checksum::Sha1 => ".sha1", + Checksum::Md5 => ".md5", + } + } + + /// The digest of `file` as lowercase hex. + pub fn hex(self, file: &FileInfo) -> String { + match self { + Checksum::Sha256 => file.sha256.clone(), + Checksum::Sha512 => base64::engine::general_purpose::STANDARD + .decode(&file.sha512) + .map(|bytes| bytes.iter().map(|b| format!("{b:02x}")).collect()) + .unwrap_or_default(), + Checksum::Sha1 => file.sha1.clone(), + Checksum::Md5 => file.md5.clone(), + } + } +} + +/// The file a sidecar path is the checksum of, and which checksum. +pub fn of(path: &str) -> Option<(&str, Checksum)> { + Checksum::ALL.into_iter().find_map(|checksum| { + let base = path.strip_suffix(checksum.extension())?; + (!base.is_empty() && !base.ends_with('/')).then_some((base, checksum)) + }) +} + +/// Whether the request's preconditions say the client's copy is current +/// (`If-None-Match`, else `If-Modified-Since`): answer 304. `modified` is in +/// seconds since the epoch. +pub fn not_modified(req: &Request, etag: &str, modified: Option) -> bool { + if let Some(value) = req.header("if-none-match") { + // Weak comparison: `W/"x"` matches `"x"`. + return value + .split(',') + .map(str::trim) + .any(|tag| tag == "*" || tag.strip_prefix("W/").unwrap_or(tag) == etag); + } + let since = req + .header("if-modified-since") + .and_then(|v| httpdate::parse_http_date(v).ok()); + match (since, modified) { + (Some(since), Some(modified)) => { + std::time::UNIX_EPOCH + std::time::Duration::from_secs(modified) <= since + } + _ => false, + } +} + +/// `Content-Disposition` of `file_path` in `repo`: a download +/// (`attachment`), or with the repository's `inline` option for display, +/// naming the file. +pub fn content_disposition(repo: &Repository, file_path: &str) -> String { + let kind = if repo.inline { "inline" } else { "attachment" }; + let name = files::file_name(file_path); + let plain = |c: char| c.is_ascii() && !c.is_ascii_control() && c != '"' && c != '\\'; + if name.chars().all(plain) { + format!("{kind}; filename=\"{name}\"") + } else { + // RFC 6266: an ASCII fallback, and the UTF-8 name. + let fallback: String = name + .chars() + .map(|c| if plain(c) { c } else { '_' }) + .collect(); + format!( + "{kind}; filename=\"{fallback}\"; filename*=UTF-8''{}", + percent_encoding::utf8_percent_encode(name, percent_encoding::NON_ALPHANUMERIC) + ) + } +} + +/// Answer `GET`/`HEAD` of `path`, the `checksum` sidecar of `base`. +pub fn respond( + repo: &Repository, + req: &Request, + path: &str, + base: &FileInfo, + checksum: Checksum, +) -> Response { + let etag = format!( + "\"{}{}\"", + base.sha256, + checksum.extension().replace('.', "-") + ); + let modified = crate::listing::epoch_seconds(&base.modified); + let validators = |response: Response| { + let response = response.with_header("etag", &etag); + match modified { + Some(seconds) => response.with_header("last-modified", &http_date(seconds)), + None => response, + } + }; + if not_modified(req, &etag, modified) { + return validators(Response::empty(304)); + } + let text = format!("{} {}\n", checksum.hex(base), files::file_name(&base.path)); + validators( + Response::text(200, &text) + .with_header("content-disposition", &content_disposition(repo, path)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use forge_plugin_api::RepositoryKind; + + fn file() -> FileInfo { + FileInfo { + path: "dir/file.tgz".into(), + size: 10, + content_type: "application/gzip".into(), + sha256: "ab".repeat(32), + // SHA-512 of nothing in particular: 64 zero bytes. + sha512: base64::engine::general_purpose::STANDARD.encode([0u8; 64]), + sha1: "cd".repeat(20), + md5: "ef".repeat(16), + modified: "2026-10-01T12:00:00.250Z".into(), + last_downloaded: None, + } + } + + #[test] + fn sidecars_name_their_file() { + assert_eq!(of("a/x.tgz.sha256"), Some(("a/x.tgz", Checksum::Sha256))); + assert_eq!(of("x.md5"), Some(("x", Checksum::Md5))); + assert_eq!(of(".sha1"), None); + assert_eq!(of("a/.sha1"), None); + assert_eq!(of("a/x.tgz"), None); + let file = file(); + assert_eq!(Checksum::Sha512.hex(&file), "00".repeat(64)); + assert_eq!(Checksum::Md5.hex(&file), "ef".repeat(16)); + } + + #[test] + fn preconditions() { + let file = file(); + let modified = crate::listing::epoch_seconds(&file.modified); + let tag = "\"x-sha256\""; + let req = Request::new("GET", "/f.sha256"); + assert!(!not_modified(&req, tag, modified)); + let req = + Request::new("GET", "/f.sha256").with_header("if-none-match", "\"y\", W/\"x-sha256\""); + assert!(not_modified(&req, tag, modified)); + // An `If-None-Match` that fails decides; the date is not consulted. + let date = http_date(modified.unwrap()); + let req = Request::new("GET", "/f.sha256") + .with_header("if-none-match", "\"other\"") + .with_header("if-modified-since", &date); + assert!(!not_modified(&req, tag, modified)); + // Sub-second precision is not part of an HTTP date. + let req = Request::new("GET", "/f.sha256").with_header("if-modified-since", &date); + assert!(not_modified(&req, tag, modified)); + let earlier = http_date(modified.unwrap() - 5); + let req = Request::new("GET", "/f.sha256").with_header("if-modified-since", &earlier); + assert!(!not_modified(&req, tag, modified)); + } + + #[test] + fn dispositions_name_the_file() { + let mut repo = Repository { + name: "r".into(), + kind: RepositoryKind::Hosted, + base_url: "https://forge.example.com/raw/r".into(), + inline: false, + }; + assert_eq!( + content_disposition(&repo, "a/b.txt"), + "attachment; filename=\"b.txt\"" + ); + repo.inline = true; + assert_eq!( + content_disposition(&repo, "a/\"ä\".txt"), + "inline; filename=\"___.txt\"; filename*=UTF-8''%22%C3%A4%22%2Etxt" + ); + } +} diff --git a/plugins/raw/tests/protocol.rs b/plugins/raw/tests/protocol.rs new file mode 100644 index 00000000..44ca8872 --- /dev/null +++ b/plugins/raw/tests/protocol.rs @@ -0,0 +1,280 @@ +//! The raw protocol against canned answers: what each request asks of the +//! host, round by round, and what it answers. + +use forge_format_raw::Raw; +use forge_plugin_api::testing::{repository, serve}; +use forge_plugin_api::{ + Action, Answer, DirectoryEntry, DirectoryPage, DirectoryQuery, FileInfo, FileSource, + FormContent, FormFile, FormPart, NewFile, Query, RepositoryKind, Request, StoreStatus, + StoredFile, +}; + +fn file(path: &str) -> FileInfo { + FileInfo { + path: path.into(), + size: 3, + content_type: "application/gzip".into(), + sha256: "ab".repeat(32), + sha512: String::new(), + sha1: "cd".repeat(20), + md5: "ef".repeat(16), + modified: "2026-10-01T12:00:00.250Z".into(), + last_downloaded: None, + } +} + +fn hosted() -> forge_plugin_api::Repository { + repository(RepositoryKind::Hosted, "/raw") +} + +#[test] +fn a_stored_file_is_served_by_the_host() { + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz"), + |q| match q { + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.outcome, Action::ServeFile("a/x.tgz".into())); +} + +#[test] +fn a_sidecar_is_computed_from_its_file() { + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz.sha1"), + |q| match q { + Query::File(p) if p == "a/x.tgz.sha1" => Answer::File(None), + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.response().status, 200); + assert_eq!(served.text(), format!("{} x.tgz\n", "cd".repeat(20))); + assert_eq!( + served.header("etag").unwrap(), + format!("\"{}-sha1\"", "ab".repeat(32)) + ); + assert_eq!( + served.header("last-modified"), + Some("Thu, 01 Oct 2026 12:00:00 GMT") + ); + assert_eq!( + served.header("content-disposition"), + Some("attachment; filename=\"x.tgz.sha1\"") + ); + // A stored file of the sidecar's name is served as it is. + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/a/x.tgz.sha1"), + |q| match q { + Query::File(p) => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }, + ); + assert_eq!(served.outcome, Action::ServeFile("a/x.tgz.sha1".into())); + // The client's copy is current. + let req = Request::new("GET", "/a/x.tgz.md5") + .with_header("if-none-match", &format!("\"{}-md5\"", "ab".repeat(32))); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::File(p) if p == "a/x.tgz" => Answer::File(Some(file(p))), + Query::File(_) => Answer::File(None), + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 304); + assert!(served.response().body.is_empty()); +} + +#[test] +fn a_proxy_fetches_what_it_has_not_cached() { + let proxy = repository(RepositoryKind::Proxy, "/raw"); + let served = serve(&Raw, &proxy, &Request::new("GET", "/x.bin"), |q| match q { + Query::File(_) => Answer::File(None), + Query::FetchFile(p) => Answer::File(Some(file(p))), + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.outcome, Action::ServeFile("x.bin".into())); + assert_eq!( + served.queries, + [ + Query::File("x.bin".into()), + Query::FetchFile("x.bin".into()) + ] + ); +} + +#[test] +fn a_directory_named_without_its_slash_redirects() { + let directory = |answer: Option| { + move |q: &Query| match q { + Query::File(_) => Answer::File(None), + Query::Directory(d) => { + assert_eq!( + d, + &DirectoryQuery { + directory: "pdfium/".into(), + after: None, + limit: 1 + } + ); + Answer::Directory(answer.clone()) + } + other => panic!("unexpected {other:?}"), + } + }; + let page = DirectoryPage { + entries: vec![DirectoryEntry::Directory("pdfium/7350/".into())], + next: None, + }; + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/pdfium"), + directory(Some(page)), + ); + assert_eq!(served.response().status, 301); + assert_eq!(served.header("location"), Some("/raw/repo/pdfium/")); + // A hosted repository asks no upstream. + assert!(!served + .queries + .iter() + .any(|q| matches!(q, Query::FetchFile(_)))); + let served = serve( + &Raw, + &hosted(), + &Request::new("GET", "/pdfium"), + directory(None), + ); + assert_eq!(served.response().status, 404); +} + +#[test] +fn the_repository_url_without_its_slash_redirects() { + let served = serve(&Raw, &hosted(), &Request::new("GET", ""), |q| { + panic!("unexpected {q:?}") + }); + assert_eq!(served.response().status, 301); + assert_eq!(served.header("location"), Some("/raw/repo/")); + let served = serve(&Raw, &hosted(), &Request::new("PUT", ""), |q| { + panic!("unexpected {q:?}") + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn listings_render_the_page_the_host_loaded() { + let req = Request::new("GET", "/a/?limit=1&format=json"); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::Directory(d) => { + assert_eq!(d.directory, "a/"); + assert_eq!(d.limit, 1); + Answer::Directory(Some(DirectoryPage { + entries: vec![DirectoryEntry::File(file("a/x.tgz"))], + next: Some("x.tgz".into()), + })) + } + other => panic!("unexpected {other:?}"), + }); + let response = served.response(); + assert_eq!(response.status, 200); + assert_eq!(served.header("content-type"), Some("application/json")); + assert_eq!( + served.header("link"), + Some("; rel=\"next\"") + ); + let page: serde_json::Value = serde_json::from_slice(&response.body).unwrap(); + assert_eq!(page["entries"][0]["name"], "x.tgz"); + assert_eq!(page["next"], "x.tgz"); + // A directory that does not exist. + let served = serve(&Raw, &hosted(), &Request::new("GET", "/b/"), |_| { + Answer::Directory(None) + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn uploads_and_deletes_go_through_the_host() { + let req = Request::new("PUT", "/a/x.tgz").with_header("content-type", "application/gzip"); + let served = serve(&Raw, &hosted(), &req, |q| match q { + Query::Store(files) => { + assert_eq!( + files, + &[NewFile { + path: "a/x.tgz".into(), + source: FileSource::Body, + content_type: Some("application/gzip".into()), + }] + ); + Answer::Stored(vec![StoredFile { + path: "a/x.tgz".into(), + size: 3, + sha256: "ab".into(), + status: StoreStatus::Created, + }]) + } + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 201); + + let served = serve(&Raw, &hosted(), &Request::new("POST", "/a/"), |q| match q { + Query::Form => Answer::Form(vec![ + FormPart { + field: "raw.directory".into(), + content: FormContent::Text("7350".into()), + }, + FormPart { + field: "file".into(), + content: FormContent::File(FormFile { + name: "x.tgz".into(), + content_type: None, + size: 3, + }), + }, + ]), + Query::Store(files) => { + assert_eq!(files[0].path, "a/7350/x.tgz"); + assert_eq!(files[0].source, FileSource::Part(1)); + Answer::Stored(vec![StoredFile { + path: files[0].path.clone(), + size: 3, + sha256: "ab".into(), + status: StoreStatus::Unchanged, + }]) + } + other => panic!("unexpected {other:?}"), + }); + assert_eq!(served.response().status, 200); + let body: serde_json::Value = serde_json::from_slice(&served.response().body).unwrap(); + assert_eq!(body["files"][0]["status"], "unchanged"); + + let served = serve(&Raw, &hosted(), &Request::new("DELETE", "/a/x.tgz"), |q| { + assert_eq!(q, &Query::Remove("a/x.tgz".into())); + Answer::Removed(false) + }); + assert_eq!(served.response().status, 404); +} + +#[test] +fn bad_paths_and_methods_are_refused() { + for (method, path, status) in [ + ("GET", "/a/../b", 400), + ("GET", "/a//b", 400), + ("GET", "/a%2fb", 400), + ("PUT", "/a/", 400), + ("POST", "/a", 400), + ("DELETE", "/a/", 400), + ("PATCH", "/a", 405), + ("GET", "/?limit=x", 400), + ("GET", "/?after=..", 400), + ] { + let served = serve(&Raw, &hosted(), &Request::new(method, path), |q| { + panic!("{method} {path}: unexpected {q:?}") + }); + assert_eq!(served.response().status, status, "{method} {path}"); + } +}