Raw repositories as a built-in format plugin over a host file store

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "crates/server",
     "crates/e2e",
     "plugins/nuget",
+    "plugins/raw",
     "xtask",
 ]
 

DEVELOPMENT.md

@@ -241,8 +241,9 @@ password: `docker login localhost:8421 -u admin@example.com`, then
 `docker push localhost:8421/docker-hosted/team/app:1`. See
 [docs/docker.md](docs/docker.md).
 
-Raw repositories (`"format":"raw"`) hold files under arbitrary paths, served
-under `/raw/`: `curl -u admin@example.com:<token> --upload-file x.tgz
+Raw repositories (`"format":"raw"`, the built-in plugin `plugins/raw`) hold
+files under arbitrary paths, served under `/raw/`: `curl -u
+admin@example.com:<token> --upload-file x.tgz
 localhost:8421/raw/binaries/tools/x.tgz`, a proxy of any HTTP(S) base URL
 (`"remote_url":"https://github.com/<owner>/<repo>/releases/download/"`), and
 groups. See [docs/raw.md](docs/raw.md).

README.md

@@ -53,10 +53,10 @@ A Cargo workspace of small crates plus the web SPA:
 | `format-oci` | `forge-oci` | Docker / OCI registry: the distribution-spec `/v2` API, manifests, registry tokens, hosted/proxy/group resolution. |
 | `plugin-api` | `forge-plugin-api` | The format-plugin contract: the `FormatPlugin` trait and its types. |
 | `plugin` | `forge-plugin` | The host side of format plugins: an installed plugin and its adapter to the repository machinery. |
-| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`), the Docker registry, raw file repositories (`/raw`), `/admin` + `/auth` REST, background workers, static SPA serving. |
+| `server` | `forge-server` | Axum HTTP server: the format plugins' feeds (NuGet at `/v3`, raw files at `/raw`), the file store of files formats, the Docker registry, `/admin` + `/auth` REST, background workers, static SPA serving. |
 | `e2e` | `forge-e2e` | End-to-end suite (testcontainers + mock upstream/IdP). |
 | `xtask` | — | The task runner (`cargo xtask …`); replaces Make/Just. |
-| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`. |
+| `plugins/` | `forge-format-*` | Format plugins, built into the server: `nuget`, `raw`. |
 | `web/` | — | TanStack Router SPA (pnpm + Vite), built into `web/dist/`. |
 
 **Every format is a plugin in `plugins/`.** A package format is a crate in

crates/core/src/format.rs

@@ -46,6 +46,13 @@ pub trait Format: Send + Sync {
     /// the `repository.format` column.
     fn id(&self) -> &'static str;
 
+    /// Whether the format's repositories hold files under paths (raw) rather
+    /// than package versions: their contents are the file store's
+    /// (`raw_file`), and none of the package seams below apply to them.
+    fn stores_files(&self) -> bool {
+        false
+    }
+
     /// Parse an uploaded package into format-neutral [`ParsedUpload`] data.
     fn parse_upload(&self, file: &mut dyn ReadSeek) -> Result<ParsedUpload, FormatError>;
 

crates/core/src/repo/proxy.rs

@@ -541,6 +541,16 @@ impl ProxyRepo {
                     );
                     return Ok((api::UpstreamAnswer::Failed(reason), origins));
                 }
+                api::UpstreamStep::Done(api::UpstreamAnswer::File(Some(url)))
+                    if !origins.permits(&url) =>
+                {
+                    tracing::warn!(format = %self.format.id(), repo = %repo.name, %url, "the format answered a file URL on an origin neither the remote_url nor a document fetched from the upstream names; not downloaded");
+                    let reason = format!(
+                        "upstream file {url}: not downloaded, its origin is neither the \
+                             remote_url's nor named by a document fetched from the upstream"
+                    );
+                    return Ok((api::UpstreamAnswer::Failed(reason), origins));
+                }
                 api::UpstreamStep::Done(api::UpstreamAnswer::Failed(reason)) if unavailable => {
                     return Err(RepoError::Upstream(reason));
                 }
@@ -669,6 +679,27 @@ impl ProxyRepo {
         }
     }
 
+    /// The upstream URL of the file at `path` of proxy `repo`
+    /// ([`api::UpstreamOp::File`], a files format), on an origin the
+    /// upstream points at; `None` when the upstream cannot have it. The
+    /// file store downloads and re-validates it.
+    pub async fn file_url(
+        &self,
+        repo: &Repository,
+        path: &str,
+    ) -> Result<Option<String>, RepoError> {
+        match self
+            .converse_resolved(repo, api::UpstreamOp::File(path.to_string()))
+            .await?
+        {
+            api::UpstreamAnswer::File(url) => Ok(url),
+            api::UpstreamAnswer::Failed(reason) => Err(RepoError::Storage(reason)),
+            _ => Err(RepoError::Storage(
+                "upstream protocol answered file with something else".into(),
+            )),
+        }
+    }
+
     /// The upstream index, resolved, for `op` on `repo`.
     async fn converse_resolved(
         &self,

crates/core/src/retention/worker.rs

@@ -231,7 +231,12 @@ impl RetentionWorker {
         if repo.format == forge_db::model::DOCKER_FORMAT {
             return self.process_images(repo, kind, run_id).await;
         }
-        if repo.format == forge_db::model::RAW_FORMAT {
+        // A files format's repositories (raw) hold files under paths.
+        if self
+            .formats
+            .resolve(&repo.format)
+            .is_some_and(|format| format.stores_files())
+        {
             return self.process_files(repo, kind, run_id).await;
         }
 

crates/db/src/model.rs

@@ -282,9 +282,6 @@ impl Repository {
     }
 }
 
-/// `repository.format` of raw (generic file) repositories.
-pub const RAW_FORMAT: &str = "raw";
-
 /// `repository.config` key of [`Repository::deployment_policy`].
 pub const DEPLOYMENT_POLICY_KEY: &str = "deployment_policy";
 
@@ -294,9 +291,9 @@ pub const CONTENT_DISPOSITION_KEY: &str = "content_disposition";
 /// `repository.config` key of [`Repository::max_age_seconds`].
 pub const MAX_AGE_SECONDS_KEY: &str = "max_age_seconds";
 
-/// How long a raw proxy serves a cached file before it asks its upstream
-/// whether the file changed, when the repository sets no `max_age_seconds`:
-/// a day.
+/// How long a proxy of files (raw) serves a cached file before it asks its
+/// upstream whether the file changed, when the repository sets no
+/// `max_age_seconds`: a day.
 pub const DEFAULT_RAW_MAX_AGE_SECONDS: u64 = 24 * 60 * 60;
 
 /// What a hosted raw repository does with an upload to a path it has
@@ -333,13 +330,11 @@ impl DeploymentPolicy {
     }
 }
 
+/// The options of a repository of a files format (raw): which repositories
+/// hold files is the format registry's to say (the server's
+/// `FormatRegistry::stores_files`).
 impl Repository {
-    /// Whether this is a raw (generic file) repository.
-    pub fn is_raw(&self) -> bool {
-        self.format == RAW_FORMAT
-    }
-
-    /// Hosted raw option (`config.deployment_policy`, default
+    /// Hosted files option (`config.deployment_policy`, default
     /// [`DeploymentPolicy::DisableRedeploy`]).
     pub fn deployment_policy(&self) -> DeploymentPolicy {
         self.config
@@ -349,7 +344,7 @@ impl Repository {
             .unwrap_or_default()
     }
 
-    /// Raw option (`config.content_disposition = "inline"`, default
+    /// Files option (`config.content_disposition = "inline"`, default
     /// `"attachment"`): files are served for display in the browser rather
     /// than as downloads.
     pub fn inline_content(&self) -> bool {
@@ -359,7 +354,7 @@ impl Repository {
             == Some("inline")
     }
 
-    /// Raw proxy option (`config.max_age_seconds`): how long a cached file
+    /// Files proxy option (`config.max_age_seconds`): how long a cached file
     /// is served before the upstream is asked whether it changed; `0` never
     /// asks (the upstream's files are immutable). Default
     /// [`DEFAULT_RAW_MAX_AGE_SECONDS`].