Accept Swift source repository PURLs in the denylist and retention rules

cmd/proxy/main.go

@@ -124,7 +124,6 @@ import (
 
 	"github.com/git-pkgs/proxy/internal/config"
 	"github.com/git-pkgs/proxy/internal/database"
-	"github.com/git-pkgs/proxy/internal/denylist"
 	"github.com/git-pkgs/proxy/internal/handler"
 	"github.com/git-pkgs/proxy/internal/memlimit"
 	"github.com/git-pkgs/proxy/internal/mirror"
@@ -552,15 +551,16 @@ func runMirror() {
 
 	// Build proxy with the same upstream clients and authentication as serve
 	proxy, images := server.NewMirrorProxy(cfg, db, store, logger)
-	proxy.Denylist, err = denylist.New(cfg.Denylist.Packages)
-	if err != nil {
-		_ = db.Close()
-		fmt.Fprintf(os.Stderr, "invalid denylist: %v\n", err)
-		os.Exit(1) //nolint:gocritic // db closed above
-	}
 	proxy.CacheMetadata = true // mirror always caches metadata
 	proxy.MetadataTTL = cfg.ParseMetadataTTL()
 	proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
+	// As serve does, resolve Swift source repositories in the denylist through
+	// the upstream, and refuse to run when one can't be.
+	if err := server.ConfigureDenylist(cfg, proxy); err != nil {
+		_ = db.Close()
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1) //nolint:gocritic // db closed above
+	}
 
 	m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency)
 	switch s := source.(type) {

cmd/proxy/mirror_swift_test.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/git-pkgs/proxy/internal/database"
+)
+
+const swiftWidgetArchive = "acme widget archive"
+
+// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package
+// registry does, with https://github.com/acme/widget as its source
+// repository, and counts archive downloads and identifier lookups. The
+// repository's identifiers are ids, or only acme.widget.
+func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) {
+	t.Helper()
+	if len(ids) == 0 {
+		ids = []string{"acme.widget"}
+	}
+	identifiers, err := json.Marshal(map[string][]string{"identifiers": ids})
+	if err != nil {
+		t.Fatal(err)
+	}
+	downloads, lookups = new(atomic.Int64), new(atomic.Int64)
+	sum := sha256.Sum256([]byte(swiftWidgetArchive))
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Version", "1")
+		switch r.URL.Path {
+		case "/identifiers":
+			lookups.Add(1)
+			if r.URL.Query().Get("url") != "https://github.com/acme/widget" {
+				http.NotFound(w, r)
+				return
+			}
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write(identifiers)
+		case "/acme/widget":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`)
+		case "/acme/widget/1.0.0/Package.swift":
+			w.Header().Set("Content-Type", "text/x-swift")
+			_, _ = io.WriteString(w, "// swift-tools-version:5.9\n")
+		case "/acme/widget/1.0.0":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:]))
+		case "/acme/widget/1.0.0.zip":
+			downloads.Add(1)
+			w.Header().Set("Content-Type", "application/zip")
+			_, _ = io.WriteString(w, swiftWidgetArchive)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv, downloads, lookups
+}
+
+// writeSwiftMirrorConfig writes a config using upstream as the Swift
+// registry, denying the denied package PURLs.
+func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string {
+	t.Helper()
+	cfg := "storage:\n" +
+		"  url: file://" + filepath.Join(dir, "artifacts") + "\n" +
+		"database:\n" +
+		"  path: " + filepath.Join(dir, "proxy.db") + "\n" +
+		"upstream:\n" +
+		"  swift: " + upstream + "\n" +
+		"  allow_loopback: true\n"
+	if len(denied) > 0 {
+		cfg += "denylist:\n  packages:\n"
+		for _, p := range denied {
+			cfg += "    - " + p + "\n"
+		}
+	}
+	path := filepath.Join(dir, "config.yaml")
+	if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
+		t.Fatalf("writing config: %v", err)
+	}
+	return path
+}
+
+func cachedArtifacts(t *testing.T, dir string) int64 {
+	t.Helper()
+	db, err := database.Open(filepath.Join(dir, "proxy.db"))
+	if err != nil {
+		t.Fatalf("opening database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+	stats, err := db.GetCacheStats()
+	if err != nil {
+		t.Fatalf("GetCacheStats: %v", err)
+	}
+	return stats.TotalArtifacts
+}
+
+// Without a denylist the fake registry mirrors the release, so the tests
+// below fail for the denylist and not for the setup.
+func TestMirrorSwiftRelease(t *testing.T) {
+	upstream, downloads, _ := swiftMirrorUpstream(t)
+	dir := t.TempDir()
+	configPath := writeSwiftMirrorConfig(t, dir, upstream.URL)
+
+	output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
+
+	if !strings.Contains(output, "1 downloaded") {
+		t.Errorf("output:\n%s", output)
+	}
+	if n := downloads.Load(); n != 1 {
+		t.Errorf("archive downloads = %d, want 1", n)
+	}
+	if n := cachedArtifacts(t, dir); n != 1 {
+		t.Errorf("cached artifacts = %d, want 1", n)
+	}
+}
+
+// A denylist entry naming the package by its source repository blocks the
+// mirror download of the registry identity it resolves to.
+func TestMirrorDenylistSwiftSourceEntry(t *testing.T) {
+	upstream, downloads, _ := swiftMirrorUpstream(t)
+	dir := t.TempDir()
+	configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
+
+	output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
+
+	if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
+		t.Errorf("output:\n%s", output)
+	}
+	if n := downloads.Load(); n != 0 {
+		t.Errorf("archive downloads = %d, want 0", n)
+	}
+	if n := cachedArtifacts(t, dir); n != 0 {
+		t.Errorf("cached artifacts = %d, want 0", n)
+	}
+}
+
+// The registry may list several identities for one repository. A source
+// entry denies all of them, not only the first.
+func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) {
+	upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget")
+	dir := t.TempDir()
+	configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0")
+
+	output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
+
+	if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") {
+		t.Errorf("output:\n%s", output)
+	}
+	if n := downloads.Load(); n != 0 {
+		t.Errorf("archive downloads = %d, want 0", n)
+	}
+}
+
+// A source repository entry the registry can't resolve stops the mirror
+// before it downloads anything, as it stops serve: left out, it would let
+// the version through.
+func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) {
+	upstream, downloads, lookups := swiftMirrorUpstream(t)
+	dir := t.TempDir()
+	configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0")
+
+	output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0")
+
+	if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") {
+		t.Errorf("output does not report the unresolved entry:\n%s", output)
+	}
+	if strings.Contains(output, "Mirror complete") {
+		t.Errorf("mirror ran:\n%s", output)
+	}
+	if lookups.Load() == 0 {
+		t.Error("the entry was not looked up")
+	}
+	if n := downloads.Load(); n != 0 {
+		t.Errorf("archive downloads = %d, want 0", n)
+	}
+}

docs/configuration.md

@@ -150,7 +150,7 @@ The package rule covers every release and asset in that repository, but each cac
 | `pub` | yes | `pkg:pub/http`, the package name as published. |
 | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. |
 | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. |
-| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. |
+| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is looked up through `upstream.swift`'s `/identifiers` before each sweep and applies to every identity listed for it; until every such key resolves, Swift artifacts aren't swept, so a rule that keeps a package is never skipped. |
 
 An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind.
 
@@ -1182,7 +1182,12 @@ Names and versions are compared the way each ecosystem treats them:
   though download URLs escape uppercase letters (`v1.0.0-!r!c1`).
 - A Swift entry names the registry scope and package, ignoring case:
   `pkg:swift/apple/swift-nio@2.62.0`. The stored form,
-  `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too.
+  `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too, and so does
+  the source repository, as SBOM tools and GHSA advisories write it:
+  `pkg:swift/github.com/apple/swift-nio@2.62.0`. The proxy looks a source
+  repository up through `upstream.swift`'s `/identifiers` when `serve` or
+  `proxy mirror` starts and denies every identity listed for it; neither runs
+  when one can't be resolved.
 - An OCI entry names the repository as it is pulled through the proxy and a
   digest or tag: `pkg:oci/library/alpine@sha256:<digest>` refuses the blob or
   manifest with that digest, including a manifest a tag resolves to, and

internal/config/retention.go

@@ -48,6 +48,7 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error
 	rules := retention.Rules{
 		Ecosystems: map[string]time.Duration{},
 		Packages:   map[string]time.Duration{},
+		Sources:    map[string]map[string]time.Duration{},
 	}
 
 	var err error
@@ -84,14 +85,21 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error
 
 	for _, key := range sortedKeys(r.Packages) {
 		field := fmt.Sprintf("storage.retention.packages[%q]", key)
-		canonical, err := retentionPackageKey(reg, key)
+		canonical, sourceOf, err := retentionPackageKey(reg, key)
 		if err != nil {
 			return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err)
 		}
 		d, err := parseRetentionDuration(field, r.Packages[key])
 		if err != nil {
 			return retention.Rules{}, err
 		}
+		if sourceOf != "" {
+			if rules.Sources[sourceOf] == nil {
+				rules.Sources[sourceOf] = map[string]time.Duration{}
+			}
+			rules.Sources[sourceOf][key] = d
+			continue
+		}
 		if prev, ok := rules.Packages[canonical]; ok && prev != d {
 			return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical)
 		}
@@ -102,38 +110,41 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error
 }
 
 // retentionPackageKey returns the stored package PURL a configured package
-// key stands for.
-func retentionPackageKey(reg *retention.Registry, key string) (string, error) {
+// key stands for, or, for a key that has to be resolved through the upstream
+// first, the ecosystem key it belongs to.
+func retentionPackageKey(reg *retention.Registry, key string) (canonical, sourceOf string, _ error) {
 	// A julia PURL needs a uuid qualifier to parse at all, and the julia
 	// handler stores none, so no key could ever match one of its packages.
 	if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") {
-		return "", fmt.Errorf("package rules are not supported for julia")
+		return "", "", fmt.Errorf("package rules are not supported for julia")
 	}
 	p, err := purl.Parse(key)
 	if err != nil {
-		return "", fmt.Errorf("not a valid package PURL: %w", err)
+		return "", "", fmt.Errorf("not a valid package PURL: %w", err)
 	}
 	if p.Version != "" {
-		return "", fmt.Errorf("package PURL must not carry a version")
+		return "", "", fmt.Errorf("package PURL must not carry a version")
 	}
 	// A rule names a whole package; qualifiers and subpaths would be
 	// dropped on the way to the stored PURL, so refuse them rather than
 	// let "?type=pom" quietly cover every file of the package.
 	if len(p.Qualifiers) > 0 || p.Subpath != "" {
-		return "", fmt.Errorf("package PURL must not carry qualifiers or a subpath")
+		return "", "", fmt.Errorf("package PURL must not carry qualifiers or a subpath")
+	}
+	if canonical, ok := reg.Canonical(p); ok {
+		return canonical, "", nil
 	}
-	canonical, ok := reg.Canonical(p)
-	if ok {
-		return canonical, nil
+	if ecosystem, ok := reg.Source(p); ok {
+		return "", ecosystem, nil
 	}
 	ecosystem, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type))
 	if !registered {
-		return "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type)
+		return "", "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type)
 	}
 	if spec, _ := reg.Lookup(ecosystem); spec.CanonicalPackage == nil {
-		return "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem)
+		return "", "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem)
 	}
-	return "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem)
+	return "", "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem)
 }
 
 func sortedKeys(m map[string]string) []string {

internal/handler/swift.go

@@ -31,10 +31,54 @@ const (
 // Source archives are stored under the registry identity,
 // pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write
 // the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored
-// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio
-// doesn't name a registry identity and is refused.
+// form. A source repository such as pkg:swift/github.com/apple/swift-nio
+// names no registry identity; it is resolved through the upstream's
+// /identifiers before it applies.
 func init() {
-	retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey})
+	retention.Default.Register(retention.Spec{
+		Key:              "swift",
+		CanonicalPackage: swiftRetentionKey,
+		SourcePackage:    func(p *purl.PURL) bool { return SwiftSourcePURL(p) != "" },
+	})
+}
+
+// SwiftSourceURL returns the repository URL of a pkg:swift name whose
+// namespace starts with a host, as in github.com/apple/swift-nio, or "" for
+// a name whose namespace is a registry scope.
+func SwiftSourceURL(name string) string {
+	if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") {
+		return "https://" + name
+	}
+	return ""
+}
+
+// SwiftSourcePURL returns the repository URL of a pkg:swift PURL that names
+// a package by its source repository, as SBOM tools and GHSA do, or "".
+func SwiftSourcePURL(p *purl.PURL) string {
+	if p == nil || p.Type != "swift" || p.Namespace == "" {
+		return ""
+	}
+	return SwiftSourceURL(p.Namespace + "/" + p.Name)
+}
+
+// ResolveSourcePURL returns p with a source repository namespace replaced by
+// each registry scope and name the upstream lists for it, and p itself when
+// it already names a registry identity. A policy for the repository applies
+// to all of them, since a client may use any.
+func (h *SwiftHandler) ResolveSourcePURL(ctx context.Context, p *purl.PURL) ([]*purl.PURL, error) {
+	source := SwiftSourcePURL(p)
+	if source == "" {
+		return []*purl.PURL{p}, nil
+	}
+	ids, err := h.ResolveIdentities(ctx, source)
+	if err != nil {
+		return nil, err
+	}
+	resolved := make([]*purl.PURL, 0, len(ids))
+	for _, id := range ids {
+		resolved = append(resolved, purl.New("swift", id.Scope, id.Name, p.Version, nil))
+	}
+	return resolved, nil
 }
 
 // swiftRetentionKey maps a package rule to the stored package PURL, refusing