diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 824f2e2c..258fba32 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -124,7 +124,6 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/memlimit" "github.com/git-pkgs/proxy/internal/mirror" @@ -552,15 +551,16 @@ func runMirror() { // Build proxy with the same upstream clients and authentication as serve proxy, images := server.NewMirrorProxy(cfg, db, store, logger) - proxy.Denylist, err = denylist.New(cfg.Denylist.Packages) - if err != nil { - _ = db.Close() - fmt.Fprintf(os.Stderr, "invalid denylist: %v\n", err) - os.Exit(1) //nolint:gocritic // db closed above - } proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + // As serve does, resolve Swift source repositories in the denylist through + // the upstream, and refuse to run when one can't be. + if err := server.ConfigureDenylist(cfg, proxy); err != nil { + _ = db.Close() + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) //nolint:gocritic // db closed above + } m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) switch s := source.(type) { diff --git a/cmd/proxy/mirror_swift_test.go b/cmd/proxy/mirror_swift_test.go new file mode 100644 index 00000000..28fa8bac --- /dev/null +++ b/cmd/proxy/mirror_swift_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +const swiftWidgetArchive = "acme widget archive" + +// swiftMirrorUpstream serves acme.widget 1.0.0 the way a Swift package +// registry does, with https://github.com/acme/widget as its source +// repository, and counts archive downloads and identifier lookups. The +// repository's identifiers are ids, or only acme.widget. +func swiftMirrorUpstream(t *testing.T, ids ...string) (srv *httptest.Server, downloads, lookups *atomic.Int64) { + t.Helper() + if len(ids) == 0 { + ids = []string{"acme.widget"} + } + identifiers, err := json.Marshal(map[string][]string{"identifiers": ids}) + if err != nil { + t.Fatal(err) + } + downloads, lookups = new(atomic.Int64), new(atomic.Int64) + sum := sha256.Sum256([]byte(swiftWidgetArchive)) + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Version", "1") + switch r.URL.Path { + case "/identifiers": + lookups.Add(1) + if r.URL.Query().Get("url") != "https://github.com/acme/widget" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(identifiers) + case "/acme/widget": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) + case "/acme/widget/1.0.0/Package.swift": + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9\n") + case "/acme/widget/1.0.0": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":"acme.widget","version":"1.0.0","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(sum[:])) + case "/acme/widget/1.0.0.zip": + downloads.Add(1) + w.Header().Set("Content-Type", "application/zip") + _, _ = io.WriteString(w, swiftWidgetArchive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, downloads, lookups +} + +// writeSwiftMirrorConfig writes a config using upstream as the Swift +// registry, denying the denied package PURLs. +func writeSwiftMirrorConfig(t *testing.T, dir, upstream string, denied ...string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " swift: " + upstream + "\n" + + " allow_loopback: true\n" + if len(denied) > 0 { + cfg += "denylist:\n packages:\n" + for _, p := range denied { + cfg += " - " + p + "\n" + } + } + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func cachedArtifacts(t *testing.T, dir string) int64 { + t.Helper() + db, err := database.Open(filepath.Join(dir, "proxy.db")) + if err != nil { + t.Fatalf("opening database: %v", err) + } + defer func() { _ = db.Close() }() + stats, err := db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + return stats.TotalArtifacts +} + +// Without a denylist the fake registry mirrors the release, so the tests +// below fail for the denylist and not for the setup. +func TestMirrorSwiftRelease(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL) + + output := runMirrorProcess(t, 0, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 1 { + t.Errorf("archive downloads = %d, want 1", n) + } + if n := cachedArtifacts(t, dir); n != 1 { + t.Errorf("cached artifacts = %d, want 1", n) + } +} + +// A denylist entry naming the package by its source repository blocks the +// mirror download of the registry identity it resolves to. +func TestMirrorDenylistSwiftSourceEntry(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } + if n := cachedArtifacts(t, dir); n != 0 { + t.Errorf("cached artifacts = %d, want 0", n) + } +} + +// The registry may list several identities for one repository. A source +// entry denies all of them, not only the first. +func TestMirrorDenylistSwiftSourceEntryEveryIdentity(t *testing.T) { + upstream, downloads, _ := swiftMirrorUpstream(t, "other.widget", "acme.widget") + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/widget@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "0 downloaded") || !strings.Contains(output, "1 failed") { + t.Errorf("output:\n%s", output) + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } +} + +// A source repository entry the registry can't resolve stops the mirror +// before it downloads anything, as it stops serve: left out, it would let +// the version through. +func TestMirrorDenylistUnresolvedSwiftSource(t *testing.T) { + upstream, downloads, lookups := swiftMirrorUpstream(t) + dir := t.TempDir() + configPath := writeSwiftMirrorConfig(t, dir, upstream.URL, "pkg:swift/github.com/acme/unknown@1.0.0") + + output := runMirrorProcess(t, 1, "-config", configPath, "pkg:swift/acme/widget@1.0.0") + + if !strings.Contains(output, "configuring denylist") || !strings.Contains(output, "github.com/acme/unknown") { + t.Errorf("output does not report the unresolved entry:\n%s", output) + } + if strings.Contains(output, "Mirror complete") { + t.Errorf("mirror ran:\n%s", output) + } + if lookups.Load() == 0 { + t.Error("the entry was not looked up") + } + if n := downloads.Load(); n != 0 { + t.Errorf("archive downloads = %d, want 0", n) + } +} diff --git a/docs/configuration.md b/docs/configuration.md index e1a4f9ef..e6c023a5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -150,7 +150,7 @@ The package rule covers every release and asset in that repository, but each cac | `pub` | yes | `pkg:pub/http`, the package name as published. | | `pypi` | yes | `pkg:pypi/requests`; case and single `_`/`-` don't matter. Dots and repeated separators are kept as written, so a project like `zope.interface` can be stored as `zope.interface` and `zope-interface`; add a rule for each. | | `rpm` | yes | `pkg:rpm/curl`, the name as packaged, case included (`pkg:rpm/NetworkManager`). A vendor namespace such as `pkg:rpm/fedora/curl` is accepted and dropped. Fedora and EPEL update repositories keep only the latest build of a package, so an evicted older one may not be fetchable again. | -| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is refused, since it doesn't name a registry identity. | +| `swift` | yes | `pkg:swift/apple/swift-nio`, with the registry scope as the namespace, or the stored form `pkg:generic/swift-registry/apple.swift-nio`; case doesn't matter. A source repository key like `pkg:swift/github.com/apple/swift-nio` is looked up through `upstream.swift`'s `/identifiers` before each sweep and applies to every identity listed for it; until every such key resolves, Swift artifacts aren't swept, so a rule that keeps a package is never skipped. | An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. Running `proxy mirror` again over a package that is still cached counts as a download and restarts its clock, so packages a scheduled mirror job keeps listing never age out. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind. @@ -1182,7 +1182,12 @@ Names and versions are compared the way each ecosystem treats them: though download URLs escape uppercase letters (`v1.0.0-!r!c1`). - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, - `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. + `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too, and so does + the source repository, as SBOM tools and GHSA advisories write it: + `pkg:swift/github.com/apple/swift-nio@2.62.0`. The proxy looks a source + repository up through `upstream.swift`'s `/identifiers` when `serve` or + `proxy mirror` starts and denies every identity listed for it; neither runs + when one can't be resolved. - An OCI entry names the repository as it is pulled through the proxy and a digest or tag: `pkg:oci/library/alpine@sha256:` refuses the blob or manifest with that digest, including a manifest a tag resolves to, and diff --git a/internal/config/retention.go b/internal/config/retention.go index 3467166f..9493d280 100644 --- a/internal/config/retention.go +++ b/internal/config/retention.go @@ -48,6 +48,7 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error rules := retention.Rules{ Ecosystems: map[string]time.Duration{}, Packages: map[string]time.Duration{}, + Sources: map[string]map[string]time.Duration{}, } var err error @@ -84,7 +85,7 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error for _, key := range sortedKeys(r.Packages) { field := fmt.Sprintf("storage.retention.packages[%q]", key) - canonical, err := retentionPackageKey(reg, key) + canonical, sourceOf, err := retentionPackageKey(reg, key) if err != nil { return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err) } @@ -92,6 +93,13 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error if err != nil { return retention.Rules{}, err } + if sourceOf != "" { + if rules.Sources[sourceOf] == nil { + rules.Sources[sourceOf] = map[string]time.Duration{} + } + rules.Sources[sourceOf][key] = d + continue + } if prev, ok := rules.Packages[canonical]; ok && prev != d { return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical) } @@ -102,38 +110,41 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error } // retentionPackageKey returns the stored package PURL a configured package -// key stands for. -func retentionPackageKey(reg *retention.Registry, key string) (string, error) { +// key stands for, or, for a key that has to be resolved through the upstream +// first, the ecosystem key it belongs to. +func retentionPackageKey(reg *retention.Registry, key string) (canonical, sourceOf string, _ error) { // A julia PURL needs a uuid qualifier to parse at all, and the julia // handler stores none, so no key could ever match one of its packages. if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") { - return "", fmt.Errorf("package rules are not supported for julia") + return "", "", fmt.Errorf("package rules are not supported for julia") } p, err := purl.Parse(key) if err != nil { - return "", fmt.Errorf("not a valid package PURL: %w", err) + return "", "", fmt.Errorf("not a valid package PURL: %w", err) } if p.Version != "" { - return "", fmt.Errorf("package PURL must not carry a version") + return "", "", fmt.Errorf("package PURL must not carry a version") } // A rule names a whole package; qualifiers and subpaths would be // dropped on the way to the stored PURL, so refuse them rather than // let "?type=pom" quietly cover every file of the package. if len(p.Qualifiers) > 0 || p.Subpath != "" { - return "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") + return "", "", fmt.Errorf("package PURL must not carry qualifiers or a subpath") + } + if canonical, ok := reg.Canonical(p); ok { + return canonical, "", nil } - canonical, ok := reg.Canonical(p) - if ok { - return canonical, nil + if ecosystem, ok := reg.Source(p); ok { + return "", ecosystem, nil } ecosystem, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type)) if !registered { - return "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) + return "", "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type) } if spec, _ := reg.Lookup(ecosystem); spec.CanonicalPackage == nil { - return "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) + return "", "", fmt.Errorf("package rules are not supported for %s; use storage.retention.ecosystems.%s", ecosystem, ecosystem) } - return "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) + return "", "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", ecosystem) } func sortedKeys(m map[string]string) []string { diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 7de0cd12..37f666e2 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -31,10 +31,54 @@ const ( // Source archives are stored under the registry identity, // pkg:generic/swift-registry/{scope}.{name}, lowercased. Package rules write // the scope as the namespace, pkg:swift/apple/swift-nio, or use the stored -// form. A source repository URL such as pkg:swift/github.com/apple/swift-nio -// doesn't name a registry identity and is refused. +// form. A source repository such as pkg:swift/github.com/apple/swift-nio +// names no registry identity; it is resolved through the upstream's +// /identifiers before it applies. func init() { - retention.Default.Register(retention.Spec{Key: "swift", CanonicalPackage: swiftRetentionKey}) + retention.Default.Register(retention.Spec{ + Key: "swift", + CanonicalPackage: swiftRetentionKey, + SourcePackage: func(p *purl.PURL) bool { return SwiftSourcePURL(p) != "" }, + }) +} + +// SwiftSourceURL returns the repository URL of a pkg:swift name whose +// namespace starts with a host, as in github.com/apple/swift-nio, or "" for +// a name whose namespace is a registry scope. +func SwiftSourceURL(name string) string { + if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { + return "https://" + name + } + return "" +} + +// SwiftSourcePURL returns the repository URL of a pkg:swift PURL that names +// a package by its source repository, as SBOM tools and GHSA do, or "". +func SwiftSourcePURL(p *purl.PURL) string { + if p == nil || p.Type != "swift" || p.Namespace == "" { + return "" + } + return SwiftSourceURL(p.Namespace + "/" + p.Name) +} + +// ResolveSourcePURL returns p with a source repository namespace replaced by +// each registry scope and name the upstream lists for it, and p itself when +// it already names a registry identity. A policy for the repository applies +// to all of them, since a client may use any. +func (h *SwiftHandler) ResolveSourcePURL(ctx context.Context, p *purl.PURL) ([]*purl.PURL, error) { + source := SwiftSourcePURL(p) + if source == "" { + return []*purl.PURL{p}, nil + } + ids, err := h.ResolveIdentities(ctx, source) + if err != nil { + return nil, err + } + resolved := make([]*purl.PURL, 0, len(ids)) + for _, id := range ids { + resolved = append(resolved, purl.New("swift", id.Scope, id.Name, p.Version, nil)) + } + return resolved, nil } // swiftRetentionKey maps a package rule to the stored package PURL, refusing diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go index d3944e3b..5326c1d3 100644 --- a/internal/handler/swift_mirror.go +++ b/internal/handler/swift_mirror.go @@ -121,37 +121,67 @@ func swiftLinkHasRel(params, rel string) bool { return false } -// ResolveIdentity returns the registry scope and name the upstream lists for -// the source repository at sourceURL. The answer is cached as the identifiers -// route caches it. +// SwiftIdentity is a package's identity in a Swift registry, scope.name. +type SwiftIdentity struct { + Scope, Name string +} + +// ResolveIdentity returns the first registry identity the upstream lists for +// the source repository at sourceURL, the one a download uses. func (h *SwiftHandler) ResolveIdentity(ctx context.Context, sourceURL string) (scope, name string, _ error) { + ids, err := h.ResolveIdentities(ctx, sourceURL) + if err != nil { + return "", "", err + } + return ids[0].Scope, ids[0].Name, nil +} + +// ResolveIdentities returns every registry identity the upstream lists for +// the source repository at sourceURL; the registry protocol allows several. +// The answer is cached as the identifiers route caches it. +func (h *SwiftHandler) ResolveIdentities(ctx context.Context, sourceURL string) ([]SwiftIdentity, error) { query := url.Values{"url": {sourceURL}}.Encode() body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "swift", swiftMetadataCacheKey("identifiers", query), h.upstreamURL+"/identifiers?"+query, swiftAcceptJSON) if errors.Is(err, ErrUpstreamNotFound) { - return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) } if err != nil { - return "", "", err + return nil, err } - return SwiftIdentityOf(body, sourceURL) + return SwiftIdentitiesOf(body, sourceURL) } // SwiftIdentityOf returns the scope and name of the first identity in body, // a registry's answer to an identifiers lookup for sourceURL. func SwiftIdentityOf(body []byte, sourceURL string) (scope, name string, _ error) { + ids, err := SwiftIdentitiesOf(body, sourceURL) + if err != nil { + return "", "", err + } + return ids[0].Scope, ids[0].Name, nil +} + +// SwiftIdentitiesOf returns every identity in body, a registry's answer to +// an identifiers lookup for sourceURL. One invalid identity fails the whole +// answer, so a policy never covers only some of them. +func SwiftIdentitiesOf(body []byte, sourceURL string) ([]SwiftIdentity, error) { var answer struct { Identifiers []string `json:"identifiers"` } if err := json.Unmarshal(body, &answer); err != nil { - return "", "", fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) + return nil, fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) } if len(answer.Identifiers) == 0 { - return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + return nil, fmt.Errorf("the Swift registry has no package for %s", sourceURL) } - scope, name, ok := strings.Cut(answer.Identifiers[0], ".") - if !ok || scope == "" || name == "" { - return "", "", fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, answer.Identifiers[0]) + ids := make([]SwiftIdentity, 0, len(answer.Identifiers)) + for _, identifier := range answer.Identifiers { + scope, name, ok := strings.Cut(identifier, ".") + if !ok || scope == "" || name == "" { + return nil, fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, identifier) + } + ids = append(ids, SwiftIdentity{Scope: scope, Name: name}) } - return scope, name, nil + return ids, nil } diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go index 4616c8f2..153bb10d 100644 --- a/internal/mirror/swift.go +++ b/internal/mirror/swift.go @@ -19,7 +19,7 @@ func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *pr } h := handler.NewSwiftHandler(m.proxy, "", upstream) var scope, name string - if source := swiftSourceURL(pv.Name); source != "" { + if source := handler.SwiftSourceURL(pv.Name); source != "" { scope, name, err = h.ResolveIdentity(ctx, source) } else { scope, name, err = swiftScopeAndName(pv.Name) @@ -45,16 +45,6 @@ func (m *Mirror) registryURL(purlType, name string) (string, error) { return "", nil } -// swiftSourceURL returns the repository URL of a pkg:swift name whose -// namespace starts with a host, as in pkg:swift/github.com/apple/swift-nio, -// or "" for a name whose namespace is a registry scope. -func swiftSourceURL(name string) string { - if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { - return "https://" + name - } - return "" -} - // swiftScopeAndName splits the name of a pkg:swift PURL whose namespace is // a registry scope, as in pkg:swift/apple/swift-nio for the identity // apple.swift-nio. diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 04353b47..0a7e1ba6 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -145,7 +145,7 @@ func (r RegistryVersions) client() *registries.Client { // swiftIdentity returns the registry identity, scope.name, of the name of a // pkg:swift PURL, looking a source repository up on the registry at baseURL. func swiftIdentity(ctx context.Context, client *registries.Client, baseURL, name string) (string, error) { - source := swiftSourceURL(name) + source := handler.SwiftSourceURL(name) if source == "" { scope, pkg, err := swiftScopeAndName(name) if err != nil { diff --git a/internal/retention/retention.go b/internal/retention/retention.go index a8ff8359..4394a7d1 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -46,6 +46,13 @@ type Spec struct { // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) + + // SourcePackage reports a package PURL that names the package by where + // its source lives, such as a Swift package by its repository. It has to + // be resolved through the upstream before it can be compared, so the + // config keeps it in Rules.Sources until then. Nil means the ecosystem + // has no such form. + SourcePackage func(p *purl.PURL) bool } // dbEcosystems returns every packages.ecosystem value that belongs to the @@ -165,6 +172,17 @@ func (r *Registry) Canonical(p *purl.PURL) (string, bool) { return "", false } +// Source returns the ecosystem key whose SourcePackage claims p. +func (r *Registry) Source(p *purl.PURL) (string, bool) { + for _, key := range r.Keys() { + s, _ := r.Lookup(key) + if s.SourcePackage != nil && s.SourcePackage(p) { + return key, true + } + } + return "", false +} + // Rules are resolved retention durations. A duration of zero means an // artifact is never evicted by age. type Rules struct { @@ -175,6 +193,40 @@ type Rules struct { Ecosystems map[string]time.Duration // Packages holds rules keyed by the package PURL stored in the cache. Packages map[string]time.Duration + // Sources holds package rules not resolved to a stored package PURL + // yet, by ecosystem key and then by the configured PURL (see + // Spec.SourcePackage). An ecosystem with any is not swept, since one of + // them may keep an artifact the sweep would otherwise clear. + Sources map[string]map[string]time.Duration +} + +// Pending reports whether the ecosystem key has package rules still to be +// resolved, and must not be swept. +func (r Rules) Pending(key string) bool { + return len(r.Sources[key]) > 0 +} + +// Resolve moves the rule for source in ecosystem key to every stored package +// PURL it resolved to. A different rule already set for any of them is a +// conflict, and the source stays pending with none of them set. +func (r Rules) Resolve(key, source string, canonical []string) error { + d, ok := r.Sources[key][source] + if !ok { + return nil + } + if len(canonical) == 0 { + return fmt.Errorf("%s resolves to no stored package", source) + } + for _, c := range canonical { + if prev, set := r.Packages[c]; set && prev != d { + return fmt.Errorf("%s resolves to %s, which another rule sets differently", source, c) + } + } + for _, c := range canonical { + r.Packages[c] = d + } + delete(r.Sources[key], source) + return nil } // For returns the retention for an artifact of the given ecosystem key and @@ -210,5 +262,10 @@ func (r Rules) MinPositive() time.Duration { for _, d := range r.Packages { consider(d) } + for _, sources := range r.Sources { + for _, d := range sources { + consider(d) + } + } return minimum } diff --git a/internal/retention/retention_test.go b/internal/retention/retention_test.go index 171fd709..50363752 100644 --- a/internal/retention/retention_test.go +++ b/internal/retention/retention_test.go @@ -192,3 +192,53 @@ func TestRulesMinPositive(t *testing.T) { t.Errorf("MinPositive() = %v, want 2h", got) } } + +func TestRulesResolveSources(t *testing.T) { + rules := Rules{ + Packages: map[string]time.Duration{"pkg:x/kept": 0}, + Sources: map[string]map[string]time.Duration{"swift": { + "pkg:swift/github.com/a/one": 0, + "pkg:swift/github.com/a/two": time.Hour, + }}, + } + if got := rules.MinPositive(); got != time.Hour { + t.Errorf("MinPositive = %v, want the pending rule's hour", got) + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/kept"}); err == nil { + t.Error("a rule resolving to a package another rule sets differently was accepted") + } + if !rules.Pending("swift") { + t.Fatal("conflicting rule no longer pending") + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/kept"}); err != nil { + t.Errorf("same duration for the same package: %v", err) + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/two", []string{"pkg:x/two"}); err != nil { + t.Fatal(err) + } + if rules.Pending("swift") || rules.Packages["pkg:x/two"] != time.Hour { + t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) + } +} + +// A source that resolves to several stored packages sets the rule for all +// of them, or, on a conflict with any, for none. +func TestRulesResolveSeveralPackages(t *testing.T) { + rules := Rules{ + Packages: map[string]time.Duration{"pkg:x/b": time.Hour}, + Sources: map[string]map[string]time.Duration{"swift": {"pkg:swift/github.com/a/one": 0}}, + } + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err == nil { + t.Error("a rule conflicting for one of its packages was accepted") + } + if _, set := rules.Packages["pkg:x/a"]; set || !rules.Pending("swift") { + t.Fatalf("conflicting rule partly applied: sources %v, packages %v", rules.Sources, rules.Packages) + } + delete(rules.Packages, "pkg:x/b") + if err := rules.Resolve("swift", "pkg:swift/github.com/a/one", []string{"pkg:x/a", "pkg:x/b"}); err != nil { + t.Fatal(err) + } + if rules.Pending("swift") || len(rules.Packages) != 2 { + t.Errorf("after resolving: sources %v, packages %v", rules.Sources, rules.Packages) + } +} diff --git a/internal/server/retention.go b/internal/server/retention.go index 520b4cc2..ecd88def 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -19,7 +19,7 @@ const retentionWindow = 5000 // follow over several intervals. A variable so tests can lower it. var retentionMaxClears = 10000 -func (s *Server) startRetentionLoop(ctx context.Context) { +func (s *Server) startRetentionLoop(ctx context.Context, resolvers map[string]sourceResolver) { reg := retention.Default rules, err := s.cfg.RetentionRules(reg) if err != nil { @@ -47,14 +47,17 @@ func (s *Server) startRetentionLoop(ctx context.Context) { ticker := time.NewTicker(s.cfg.ParseRetentionSweepInterval()) defer ticker.Stop() - s.sweepRetention(ctx, reg, rules) - + sweep := func() { + resolveRetentionSources(ctx, s.logger, resolvers, rules) + s.sweepRetention(ctx, reg, rules) + } + sweep() for { select { case <-ctx.Done(): return case <-ticker.C: - s.sweepRetention(ctx, reg, rules) + sweep() } } } @@ -147,7 +150,7 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registry, rules retention.Rules, now time.Time, c database.RetentionCandidate) bool { key, ok := reg.KeyForDBEcosystem(c.Ecosystem) - if !ok { + if !ok || rules.Pending(key) { return false } d := rules.For(key, c.PackagePURL) diff --git a/internal/server/retention_swift_test.go b/internal/server/retention_swift_test.go index 441320a4..3788d4bf 100644 --- a/internal/server/retention_swift_test.go +++ b/internal/server/retention_swift_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -15,6 +16,7 @@ import ( "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/registries/fetch" @@ -154,18 +156,144 @@ func TestSwiftRetentionEvictsAndRefetches(t *testing.T) { } } -// A source repository key parses as a swift PURL but names no registry -// identity, so it is refused at validation. A scope/name key is accepted. -func TestSwiftRetentionRefusesRepositoryKeys(t *testing.T) { +// A source repository key is accepted at validation and kept pending until +// it is resolved through the upstream. A scope/name key resolves offline. +func TestSwiftRetentionSourceKeysPending(t *testing.T) { cfg := config.Default() - cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/apple/swift-nio": "0"} - if _, err := cfg.RetentionRules(retention.Default); err != nil { - t.Fatalf("RetentionRules(pkg:swift/apple/swift-nio): %v", err) + cfg.Storage.Retention.Packages = map[string]string{ + "pkg:swift/apple/swift-nio": "0", + "pkg:swift/github.com/apple/swift-log": "0", } + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatalf("RetentionRules: %v", err) + } + if _, ok := rules.Packages["pkg:generic/swift-registry/apple.swift-nio"]; !ok { + t.Errorf("scope/name key not resolved offline: %v", rules.Packages) + } + if _, ok := rules.Sources["swift"]["pkg:swift/github.com/apple/swift-log"]; !ok || !rules.Pending("swift") { + t.Errorf("source key not pending: %v", rules.Sources) + } +} + +// swiftIdentifiers answers /identifiers for the swift-nio and swift-log +// repositories while up is set, and fails otherwise. +func swiftIdentifiers(up *atomic.Bool, next http.Handler) http.Handler { + return swiftIdentifiersFor(up, map[string][]string{ + "https://github.com/apple/swift-nio": {"apple.swift-nio"}, + "https://github.com/apple/swift-log": {"apple.swift-log"}, + }, next) +} + +// swiftIdentifiersFor answers /identifiers from ids, by repository URL, while +// up is set, and fails otherwise. +func swiftIdentifiersFor(up *atomic.Bool, ids map[string][]string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/identifiers" { + next.ServeHTTP(w, r) + return + } + list, ok := ids[r.URL.Query().Get("url")] + switch { + case !up.Load(): + http.Error(w, "down", http.StatusBadGateway) + case !ok: + http.NotFound(w, r) + default: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string][]string{"identifiers": list}) + } + }) +} + +// A never-evict rule keyed by the source repository keeps Swift from being +// swept while the upstream can't resolve it, and applies once it can. +func TestSwiftRetentionSourceRuleDefersSweep(t *testing.T) { + archives, _ := swiftRetentionUpstream(t) + var up atomic.Bool + identifiers := httptest.NewServer(swiftIdentifiers(&up, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + srv, db := newSwiftRetentionProxy(t, archives) + const nio, log = "swift-nio-2.65.0.zip", "swift-log-1.5.4.zip" + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + swiftGet(t, srv.URL+"/apple/swift-log/1.5.4.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + swiftBackdate(t, db, log, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} - _, err := cfg.RetentionRules(retention.Default) - if err == nil || !strings.Contains(err.Error(), "does not match how swift packages are stored") { - t.Errorf("RetentionRules(pkg:swift/github.com/apple/swift-nio) error = %v, want the key refused", err) + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatal(err) + } + proxy := handler.NewProxy(db, nil, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + resolvers := map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL), retention.Default), + } + + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid || !swiftArtifact(t, db, log).StoragePath.Valid { + t.Fatal("swift swept while a package rule was unresolved") + } + + up.Store(true) + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + if rules.Pending("swift") { + t.Fatalf("rule still pending once the upstream answers: %v", rules.Sources) + } + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Error("swift-nio evicted despite its never-evict rule") + } + if swiftArtifact(t, db, log).StoragePath.Valid { + t.Error("swift-log kept past the swift rule") + } +} + +// Denylist entries naming a source repository are resolved to the registry +// identity with their version; one the upstream can't resolve is an error. +func TestResolveDenylistSources(t *testing.T) { + var up atomic.Bool + up.Store(true) + identifiers := httptest.NewServer(swiftIdentifiers(&up, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + db, store := setupEvictionTest(t) + proxy := handler.NewProxy(db, store, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + h := handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL) + + got, err := resolveDenylistSources(context.Background(), h, []string{ + "pkg:npm/left-pad@1.3.0", + "pkg:swift/github.com/apple/swift-nio@2.62.0", + "pkg:swift/apple/swift-log@1.5.4", + }) + if err != nil { + t.Fatal(err) + } + want := []string{"pkg:npm/left-pad@1.3.0", "pkg:swift/apple/swift-nio@2.62.0", "pkg:swift/apple/swift-log@1.5.4"} + if fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("resolved %v, want %v", got, want) + } + policy, err := denylist.New(got) + if err != nil { + t.Fatal(err) + } + if !policy.Denied("pkg:generic/swift-registry/apple.swift-nio@2.62.0") { + t.Error("the resolved entry doesn't deny the stored archive") + } + + for _, entry := range []string{"pkg:swift/github.com/unknown/repo@1.0.0"} { + if _, err := resolveDenylistSources(context.Background(), h, []string{entry}); err == nil { + t.Errorf("%s: resolved a repository the upstream doesn't know", entry) + } + } + up.Store(false) + if _, err := resolveDenylistSources(context.Background(), h, []string{"pkg:swift/github.com/apple/swift-log@1.5.4"}); err == nil { + t.Error("resolved with the upstream down") } } @@ -210,3 +338,40 @@ func TestSwiftRetentionPackageRules(t *testing.T) { } } } + +// The registry may list several identities for one repository. A source rule +// covers all of them, so the stored archive is kept even when its identity +// is not the first one listed. +func TestSwiftRetentionSourceRuleCoversEveryIdentity(t *testing.T) { + archives, _ := swiftRetentionUpstream(t) + var up atomic.Bool + up.Store(true) + identifiers := httptest.NewServer(swiftIdentifiersFor(&up, map[string][]string{ + "https://github.com/apple/swift-nio": {"other.swift-nio", "apple.swift-nio"}, + }, http.NotFoundHandler())) + t.Cleanup(identifiers.Close) + srv, db := newSwiftRetentionProxy(t, archives) + const nio = "swift-nio-2.65.0.zip" + swiftGet(t, srv.URL+"/apple/swift-nio/2.65.0.zip") + old := time.Now().Add(-30 * retentionDay) + swiftBackdate(t, db, nio, old, old) + + cfg := config.Default() + cfg.Storage.Retention.Ecosystems = map[string]string{"swift": "1d"} + cfg.Storage.Retention.Packages = map[string]string{"pkg:swift/github.com/apple/swift-nio": "0"} + rules, err := cfg.RetentionRules(retention.Default) + if err != nil { + t.Fatal(err) + } + proxy := handler.NewProxy(db, nil, nil, fetch.NewResolver(), discardLogger()) + proxy.HTTPClient = identifiers.Client() + resolvers := map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, "http://proxy.test", identifiers.URL), retention.Default), + } + + resolveRetentionSources(context.Background(), discardLogger(), resolvers, rules) + sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now()) + if !swiftArtifact(t, db, nio).StoragePath.Valid { + t.Error("apple.swift-nio evicted although its repository has a never-evict rule") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68b..69363ca8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -75,7 +75,6 @@ import ( "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" - "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/gitcache" "github.com/git-pkgs/proxy/internal/handler" @@ -259,11 +258,6 @@ func (s *Server) serve(listener net.Listener) error { if s.cooldown, err = newCooldownView(s.cfg.Cooldown, cooldownPolicy); err != nil { return err } - policy, err := denylist.New(s.cfg.Denylist.Packages) - if err != nil { - return fmt.Errorf("configuring denylist: %w", err) - } - proxy.Denylist = policy scanGroup, err := configureScanning(proxy, s.cfg.Scanning, s.cfg.BaseURL, s.logger) if err != nil { return fmt.Errorf("configuring scanners: %w", err) @@ -282,6 +276,9 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts + if err := ConfigureDenylist(s.cfg, proxy); err != nil { + return err + } if len(s.cfg.Upstream.Git) > 0 { upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) @@ -429,7 +426,8 @@ func (s *Server) serve(listener net.Listener) error { s.background.Go(func() { s.updateCacheStatsMetrics(bgCtx) }) s.background.Go(func() { backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) }) s.background.Go(func() { s.startEvictionLoop(bgCtx) }) - s.background.Go(func() { s.startRetentionLoop(bgCtx) }) + retentionResolvers := s.retentionResolvers(proxy) + s.background.Go(func() { s.startRetentionLoop(bgCtx, retentionResolvers) }) s.background.Go(func() { s.startReclaimLoop(bgCtx) }) if listener != nil { diff --git a/internal/server/swift_sources.go b/internal/server/swift_sources.go new file mode 100644 index 00000000..b944e6a0 --- /dev/null +++ b/internal/server/swift_sources.go @@ -0,0 +1,136 @@ +package server + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" +) + +// A Swift package can be configured by its source repository, as SBOM tools +// and GHSA name it (pkg:swift/github.com/apple/swift-nio), as well as by its +// registry identity. The proxy stores and compares registry identities, so +// source repositories are resolved through the upstream's /identifiers: for +// the denylist at startup, which fails when one can't be, and for retention +// before each sweep, which leaves Swift unswept until all are. + +// sourceResolveTimeout bounds resolving the denylist's source repositories at +// startup. +const sourceResolveTimeout = time.Minute + +// sourceResolver resolves a package PURL that names a package by its source +// to the stored package PURLs retention compares. +type sourceResolver func(ctx context.Context, p *purl.PURL) ([]string, error) + +// swiftRetentionResolver resolves Swift source repositories through h, to +// every identity the upstream lists. +func swiftRetentionResolver(h *handler.SwiftHandler, reg *retention.Registry) sourceResolver { + return func(ctx context.Context, p *purl.PURL) ([]string, error) { + resolved, err := h.ResolveSourcePURL(ctx, p) + if err != nil { + return nil, err + } + canonical := make([]string, 0, len(resolved)) + for _, r := range resolved { + c, ok := reg.Canonical(r) + if !ok { + return nil, fmt.Errorf("%s resolved to %s, which names no stored package", p, r) + } + canonical = append(canonical, c) + } + return canonical, nil + } +} + +// retentionResolvers returns how the retention loop resolves package rules +// that name a package by its source, by ecosystem key. +func (s *Server) retentionResolvers(proxy *handler.Proxy) map[string]sourceResolver { + return map[string]sourceResolver{ + "swift": swiftRetentionResolver(handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift), retention.Default), + } +} + +// resolveDenylistSources returns entries with each Swift source repository +// replaced by every registry identity the upstream lists for it, keeping the +// version. An entry that can't be resolved is an error: left out, it would +// let the version through. +func resolveDenylistSources(ctx context.Context, h *handler.SwiftHandler, entries []string) ([]string, error) { + resolved := make([]string, 0, len(entries)) + for _, entry := range entries { + p, err := purl.Parse(entry) + if err != nil || handler.SwiftSourcePURL(p) == "" { + resolved = append(resolved, entry) + continue + } + identities, err := h.ResolveSourcePURL(ctx, p) + if err != nil { + return nil, fmt.Errorf("resolving denylist package %q: %w", entry, err) + } + for _, identity := range identities { + resolved = append(resolved, identity.String()) + } + } + return resolved, nil +} + +// resolveRetentionSources resolves the package rules in rules.Sources it can, +// moving them to rules.Packages. A rule that fails stays pending, so its +// ecosystem isn't swept, and is tried again before the next sweep. +func resolveRetentionSources(ctx context.Context, logger *slog.Logger, resolvers map[string]sourceResolver, rules retention.Rules) { + for _, key := range slices.Sorted(maps.Keys(rules.Sources)) { + resolve := resolvers[key] + for _, source := range slices.Sorted(maps.Keys(rules.Sources[key])) { + if resolve == nil { + logger.Error("retention: no way to resolve package rule, not sweeping "+key, "package", source) + continue + } + p, err := purl.Parse(source) + if err != nil { + continue // config validation refuses these + } + canonical, err := resolve(ctx, p) + if err == nil { + err = rules.Resolve(key, source, canonical) + } + if err != nil { + logger.Warn("retention: could not resolve package rule, not sweeping "+key+" until it is", + "package", source, "error", err) + continue + } + logger.Info("retention: resolved package rule", "package", source, "stored", strings.Join(canonical, ", ")) + } + } +} + +// ConfigureDenylist sets proxy's denylist from cfg, resolving Swift source +// repositories through the upstream first. serve and proxy mirror call it +// once proxy can fetch metadata. An entry that can't be resolved is an +// error, so neither starts with a denylist that lets its versions through. +func ConfigureDenylist(cfg *config.Config, proxy *handler.Proxy) error { + entries := cfg.Denylist.Packages + // Report a malformed entry before asking the upstream about any. + if _, err := denylist.New(entries); err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), sourceResolveTimeout) + defer cancel() + entries, err := resolveDenylistSources(ctx, handler.NewSwiftHandler(proxy, cfg.BaseURL, cfg.Upstream.Swift), entries) + if err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + policy, err := denylist.New(entries) + if err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + proxy.Denylist = policy + return nil +}