Add registry redirected urls to registryRedirectHosts in egress allowlist derivation

.github/skills/add-egress-allowlist-domain/SKILL.md

@@ -20,7 +20,7 @@ Ask the requester for the host, the package ecosystem, and — if they have it 
 |---|---|---|
 | **Public, provider-controlled package infrastructure** | public registry, mirror, CDN, checksum/CRL endpoint, public VCS forge | **Add to the YAML.** Continue to step 2. |
 | **Private, internal, or org-specific registry** | `artifacts.acme-corp.internal`, `acme.jfrog.io`, a self-hosted Nexus/Artifactory | **Do not add.** Tell the user to declare it under `registries:` in their `dependabot.yml`. Those hosts are allowlisted per-job automatically by `internal/handlers/egress_dynamic_hosts.go`. Stop here. |
-| **Shared multi-tenant host where the tenant is in the URL path** | `dl.cloudsmith.io`, generic object-store download hosts | **Do not add.** The handler authorizes the **hostname only** — it never constrains path or method — so allowing the host grants every tenant's content to every job. Explain this and stop. |
+| **Shared multi-tenant host where the tenant is in the URL path** | `dl.cloudsmith.io`, generic object-store download hosts | **Do not add to the static sections.** The handler authorizes the **hostname only** — it never constrains path or method — so allowing the host grants every tenant's content to every job. If the host is the download target a configured registry *redirects* to, add it under `registry_redirect_derivations:` instead, where it is allowed only for jobs holding that registry's credential. Otherwise explain this and stop. |
 | **User-uploadable file hosting** | `downloads.sourceforge.net`, arbitrary release-file mirrors | **Do not add** without explicit maintainer sign-off. Flag it and ask. |
 | **Documentation, changelog, or homepage host** | project docs sites, blog domains | **Usually don't add.** These fail gracefully — `dependabot-core`'s metadata finder treats a non-200 as "no metadata", so the only loss is a missing changelog link in the PR body. Say so and ask whether it's worth it. |
 
@@ -107,8 +107,9 @@ Then choose the section:
 - `github_infra_domains` — GitHub/Dependabot infrastructure only. Don't add third-party hosts here.
 - `shared_registry_domains` — hosts genuinely used by more than one ecosystem.
 - `ecosystem_default_domains.<ecosystem>` — the normal case.
+- `registry_redirect_derivations` — **not** a static section: the host is allowed only for jobs whose credentials name the matching registry. Use it when a configured registry redirects downloads to a provider-owned storage host that step 1 ruled out of the static sections. Entries are `credential_host` (exact, or leading dot for any subdomain) plus `derived` (exact hosts only; globs are rejected at startup).
 
-Add the host to exactly one section. Every section is applied to every job, so listing it twice is redundant, not safer.
+Add the host to exactly one section. Every static section is applied to every job, so listing it twice is redundant, not safer.
 
 Keep entries in the existing grouping and ordering of that section, and add a brief comment saying what the host serves when it isn't self-evident.
 

internal/handlers/egress_allowlist_defaults.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path"
 	"slices"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -18,9 +19,10 @@ var egressDefaultsYAML []byte
 
 // egressDefaults is the parsed representation of egress_allowlist_defaults.yaml.
 type egressDefaults struct {
-	GithubInfraDomains      []string            `yaml:"github_infra_domains"`
-	SharedRegistryDomains   []string            `yaml:"shared_registry_domains"`
-	EcosystemDefaultDomains map[string][]string `yaml:"ecosystem_default_domains"`
+	GithubInfraDomains         []string                     `yaml:"github_infra_domains"`
+	SharedRegistryDomains      []string                     `yaml:"shared_registry_domains"`
+	EcosystemDefaultDomains    map[string][]string          `yaml:"ecosystem_default_domains"`
+	RegistryRedirectDerivation []registryRedirectDerivation `yaml:"registry_redirect_derivations"`
 }
 
 var (
@@ -57,6 +59,7 @@ func init() {
 	githubInfraDomains = defaults.GithubInfraDomains
 	sharedRegistryDomains = defaults.SharedRegistryDomains
 	ecosystemDefaultDomains = defaults.EcosystemDefaultDomains
+	registryRedirectDerivations = defaults.RegistryRedirectDerivation
 
 	seen := make(map[string]struct{})
 	for _, hosts := range ecosystemDefaultDomains {
@@ -75,6 +78,37 @@ func init() {
 	validateGlobDefaults(githubInfraDomains)
 	validateGlobDefaults(sharedRegistryDomains)
 	validateGlobDefaults(allEcosystemDomains)
+
+	if err := validateRegistryRedirectDerivations(registryRedirectDerivations); err != nil {
+		panic(fmt.Sprintf("invalid registry_redirect_derivations in egress_allowlist_defaults.yaml: %v", err))
+	}
+}
+
+// validateRegistryRedirectDerivations rejects derivations that dynamic-host
+// matching cannot honour. Derived hosts are matched exactly, so a glob or
+// leading-dot entry there would silently never match; a glob in credentialHost
+// would be compared literally and likewise never fire.
+func validateRegistryRedirectDerivations(derivations []registryRedirectDerivation) error {
+	for _, d := range derivations {
+		if d.CredentialHost == "" || d.CredentialHost == "." {
+			return fmt.Errorf("credential_host must not be empty")
+		}
+		if isGlobPattern(d.CredentialHost) {
+			return fmt.Errorf("credential_host %q must be an exact host or a leading-dot domain, not a glob", d.CredentialHost)
+		}
+		if len(d.Derived) == 0 {
+			return fmt.Errorf("credential_host %q has no derived hosts", d.CredentialHost)
+		}
+		for _, host := range d.Derived {
+			if host == "" {
+				return fmt.Errorf("credential_host %q has an empty derived host", d.CredentialHost)
+			}
+			if isGlobPattern(host) || strings.HasPrefix(host, ".") {
+				return fmt.Errorf("derived host %q (credential_host %q) must be an exact host: derived hosts are matched exactly", host, d.CredentialHost)
+			}
+		}
+	}
+	return nil
 }
 
 // validateGlobDefaults panics if any glob entry is not a valid path.Match

internal/handlers/egress_allowlist_defaults.yaml

@@ -151,6 +151,38 @@ shared_registry_domains:
   - mirrors.cloud.tencent.com
   - mirrors.huaweicloud.com
 
+# registry_redirect_derivations are storage hosts that a configured registry
+# 302-redirects package downloads to, but that appear in no credential field.
+# Unlike every other section here, these are NOT applied to every job: each is
+# allowed only for a job whose own credentials name the matching registry.
+#
+# Use this section when the target is a single bucket or CDN shared by all of a
+# provider's tenants, with the tenant in the URL path. Such a host must never go
+# into the sections above, where it would expose one tenant's content to every
+# job in the world (see add-egress-allowlist-domain).
+#
+#   credential_host: matches a job's credential host exactly, or, with a leading
+#                    dot, any subdomain of that domain but not the apex.
+#   derived:         exact hosts to allow for that job. Glob and leading-dot
+#                    forms are rejected at startup: dynamic hosts are matched
+#                    exactly so that a crafted credential cannot widen them.
+#
+# Every derived host is a constant. Targets whose name embeds a value taken from
+# the credential (e.g. the private ECR layer bucket, which interpolates the
+# region) stay in registryRedirectHosts in Go.
+registry_redirect_derivations:
+  # packagecloud.io redirects downloads to one CloudFront distribution it owns.
+  - credential_host: packagecloud.io
+    derived:
+      - d3fo0g5hm7lbuv.cloudfront.net
+  # Every Gemfury endpoint (pypi., npm., gem., ...) redirects downloads to
+  # pre-signed URLs on one Gemfury-owned bucket, reachable on both its dualstack
+  # and plain Transfer Acceleration endpoints.
+  - credential_host: .fury.io
+    derived:
+      - gemfury.s3-accelerate.dualstack.amazonaws.com
+      - gemfury.s3-accelerate.amazonaws.com
+
 # ecosystem_default_domains lists the public registry and CDN hosts each
 # Dependabot ecosystem needs. The map is kept keyed by ecosystem for provenance
 # and documentation, but the handler applies the UNION of all values to every

internal/handlers/egress_allowlist_test.go

@@ -988,6 +988,46 @@ func TestEgressDefaults_LoadedFromYAML(t *testing.T) {
 	assert.Contains(t, allEcosystemDomains, "pypi.org")
 }
 
+func TestEgressDefaults_RegistryRedirectDerivationsLoadedFromYAML(t *testing.T) {
+	require.NotEmpty(t, registryRedirectDerivations, "derivations loaded from YAML")
+
+	// The embedded defaults must satisfy the rules enforced at startup.
+	assert.NoError(t, validateRegistryRedirectDerivations(registryRedirectDerivations))
+
+	assert.Equal(t, []string{"d3fo0g5hm7lbuv.cloudfront.net"},
+		registryRedirectHosts([]string{"packagecloud.io"}))
+	assert.Equal(t,
+		[]string{
+			"gemfury.s3-accelerate.dualstack.amazonaws.com",
+			"gemfury.s3-accelerate.amazonaws.com",
+		},
+		registryRedirectHosts([]string{"pypi.fury.io"}))
+}
+
+func TestValidateRegistryRedirectDerivations_RejectsUnmatchableEntries(t *testing.T) {
+	// Derived hosts join the dynamic hosts, which are matched exactly, so a glob
+	// or leading-dot entry would silently never match.
+	cases := map[string]registryRedirectDerivation{
+		"empty credential host":     {CredentialHost: "", Derived: []string{"cdn.example.com"}},
+		"bare dot credential host":  {CredentialHost: ".", Derived: []string{"cdn.example.com"}},
+		"glob credential host":      {CredentialHost: "*.example.com", Derived: []string{"cdn.example.com"}},
+		"no derived hosts":          {CredentialHost: "example.com"},
+		"empty derived host":        {CredentialHost: "example.com", Derived: []string{""}},
+		"glob derived host":         {CredentialHost: "example.com", Derived: []string{"*.cdn.example.com"}},
+		"leading-dot derived host":  {CredentialHost: "example.com", Derived: []string{".cdn.example.com"}},
+		"glob in second derivation": {CredentialHost: "example.com", Derived: []string{"cdn.example.com", "cdn[.example.com"}},
+	}
+	for name, derivation := range cases {
+		assert.Errorf(t, validateRegistryRedirectDerivations([]registryRedirectDerivation{derivation}),
+			"must be rejected: %s", name)
+	}
+
+	assert.NoError(t, validateRegistryRedirectDerivations([]registryRedirectDerivation{
+		{CredentialHost: "example.com", Derived: []string{"cdn.example.com"}},
+		{CredentialHost: ".example.org", Derived: []string{"cdn1.example.org", "cdn2.example.org"}},
+	}))
+}
+
 // TestEgressDefaults_NoRedundantEntries pins the YAML source, not the computed
 // union. The union builder deduplicates, so a host listed twice in the file is
 // absorbed silently and TestEgressDefaults_LoadedFromYAML still passes. That

internal/handlers/egress_dynamic_hosts.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/dependabot/proxy/internal/config"
+	"github.com/dependabot/proxy/internal/helpers"
 )
 
 // ecrHostPattern matches the canonical private ECR registry host,
@@ -18,26 +19,56 @@ import (
 // here — it has no capture group, and its "*" spans dots.
 var ecrHostPattern = regexp.MustCompile(`^[0-9]{12}\.dkr\.ecr\.([a-z0-9-]+)\.amazonaws\.com$`)
 
+// registryRedirectDerivation maps a credential host to the fixed storage hosts
+// that registry redirects downloads to. The derived hosts are constants, so a
+// crafted credential cannot widen the destination.
+type registryRedirectDerivation struct {
+	// CredentialHost matches exactly, or, with a leading dot, any subdomain of
+	// that domain but not the apex.
+	CredentialHost string   `yaml:"credential_host"`
+	Derived        []string `yaml:"derived"`
+}
+
+// registryRedirectDerivations is loaded from the registry_redirect_derivations
+// section of egress_allowlist_defaults.yaml, which documents the rules for
+// adding one. Derivations whose target embeds a credential-derived value (ECR)
+// stay in registryRedirectHosts below.
+var registryRedirectDerivations []registryRedirectDerivation
+
 // registryRedirectHosts returns storage backends that a configured registry
 // redirects to on download but that appear in no credential field.
 //
-// Private ECR 307-redirects layer downloads to a per-region, AWS-owned S3
-// bucket. It is derived per job rather than globbed into the static defaults
-// because "prod-<anything>-starport-layer-bucket" is a claimable S3 name, so a
-// glob would hand every job an attacker-registrable destination.
+// Private ECR is the exception to registryRedirectDerivations: its layer bucket
+// embeds the region, so the host is interpolated from the job's own credential
+// rather than globbed into the static defaults, where
+// "prod-<anything>-starport-layer-bucket" would be an attacker-registrable name.
 func registryRedirectHosts(credHosts []string) []string {
 	var hosts []string
 	for _, h := range credHosts {
-		m := ecrHostPattern.FindStringSubmatch(h)
-		if m == nil {
-			continue
+		// hostFromValue lower-cases; drop the absolute-DNS trailing dot too.
+		h = strings.TrimSuffix(h, ".")
+
+		for _, derivation := range registryRedirectDerivations {
+			if derivation.matches(h) {
+				hosts = append(hosts, derivation.Derived...)
+			}
+		}
+
+		if m := ecrHostPattern.FindStringSubmatch(h); m != nil {
+			region := m[1]
+			hosts = append(hosts, fmt.Sprintf("prod-%s-starport-layer-bucket.s3.%s.amazonaws.com", region, region))
 		}
-		region := m[1]
-		hosts = append(hosts, fmt.Sprintf("prod-%s-starport-layer-bucket.s3.%s.amazonaws.com", region, region))
 	}
 	return hosts
 }
 
+func (d registryRedirectDerivation) matches(host string) bool {
+	if strings.HasPrefix(d.CredentialHost, ".") {
+		return strings.HasSuffix(host, d.CredentialHost)
+	}
+	return helpers.AreHostnamesEqual(host, d.CredentialHost)
+}
+
 // credentialHostKeys are the credential fields that carry a registry host or
 // URL. The host of each is added to the per-job allowlist so that the private
 // registries a job is configured to use are never treated as exfiltration.

internal/handlers/egress_dynamic_hosts_test.go

@@ -154,3 +154,137 @@ func TestRegistryRedirectHosts_NotAddedWithoutECRCredential(t *testing.T) {
 	assert.NotNil(t, egressResult(t, h, "https://prod-eu-west-1-starport-layer-bucket.s3.eu-west-1.amazonaws.com/loot"),
 		"layer bucket must not be allowed for a job with no ECR credential")
 }
+
+func TestRegistryRedirectHosts_PackagecloudDownloadCDNDerived(t *testing.T) {
+	// packagecloud.io 302-redirects package downloads to a CloudFront
+	// distribution that appears in no credential field.
+	creds := config.Credentials{
+		{"type": "python_index", "index-url": "https://packagecloud.io/acme/repo/pypi/simple"},
+	}
+	h := newEgressHandlerWithCreds(creds)
+
+	assert.Nil(t, egressResult(t, h, "https://packagecloud.io/acme/repo/pypi/simple/pkg/"),
+		"the configured packagecloud registry must be allowed")
+	assert.Nil(t, egressResult(t, h, "https://d3fo0g5hm7lbuv.cloudfront.net/1358/1665/blobs/pkg.whl?Expires=1&Signature=x"),
+		"the packagecloud download CDN must be allowed for a job with a packagecloud credential")
+
+	for _, blocked := range []string{
+		// Dynamic hosts are matched exactly, so no child or lookalike widens it.
+		"https://evil.d3fo0g5hm7lbuv.cloudfront.net/loot",
+		"https://d3fo0g5hm7lbuv.cloudfront.net.attacker.com/loot",
+		// The shared CloudFront namespace stays closed.
+		"https://cloudfront.net/loot",
+		"https://d1ii4ma7ymllif.cloudfront.net/loot",
+	} {
+		assert.NotNil(t, egressResult(t, h, blocked), "must remain blocked: "+blocked)
+	}
+}
+
+func TestRegistryRedirectHosts_PackagecloudMatchedExactly(t *testing.T) {
+	// Only packagecloud.io itself opens the CDN, not a lookalike.
+	assert.Equal(t,
+		[]string{"d3fo0g5hm7lbuv.cloudfront.net"},
+		registryRedirectHosts([]string{"packagecloud.io"}))
+	assert.Equal(t,
+		[]string{"d3fo0g5hm7lbuv.cloudfront.net"},
+		registryRedirectHosts([]string{"packagecloud.io."}),
+		"an absolute DNS name derives like its relative form")
+
+	for _, host := range []string{
+		"packagecloud.io.attacker.com",
+		"evil.packagecloud.io",
+		"packagecloud.com",
+		"notpackagecloud.io",
+	} {
+		assert.Empty(t, registryRedirectHosts([]string{host}), "must derive nothing from %q", host)
+	}
+}
+
+func TestRegistryRedirectHosts_PackagecloudCDNNotAllowedWithoutCredential(t *testing.T) {
+	h := newEgressHandlerWithCreds(config.Credentials{
+		{"type": "python_index", "index-url": "https://pypi.internal.example.com/simple"},
+	})
+	assert.NotNil(t, egressResult(t, h, "https://d3fo0g5hm7lbuv.cloudfront.net/loot"),
+		"download CDN must not be allowed for a job with no packagecloud credential")
+}
+
+func TestRegistryRedirectHosts_GemfuryStorageDerived(t *testing.T) {
+	// Gemfury endpoints redirect downloads to one Gemfury-owned bucket,
+	// reachable on both its dualstack and plain accelerate endpoints.
+	creds := config.Credentials{
+		{"type": "python_index", "index-url": "https://pypi.fury.io/acme/"},
+	}
+	h := newEgressHandlerWithCreds(creds)
+
+	assert.Nil(t, egressResult(t, h, "https://pypi.fury.io/acme/-/ver_x/pkg-1.0.0-py3-none-any.whl"),
+		"the Gemfury registry itself must be allowed")
+	for _, allowed := range []string{
+		"https://gemfury.s3-accelerate.dualstack.amazonaws.com/gems/x/pkg_whl?X-Amz-Signature=x",
+		"https://gemfury.s3-accelerate.amazonaws.com/gems/x/pkg_whl?X-Amz-Signature=x",
+	} {
+		assert.Nil(t, egressResult(t, h, allowed), "Gemfury storage must be allowed: "+allowed)
+	}
+
+	for _, blocked := range []string{
+		// Dynamic hosts are matched exactly, so no child or lookalike widens it.
+		"https://evil.gemfury.s3-accelerate.dualstack.amazonaws.com/loot",
+		"https://gemfuryx.s3-accelerate.dualstack.amazonaws.com/loot",
+		// Only the accelerate endpoints are opened.
+		"https://gemfury.s3.amazonaws.com/loot",
+		// The shared parent namespace stays closed.
+		"https://attacker.s3-accelerate.dualstack.amazonaws.com/loot",
+		"https://attacker.s3-accelerate.amazonaws.com/loot",
+	} {
+		assert.NotNil(t, egressResult(t, h, blocked), "must remain blocked: "+blocked)
+	}
+}
+
+func TestRegistryRedirectHosts_OnlyGemfuryHosts(t *testing.T) {
+	gemfuryStorage := []string{
+		"gemfury.s3-accelerate.dualstack.amazonaws.com",
+		"gemfury.s3-accelerate.amazonaws.com",
+	}
+
+	for _, host := range []string{
+		"pypi.fury.io",
+		"npm.fury.io",
+		"npm-proxy.fury.io",
+		"gem.fury.io",
+		"repo.fury.io",
+		// An absolute DNS name derives like its relative form.
+		"pypi.fury.io.",
+	} {
+		assert.Equal(t, gemfuryStorage, registryRedirectHosts([]string{host}),
+			"must derive the Gemfury storage hosts from %q", host)
+	}
+
+	for _, host := range []string{
+		"fury.io",               // apex is not a registry endpoint
+		"pypi.fury.io.evil.com", // suffix must be fury.io
+		"pypifury.io",
+		"evil.com",
+	} {
+		assert.Empty(t, registryRedirectHosts([]string{host}), "must derive nothing from %q", host)
+	}
+
+	// Several Gemfury credentials yield one entry per storage host.
+	assert.Equal(t,
+		append([]string{"pypi.fury.io", "npm.fury.io"}, gemfuryStorage...),
+		dynamicHosts(config.Credentials{
+			{"type": "python_index", "index-url": "https://pypi.fury.io/acme/"},
+			{"type": "npm_registry", "registry": "https://npm.fury.io/acme/"},
+		}))
+}
+
+func TestRegistryRedirectHosts_NotAddedWithoutGemfuryCredential(t *testing.T) {
+	h := newEgressHandlerWithCreds(config.Credentials{
+		{"type": "python_index", "index-url": "https://pypi.internal.example.com/simple"},
+	})
+	for _, blocked := range []string{
+		"https://gemfury.s3-accelerate.dualstack.amazonaws.com/gems/x/loot",
+		"https://gemfury.s3-accelerate.amazonaws.com/gems/x/loot",
+	} {
+		assert.NotNil(t, egressResult(t, h, blocked),
+			"Gemfury storage must not be allowed for a job with no Gemfury credential: "+blocked)
+	}
+}