From 5e3859cd48be10152f0da7401978af4a52e2b24d Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:02:26 -0300 Subject: [PATCH 1/2] feat(agentconfig): secret detection, redaction and digests Fourth layer of the agent remote-configuration stack (split from #465): secret detection by key name and by value (linear URL-password scan, token patterns, scheme-less DSNs), Redact/RedactDocument/ScrubSecretText, and Digest over the redacted canonical config. Co-Authored-By: Claude Opus 5.5 --- pkg/agentconfig/digest.go | 24 ++ pkg/agentconfig/document.go | 68 ++++++ pkg/agentconfig/redact.go | 155 ++++++++++++ pkg/agentconfig/redact_test.go | 181 ++++++++++++++ pkg/agentconfig/sensitive.go | 295 +++++++++++++++++++++++ pkg/agentconfig/sensitive_test.go | 377 ++++++++++++++++++++++++++++++ 6 files changed, 1100 insertions(+) create mode 100644 pkg/agentconfig/digest.go create mode 100644 pkg/agentconfig/document.go create mode 100644 pkg/agentconfig/redact.go create mode 100644 pkg/agentconfig/redact_test.go create mode 100644 pkg/agentconfig/sensitive.go create mode 100644 pkg/agentconfig/sensitive_test.go diff --git a/pkg/agentconfig/digest.go b/pkg/agentconfig/digest.go new file mode 100644 index 00000000..16c9257f --- /dev/null +++ b/pkg/agentconfig/digest.go @@ -0,0 +1,24 @@ +package agentconfig + +import ( + "crypto/sha256" + "encoding/hex" +) + +// DigestPrefix prefixes config digests. +const DigestPrefix = "sha256:" + +// Digest returns "sha256:" + hex(sha256(canonical(Redact(c, opts...) with API = nil))). +// Callers pass the SAME options they used to redact the reported effective config, so the +// digest matches the reported document (R55). Canonical JSON is described on CanonicalJSON. +func Digest(c Config, opts ...RedactOption) string { + r := Redact(c, opts...) + r.API = nil + raw, err := CanonicalJSON(r) + if err != nil { + // Redact normalizes every free-form value, so the redacted config always encodes. + raw = nil + } + sum := sha256.Sum256(raw) + return DigestPrefix + hex.EncodeToString(sum[:]) +} diff --git a/pkg/agentconfig/document.go b/pkg/agentconfig/document.go new file mode 100644 index 00000000..85339a5e --- /dev/null +++ b/pkg/agentconfig/document.go @@ -0,0 +1,68 @@ +package agentconfig + +import ( + "encoding/json" + "fmt" +) + +// RedactDocument re-applies Redact's rules to a reported config document (base or +// effective) WITHOUT decoding it into Config, so fields a newer agent sends are preserved +// (R51). It removes api.auth.client_secret, masks api.url, plugins.*.source and +// plugins.*.policies entries when they hold a secret by content, and masks plugins.*.config +// and plugins.*.policy_data exactly as Redact does (key names, content and placeholder +// rules). It cannot know the agent's env-sourced pointers (R55), so it is best +// effort; on a document Redact produced with the same rules it is a no-op. changed reports +// whether anything was altered. The input must be a JSON object. +func RedactDocument(doc json.RawMessage) (out json.RawMessage, changed bool, err error) { + v, err := decodeAny(doc) + if err != nil { + return nil, false, fmt.Errorf("redact document: %w", err) + } + obj, ok := v.(map[string]any) + if !ok { + return nil, false, fmt.Errorf("redact document: must be a JSON object") + } + before, err := encodeCanonical(obj) + if err != nil { + return nil, false, err + } + + var o redactOpts + if api, ok := obj["api"].(map[string]any); ok { + if auth, ok := api["auth"].(map[string]any); ok { + delete(auth, "client_secret") + } + if u, ok := api["url"].(string); ok && containsSecretValue(u) { + api["url"] = MaskedValue + } + } + if plugins, ok := obj["plugins"].(map[string]any); ok { + for name, raw := range plugins { + p, ok := raw.(map[string]any) + if !ok { + continue + } + if src, ok := p["source"].(string); ok { + p["source"] = maskSecretText(src) + } + if policies, ok := p["policies"].([]any); ok { + for i, e := range policies { + if entry, ok := e.(string); ok { + policies[i] = maskSecretText(entry) + } + } + } + if cfg, ok := p["config"].(map[string]any); ok { + p["config"] = o.redactMap(Pointer("plugins", name, "config"), cfg) + } + if data, ok := p["policy_data"].(map[string]any); ok { + p["policy_data"] = o.redactMap(Pointer("plugins", name, "policy_data"), data) + } + } + } + after, err := encodeCanonical(obj) + if err != nil { + return nil, false, err + } + return after, string(before) != string(after), nil +} diff --git a/pkg/agentconfig/redact.go b/pkg/agentconfig/redact.go new file mode 100644 index 00000000..d645b499 --- /dev/null +++ b/pkg/agentconfig/redact.go @@ -0,0 +1,155 @@ +package agentconfig + +import ( + "strconv" +) + +// RedactOption configures Redact and Digest. +type RedactOption func(*redactOpts) + +type redactOpts struct { + masked map[string]bool +} + +// WithMaskedPointers masks the values at these RFC 6901 pointers. The agent passes the +// plugin config values that came from viper env (CCF_PLUGINS_

_CONFIG_) rather than from +// placeholders (R25, R44). Pass the SAME pointers to Redact and Digest (R55). +func WithMaskedPointers(ptrs ...string) RedactOption { + return func(o *redactOpts) { + if o.masked == nil { + o.masked = map[string]bool{} + } + for _, p := range ptrs { + o.masked[p] = true + } + } +} + +// Redact returns a deep copy of c with api.auth.client_secret cleared and secret-like values +// replaced by MaskedValue. Apply it to the UNRESOLVED config (placeholders intact). It is +// idempotent, and Digest hashes its output, so both always apply the same rules (R55). +// +// A value is masked whole: the result is exactly MaskedValue, never a partially masked +// string, so a redacted view can never be resubmitted (ValidateOverlay rejects MaskedValue, +// O10). The rules, under plugins.*.config and plugins.*.policy_data (any depth): +// +// For a string value, let literal be the value with its ${env:NAME} placeholders removed. +// 1. A value whose literal is empty or only whitespace and the separators ":;,|/@=&" +// (placeholder-only, e.g. "${env:PASS}" or "${env:USER}:${env:PASS}") is kept verbatim. +// 2. Else it is masked when it is at a pointer given to WithMaskedPointers, or its key is +// secret-like (isSecretKey: e.g. password, passphrase, secret, token, credential, +// api_key, private_key, dsn, connection_string, auth, cookie, session_id; see +// secretKeyStems and secretKeyWords). Keys that only describe a secret are not +// secret-like (e.g. secret_name, token_url, password_file, api_key_id, max_tokens; see +// isNonSecretKeyName). Under a secret-like key, literal text mixed with a placeholder +// ("lit${env:X}") is masked. +// 3. Else it is masked when its literal contains a secret by content, whatever the key +// (containsSecretValue): a URL with a password in its userinfo (also inside a longer +// string such as a DSN), a PEM private key, a password=... assignment, or a +// high-confidence provider token (AWS access key ID, GitHub, GitLab, Slack, Google API +// key, Stripe, JWT, SendGrid, npm, PyPI, OpenAI, Anthropic, Hugging Face, +// DigitalOcean, Shopify, Terraform Cloud, Vault, Azure AD client secret, age), or a +// scheme-less MySQL DSN with a password (user:pass@tcp(host)/db). +// +// A non-string value (number, object, array) at a masked pointer or under a secret-like key +// is masked whole; booleans and null are never secret and are kept unless at a masked +// pointer. Strings nested in kept objects and arrays get the same rules, with the nearest +// enclosing object key as their key. api.url, plugins.*.source and each plugins.*.policies +// entry are masked when they contain a secret by content (rule 3 only; no key rule). Map +// keys and labels are never masked. +func Redact(c Config, opts ...RedactOption) Config { + var o redactOpts + for _, opt := range opts { + opt(&o) + } + out := c.clone() + if out.API != nil { + if out.API.Auth != nil { + out.API.Auth.ClientSecret = "" + } + if containsSecretValue(out.API.URL) { + out.API.URL = MaskedValue + } + } + for pluginName, p := range out.Plugins { + if p == nil { + continue + } + p.Source = maskSecretText(p.Source) + for i, e := range p.Policies { + p.Policies[i] = maskSecretText(e) + } + for key, value := range p.Config { + ptr := Pointer("plugins", pluginName, "config", key) + if o.shouldMask(ptr, key, value) { + p.Config[key] = MaskedValue + } + } + if p.PolicyData != nil { + p.PolicyData = o.redactMap(Pointer("plugins", pluginName, "policy_data"), p.PolicyData) + } + } + return out +} + +// maskSecretText returns MaskedValue when s contains a secret by content, else s. +func maskSecretText(s string) string { + masked, _ := ScrubSecretText(s) + return masked +} + +// shouldMask applies the mask rule (see Redact) to one string value. +func (o redactOpts) shouldMask(ptr, key, value string) bool { + literal, hasRef := envLiteral(value) + if hasRef && isPlaceholderOnly(literal) { + return false + } + if o.masked[ptr] || isSecretKey(key) { + return true + } + return containsSecretValue(literal) +} + +// redactMap redacts the entries of a free-form object. +func (o redactOpts) redactMap(ptr string, m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, val := range m { + out[k] = o.redactTree(appendPointer(ptr, k), k, val) + } + return out +} + +// redactTree returns a redacted copy of a free-form value. key is the nearest enclosing map +// key. +func (o redactOpts) redactTree(ptr, key string, v any) any { + switch t := v.(type) { + case nil: + return nil + case string: + if o.shouldMask(ptr, key, t) { + return MaskedValue + } + return t + case bool: + if o.masked[ptr] { + return MaskedValue + } + return t + } + // Any other value at a masked pointer or under a secret-like key is masked whole. + if o.masked[ptr] || (key != "" && isSecretKey(key)) { + return MaskedValue + } + switch t := v.(type) { + case map[string]any: + return o.redactMap(ptr, t) + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = o.redactTree(appendPointer(ptr, strconv.Itoa(i)), key, val) + } + return out + default: + return t + } +} diff --git a/pkg/agentconfig/redact_test.go b/pkg/agentconfig/redact_test.go new file mode 100644 index 00000000..9a7d10d0 --- /dev/null +++ b/pkg/agentconfig/redact_test.go @@ -0,0 +1,181 @@ +package agentconfig + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func redactBase() Config { + return Config{ + Daemon: true, + API: &APIConfig{ + URL: "https://api.example.com", + Auth: &APIAuth{ClientID: "0b3c1b8a-7c8e-4d53-9a52-9a3c1d1f2e10", ClientSecret: "s3cret"}, + }, + Plugins: map[string]*Plugin{ + "local-ssh": { + Source: srcSSH, + Config: map[string]string{ + "host": "localhost", + "user": "root", + "password": "hunter2", + "api_key": "k", + "AuthHeader": "Bearer x", + "db_password": "${env:DB_PASSWORD}", + "dsn": "postgres://u:${env:PG_PASS}@h/db", + }, + Labels: map[string]string{"token": "label-values-are-not-redacted"}, + PolicyData: map[string]any{ + "threshold": json.Number("5"), + "db": map[string]any{ + "host": "h", + "password": "p", + "port": json.Number("5432"), + }, + "credentials": map[string]any{"user": "a", "pass": "b"}, + "tokens": []any{"t1", "t2"}, + "secret_n": json.Number("42"), + "items": []any{map[string]any{"passwd": "x", "name": "n"}}, + "env_token": "${env:TOKEN}", + "nothing": nil, + }, + }, + "nil-plugin": nil, + }, + } +} + +func TestRedact(t *testing.T) { + in := redactBase() + out := Redact(in) + + assert.Equal(t, "••••", MaskedValue, "the mask is exactly four bullets") + + require.NotNil(t, out.API) + assert.Equal(t, "https://api.example.com", out.API.URL) + assert.Equal(t, "0b3c1b8a-7c8e-4d53-9a52-9a3c1d1f2e10", out.API.Auth.ClientID) + assert.Empty(t, out.API.Auth.ClientSecret, "client secret cleared") + + cfg := out.Plugins["local-ssh"].Config + assert.Equal(t, "localhost", cfg["host"]) + assert.Equal(t, "root", cfg["user"]) + assert.Equal(t, MaskedValue, cfg["password"]) + assert.Equal(t, MaskedValue, cfg["api_key"]) + assert.Equal(t, MaskedValue, cfg["AuthHeader"], "case-insensitive key match") + assert.Equal(t, "${env:DB_PASSWORD}", cfg["db_password"], "env placeholders kept verbatim") + assert.Equal(t, MaskedValue, cfg["dsn"], "literal text mixed with a placeholder under a secret-like key is masked") + + assert.Equal(t, "label-values-are-not-redacted", out.Plugins["local-ssh"].Labels["token"], "labels are not in the mask scope") + assert.Equal(t, srcSSH, out.Plugins["local-ssh"].Source) + assert.Nil(t, out.Plugins["nil-plugin"]) + + pd := out.Plugins["local-ssh"].PolicyData + assert.Equal(t, json.Number("5"), pd["threshold"]) + assert.Equal(t, map[string]any{"host": "h", "password": MaskedValue, "port": json.Number("5432")}, pd["db"]) + assert.Equal(t, MaskedValue, pd["credentials"], "non-string value under a matching key masked whole") + assert.Equal(t, MaskedValue, pd["tokens"], "array under a matching key masked whole") + assert.Equal(t, MaskedValue, pd["secret_n"], "number under a matching key masked") + assert.Equal(t, []any{map[string]any{"passwd": MaskedValue, "name": "n"}}, pd["items"]) + assert.Equal(t, "${env:TOKEN}", pd["env_token"]) + assert.Contains(t, pd, "nothing") + assert.Nil(t, pd["nothing"]) + + assert.Equal(t, redactBase(), in, "input not mutated") +} + +func TestRedactWithMaskedPointers(t *testing.T) { + out := Redact(redactBase(), WithMaskedPointers( + "/plugins/local-ssh/config/user", + "/plugins/local-ssh/config/db_password", // placeholder wins over the pointer + "/plugins/local-ssh/policy_data/db/host", + "/plugins/local-ssh/policy_data/threshold", + ), WithMaskedPointers("/plugins/local-ssh/config/host")) + + cfg := out.Plugins["local-ssh"].Config + assert.Equal(t, MaskedValue, cfg["user"]) + assert.Equal(t, MaskedValue, cfg["host"], "options accumulate") + assert.Equal(t, "${env:DB_PASSWORD}", cfg["db_password"]) + pd := out.Plugins["local-ssh"].PolicyData + assert.Equal(t, MaskedValue, pd["db"].(map[string]any)["host"]) + assert.Equal(t, MaskedValue, pd["threshold"], "non-string at a masked pointer") +} + +func TestRedactPointerEscaping(t *testing.T) { + c := Config{Plugins: map[string]*Plugin{"p": {Source: srcSSH, Config: map[string]string{"a/b": "v", "a~b": "w"}}}} + out := Redact(c, WithMaskedPointers(Pointer("plugins", "p", "config", "a/b"))) + assert.Equal(t, MaskedValue, out.Plugins["p"].Config["a/b"]) + assert.Equal(t, "w", out.Plugins["p"].Config["a~b"]) +} + +func TestRedactIdempotent(t *testing.T) { + opts := []RedactOption{WithMaskedPointers("/plugins/local-ssh/config/user")} + once := Redact(redactBase(), opts...) + twice := Redact(once, opts...) + assert.Equal(t, once, twice) + assert.Equal(t, Digest(once, opts...), Digest(twice, opts...)) +} + +func TestRedactYAMLMaps(t *testing.T) { + c := Config{Plugins: map[string]*Plugin{"p": {Source: srcSSH, PolicyData: map[string]any{"m": map[any]any{"token": "x", 1: "y"}}}}} + out := Redact(c) + assert.Equal(t, map[string]any{"token": MaskedValue, "1": "y"}, out.Plugins["p"].PolicyData["m"]) + _, err := json.Marshal(out) + assert.NoError(t, err) +} + +func TestDigest(t *testing.T) { + d := Digest(redactBase()) + assert.True(t, strings.HasPrefix(d, "sha256:"), d) + assert.Len(t, d, len("sha256:")+64) + + t.Run("stable", func(t *testing.T) { + opts := []RedactOption{WithMaskedPointers("/plugins/local-ssh/config/user")} + first := Digest(redactBase(), opts...) + for range 20 { + assert.Equal(t, first, Digest(redactBase(), opts...)) + } + }) + t.Run("differs when options differ", func(t *testing.T) { + assert.NotEqual(t, Digest(redactBase()), Digest(redactBase(), WithMaskedPointers("/plugins/local-ssh/config/user"))) + }) + t.Run("equal when masking makes documents equal", func(t *testing.T) { + a := redactBase() + b := redactBase() + b.Plugins["local-ssh"].Config["password"] = "different" + assert.Equal(t, Digest(a), Digest(b)) + }) + t.Run("differs on a real change", func(t *testing.T) { + b := redactBase() + b.Plugins["local-ssh"].Config["host"] = "other" + assert.NotEqual(t, Digest(redactBase()), Digest(b)) + }) + t.Run("ignores api block", func(t *testing.T) { + b := redactBase() + b.API = &APIConfig{URL: "https://other.example.com", Auth: &APIAuth{ClientID: "x", ClientSecret: "y"}} + assert.Equal(t, Digest(redactBase()), Digest(b)) + b.API = nil + assert.Equal(t, Digest(redactBase()), Digest(b)) + }) + t.Run("matches the canonical redacted document", func(t *testing.T) { + r := Redact(redactBase()) + r.API = nil + raw, err := CanonicalJSON(r) + require.NoError(t, err) + assert.NotContains(t, string(raw), `&`, "no HTML escaping") + assert.False(t, strings.HasSuffix(string(raw), "\n")) + // Digest of a round-tripped document is the same. + back, err := DecodeConfig(raw) + require.NoError(t, err) + assert.Equal(t, Digest(redactBase()), Digest(back)) + }) +} + +func TestCanonicalJSON(t *testing.T) { + raw, err := CanonicalJSON(map[string]any{"b": "<&>", "a": json.Number("12345678901234567890"), "c": []any{}}) + require.NoError(t, err) + assert.Equal(t, `{"a":12345678901234567890,"b":"<&>","c":[]}`, string(raw)) +} diff --git a/pkg/agentconfig/sensitive.go b/pkg/agentconfig/sensitive.go new file mode 100644 index 00000000..319bae95 --- /dev/null +++ b/pkg/agentconfig/sensitive.go @@ -0,0 +1,295 @@ +package agentconfig + +import ( + "net/url" + "regexp" + "slices" + "strings" + "unicode" +) + +// Secret detection for Redact, Digest and RedactDocument. Two independent checks decide +// whether a value is secret-like: its key name (isSecretKey) and its content +// (containsSecretValue). Both are deliberately free of entropy heuristics, since +// policy_data legitimately holds hashes and IDs. + +// secretKeyStems mark a key as secret wherever they occur in it. They are matched against +// the key lowercased with every non-alphanumeric character removed, so "db_password", +// "dbPassword", "dbpassword" (viper lowercases keys) and "API-Key" all match. Matching +// anywhere is deliberate: a false positive only hides a value from reports and digests; a +// miss leaks a secret. Keys that only describe a secret (nonSecretKeyLastWords, +// nonSecretKeyFirstWords) are exempt, so "max_tokens", "tokens_per_minute", "secret_name", +// "token_url" and "password_file" are not masked by key; their values still get the +// content check (containsSecretValue). +var secretKeyStems = []string{ + "password", "passwd", "passphrase", + "secret", "token", "credential", + "apikey", "privatekey", "accesskey", + "connectionstring", "connstring", "connstr", + "sessionid", "authorization", "authheader", +} + +// secretKeyWords mark a key as secret when they are its last word, or the suffix of its +// last word ("sshkey", "dbpass", "basicauth", "sentrydsn" as viper lowercases them), after +// dropping trailing format words (secretKeyFormatWords). Words are split at +// non-alphanumeric characters and camelCase boundaries. So "api_key", "client_key", +// "private_key_pem", "db_pass", "auth", "oauth" and "session_cookie" match, while +// "keyword", "key_id", "key_file", "pass_rate", "auth_method" and "cookie_secure" do not. +var secretKeyWords = []string{"key", "keys", "pass", "pwd", "auth", "dsn", "cookie", "cookies"} + +// secretKeyFormatWords are trailing words that name an encoding of the value rather than +// what it is: "signing_key_b64" is a key. +var secretKeyFormatWords = []string{"b64", "base64", "hex", "pem", "der", "raw", "data", "value", "json", "string", "str"} + +// secretKeyWordExceptions are ordinary words that end in a secretKeyWords entry. +var secretKeyWordExceptions = []string{ + "monkey", "monkeys", "donkey", "donkeys", "turkey", "turkeys", "hockey", "jockey", + "jockeys", "whiskey", "lackey", "turnkey", "hotkey", "hotkeys", + "bypass", "compass", "encompass", "surpass", "overpass", "underpass", "trespass", +} + +// nonSecretKeyLastWords end a key that describes a secret rather than holds one: its name, +// location, identifier, size or switch ("secret_name", "token_url", "password_file", +// "api_key_id", "token_ttl", "auth_enabled"). +var nonSecretKeyLastWords = []string{ + "name", "url", "uri", "file", "path", "dir", "id", "count", "limit", "ttl", "size", + "length", "enabled", +} + +// nonSecretKeyFirstWords start a key that bounds a quantity ("max_tokens", "min_key_size"). +var nonSecretKeyFirstWords = []string{"max", "min"} + +// isNonSecretKeyName reports whether a key (split by keyWords) describes a secret rather +// than holds one: it starts with max/min or "tokens_per", or its last word is in +// nonSecretKeyLastWords. A key whose secret stem needs that last word still names a secret +// ("session_id" is the stem "sessionid"). +func isNonSecretKeyName(words []string) bool { + if len(words) < 2 { + return false + } + if slices.Contains(nonSecretKeyFirstWords, words[0]) || (words[0] == "tokens" && words[1] == "per") { + return true + } + if !slices.Contains(nonSecretKeyLastWords, words[len(words)-1]) { + return false + } + joined := strings.Join(words, "") + prefix := strings.Join(words[:len(words)-1], "") + for _, stem := range secretKeyStems { + if strings.Contains(joined, stem) && !strings.Contains(prefix, stem) { + return false + } + } + return true +} + +// isSecretKey reports whether a config or policy_data key names a secret. Keys that name +// public or non-secret material, such as "cert", "certificate", "signature" or "session", +// do not match; private key material is caught by its content (containsSecretValue). +func isSecretKey(key string) bool { + words := keyWords(key) + if len(words) == 0 || isNonSecretKeyName(words) { + return false + } + joined := strings.Join(words, "") + for _, stem := range secretKeyStems { + if strings.Contains(joined, stem) { + return true + } + } + for len(words) > 1 && slices.Contains(secretKeyFormatWords, words[len(words)-1]) { + words = words[:len(words)-1] + } + last := words[len(words)-1] + if slices.Contains(secretKeyWordExceptions, last) { + return false + } + for _, w := range secretKeyWords { + if strings.HasSuffix(last, w) { + return true + } + } + return false +} + +// keyWords splits a key into lowercase words at non-alphanumeric characters and camelCase +// boundaries ("APIKey" -> "api", "key"; "authHeader" -> "auth", "header"). +func keyWords(key string) []string { + var words []string + var cur []rune + flush := func() { + if len(cur) > 0 { + words = append(words, strings.ToLower(string(cur))) + cur = cur[:0] + } + } + runes := []rune(key) + for i, r := range runes { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + flush() + continue + } + if unicode.IsUpper(r) && len(cur) > 0 { + prev := cur[len(cur)-1] + nextLower := i+1 < len(runes) && unicode.IsLower(runes[i+1]) + if unicode.IsLower(prev) || unicode.IsDigit(prev) || (unicode.IsUpper(prev) && nextLower) { + flush() + } + } + cur = append(cur, r) + } + flush() + return words +} + +// secretValuePattern is a high-confidence secret format. +type secretValuePattern struct { + name string + re *regexp.Regexp +} + +// secretValuePatterns are formats that identify a secret by content alone. The provider +// token formats are adapted from gitleaks' default rules +// (https://github.com/gitleaks/gitleaks, config/gitleaks.toml; MIT License, Copyright (c) +// 2019 Zachary Rice). Only formats with a distinctive prefix or structure are included. +var secretValuePatterns = []secretValuePattern{ + {"private-key", regexp.MustCompile(`-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----`)}, + {"password-assignment", regexp.MustCompile(`(?i)(?:^|[^a-z0-9_])(?:password|passwd|pwd)\s*=\s*[^\s;&,'"]`)}, + {"aws-access-key-id", regexp.MustCompile(`\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}\b`)}, + {"github-token", regexp.MustCompile(`\bgh[pousr]_[0-9A-Za-z]{36}\b`)}, + {"github-fine-grained-pat", regexp.MustCompile(`\bgithub_pat_[0-9A-Za-z_]{82}\b`)}, + {"gitlab-token", regexp.MustCompile(`\bglpat-[0-9A-Za-z_-]{20,}`)}, + {"slack-token", regexp.MustCompile(`\bxox[abeoprs]-[0-9]+-[0-9A-Za-z-]{8,}`)}, + {"slack-app-token", regexp.MustCompile(`(?i)\bxapp-[0-9]+-[A-Z0-9]+-[0-9]+-[a-z0-9]+`)}, + {"slack-webhook-url", regexp.MustCompile(`hooks\.slack\.com/(?:services|workflows|triggers)/[A-Za-z0-9+/]{43,56}`)}, + {"google-api-key", regexp.MustCompile(`\bAIza[0-9A-Za-z_-]{35}(?:[^0-9A-Za-z_-]|$)`)}, + {"stripe-key", regexp.MustCompile(`\b(?:sk|rk)_(?:live|test|prod)_[0-9A-Za-z]{10,99}\b`)}, + {"jwt", regexp.MustCompile(`\bey[A-Za-z0-9_-]{17,}\.ey[A-Za-z0-9_/\\-]{17,}\.`)}, + {"sendgrid-api-key", regexp.MustCompile(`\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}`)}, + {"npm-token", regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}\b`)}, + {"pypi-token", regexp.MustCompile(`pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}`)}, + {"openai-api-key", regexp.MustCompile(`\bsk-[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,}`)}, + {"anthropic-api-key", regexp.MustCompile(`\bsk-ant-(?:api|admin)[0-9]{2}-[A-Za-z0-9_-]{80,}`)}, + {"huggingface-token", regexp.MustCompile(`\bhf_[A-Za-z]{34}\b`)}, + {"digitalocean-token", regexp.MustCompile(`\bdo[opr]_v1_[a-f0-9]{64}\b`)}, + {"shopify-token", regexp.MustCompile(`\bshp(?:at|ca|pa|ss)_[a-fA-F0-9]{32}\b`)}, + {"terraform-cloud-token", regexp.MustCompile(`\b[A-Za-z0-9]{14}\.atlasv1\.[A-Za-z0-9_=-]{60,70}`)}, + {"vault-token", regexp.MustCompile(`\bhv[sb]\.[A-Za-z0-9_-]{90,}`)}, + {"azure-ad-client-secret", regexp.MustCompile(`(?:^|[\\'"\x60\s>=:(,)])[A-Za-z0-9_~.]{3}[0-9]Q~[A-Za-z0-9_~.-]{31,34}(?:$|[\\'"\x60\s<),])`)}, + {"age-secret-key", regexp.MustCompile(`AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58}`)}, + // Go MySQL driver DSN without a scheme: user:password@tcp(host:port)/db. + {"mysql-dsn", regexp.MustCompile(`[^\s:@/]*:[^\s@]+@(?:tcp[46]?|udp[46]?|unix)\(`)}, +} + +// urlSchemePattern finds the start of each URL in a string. +var urlSchemePattern = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.\-]*://`) + +// urlUserinfoPattern recognizes "scheme://user:password@" in a URL net/url cannot parse +// (an unescaped '/', '#', '?' or '@' in the password is common in hand-written DSNs) or +// parses as host:port (a password starting with digits then '/'). A password ending in '/' +// is not matched, so "https://host:443/users/@me" (a path) is not taken for userinfo. +var urlUserinfoPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9+.\-]*://[^\s/@:]*:[^\s@]*[^\s@/]@`) + +// Bounds on the URL password scan (hasURLPassword), so its cost is linear in the input. +const ( + // maxURLScanBytes is the prefix of a value scanned for URLs with a password. The token + // patterns (RE2, linear) still run on the whole value. + maxURLScanBytes = 64 << 10 + // maxURLCandidates caps the URLs inspected in one value. + maxURLCandidates = 64 + // maxURLCandidateBytes caps the length of one URL candidate. + maxURLCandidateBytes = 2048 +) + +// ScrubSecretText returns MaskedValue and true when s contains a secret by content (the +// rule Redact applies whatever the key: a URL with a password, a DSN, a PEM private key, a +// password=... assignment or a provider token), else s and false. It masks the whole string. +// Use it on free text such as error messages and plugin sources. +func ScrubSecretText(s string) (string, bool) { + if containsSecretValue(s) { + return MaskedValue, true + } + return s, false +} + +// containsSecretValue reports whether s holds a secret whatever its key: a URL with a +// non-empty password in its userinfo (anywhere in s, so DSNs and command lines count), a +// PEM private key, a password=... assignment, or a high-confidence provider token +// (secretValuePatterns). +func containsSecretValue(s string) bool { + if s == "" || s == MaskedValue { + return false + } + if hasURLPassword(s) { + return true + } + for _, p := range secretValuePatterns { + if p.re.MatchString(s) { + return true + } + } + return false +} + +// hasURLPassword reports whether any URL in s carries a non-empty password. Only the first +// maxURLScanBytes of s and its first maxURLCandidates URLs are inspected, and each candidate +// ends at the next URL, so the work is linear in len(s). +func hasURLPassword(s string) bool { + if len(s) > maxURLScanBytes { + s = s[:maxURLScanBytes] + } + locs := urlSchemePattern.FindAllStringIndex(s, maxURLCandidates) + for i, loc := range locs { + end := len(s) + if i+1 < len(locs) { + end = locs[i+1][0] + } + end = min(end, loc[0]+maxURLCandidateBytes) + candidate := s[loc[0]:end] + if t := strings.IndexFunc(candidate, isURLTerminator); t >= 0 { + candidate = candidate[:t] + } + if u, err := url.Parse(candidate); err == nil && u.User != nil { + if password, ok := u.User.Password(); ok && password != "" { + return true + } + } + // Also when net/url parses the candidate without a password: it reads + // "https://user:5678/abc@host" as host "user" and port 5678. An image digest after + // the '@' ("oci://registry:5000/plugin@sha256:...") is a reference, not userinfo. + if m := urlUserinfoPattern.FindStringIndex(candidate); m != nil && !isDigestReference(candidate[m[1]:]) { + return true + } + } + return false +} + +// isDigestReference reports whether s starts with a content digest ("sha256:..."). +func isDigestReference(s string) bool { + return strings.HasPrefix(s, "sha256:") || strings.HasPrefix(s, "sha384:") || strings.HasPrefix(s, "sha512:") +} + +func isURLTerminator(r rune) bool { + return unicode.IsSpace(r) || strings.ContainsRune("\"'`<>", r) +} + +// placeholderSeparators may join placeholders in a value that is still placeholder-only, +// e.g. "${env:USER}:${env:PASS}" or "${env:A}, ${env:B}". +const placeholderSeparators = ":;,|/@=&" + +// envLiteral returns s with every ${env:NAME} placeholder removed, and whether s had any. +func envLiteral(s string) (literal string, hasRef bool) { + if !strings.Contains(s, "${env:") { + return s, false + } + literal = EnvRefPattern.ReplaceAllString(s, "") + return literal, literal != s +} + +// isPlaceholderOnly reports whether the literal part of a value (envLiteral) carries no +// content: only whitespace and placeholderSeparators. +func isPlaceholderOnly(literal string) bool { + return strings.IndexFunc(literal, func(r rune) bool { + return !unicode.IsSpace(r) && !strings.ContainsRune(placeholderSeparators, r) + }) < 0 +} diff --git a/pkg/agentconfig/sensitive_test.go b/pkg/agentconfig/sensitive_test.go new file mode 100644 index 00000000..32bc4e2a --- /dev/null +++ b/pkg/agentconfig/sensitive_test.go @@ -0,0 +1,377 @@ +package agentconfig + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fake token fixtures are assembled at run time so the literal source never looks like a +// live credential to secret scanners. +func fake(parts ...string) string { return strings.Join(parts, "") } + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func TestIsSecretKey(t *testing.T) { + secret := []string{ + // stems, anywhere in the key + "password", "db_password", "dbPassword", "dbpassword", "DB-PASSWORD", "passwd", + "passphrase", "ssh_key_passphrase", "secret", "client_secret", "clientsecret", + "token", "access_token", "accessToken", "tokens", "credentials", "credentials_json", + "api_key", "apiKey", "APIKey", "apikey", "x-api-key", "private_key", "privateKey", + "access_key", "aws_secret_access_key", "connection_string", + "connectionString", "conn_str", "connstr", "session_id", "sessionid", "SessionID", + "authorization", "Authorization", "auth_header", "AuthHeader", "authheader", + // last-word matches + "key", "keys", "client_key", "tls.key", "ssh-key", "sshkey", "signing_key_b64", + "encryption_key_hex", "private_key_pem", "passkey", "pass", "db_pass", "dbpass", + "pwd", "db_pwd", "auth", "basic_auth", "basicauth", "oauth", "proxyAuth", "dsn", + "sentry_dsn", "sentrydsn", "cookie", "cookies", "session_cookie", "setCookie", + // a stem that needs the last word still names a secret + "session_id", "sessionId", + } + for _, k := range secret { + assert.True(t, isSecretKey(k), "%q should be secret-like", k) + } + notSecret := []string{ + "", "host", "user", "username", "port", "keyword", "keywords", "monkey", "monkeys", + "turkey", "hockey", "whiskey", "hotkey", "turnkey", "bypass", "compass", + "key_id", "kms_key_id", "key_file", "keyspace", "keyboard", "pass_rate", + "min_pass_percentage", "passthrough", "author", "authority", "authentication_method", + "auth_method", "authz_mode", "cookie_secure", "cookie_name", "dsn_timeout", + "session", "session_timeout", "cert", "certificate", "client_cert", "ca_cert", + "signature", "sig", "threshold", "keyed", "monkey_patch", + // keys that describe a secret rather than hold one (the value is still content-checked) + "tokens_per_minute", "max_tokens", "maxTokens", "min_key_size", "secret_name", + "password_file", "private_key_path", "token_url", "auth_uri", "secrets_dir", + "api_key_id", "token_count", "token_limit", "token_ttl", "secret_size", + "password_length", "auth_enabled", + } + for _, k := range notSecret { + assert.False(t, isSecretKey(k), "%q should not be secret-like", k) + } +} + +func TestKeyWords(t *testing.T) { + tests := map[string][]string{ + "api_key": {"api", "key"}, + "APIKey": {"api", "key"}, + "apiKeyID": {"api", "key", "id"}, + "authHeader": {"auth", "header"}, + "x-api-key": {"x", "api", "key"}, + "tls.key": {"tls", "key"}, + "OAuth2Token": {"o", "auth2", "token"}, + "key2": {"key2"}, + "HTTPServerURL": {"http", "server", "url"}, + "__": nil, + "snake_CASE_key": {"snake", "case", "key"}, + "päss wörd": {"päss", "wörd"}, + "aws_access_key1": {"aws", "access", "key1"}, + } + for in, want := range tests { + assert.Equal(t, want, keyWords(in), in) + } +} + +func TestContainsSecretValue(t *testing.T) { + alnum36 := strings.Repeat("a1B2", 9) + positives := map[string]string{ + "url password": "postgres://user:hunter2@db.example.com:5432/app", + "url empty user": "redis://:hunter2@cache:6379/0", + "url in a longer string": "--dsn=postgres://user:hunter2@db/app --verbose", + "url in a jdbc dsn": "jdbc:postgresql://user:hunter2@db/app", + "url unparseable password": "postgres://user:pa/ss#w?rd@db/app", + "url password with at": "mongodb://user:p@ss@db/app", + "url escaped password": "amqp://user:p%40ss@mq/vhost", + "second url has a password": "https://example.com/a https://u:p@example.org", + "pem private key": "-----BEGIN PRIVATE KEY-----\nMIIEv...\n-----END PRIVATE KEY-----", + "pem rsa private key": "-----BEGIN RSA PRIVATE KEY-----", + "pem openssh private key": "x -----BEGIN OPENSSH PRIVATE KEY----- y", + "pem pgp private key block": "-----BEGIN PGP PRIVATE KEY BLOCK-----", + "libpq password": "host=db user=app password=hunter2 sslmode=require", + "odbc pwd": "Server=db;Uid=app;Pwd=hunter2;", + "jdbc query password": "jdbc:mysql://db/app?user=app&password=hunter2", + "aws access key id": fake("AKIA", "IOSFODNN7EXAMPLE"), + "aws session key id": fake("ASIA", "IOSFODNN7EXAMPLE"), + "github pat": fake("gh", "p_", alnum36), + "github oauth": fake("gh", "o_", alnum36), + "github user-to-server": fake("gh", "u_", alnum36), + "github server-to-server": fake("gh", "s_", alnum36), + "github refresh": fake("gh", "r_", alnum36), + "github fine-grained pat": fake("github", "_pat_", strings.Repeat("a1B2_", 16), "ab"), + "gitlab pat": fake("gl", "pat-", strings.Repeat("aB3-", 5)), + "slack bot token": fake("xo", "xb-", "1234567890-1234567890-", strings.Repeat("aB3", 8)), + "slack user token": fake("xo", "xp-", "1234567890-1234567890-1234567890-", strings.Repeat("ab12", 8)), + "slack app token": fake("xa", "pp-1-A0123BCDEF-1234567890-", strings.Repeat("ab12", 16)), + "slack webhook": fake("https://hooks.", "slack.com/services/", strings.Repeat("A1b2", 11)), + "google api key": fake("AI", "za", strings.Repeat("Sy0_-", 7)), + "stripe live key": fake("sk", "_live_", strings.Repeat("a1B2", 6)), + "stripe restricted key": fake("rk", "_live_", strings.Repeat("a1B2", 6)), + "stripe test key": fake("sk", "_test_", strings.Repeat("a1B2", 6)), + "jwt": fake("ey", "JhbGciOiJIUzI1NiJ9", ".", "ey", "JzdWIiOiIxMjM0NTY3ODkwIn0", ".", "dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"), + "jwt in a header": fake("Bearer ey", "JhbGciOiJIUzI1NiJ9", ".", "ey", "JzdWIiOiIxMjM0NTY3ODkwIn0", "."), + "sendgrid": fake("SG", ".", strings.Repeat("a", 22), ".", strings.Repeat("b", 43)), + "npm": fake("np", "m_", alnum36), + "pypi": fake("pypi-", "AgEIcHlwaS5vcmc", strings.Repeat("ab_-", 13)), + "openai": fake("sk-proj-", strings.Repeat("a", 24), "T3Blbk", "FJ", strings.Repeat("b", 24)), + "anthropic": fake("sk-", "ant-api03-", strings.Repeat("aB3_", 22)), + "huggingface": fake("hf", "_", strings.Repeat("abcd", 8), "ab"), + "digitalocean": fake("do", "p_v1_", strings.Repeat("0a", 32)), + "shopify": fake("shp", "at_", strings.Repeat("0a", 16)), + "terraform cloud": fake(strings.Repeat("a", 14), ".atlas", "v1.", strings.Repeat("ab", 32)), + "vault": fake("hv", "s.", strings.Repeat("aB3_", 25)), + "azure ad client secret": fake("abc", "8Q~", strings.Repeat("aB3.", 8), "xy"), + "age secret key": fake("AGE-SECRET", "-KEY-1", strings.Repeat("QPZRY9X8GF", 5), "2TVDW0S3"), + "token inside a longer value": fake("token for ci: gh", "p_", alnum36, " (rotate monthly)"), + "url digits password parsed as a port": "https://user:5678/abc@host", + "mysql dsn without a scheme": "user:hunter2@tcp(db:3306)/app", + "mysql dsn unix socket": "app:hunter2@unix(/var/run/mysqld.sock)/app", + } + for name, v := range positives { + assert.True(t, containsSecretValue(v), "%s: %q", name, v) + } + + negatives := map[string]string{ + "empty": "", + "masked": MaskedValue, + "plain": "localhost", + "url without userinfo": "https://api.example.com:8443/v1?x=1", + "url user only": "ssh://git@github.com/org/repo.git", + "url empty password": "redis://user:@cache:6379", + "url port then at in path": "https://example.com:443/users/@me", + "scp-like git": "git@github.com:org/repo.git", + "mailto": "mailto:someone@example.com", + "sha256 hash": strings.Repeat("0123456789abcdef", 4), + "uuid": "0b3c1b8a-7c8e-4d53-9a52-9a3c1d1f2e10", + "image digest": "sha256:" + strings.Repeat("ab", 32), + "base64 blob": "SGVsbG8sIFdvcmxkIQ==", + "password word": "the password must be rotated", + "password key empty": "password=", + "password placeholder gap": "password=;host=db", + "public cert": "-----BEGIN CERTIFICATE-----\nMIIB...\n-----END CERTIFICATE-----", + "public key": "-----BEGIN PUBLIC KEY-----", + "short ghp prefix": "ghp_short", + "akia too short": "AKIA123", + "one dot ey": "eyJhbGciOiJIUzI1NiJ9.notjwt", + "oci ref": "ghcr.io/compliance-framework/plugin-local-ssh:v1.2.3", + "cron": "*/5 * * * *", + "keyword like value": "monkey", + "oci digest with a port": "oci://registry.local:5000/plugins/ssh@sha256:" + strings.Repeat("ab", 32), + "mysql dsn no password": "user@tcp(db:3306)/app", + "mysql dsn empty password": "user:@tcp(db:3306)/app", + } + for name, v := range negatives { + assert.False(t, containsSecretValue(v), "%s: %q", name, v) + } +} + +// The URL scan is linear: a long value of back-to-back schemes with no terminator used to +// re-parse the rest of the string for every match. +func TestContainsSecretValueLinear(t *testing.T) { + adversarial := strings.Repeat("a://", (1<<20)/4) + start := time.Now() + assert.False(t, containsSecretValue(adversarial)) + assert.Less(t, time.Since(start), 2*time.Second) + + // A real URL password within the scanned prefix is still detected. + assert.True(t, containsSecretValue(strings.Repeat("a://", 50)+" https://u:pw@h/x")) + assert.True(t, containsSecretValue(strings.Repeat("x", 1000)+"https://u:pw@h/x")) +} + +func TestScrubSecretText(t *testing.T) { + got, scrubbed := ScrubSecretText("dial postgres://app:hunter2@db:5432/app: connection refused") + assert.True(t, scrubbed) + assert.Equal(t, MaskedValue, got) + + got, scrubbed = ScrubSecretText("dial tcp db:5432: connection refused") + assert.False(t, scrubbed) + assert.Equal(t, "dial tcp db:5432: connection refused", got) + + got, scrubbed = ScrubSecretText(MaskedValue) + assert.False(t, scrubbed) + assert.Equal(t, MaskedValue, got) +} + +func TestRedactSourcesAndPolicies(t *testing.T) { + c := Config{Plugins: map[string]*Plugin{"p": { + Source: "https://u:pw@plugins.example.com/ssh.tar.gz", + Policies: []string{srcPolicies, "https://ci:hunter2@policies.example.com/p.tar.gz"}, + }, "q": { + Source: srcSSH, + Policies: []string{srcCommon}, + Labels: map[string]string{"token": "x"}, + }}} + out := Redact(c) + assert.Equal(t, MaskedValue, out.Plugins["p"].Source) + assert.Equal(t, []string{srcPolicies, MaskedValue}, out.Plugins["p"].Policies) + assert.Equal(t, srcSSH, out.Plugins["q"].Source, "no key rule on sources") + assert.Equal(t, []string{srcCommon}, out.Plugins["q"].Policies) + assert.Equal(t, "x", out.Plugins["q"].Labels["token"], "labels are never masked") + assert.Equal(t, "https://u:pw@plugins.example.com/ssh.tar.gz", c.Plugins["p"].Source, "input not mutated") + + raw, err := json.Marshal(c) + require.NoError(t, err) + doc, changed, err := RedactDocument(raw) + require.NoError(t, err) + assert.True(t, changed) + want, err := CanonicalJSON(Redact(c)) + require.NoError(t, err) + assert.JSONEq(t, string(want), string(doc), "RedactDocument applies the same rules") +} + +func TestRedactValueAndPlaceholderRules(t *testing.T) { + ghp := fake("gh", "p_", strings.Repeat("a1B2", 9)) + c := Config{ + API: &APIConfig{URL: "https://agent:hunter2@api.example.com", Auth: &APIAuth{ClientID: "id", ClientSecret: "s"}}, + Plugins: map[string]*Plugin{"p": { + Source: srcSSH, + Config: map[string]string{ + // content rules, whatever the key + "url": "postgres://user:hunter2@db/app", + "url_user_only": "postgres://user@db/app", + "args": "--token-file /x " + ghp, + "ca": "-----BEGIN EC PRIVATE KEY-----\nabc\n-----END EC PRIVATE KEY-----", + "conn": "host=db password=hunter2", + "host": "localhost", + // placeholder rules + "only_ref": "${env:PG_PASS}", + "refs_and_seps": "${env:USER}:${env:PASS}", + "refs_ws": " ${env:A} , ${env:B} ", + "password": "lit${env:X}", + "api_key": "${env:A}${env:B}x", + "url_with_ref": "postgres://user:${env:PG_PASS}@db/app", + "url_mixed_ref": "postgres://user:lit${env:PG_PASS}@db/app", + "ref_and_token": "${env:A} " + ghp, + "plain_ref_literal": "prefix-${env:REGION}-suffix", + }, + PolicyData: map[string]any{ + "enabled": true, + "pass": false, + "key": nil, + "pass_rate": json.Number("0.9"), + "signing_key": json.Number("1234"), + "allowed": []any{"ok", "postgres://u:p@h/db", map[string]any{"note": ghp, "id": "x"}}, + "nested": map[string]any{"connection_string": "Server=x", "public": "y"}, + "credentials": true, + }, + }}, + } + out := Redact(c) + + assert.Equal(t, MaskedValue, out.API.URL, "api.url with a password is masked whole") + assert.Empty(t, out.API.Auth.ClientSecret) + + cfg := out.Plugins["p"].Config + want := map[string]string{ + "url": MaskedValue, + "url_user_only": "postgres://user@db/app", + "args": MaskedValue, + "ca": MaskedValue, + "conn": MaskedValue, + "host": "localhost", + "only_ref": "${env:PG_PASS}", + "refs_and_seps": "${env:USER}:${env:PASS}", + "refs_ws": " ${env:A} , ${env:B} ", + "password": MaskedValue, + "api_key": MaskedValue, + "url_with_ref": "postgres://user:${env:PG_PASS}@db/app", + "url_mixed_ref": MaskedValue, + "ref_and_token": MaskedValue, + "plain_ref_literal": "prefix-${env:REGION}-suffix", + } + assert.Equal(t, want, cfg) + for k, v := range cfg { + assert.True(t, v == MaskedValue || !strings.Contains(v, MaskedValue), "%s: masked whole, never partially", k) + } + + pd := out.Plugins["p"].PolicyData + assert.Equal(t, true, pd["enabled"]) + assert.Equal(t, false, pd["pass"], "booleans are never secret") + assert.Equal(t, true, pd["credentials"], "booleans are never secret") + assert.Nil(t, pd["key"]) + assert.Equal(t, json.Number("0.9"), pd["pass_rate"], "pass is only secret as the last word") + assert.Equal(t, MaskedValue, pd["signing_key"], "number under a secret-like key") + assert.Equal(t, []any{"ok", MaskedValue, map[string]any{"note": MaskedValue, "id": "x"}}, pd["allowed"]) + assert.Equal(t, map[string]any{"connection_string": MaskedValue, "public": "y"}, pd["nested"]) + + t.Run("masked pointer masks a boolean", func(t *testing.T) { + got := Redact(c, WithMaskedPointers("/plugins/p/policy_data/enabled")) + assert.Equal(t, MaskedValue, got.Plugins["p"].PolicyData["enabled"]) + }) + t.Run("masked pointer masks mixed placeholder values", func(t *testing.T) { + got := Redact(c, WithMaskedPointers("/plugins/p/config/plain_ref_literal", "/plugins/p/config/only_ref")) + assert.Equal(t, MaskedValue, got.Plugins["p"].Config["plain_ref_literal"]) + assert.Equal(t, "${env:PG_PASS}", got.Plugins["p"].Config["only_ref"], "placeholder-only wins") + }) + t.Run("idempotent", func(t *testing.T) { + assert.Equal(t, out, Redact(out)) + assert.Equal(t, Digest(c), Digest(out)) + }) + t.Run("input not mutated", func(t *testing.T) { + assert.Equal(t, "https://agent:hunter2@api.example.com", c.API.URL) + assert.Equal(t, "postgres://user:hunter2@db/app", c.Plugins["p"].Config["url"]) + }) +} + +func TestRedactDocumentParity(t *testing.T) { + ghp := fake("gh", "p_", strings.Repeat("a1B2", 9)) + c := redactBase() + c.API.URL = "https://agent:hunter2@api.example.com" + p := c.Plugins["local-ssh"] + p.Config["url"] = "postgres://user:hunter2@db/app" + p.Config["mixed"] = "lit${env:X}" + p.Config["password_mixed"] = "lit${env:X}" + p.Config["refs"] = "${env:A}:${env:B}" + p.PolicyData["list"] = []any{ghp, "plain", true} + p.PolicyData["flag"] = true + + raw, err := json.Marshal(c) + require.NoError(t, err) + doc, changed, err := RedactDocument(raw) + require.NoError(t, err) + assert.True(t, changed) + + want, err := CanonicalJSON(Redact(c)) + require.NoError(t, err) + // Redact keeps an empty client_secret field that RedactDocument deletes; compare the rest. + var gotObj, wantObj map[string]any + require.NoError(t, json.Unmarshal(doc, &gotObj)) + require.NoError(t, json.Unmarshal(want, &wantObj)) + assert.Equal(t, wantObj["plugins"], gotObj["plugins"], "same rules on the raw document") + assert.Equal(t, MaskedValue, gotObj["api"].(map[string]any)["url"]) + assert.NotContains(t, gotObj["api"].(map[string]any)["auth"], "client_secret") + + again, changed, err := RedactDocument(doc) + require.NoError(t, err) + assert.False(t, changed, "no-op on a redacted document") + assert.JSONEq(t, string(doc), string(again)) + + agentRedacted, err := json.Marshal(Redact(c)) + require.NoError(t, err) + _, changed, err = RedactDocument(agentRedacted) + require.NoError(t, err) + assert.False(t, changed, "no-op on a document Redact produced") +} + +func TestDigestStableForNonSecretConfig(t *testing.T) { + c := Config{Plugins: map[string]*Plugin{"p": { + Source: srcSSH, + Config: map[string]string{"host": "localhost", "port": "22", "user": "root", "region": "${env:REGION}"}, + PolicyData: map[string]any{"threshold": json.Number("5"), "enabled": true, "ids": []any{"a", "b"}}, + }}} + // Nothing in c is secret-like, so redaction leaves it as is and the digest is that of + // the config itself. + assert.Equal(t, c.clone(), Redact(c)) + raw, err := CanonicalJSON(c) + require.NoError(t, err) + assert.Equal(t, DigestPrefix+sha256Hex(raw), Digest(c)) +} From 0ffe5eba15c14a93ac92385e1f0c1f2afde0d507 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:15:35 -0300 Subject: [PATCH 2/2] fix(agentconfig): scale the linear-scan time budget under the race detector TestContainsSecretValueLinear asserted a fixed 2s budget, which the race detector's instrumentation exceeds (about 5s). Keep the linearity check but scale the budget x10 when built with -race (raceEnabled, set by build tag). Co-Authored-By: Claude Opus 5.5 --- pkg/agentconfig/race_disabled_test.go | 6 ++++++ pkg/agentconfig/race_enabled_test.go | 7 +++++++ pkg/agentconfig/sensitive_test.go | 5 +++-- pkg/agentconfig/timebudget_test.go | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 pkg/agentconfig/race_disabled_test.go create mode 100644 pkg/agentconfig/race_enabled_test.go create mode 100644 pkg/agentconfig/timebudget_test.go diff --git a/pkg/agentconfig/race_disabled_test.go b/pkg/agentconfig/race_disabled_test.go new file mode 100644 index 00000000..c967ac1b --- /dev/null +++ b/pkg/agentconfig/race_disabled_test.go @@ -0,0 +1,6 @@ +//go:build !race + +package agentconfig + +// raceEnabled reports whether the tests run under the race detector (see race_enabled_test.go). +const raceEnabled = false diff --git a/pkg/agentconfig/race_enabled_test.go b/pkg/agentconfig/race_enabled_test.go new file mode 100644 index 00000000..6a7aefd9 --- /dev/null +++ b/pkg/agentconfig/race_enabled_test.go @@ -0,0 +1,7 @@ +//go:build race + +package agentconfig + +// raceEnabled reports whether the tests run under the race detector, which slows tight loops +// by roughly an order of magnitude. Timing budgets scale with it (see timeBudget). +const raceEnabled = true diff --git a/pkg/agentconfig/sensitive_test.go b/pkg/agentconfig/sensitive_test.go index 32bc4e2a..8c5c1ef4 100644 --- a/pkg/agentconfig/sensitive_test.go +++ b/pkg/agentconfig/sensitive_test.go @@ -175,12 +175,13 @@ func TestContainsSecretValue(t *testing.T) { } // The URL scan is linear: a long value of back-to-back schemes with no terminator used to -// re-parse the rest of the string for every match. +// re-parse the rest of the string for every match. The budget only has to tell linear from +// quadratic, so it scales under the race detector's instrumentation (timeBudget). func TestContainsSecretValueLinear(t *testing.T) { adversarial := strings.Repeat("a://", (1<<20)/4) start := time.Now() assert.False(t, containsSecretValue(adversarial)) - assert.Less(t, time.Since(start), 2*time.Second) + assert.Less(t, time.Since(start), timeBudget(2*time.Second)) // A real URL password within the scanned prefix is still detected. assert.True(t, containsSecretValue(strings.Repeat("a://", 50)+" https://u:pw@h/x")) diff --git a/pkg/agentconfig/timebudget_test.go b/pkg/agentconfig/timebudget_test.go new file mode 100644 index 00000000..80759ffc --- /dev/null +++ b/pkg/agentconfig/timebudget_test.go @@ -0,0 +1,15 @@ +package agentconfig + +import "time" + +// raceSlowdown is how much a wall-clock budget grows under the race detector. +const raceSlowdown = 10 + +// timeBudget scales a wall-clock budget for the instrumentation in use, so a linearity check +// keeps failing on quadratic behaviour without failing on a slower, instrumented build. +func timeBudget(d time.Duration) time.Duration { + if raceEnabled { + return d * raceSlowdown + } + return d +}