feat(agentconfig): secret detection, redaction and digests (4/15)

pkg/agentconfig/digest.go

@@ -0,0 +1,24 @@
+package agentconfig
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+)
+
+// DigestPrefix prefixes config digests.
+const DigestPrefix = "sha256:"
+
+// Digest returns "sha256:" + hex(sha256(canonical(Redact(c, opts...) with API = nil))).
+// Callers pass the SAME options they used to redact the reported effective config, so the
+// digest matches the reported document (R55). Canonical JSON is described on CanonicalJSON.
+func Digest(c Config, opts ...RedactOption) string {
+	r := Redact(c, opts...)
+	r.API = nil
+	raw, err := CanonicalJSON(r)
+	if err != nil {
+		// Redact normalizes every free-form value, so the redacted config always encodes.
+		raw = nil
+	}
+	sum := sha256.Sum256(raw)
+	return DigestPrefix + hex.EncodeToString(sum[:])
+}

pkg/agentconfig/document.go

@@ -0,0 +1,68 @@
+package agentconfig
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// RedactDocument re-applies Redact's rules to a reported config document (base or
+// effective) WITHOUT decoding it into Config, so fields a newer agent sends are preserved
+// (R51). It removes api.auth.client_secret, masks api.url, plugins.*.source and
+// plugins.*.policies entries when they hold a secret by content, and masks plugins.*.config
+// and plugins.*.policy_data exactly as Redact does (key names, content and placeholder
+// rules). It cannot know the agent's env-sourced pointers (R55), so it is best
+// effort; on a document Redact produced with the same rules it is a no-op. changed reports
+// whether anything was altered. The input must be a JSON object.
+func RedactDocument(doc json.RawMessage) (out json.RawMessage, changed bool, err error) {
+	v, err := decodeAny(doc)
+	if err != nil {
+		return nil, false, fmt.Errorf("redact document: %w", err)
+	}
+	obj, ok := v.(map[string]any)
+	if !ok {
+		return nil, false, fmt.Errorf("redact document: must be a JSON object")
+	}
+	before, err := encodeCanonical(obj)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var o redactOpts
+	if api, ok := obj["api"].(map[string]any); ok {
+		if auth, ok := api["auth"].(map[string]any); ok {
+			delete(auth, "client_secret")
+		}
+		if u, ok := api["url"].(string); ok && containsSecretValue(u) {
+			api["url"] = MaskedValue
+		}
+	}
+	if plugins, ok := obj["plugins"].(map[string]any); ok {
+		for name, raw := range plugins {
+			p, ok := raw.(map[string]any)
+			if !ok {
+				continue
+			}
+			if src, ok := p["source"].(string); ok {
+				p["source"] = maskSecretText(src)
+			}
+			if policies, ok := p["policies"].([]any); ok {
+				for i, e := range policies {
+					if entry, ok := e.(string); ok {
+						policies[i] = maskSecretText(entry)
+					}
+				}
+			}
+			if cfg, ok := p["config"].(map[string]any); ok {
+				p["config"] = o.redactMap(Pointer("plugins", name, "config"), cfg)
+			}
+			if data, ok := p["policy_data"].(map[string]any); ok {
+				p["policy_data"] = o.redactMap(Pointer("plugins", name, "policy_data"), data)
+			}
+		}
+	}
+	after, err := encodeCanonical(obj)
+	if err != nil {
+		return nil, false, err
+	}
+	return after, string(before) != string(after), nil
+}

pkg/agentconfig/race_disabled_test.go

@@ -0,0 +1,6 @@
+//go:build !race
+
+package agentconfig
+
+// raceEnabled reports whether the tests run under the race detector (see race_enabled_test.go).
+const raceEnabled = false

pkg/agentconfig/race_enabled_test.go

@@ -0,0 +1,7 @@
+//go:build race
+
+package agentconfig
+
+// raceEnabled reports whether the tests run under the race detector, which slows tight loops
+// by roughly an order of magnitude. Timing budgets scale with it (see timeBudget).
+const raceEnabled = true

pkg/agentconfig/redact.go

@@ -0,0 +1,155 @@
+package agentconfig
+
+import (
+	"strconv"
+)
+
+// RedactOption configures Redact and Digest.
+type RedactOption func(*redactOpts)
+
+type redactOpts struct {
+	masked map[string]bool
+}
+
+// WithMaskedPointers masks the values at these RFC 6901 pointers. The agent passes the
+// plugin config values that came from viper env (CCF_PLUGINS_<P>_CONFIG_<K>) rather than from
+// placeholders (R25, R44). Pass the SAME pointers to Redact and Digest (R55).
+func WithMaskedPointers(ptrs ...string) RedactOption {
+	return func(o *redactOpts) {
+		if o.masked == nil {
+			o.masked = map[string]bool{}
+		}
+		for _, p := range ptrs {
+			o.masked[p] = true
+		}
+	}
+}
+
+// Redact returns a deep copy of c with api.auth.client_secret cleared and secret-like values
+// replaced by MaskedValue. Apply it to the UNRESOLVED config (placeholders intact). It is
+// idempotent, and Digest hashes its output, so both always apply the same rules (R55).
+//
+// A value is masked whole: the result is exactly MaskedValue, never a partially masked
+// string, so a redacted view can never be resubmitted (ValidateOverlay rejects MaskedValue,
+// O10). The rules, under plugins.*.config and plugins.*.policy_data (any depth):
+//
+// For a string value, let literal be the value with its ${env:NAME} placeholders removed.
+//  1. A value whose literal is empty or only whitespace and the separators ":;,|/@=&"
+//     (placeholder-only, e.g. "${env:PASS}" or "${env:USER}:${env:PASS}") is kept verbatim.
+//  2. Else it is masked when it is at a pointer given to WithMaskedPointers, or its key is
+//     secret-like (isSecretKey: e.g. password, passphrase, secret, token, credential,
+//     api_key, private_key, dsn, connection_string, auth, cookie, session_id; see
+//     secretKeyStems and secretKeyWords). Keys that only describe a secret are not
+//     secret-like (e.g. secret_name, token_url, password_file, api_key_id, max_tokens; see
+//     isNonSecretKeyName). Under a secret-like key, literal text mixed with a placeholder
+//     ("lit${env:X}") is masked.
+//  3. Else it is masked when its literal contains a secret by content, whatever the key
+//     (containsSecretValue): a URL with a password in its userinfo (also inside a longer
+//     string such as a DSN), a PEM private key, a password=... assignment, or a
+//     high-confidence provider token (AWS access key ID, GitHub, GitLab, Slack, Google API
+//     key, Stripe, JWT, SendGrid, npm, PyPI, OpenAI, Anthropic, Hugging Face,
+//     DigitalOcean, Shopify, Terraform Cloud, Vault, Azure AD client secret, age), or a
+//     scheme-less MySQL DSN with a password (user:pass@tcp(host)/db).
+//
+// A non-string value (number, object, array) at a masked pointer or under a secret-like key
+// is masked whole; booleans and null are never secret and are kept unless at a masked
+// pointer. Strings nested in kept objects and arrays get the same rules, with the nearest
+// enclosing object key as their key. api.url, plugins.*.source and each plugins.*.policies
+// entry are masked when they contain a secret by content (rule 3 only; no key rule). Map
+// keys and labels are never masked.
+func Redact(c Config, opts ...RedactOption) Config {
+	var o redactOpts
+	for _, opt := range opts {
+		opt(&o)
+	}
+	out := c.clone()
+	if out.API != nil {
+		if out.API.Auth != nil {
+			out.API.Auth.ClientSecret = ""
+		}
+		if containsSecretValue(out.API.URL) {
+			out.API.URL = MaskedValue
+		}
+	}
+	for pluginName, p := range out.Plugins {
+		if p == nil {
+			continue
+		}
+		p.Source = maskSecretText(p.Source)
+		for i, e := range p.Policies {
+			p.Policies[i] = maskSecretText(e)
+		}
+		for key, value := range p.Config {
+			ptr := Pointer("plugins", pluginName, "config", key)
+			if o.shouldMask(ptr, key, value) {
+				p.Config[key] = MaskedValue
+			}
+		}
+		if p.PolicyData != nil {
+			p.PolicyData = o.redactMap(Pointer("plugins", pluginName, "policy_data"), p.PolicyData)
+		}
+	}
+	return out
+}
+
+// maskSecretText returns MaskedValue when s contains a secret by content, else s.
+func maskSecretText(s string) string {
+	masked, _ := ScrubSecretText(s)
+	return masked
+}
+
+// shouldMask applies the mask rule (see Redact) to one string value.
+func (o redactOpts) shouldMask(ptr, key, value string) bool {
+	literal, hasRef := envLiteral(value)
+	if hasRef && isPlaceholderOnly(literal) {
+		return false
+	}
+	if o.masked[ptr] || isSecretKey(key) {
+		return true
+	}
+	return containsSecretValue(literal)
+}
+
+// redactMap redacts the entries of a free-form object.
+func (o redactOpts) redactMap(ptr string, m map[string]any) map[string]any {
+	out := make(map[string]any, len(m))
+	for k, val := range m {
+		out[k] = o.redactTree(appendPointer(ptr, k), k, val)
+	}
+	return out
+}
+
+// redactTree returns a redacted copy of a free-form value. key is the nearest enclosing map
+// key.
+func (o redactOpts) redactTree(ptr, key string, v any) any {
+	switch t := v.(type) {
+	case nil:
+		return nil
+	case string:
+		if o.shouldMask(ptr, key, t) {
+			return MaskedValue
+		}
+		return t
+	case bool:
+		if o.masked[ptr] {
+			return MaskedValue
+		}
+		return t
+	}
+	// Any other value at a masked pointer or under a secret-like key is masked whole.
+	if o.masked[ptr] || (key != "" && isSecretKey(key)) {
+		return MaskedValue
+	}
+	switch t := v.(type) {
+	case map[string]any:
+		return o.redactMap(ptr, t)
+	case []any:
+		out := make([]any, len(t))
+		for i, val := range t {
+			out[i] = o.redactTree(appendPointer(ptr, strconv.Itoa(i)), key, val)
+		}
+		return out
+	default:
+		return t
+	}
+}