From 79797a5364397ac3ce3516dbbd64a306438471a2 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:01:54 -0300 Subject: [PATCH 1/3] feat(agentconfig): config model and RFC 7396 overlay merge First layer of the agent remote-configuration stack (split from #465): the declared agent config types, JSON helpers and canonical encoding, RFC 6901 pointers, and Merge/MergePatch/StripLocked of an API overlay onto the agent's file config. Co-Authored-By: Claude Opus 5.5 --- pkg/agentconfig/doc.go | 16 ++ pkg/agentconfig/helpers_test.go | 3 + pkg/agentconfig/jsonutil.go | 261 ++++++++++++++++++++++++ pkg/agentconfig/mergepatch.go | 105 ++++++++++ pkg/agentconfig/mergepatch_test.go | 305 +++++++++++++++++++++++++++++ pkg/agentconfig/pointer.go | 46 +++++ pkg/agentconfig/types.go | 116 +++++++++++ 7 files changed, 852 insertions(+) create mode 100644 pkg/agentconfig/doc.go create mode 100644 pkg/agentconfig/helpers_test.go create mode 100644 pkg/agentconfig/jsonutil.go create mode 100644 pkg/agentconfig/mergepatch.go create mode 100644 pkg/agentconfig/mergepatch_test.go create mode 100644 pkg/agentconfig/pointer.go create mode 100644 pkg/agentconfig/types.go diff --git a/pkg/agentconfig/doc.go b/pkg/agentconfig/doc.go new file mode 100644 index 00000000..1a8fba0a --- /dev/null +++ b/pkg/agentconfig/doc.go @@ -0,0 +1,16 @@ +// Package agentconfig is the configuration model shared by the CCF agent and the API for +// remote agent configuration: the declared config types, RFC 7396 merge of an API-stored +// overlay onto the agent's file config, overlay validation, change-safety classification, +// redaction, digests, opaque ETags, and the agent<->API wire types. +// +// The package does no I/O and never imports OPA, so importing agentconfig (as sdk/ does) +// stays light. +// +// Conventions: +// - Config documents are snake_case JSON and are treated as opaque by API envelopes. +// - Every path that addresses a config document is an RFC 6901 JSON Pointer (see Pointer). +// - Overlay null semantics follow RFC 7396: omitting a key keeps the file's value, while +// null deletes the key from the effective config so the agent's default applies. +// - Only ValidateOverlay decodes strictly. Merge, Validate, ValidateEditable, Classify, +// Redact and Digest never reject unknown fields or weakly-typed values in a base. +package agentconfig diff --git a/pkg/agentconfig/helpers_test.go b/pkg/agentconfig/helpers_test.go new file mode 100644 index 00000000..8a981e38 --- /dev/null +++ b/pkg/agentconfig/helpers_test.go @@ -0,0 +1,3 @@ +package agentconfig + +func strPtr(s string) *string { return &s } diff --git a/pkg/agentconfig/jsonutil.go b/pkg/agentconfig/jsonutil.go new file mode 100644 index 00000000..6d13ea58 --- /dev/null +++ b/pkg/agentconfig/jsonutil.go @@ -0,0 +1,261 @@ +package agentconfig + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strings" +) + +// decodeAny decodes a single JSON value with UseNumber, so integers round-trip exactly. +// Empty or whitespace-only input decodes to nil (JSON null). Trailing data is an error. +func decodeAny(data []byte) (any, error) { + if len(bytes.TrimSpace(data)) == 0 { + return nil, nil + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return nil, errors.New("unexpected data after the JSON value") + } + return v, nil +} + +// encodeCanonical encodes v without HTML escaping and without a trailing newline. Map keys +// are sorted by encoding/json, so the output is canonical for decoded (map/slice/Number) +// values. +func encodeCanonical(v any) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil +} + +// CanonicalJSON returns the canonical encoding of v: json.Marshal, decode to any with +// UseNumber, re-encode with SetEscapeHTML(false) and no trailing newline. Object keys are +// sorted. It is the encoding Digest hashes. +// +// It intentionally duplicates the API's internal/artifact.CanonicalJSON: artifact forms are +// pinned by golden tests, and this public package must not import internal/. +func CanonicalJSON(v any) ([]byte, error) { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + decoded, err := decodeAny(raw) + if err != nil { + return nil, err + } + return encodeCanonical(decoded) +} + +// decodeConfig decodes a JSON document into a Config non-strictly (unknown fields are +// ignored) with UseNumber for the free-form maps. +func decodeConfig(data []byte) (Config, error) { + var c Config + if len(bytes.TrimSpace(data)) == 0 || bytes.Equal(bytes.TrimSpace(data), []byte("null")) { + return c, nil + } + v, err := decodeAny(data) + if err != nil { + return Config{}, err + } + if obj, ok := v.(map[string]any); ok { + dropMiscasedKeys(obj) + if data, err = encodeCanonical(obj); err != nil { + return Config{}, err + } + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&c); err != nil { + return Config{}, err + } + return c, nil +} + +// Schema field names per object level, for dropMiscasedKeys. +var ( + rootFields = []string{"daemon", "verbosity", "api", "remote_config", "plugins", "agent_evidence"} + apiFields = []string{"url", "auth"} + apiAuthFields = []string{"client_id", "client_secret"} + remoteConfigFields = []string{"mode", "poll_interval", "trusted_sources", "overridable_config_flags", "allow_local_sources"} + evidenceFields = []string{"enabled", "emit_on_run_completion", "interval"} + pluginFields = []string{"enabled", "protocol_version", "schedule", "source", "policies", "config", "labels", "policy_data", "policy_behavior"} +) + +// dropMiscasedKeys removes keys that encoding/json would match case-insensitively to a schema +// field without being that exact field name (e.g. "Plugins" or "plugins.x.Source"). Without +// this a document could smuggle a value in through a differently-cased key that no strict +// check looks at. Other unknown keys are left alone (the non-strict decode ignores them). +func dropMiscasedKeys(root map[string]any) { + dropMiscased(root, rootFields) + if api, ok := root["api"].(map[string]any); ok { + dropMiscased(api, apiFields) + if auth, ok := api["auth"].(map[string]any); ok { + dropMiscased(auth, apiAuthFields) + } + } + if rc, ok := root["remote_config"].(map[string]any); ok { + dropMiscased(rc, remoteConfigFields) + } + if ev, ok := root["agent_evidence"].(map[string]any); ok { + dropMiscased(ev, evidenceFields) + } + if plugins, ok := root["plugins"].(map[string]any); ok { + for _, raw := range plugins { + if p, ok := raw.(map[string]any); ok { + dropMiscased(p, pluginFields) + } + } + } +} + +func dropMiscased(obj map[string]any, fields []string) { + for k := range obj { + if slices.Contains(fields, k) { + continue + } + for _, f := range fields { + if strings.EqualFold(k, f) { + delete(obj, k) + break + } + } + } +} + +// DecodeConfig decodes a reported config document (base or effective) non-strictly: unknown +// fields are ignored so a newer agent's fields never make the API reject a report (R51). +func DecodeConfig(data []byte) (Config, error) { + c, err := decodeConfig(data) + if err != nil { + return Config{}, fmt.Errorf("decode config: %w", err) + } + return c, nil +} + +// clone deep-copies a Config. Free-form values (policy_data) are normalized +// on the way: map[any]any (as produced by some YAML decoders) becomes map[string]any, so the +// copy is always JSON-encodable. +func (c Config) clone() Config { + out := c + if c.API != nil { + api := *c.API + if c.API.Auth != nil { + auth := *c.API.Auth + api.Auth = &auth + } + out.API = &api + } + if c.RemoteConfig != nil { + rc := *c.RemoteConfig + rc.TrustedSources = cloneStrings(c.RemoteConfig.TrustedSources) + rc.OverridableConfigFlags = cloneStrings(c.RemoteConfig.OverridableConfigFlags) + out.RemoteConfig = &rc + } + if c.AgentEvidence != nil { + ev := *c.AgentEvidence + ev.Enabled = cloneBoolPtr(c.AgentEvidence.Enabled) + ev.EmitOnRunCompletion = cloneBoolPtr(c.AgentEvidence.EmitOnRunCompletion) + out.AgentEvidence = &ev + } + if c.Plugins != nil { + out.Plugins = make(map[string]*Plugin, len(c.Plugins)) + for name, p := range c.Plugins { + out.Plugins[name] = p.clone() + } + } + return out +} + +func (p *Plugin) clone() *Plugin { + if p == nil { + return nil + } + out := *p + out.Enabled = cloneBoolPtr(p.Enabled) + if p.Schedule != nil { + s := *p.Schedule + out.Schedule = &s + } + out.Policies = cloneStrings(p.Policies) + out.Config = cloneStringMap(p.Config) + out.Labels = cloneStringMap(p.Labels) + if p.PolicyData != nil { + out.PolicyData = deepCopyAny(map[string]any(p.PolicyData)).(map[string]any) + } + if p.PolicyBehavior != nil { + out.PolicyBehavior = make(map[string][]string, len(p.PolicyBehavior)) + for k, v := range p.PolicyBehavior { + out.PolicyBehavior[k] = cloneStrings(v) + } + } + return &out +} + +func cloneStrings(s []string) []string { + if s == nil { + return nil + } + return append(make([]string, 0, len(s)), s...) +} + +func cloneStringMap(m map[string]string) map[string]string { + if m == nil { + return nil + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +func cloneBoolPtr(b *bool) *bool { + if b == nil { + return nil + } + v := *b + return &v +} + +// deepCopyAny deep-copies a free-form value, converting map[any]any to map[string]any. +func deepCopyAny(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[k] = deepCopyAny(val) + } + return out + case map[any]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[fmt.Sprint(k)] = deepCopyAny(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = deepCopyAny(val) + } + return out + case []string: + return cloneStrings(t) + case map[string]string: + return cloneStringMap(t) + default: + return v + } +} diff --git a/pkg/agentconfig/mergepatch.go b/pkg/agentconfig/mergepatch.go new file mode 100644 index 00000000..464842c2 --- /dev/null +++ b/pkg/agentconfig/mergepatch.go @@ -0,0 +1,105 @@ +package agentconfig + +import ( + "encoding/json" + "errors" + "fmt" + "slices" +) + +// MergePatch applies an RFC 7396 JSON merge patch. target may be nil/empty (treated as +// null). A non-object patch replaces the target. Objects merge recursively, null deletes a +// key and arrays replace wholesale. Numbers are preserved exactly. The result is canonical +// JSON (sorted keys, no HTML escaping). +func MergePatch(target, patch []byte) ([]byte, error) { + t, err := decodeAny(target) + if err != nil { + return nil, fmt.Errorf("merge patch: decode target: %w", err) + } + p, err := decodeAny(patch) + if err != nil { + return nil, fmt.Errorf("merge patch: decode patch: %w", err) + } + return encodeCanonical(mergeValue(t, p)) +} + +// mergeValue implements the RFC 7396 MergePatch pseudo-code on decoded values. +func mergeValue(target, patch any) any { + patchObj, ok := patch.(map[string]any) + if !ok { + return patch + } + targetObj, ok := target.(map[string]any) + if !ok { + targetObj = map[string]any{} + } + for k, v := range patchObj { + if v == nil { + delete(targetObj, k) + continue + } + targetObj[k] = mergeValue(targetObj[k], v) + } + return targetObj +} + +// StripLocked returns the overlay without the top-level LockedKeys and the sorted list of +// removed keys. A nil/empty/"null" overlay is returned as "{}". A non-object overlay is an +// error. +func StripLocked(overlay []byte) (stripped []byte, removed []string, err error) { + v, err := decodeAny(overlay) + if err != nil { + return nil, nil, fmt.Errorf("strip locked keys: %w", err) + } + if v == nil { + return []byte("{}"), nil, nil + } + obj, ok := v.(map[string]any) + if !ok { + return nil, nil, errors.New("strip locked keys: overlay must be a JSON object") + } + for _, k := range LockedKeys { + if _, present := obj[k]; present { + delete(obj, k) + removed = append(removed, k) + } + } + slices.Sort(removed) + out, err := encodeCanonical(obj) + if err != nil { + return nil, nil, err + } + return out, removed, nil +} + +// Merge computes the effective config: StripLocked(overlay), then MergePatch onto +// json.Marshal(base), then a non-strict decode into Config (so fields a newer base carries +// survive). Locked keys always come from base, as defence in depth (R23). +// +// Merge does no env resolution, applies no defaults and does no validation. It errors when +// the overlay is not an object or the merged document does not fit the Config types (for +// example a number where a string map value is expected); ValidateOverlay reports those +// problems with pointers. +func Merge(base Config, overlay json.RawMessage) (Config, error) { + stripped, _, err := StripLocked(overlay) + if err != nil { + return Config{}, err + } + b := base.clone() + baseJSON, err := json.Marshal(b) + if err != nil { + return Config{}, fmt.Errorf("merge: encode base: %w", err) + } + merged, err := MergePatch(baseJSON, stripped) + if err != nil { + return Config{}, err + } + out, err := decodeConfig(merged) + if err != nil { + return Config{}, fmt.Errorf("merge: decode effective config: %w", err) + } + out.API = b.API + out.Daemon = b.Daemon + out.RemoteConfig = b.RemoteConfig + return out, nil +} diff --git a/pkg/agentconfig/mergepatch_test.go b/pkg/agentconfig/mergepatch_test.go new file mode 100644 index 00000000..df54d8bc --- /dev/null +++ b/pkg/agentconfig/mergepatch_test.go @@ -0,0 +1,305 @@ +package agentconfig + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMergePatchRFC7396AppendixA runs every example of RFC 7396 Appendix A. +func TestMergePatchRFC7396AppendixA(t *testing.T) { + tests := []struct { + target, patch, want string + }{ + {`{"a":"b"}`, `{"a":"c"}`, `{"a":"c"}`}, + {`{"a":"b"}`, `{"b":"c"}`, `{"a":"b","b":"c"}`}, + {`{"a":"b"}`, `{"a":null}`, `{}`}, + {`{"a":"b","b":"c"}`, `{"a":null}`, `{"b":"c"}`}, + {`{"a":["b"]}`, `{"a":"c"}`, `{"a":"c"}`}, + {`{"a":"c"}`, `{"a":["b"]}`, `{"a":["b"]}`}, + {`{"a":{"b":"c"}}`, `{"a":{"b":"d","c":null}}`, `{"a":{"b":"d"}}`}, + {`{"a":[{"b":"c"}]}`, `{"a":[1]}`, `{"a":[1]}`}, + {`["a","b"]`, `["c","d"]`, `["c","d"]`}, + {`{"a":"b"}`, `["c"]`, `["c"]`}, + {`{"a":"foo"}`, `null`, `null`}, + {`{"a":"foo"}`, `"bar"`, `"bar"`}, + {`{"e":null}`, `{"a":1}`, `{"a":1,"e":null}`}, + {`[1,2]`, `{"a":"b","c":null}`, `{"a":"b"}`}, + {`{}`, `{"a":{"bb":{"ccc":null}}}`, `{"a":{"bb":{}}}`}, + } + for _, tt := range tests { + t.Run(tt.target+"+"+tt.patch, func(t *testing.T) { + got, err := MergePatch([]byte(tt.target), []byte(tt.patch)) + require.NoError(t, err) + assert.JSONEq(t, tt.want, string(got)) + }) + } +} + +func TestMergePatch(t *testing.T) { + tests := []struct { + name string + target, patch string + want string + wantExact bool // compare bytes, not JSONEq + }{ + {name: "nil target is null", target: "", patch: `{"a":1}`, want: `{"a":1}`}, + {name: "null target", target: "null", patch: `{"a":{"b":null,"c":2}}`, want: `{"a":{"c":2}}`}, + {name: "nested null delete", target: `{"a":{"b":{"c":1,"d":2},"e":3}}`, patch: `{"a":{"b":{"c":null}}}`, want: `{"a":{"b":{"d":2},"e":3}}`}, + {name: "deep null delete of object", target: `{"a":{"b":{"c":1}},"x":1}`, patch: `{"a":{"b":null}}`, want: `{"a":{},"x":1}`}, + {name: "arrays replaced wholesale", target: `{"p":["a","b","c"]}`, patch: `{"p":["z"]}`, want: `{"p":["z"]}`}, + {name: "array replaced by empty array", target: `{"p":["a"]}`, patch: `{"p":[]}`, want: `{"p":[]}`}, + {name: "object replaces scalar", target: `{"a":1}`, patch: `{"a":{"b":1}}`, want: `{"a":{"b":1}}`}, + {name: "large integers preserved", target: `{"n":12345678901234567890,"f":1.10}`, patch: `{"m":9007199254740993}`, want: `{"f":1.10,"m":9007199254740993,"n":12345678901234567890}`, wantExact: true}, + {name: "output is canonical: sorted, no HTML escaping", target: `{"b":"","a":"&"}`, patch: `{}`, want: `{"a":"&","b":""}`, wantExact: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MergePatch([]byte(tt.target), []byte(tt.patch)) + require.NoError(t, err) + if tt.wantExact { + assert.Equal(t, tt.want, string(got)) + } else { + assert.JSONEq(t, tt.want, string(got)) + } + }) + } +} + +func TestMergePatchErrors(t *testing.T) { + _, err := MergePatch([]byte(`{`), []byte(`{}`)) + assert.Error(t, err) + _, err = MergePatch([]byte(`{}`), []byte(`{"a":`)) + assert.Error(t, err) + _, err = MergePatch([]byte(`{}`), []byte(`{} {}`)) + assert.Error(t, err, "trailing data") +} + +func TestStripLocked(t *testing.T) { + tests := []struct { + name string + overlay string + want string + wantRemoved []string + wantErr bool + }{ + {name: "nil", overlay: "", want: `{}`}, + {name: "whitespace", overlay: " \n", want: `{}`}, + {name: "null", overlay: "null", want: `{}`}, + {name: "empty object", overlay: `{}`, want: `{}`}, + {name: "no locked keys", overlay: `{"verbosity":1,"plugins":{"x":{"source":"s"}}}`, want: `{"plugins":{"x":{"source":"s"}},"verbosity":1}`}, + {name: "api only", overlay: `{"api":{"url":"http://evil"},"verbosity":2}`, want: `{"verbosity":2}`, wantRemoved: []string{"api"}}, + { + name: "all locked keys, including null values", + overlay: `{"remote_config":null,"daemon":true,"api":null,"plugins":{}}`, + want: `{"plugins":{}}`, + wantRemoved: []string{"api", "daemon", "remote_config"}, + }, + {name: "nested keys named like locked keys are kept", overlay: `{"plugins":{"api":{"config":{"daemon":"1"}}}}`, want: `{"plugins":{"api":{"config":{"daemon":"1"}}}}`}, + {name: "array overlay", overlay: `[1]`, wantErr: true}, + {name: "string overlay", overlay: `"x"`, wantErr: true}, + {name: "invalid json", overlay: `{"a"`, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, removed, err := StripLocked([]byte(tt.overlay)) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.JSONEq(t, tt.want, string(got)) + assert.Equal(t, tt.wantRemoved, removed) + }) + } +} + +func lockedBase() Config { + return Config{ + Daemon: true, + Verbosity: 1, + API: &APIConfig{ + URL: "https://api.example.com", + Auth: &APIAuth{ClientID: "0b3c1b8a-7c8e-4d53-9a52-9a3c1d1f2e10", ClientSecret: "s3cret"}, + }, + RemoteConfig: &RemoteConfig{ + Mode: ModeApplySafe, + PollInterval: "30s", + TrustedSources: []string{"ghcr.io/compliance-framework/*"}, + OverridableConfigFlags: []string{"local-ssh:port"}, + AllowLocalSources: true, + }, + Plugins: map[string]*Plugin{ + "local-ssh": { + Source: "ghcr.io/compliance-framework/plugin-local-ssh:v1.0.0", + Schedule: strPtr("*/5 * * * *"), + Policies: []string{"ghcr.io/compliance-framework/plugin-local-ssh-policies:v1.0.0"}, + Config: map[string]string{"host": "localhost", "port": "22"}, + }, + }, + } +} + +func TestMergeLockedKeysAlwaysFromBase(t *testing.T) { + overlays := []string{ + `{"api":{"url":"https://evil.example.com","auth":{"client_id":"x","client_secret":"y"}}}`, + `{"api":null}`, + `{"daemon":false}`, + `{"daemon":null}`, + `{"remote_config":{"mode":"apply_all","allow_local_sources":true,"trusted_sources":["*"]}}`, + `{"remote_config":null}`, + `{"api":null,"daemon":false,"remote_config":null,"verbosity":2}`, + } + for _, o := range overlays { + t.Run(o, func(t *testing.T) { + base := lockedBase() + eff, err := Merge(base, json.RawMessage(o)) + require.NoError(t, err) + assert.Equal(t, base.API, eff.API) + assert.Equal(t, base.Daemon, eff.Daemon) + assert.Equal(t, base.RemoteConfig, eff.RemoteConfig) + assert.Equal(t, lockedBase(), base, "base must not be mutated") + }) + } +} + +func TestMerge(t *testing.T) { + t.Run("empty and null overlays keep base", func(t *testing.T) { + for _, o := range []string{"", "null", "{}"} { + base := lockedBase() + eff, err := Merge(base, json.RawMessage(o)) + require.NoError(t, err, o) + assert.Equal(t, base, eff, o) + } + }) + + t.Run("overlay changes plugin fields and adds a plugin", func(t *testing.T) { + eff, err := Merge(lockedBase(), json.RawMessage(`{ + "verbosity": 2, + "plugins": { + "local-ssh": {"config": {"port": "2222", "host": null}, "schedule": null, "labels": {"env": "prod"}}, + "new": {"source": "ghcr.io/x/y:v1"} + } + }`)) + require.NoError(t, err) + assert.Equal(t, int32(2), eff.Verbosity) + ssh := eff.Plugins["local-ssh"] + require.NotNil(t, ssh) + assert.Equal(t, map[string]string{"port": "2222"}, ssh.Config) + assert.Nil(t, ssh.Schedule, "schedule: null deletes the key") + assert.Equal(t, map[string]string{"env": "prod"}, ssh.Labels) + assert.Equal(t, lockedBase().Plugins["local-ssh"].Policies, ssh.Policies) + require.NotNil(t, eff.Plugins["new"]) + assert.Equal(t, "ghcr.io/x/y:v1", eff.Plugins["new"].Source) + }) + + t.Run("plugin null deletes it", func(t *testing.T) { + eff, err := Merge(lockedBase(), json.RawMessage(`{"plugins":{"local-ssh":null}}`)) + require.NoError(t, err) + assert.NotContains(t, eff.Plugins, "local-ssh") + }) + + t.Run("policies array replaced", func(t *testing.T) { + eff, err := Merge(lockedBase(), json.RawMessage(`{"plugins":{"local-ssh":{"policies":["./policies/x"]}}}`)) + require.NoError(t, err) + assert.Equal(t, []string{"./policies/x"}, eff.Plugins["local-ssh"].Policies) + }) + + t.Run("protocol_version null resets to auto", func(t *testing.T) { + base := lockedBase() + base.Plugins["local-ssh"].ProtocolVersion = 2 + eff, err := Merge(base, json.RawMessage(`{"plugins":{"local-ssh":{"protocol_version":null}}}`)) + require.NoError(t, err) + assert.Equal(t, int32(0), eff.Plugins["local-ssh"].ProtocolVersion) + assert.Equal(t, int32(2), base.Plugins["local-ssh"].ProtocolVersion, "base not mutated") + }) + + t.Run("policy_data numbers preserved exactly", func(t *testing.T) { + base := lockedBase() + base.Plugins["local-ssh"].PolicyData = map[string]any{"keep": json.Number("12345678901234567890")} + eff, err := Merge(base, json.RawMessage(`{"plugins":{"local-ssh":{"policy_data":{"big":9007199254740993,"f":0.1}}}}`)) + require.NoError(t, err) + pd := eff.Plugins["local-ssh"].PolicyData + assert.Equal(t, json.Number("12345678901234567890"), pd["keep"]) + assert.Equal(t, json.Number("9007199254740993"), pd["big"]) + assert.Equal(t, json.Number("0.1"), pd["f"]) + }) + + t.Run("yaml-decoded base values are normalized", func(t *testing.T) { + base := lockedBase() + base.Plugins["local-ssh"].PolicyData = map[string]any{"m": map[any]any{"a": 1}} + eff, err := Merge(base, json.RawMessage(`{}`)) + require.NoError(t, err) + assert.Equal(t, map[string]any{"a": json.Number("1")}, eff.Plugins["local-ssh"].PolicyData["m"]) + }) + + t.Run("unknown overlay keys are ignored (non-strict)", func(t *testing.T) { + eff, err := Merge(lockedBase(), json.RawMessage(`{"future_field":{"x":1},"plugins":{"local-ssh":{"future":true}}}`)) + require.NoError(t, err) + assert.Equal(t, lockedBase().Plugins, eff.Plugins) + }) + + t.Run("errors", func(t *testing.T) { + for _, o := range []string{`[1]`, `"x"`, `{"a"`, `{"plugins":{"x":{"config":{"port":2222}}}}`} { + _, err := Merge(lockedBase(), json.RawMessage(o)) + assert.Error(t, err, o) + } + }) +} + +// FuzzMergeNeverChangesLocked checks that no overlay can change api, daemon or remote_config. +func FuzzMergeNeverChangesLocked(f *testing.F) { + seeds := []string{ + ``, + `null`, + `{}`, + `{"api":{"url":"https://evil"}}`, + `{"api":{"auth":{"client_id":"a","client_secret":"b"}}}`, + `{"api":null}`, + `{"daemon":false}`, + `{"daemon":null}`, + `{"remote_config":{"mode":"apply_all","allow_local_sources":true}}`, + `{"remote_config":{"trusted_sources":["*"],"overridable_config_flags":["*"]}}`, + `{"remote_config":null,"api":null,"daemon":false}`, + `{"verbosity":2,"plugins":{"x":{"source":"ghcr.io/x/y:v1"}}}`, + `{"plugins":{"api":{"config":{"daemon":"x"}}}}`, + `{"API":{"url":"case"},"Daemon":false,"Remote_Config":{"mode":"apply_all"}}`, + `{"api":1,"api":{"url":"dup"}}`, + `[1,2]`, + } + for _, s := range seeds { + f.Add([]byte(s)) + } + f.Fuzz(func(t *testing.T, overlay []byte) { + base := lockedBase() + eff, err := Merge(base, overlay) + if err != nil { + return + } + want := lockedBase() + if !assert.Equal(t, want.API, eff.API) || + !assert.Equal(t, want.Daemon, eff.Daemon) || + !assert.Equal(t, want.RemoteConfig, eff.RemoteConfig) { + t.Fatalf("overlay %q changed a locked key", overlay) + } + assert.Equal(t, want, base, "base must not be mutated") + }) +} + +// encoding/json matches field names case-insensitively; Merge must not let a differently +// cased key ("Plugins", "Source") smuggle values past the exact-name checks. +func TestMergeIgnoresMiscasedKeys(t *testing.T) { + base := Config{Plugins: map[string]*Plugin{"ssh": {Source: "ghcr.io/x/ssh:v1"}}} + eff, err := Merge(base, json.RawMessage(`{"Plugins":{"evil":{"source":"/tmp/evil"}},"plugins":{"ssh":{"Source":"/tmp/other"}}}`)) + if err != nil { + t.Fatalf("Merge: %v", err) + } + if _, ok := eff.Plugins["evil"]; ok { + t.Errorf("miscased root key added a plugin: %+v", eff.Plugins) + } + if got := eff.Plugins["ssh"].Source; got != "ghcr.io/x/ssh:v1" { + t.Errorf("miscased plugin key changed the source to %q", got) + } +} diff --git a/pkg/agentconfig/pointer.go b/pkg/agentconfig/pointer.go new file mode 100644 index 00000000..648181f9 --- /dev/null +++ b/pkg/agentconfig/pointer.go @@ -0,0 +1,46 @@ +package agentconfig + +import "strings" + +var ( + pointerEscaper = strings.NewReplacer("~", "~0", "/", "~1") + pointerUnescaper = strings.NewReplacer("~1", "/", "~0", "~") +) + +// EscapePointerToken escapes one reference token for an RFC 6901 JSON Pointer +// ("~" -> "~0", "/" -> "~1"). +func EscapePointerToken(token string) string { + return pointerEscaper.Replace(token) +} + +// UnescapePointerToken reverses EscapePointerToken. +func UnescapePointerToken(token string) string { + return pointerUnescaper.Replace(token) +} + +// Pointer builds an RFC 6901 JSON Pointer from unescaped segments, e.g. +// Pointer("plugins", "x", "config", "a/b") == "/plugins/x/config/a~1b". No segments yields +// "" (the whole document). +func Pointer(segments ...string) string { + if len(segments) == 0 { + return "" + } + var b strings.Builder + for _, s := range segments { + b.WriteByte('/') + b.WriteString(EscapePointerToken(s)) + } + return b.String() +} + +// SplitPointer splits an RFC 6901 JSON Pointer into unescaped segments. "" yields nil. +func SplitPointer(ptr string) []string { + if ptr == "" { + return nil + } + parts := strings.Split(strings.TrimPrefix(ptr, "/"), "/") + for i, p := range parts { + parts[i] = UnescapePointerToken(p) + } + return parts +} diff --git a/pkg/agentconfig/types.go b/pkg/agentconfig/types.go new file mode 100644 index 00000000..b3c29e38 --- /dev/null +++ b/pkg/agentconfig/types.go @@ -0,0 +1,116 @@ +package agentconfig + +import ( + "regexp" + "strings" + "time" +) + +// Apply modes (remote_config.mode). +const ( + ModeOff = "off" + ModeReport = "report" + ModeApplySafe = "apply_safe" + ModeApplyAll = "apply_all" +) + +const ( + // MaskedValue replaces redacted values. It is exactly this string (R25) and the API + // rejects it on write, so a redacted view can never round-trip into an overlay. + MaskedValue = "••••" + + // MaxOverlayBytes bounds the compact JSON of an overlay. + MaxOverlayBytes = 256 << 10 + // MaxReportBytes bounds a config report body. + MaxReportBytes = 4 << 20 + + // DefaultPollInterval is the remote_config.poll_interval default. + DefaultPollInterval = 60 * time.Second + // MinPollInterval is the smallest accepted remote_config.poll_interval. + MinPollInterval = 15 * time.Second +) + +// LockedKeys are the top-level keys an overlay may never set (D3). They always come from the +// agent's local config. +var LockedKeys = []string{"api", "daemon", "remote_config"} + +// PluginNamePattern is the name pattern for plugins named in an overlay (R28, O6). Viper +// lowercases file plugin names, so upper case would silently create a second plugin. +// ValidateOverlay applies it to every plugin an overlay sets, including file plugins it only +// changes, so a file plugin whose name does not match cannot be changed remotely. +var PluginNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`) + +// Config is the declared form of the agent configuration (file, overlay and effective). +// The agent keeps its private runtime structs and converts once from this form (R4). +type Config struct { + Daemon bool `json:"daemon" mapstructure:"daemon"` + Verbosity int32 `json:"verbosity" mapstructure:"verbosity"` // 0/1/2 = Info/Debug/Trace + API *APIConfig `json:"api,omitempty" mapstructure:"api"` + RemoteConfig *RemoteConfig `json:"remote_config,omitempty" mapstructure:"remote_config"` + Plugins map[string]*Plugin `json:"plugins" mapstructure:"plugins"` + AgentEvidence *EvidenceConfig `json:"agent_evidence,omitempty" mapstructure:"agent_evidence"` +} + +// APIConfig is the agent's API connection block. It is a locked key. +type APIConfig struct { + URL string `json:"url" mapstructure:"url"` + Auth *APIAuth `json:"auth,omitempty" mapstructure:"auth"` +} + +// APIAuth holds the agent service-account credentials. +type APIAuth struct { + ClientID string `json:"client_id" mapstructure:"client_id"` + ClientSecret string `json:"client_secret,omitempty" mapstructure:"client_secret"` +} + +// HasAuth reports whether both client_id and client_secret are set (non-blank). +func (a *APIConfig) HasAuth() bool { + return a != nil && a.Auth != nil && + strings.TrimSpace(a.Auth.ClientID) != "" && + strings.TrimSpace(a.Auth.ClientSecret) != "" +} + +// HasPartialAuth reports whether exactly one of client_id and client_secret is set. +func (a *APIConfig) HasPartialAuth() bool { + if a == nil || a.Auth == nil { + return false + } + return (strings.TrimSpace(a.Auth.ClientID) == "") != (strings.TrimSpace(a.Auth.ClientSecret) == "") +} + +// RemoteConfig is the agent's remote-configuration policy. It is set locally only (file, +// host env, CLI), never remotely (R30). +type RemoteConfig struct { + // Mode is off, report, apply_safe or apply_all. Unset means report for an agent with + // credentials (it reports but never applies), off without (see Normalize). + Mode string `json:"mode,omitempty" mapstructure:"mode"` + PollInterval string `json:"poll_interval,omitempty" mapstructure:"poll_interval"` + TrustedSources []string `json:"trusted_sources" mapstructure:"trusted_sources"` // default [] + OverridableConfigFlags []string `json:"overridable_config_flags" mapstructure:"overridable_config_flags"` // default [] + AllowLocalSources bool `json:"allow_local_sources" mapstructure:"allow_local_sources"` // default false +} + +// EvidenceConfig controls the agent's own evidence. +type EvidenceConfig struct { + Enabled *bool `json:"enabled,omitempty" mapstructure:"enabled"` + EmitOnRunCompletion *bool `json:"emit_on_run_completion,omitempty" mapstructure:"emit_on_run_completion"` + Interval string `json:"interval,omitempty" mapstructure:"interval"` +} + +// Plugin is one plugin entry. +type Plugin struct { + Enabled *bool `json:"enabled,omitempty" mapstructure:"enabled"` + ProtocolVersion int32 `json:"protocol_version,omitempty" mapstructure:"protocol_version"` // 0 = auto (R9) + Schedule *string `json:"schedule,omitempty" mapstructure:"schedule"` + Source string `json:"source" mapstructure:"source"` + Policies []string `json:"policies,omitempty" mapstructure:"policies"` + Config map[string]string `json:"config,omitempty" mapstructure:"config"` + Labels map[string]string `json:"labels,omitempty" mapstructure:"labels"` + PolicyData map[string]any `json:"policy_data,omitempty" mapstructure:"policy_data"` + PolicyBehavior map[string][]string `json:"policy_behavior,omitempty" mapstructure:"policy_behavior"` +} + +// IsEnabled reports whether the plugin runs; a nil Enabled means true. +func (p *Plugin) IsEnabled() bool { + return p != nil && (p.Enabled == nil || *p.Enabled) +} From 93c39e17fc507857cd1ece2e37527d7b2e1d7734 Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:13:14 -0300 Subject: [PATCH 2/3] docs(agentconfig): say where the design IDs cited in comments are defined Co-Authored-By: Claude Opus 5.5 --- pkg/agentconfig/doc.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/agentconfig/doc.go b/pkg/agentconfig/doc.go index 1a8fba0a..d4420b39 100644 --- a/pkg/agentconfig/doc.go +++ b/pkg/agentconfig/doc.go @@ -13,4 +13,10 @@ // null deletes the key from the effective config so the agent's default applies. // - Only ValidateOverlay decodes strictly. Merge, Validate, ValidateEditable, Classify, // Redact and Digest never reject unknown fields or weakly-typed values in a base. +// +// Design references: comments across this package, the agentcfg service and the agent config +// handlers cite design IDs. They are defined in the compliance-framework/local-dev repository: +// - docs/agent-remote-config-design.md: D decisions (§2) and R resolutions (§12, §13). +// - docs/agent-remote-config-lld-api.md: A work packages and the O overlay validation +// rules (A1). package agentconfig From abe743748bc0dd35433d7263710110228dac89de Mon Sep 17 00:00:00 2001 From: "ccf-lisa[bot]" <286799724+ccf-lisa[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:37:32 -0300 Subject: [PATCH 3/3] perf(agentconfig): decode config documents in place decodeAny and decodeConfig decoded through a json.Decoder, which copies its input through a buffer it grows by doubling: about 4 MiB of allocations per decode of a 1 MiB document, and a config save decodes a reported base about six times. Decode with json.Unmarshal instead, which reads the bytes in place, keeping the Decoder's results exactly: - decodeAny decodes into numberValue, which builds the same map[string]any, []any and json.Number values as a UseNumber Decoder. Documents nested deeper than 12 levels (Unmarshal re-scans each level) and input Unmarshal rejects still go through the Decoder, so its errors are unchanged. - decodeConfig unmarshals the struct and takes policy_data, the one free-form field, from the UseNumber decoding it already has. Co-Authored-By: Claude Opus 5.5 --- pkg/agentconfig/jsonutil.go | 105 +++++++++++++++++++++++++++++++ pkg/agentconfig/jsonutil_test.go | 93 +++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 pkg/agentconfig/jsonutil_test.go diff --git a/pkg/agentconfig/jsonutil.go b/pkg/agentconfig/jsonutil.go index 6d13ea58..32c9cd99 100644 --- a/pkg/agentconfig/jsonutil.go +++ b/pkg/agentconfig/jsonutil.go @@ -12,10 +12,21 @@ import ( // decodeAny decodes a single JSON value with UseNumber, so integers round-trip exactly. // Empty or whitespace-only input decodes to nil (JSON null). Trailing data is an error. +// +// A document nested at most numberValueMaxDepth deep is decoded in place with json.Unmarshal +// (see numberValue): a json.Decoder copies its input through a buffer it grows by doubling, +// several times the size of a large document. Deeper documents, and input Unmarshal +// rejects, go through the Decoder, so the errors stay the Decoder's. func decodeAny(data []byte) (any, error) { if len(bytes.TrimSpace(data)) == 0 { return nil, nil } + if nestedWithin(data, numberValueMaxDepth) { + var nv numberValue + if err := json.Unmarshal(data, &nv); err == nil { + return nv.v, nil + } + } dec := json.NewDecoder(bytes.NewReader(data)) dec.UseNumber() var v any @@ -28,6 +39,75 @@ func decodeAny(data []byte) (any, error) { return v, nil } +// numberValueMaxDepth bounds the nesting numberValue decodes. json.Unmarshal re-scans an +// Unmarshaler's value at every level, so numberValue costs about twice the document size +// per level of nesting. +const numberValueMaxDepth = 12 + +// numberValue decodes a JSON value exactly as a json.Decoder with UseNumber decodes it into +// an any (objects as map[string]any, arrays as []any, numbers as json.Number), but with +// json.Unmarshal, which reads the input in place. +type numberValue struct{ v any } + +// nestedWithin reports whether the objects and arrays of data (assumed to be JSON) nest at +// most limit deep. It skips string contents, so brackets inside strings do not count. +func nestedWithin(data []byte, limit int) bool { + depth := 0 + inString, escaped := false, false + for _, c := range data { + switch { + case inString: + switch { + case escaped: + escaped = false + case c == '\\': + escaped = true + case c == '"': + inString = false + } + case c == '"': + inString = true + case c == '{' || c == '[': + if depth++; depth > limit { + return false + } + case c == '}' || c == ']': + depth-- + } + } + return true +} + +func (n *numberValue) UnmarshalJSON(b []byte) error { + switch b[0] { + case '{': + var m map[string]numberValue + if err := json.Unmarshal(b, &m); err != nil { + return err + } + out := make(map[string]any, len(m)) + for k, e := range m { + out[k] = e.v + } + n.v = out + case '[': + var s []numberValue + if err := json.Unmarshal(b, &s); err != nil { + return err + } + out := make([]any, len(s)) + for i, e := range s { + out[i] = e.v + } + n.v = out + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + n.v = json.Number(b) + default: // string, true, false, null + return json.Unmarshal(b, &n.v) + } + return nil +} + // encodeCanonical encodes v without HTML escaping and without a trailing newline. Map keys // are sorted by encoding/json, so the output is canonical for decoded (map/slice/Number) // values. @@ -75,6 +155,15 @@ func decodeConfig(data []byte) (Config, error) { if data, err = encodeCanonical(obj); err != nil { return Config{}, err } + // json.Unmarshal reads data in place, where a Decoder copies it through a growing + // buffer. It decodes like a UseNumber Decoder except for the numbers of policy_data, + // the one free-form field, which usePolicyDataNumbers takes from obj. Input it + // rejects goes through the Decoder below, so the errors stay the Decoder's. + if err := json.Unmarshal(data, &c); err == nil { + usePolicyDataNumbers(&c, obj) + return c, nil + } + c = Config{} } dec := json.NewDecoder(bytes.NewReader(data)) dec.UseNumber() @@ -84,6 +173,22 @@ func decodeConfig(data []byte) (Config, error) { return c, nil } +// usePolicyDataNumbers replaces each decoded plugin's policy_data with its value in obj, the +// UseNumber decoding of the same document, so its numbers are json.Number as a UseNumber +// Decoder leaves them (json.Unmarshal makes them float64). +func usePolicyDataNumbers(c *Config, obj map[string]any) { + plugins, _ := obj["plugins"].(map[string]any) + for name, p := range c.Plugins { + if p == nil || p.PolicyData == nil { + continue + } + raw, _ := plugins[name].(map[string]any) + if pd, ok := raw["policy_data"].(map[string]any); ok { + p.PolicyData = pd + } + } +} + // Schema field names per object level, for dropMiscasedKeys. var ( rootFields = []string{"daemon", "verbosity", "api", "remote_config", "plugins", "agent_evidence"} diff --git a/pkg/agentconfig/jsonutil_test.go b/pkg/agentconfig/jsonutil_test.go new file mode 100644 index 00000000..e7b33d22 --- /dev/null +++ b/pkg/agentconfig/jsonutil_test.go @@ -0,0 +1,93 @@ +package agentconfig + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// decodeAnyWithDecoder is the reference decodeAny must match: a json.Decoder with UseNumber. +func decodeAnyWithDecoder(data []byte) (any, error) { + if len(bytes.TrimSpace(data)) == 0 { + return nil, nil + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return nil, errors.New("unexpected data after the JSON value") + } + return v, nil +} + +func TestDecodeAnyMatchesDecoder(t *testing.T) { + deep := strings.Repeat("[", numberValueMaxDepth+1) + "1" + strings.Repeat("]", numberValueMaxDepth+1) + atLimit := strings.Repeat(`{"a":`, numberValueMaxDepth) + "1.50" + strings.Repeat("}", numberValueMaxDepth) + for _, in := range []string{ + `{"a":1,"b":-0.5e+10,"c":12345678901234567890123,"d":1.0,"e":[1,2.50,[]],"f":{}}`, + `{"s":"x\"}]{[\\","u":"é😀","n":null,"t":true,"f":false}`, + ` [ {"a" : [ ] } , "x" , 0 ] `, + `{"dup":1,"dup":2}`, + "\"\xff\xfe invalid utf-8\"", + `"plain"`, `12`, `-0`, `true`, `null`, + deep, atLimit, + `{"a":1} x`, `{"a":1}}`, `{"a":1} {}`, `{"a":}`, `[1,]`, `{"a" 1}`, `01`, `{`, + } { + want, wantErr := decodeAnyWithDecoder([]byte(in)) + got, err := decodeAny([]byte(in)) + if wantErr != nil { + require.Error(t, err, in) + assert.Equal(t, wantErr.Error(), err.Error(), in) + continue + } + require.NoError(t, err, in) + assert.Equal(t, want, got, in) + } +} + +func TestNestedWithin(t *testing.T) { + assert.True(t, nestedWithin([]byte(`{"a":[{"b":1}]}`), 3)) + assert.False(t, nestedWithin([]byte(`{"a":[{"b":1}]}`), 2)) + assert.True(t, nestedWithin([]byte(`{"a":"[[[[\"[[["}`), 1), "brackets in strings do not count") + assert.True(t, nestedWithin([]byte(`["\\",[1]]`), 2), "an escaped backslash ends the escape") +} + +// DecodeConfig keeps the exact number literals of policy_data, as a UseNumber Decoder does, +// and decodes everything else as before. +func TestDecodeConfigPolicyDataNumbers(t *testing.T) { + raw := []byte(`{"daemon":true,"verbosity":2,"Verbosity":1,"plugins":{ + "p":{"source":"s","protocol_version":2,"policy_data":{"big":12345678901234567890123,"f":1.50,"nested":{"l":[1,{"x":2.0}]}},"Policy_Data":{"x":1}}, + "q":{"source":"s"}}}`) + got, err := DecodeConfig(raw) + require.NoError(t, err) + + var want Config + obj, err := decodeAnyWithDecoder(raw) + require.NoError(t, err) + dropMiscasedKeys(obj.(map[string]any)) + canonical, err := encodeCanonical(obj) + require.NoError(t, err) + dec := json.NewDecoder(bytes.NewReader(canonical)) + dec.UseNumber() + require.NoError(t, dec.Decode(&want)) + + assert.Equal(t, want, got) + assert.Equal(t, json.Number("12345678901234567890123"), got.Plugins["p"].PolicyData["big"]) + assert.Equal(t, json.Number("1.50"), got.Plugins["p"].PolicyData["f"]) + assert.Nil(t, got.Plugins["q"].PolicyData) + + // A document Unmarshal rejects keeps the Decoder's error. + _, err = DecodeConfig([]byte(`{"plugins":{"p":{"policy_data":[1]}}}`)) + require.Error(t, err) + var typeErr *json.UnmarshalTypeError + assert.ErrorAs(t, err, &typeErr) +}