feat(agentcfg): append-only agent config revision store (7/15)

.env.example

@@ -53,6 +53,18 @@ CCF_PLAYBACK_TIMEOUT="5s"
 CCF_PLAYBACK_MAX_BYTES=2097152
 CCF_PLAYBACK_MAX_CONCURRENT=4
 
+# Agent remote configuration: instance freshness, retention, pruning (River job; needs
+# CCF_WORKER_ENABLED) and the per-agent instance cap.
+CCF_AGENT_INSTANCE_STALE_AFTER="10m"
+CCF_AGENT_INSTANCE_RETENTION="720h"
+CCF_AGENT_INSTANCE_ONESHOT_RETENTION="24h"
+CCF_AGENT_INSTANCE_PRUNE_ENABLED=true
+# The prune job is deduplicated per hour: it runs at most hourly, whatever the schedule.
+CCF_AGENT_INSTANCE_PRUNE_SCHEDULE="0 17 * * * *"
+# Cap on non-prunable instances per agent. When full, a new instance replaces the oldest
+# stale one (not seen within CCF_AGENT_INSTANCE_STALE_AFTER); 409 only when all are fresh.
+CCF_AGENT_MAX_INSTANCES=500
+
 # Policy evaluation artifacts: POST /api/agent/artifacts (see docs/artifacts.md)
 CCF_ARTIFACT_MAX_BYTES=16777216
 CCF_ARTIFACT_MAX_CONCURRENT=8

cmd/root.go

@@ -99,6 +99,12 @@ func bindEnvironmentVariables() {
 	viper.MustBindEnv("playback_timeout")
 	viper.MustBindEnv("playback_max_bytes")
 	viper.MustBindEnv("playback_max_concurrent")
+	viper.MustBindEnv("agent_instance_stale_after")
+	viper.MustBindEnv("agent_instance_retention")
+	viper.MustBindEnv("agent_instance_oneshot_retention")
+	viper.MustBindEnv("agent_instance_prune_enabled")
+	viper.MustBindEnv("agent_instance_prune_schedule")
+	viper.MustBindEnv("agent_max_instances")
 	viper.MustBindEnv("artifact_max_bytes")
 	viper.MustBindEnv("artifact_max_concurrent")
 }

internal/config/agents.go

@@ -0,0 +1,72 @@
+package config
+
+import (
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+// AgentsConfig tunes agent remote configuration: instance freshness, retention, pruning and
+// the per-agent instance cap (R14, R37).
+type AgentsConfig struct {
+	// InstanceStaleAfter: an instance is fresh when seen (heartbeat or report) within this
+	// window. CCF_AGENT_INSTANCE_STALE_AFTER, default 10m.
+	InstanceStaleAfter time.Duration `json:"instanceStaleAfter"`
+	// InstanceRetention: daemon (or unknown) instances not seen for this long are pruned.
+	// CCF_AGENT_INSTANCE_RETENTION, default 720h.
+	InstanceRetention time.Duration `json:"instanceRetention"`
+	// OneShotInstanceRetention: daemon=false instances not seen for this long are pruned.
+	// CCF_AGENT_INSTANCE_ONESHOT_RETENTION, default 24h.
+	OneShotInstanceRetention time.Duration `json:"oneShotInstanceRetention"`
+	// InstancePruneEnabled schedules the prune job (needs the worker service).
+	// CCF_AGENT_INSTANCE_PRUNE_ENABLED, default true.
+	InstancePruneEnabled bool `json:"instancePruneEnabled"`
+	// InstancePruneSchedule is the River (6-field, seconds first) cron of the prune job.
+	// CCF_AGENT_INSTANCE_PRUNE_SCHEDULE, default "0 17 * * * *" (hourly). The job is
+	// deduplicated per hour, so it runs at most hourly: a more frequent schedule is not
+	// honored.
+	InstancePruneSchedule string `json:"instancePruneSchedule"`
+	// MaxInstancesPerAgent caps the non-prunable instances of one agent. When the cap is
+	// reached, a new instance replaces the oldest stale one (not seen within
+	// InstanceStaleAfter); only when every counted instance is fresh does a report from a new
+	// instance get 409 (and its heartbeats are not recorded). CCF_AGENT_MAX_INSTANCES,
+	// default 500.
+	MaxInstancesPerAgent int `json:"maxInstancesPerAgent"`
+}
+
+// DefaultAgentsConfig returns the defaults.
+func DefaultAgentsConfig() *AgentsConfig {
+	return &AgentsConfig{
+		InstanceStaleAfter:       10 * time.Minute,
+		InstanceRetention:        720 * time.Hour,
+		OneShotInstanceRetention: 24 * time.Hour,
+		InstancePruneEnabled:     true,
+		InstancePruneSchedule:    "0 17 * * * *",
+		MaxInstancesPerAgent:     500,
+	}
+}
+
+// LoadAgentsConfig reads the CCF_AGENT_* settings, falling back to the default for any value
+// that is unset or not positive.
+func LoadAgentsConfig() *AgentsConfig {
+	cfg := DefaultAgentsConfig()
+	if d := viper.GetDuration("agent_instance_stale_after"); d > 0 {
+		cfg.InstanceStaleAfter = d
+	}
+	if d := viper.GetDuration("agent_instance_retention"); d > 0 {
+		cfg.InstanceRetention = d
+	}
+	if d := viper.GetDuration("agent_instance_oneshot_retention"); d > 0 {
+		cfg.OneShotInstanceRetention = d
+	}
+	if viper.IsSet("agent_instance_prune_enabled") {
+		cfg.InstancePruneEnabled = viper.GetBool("agent_instance_prune_enabled")
+	}
+	if s := viper.GetString("agent_instance_prune_schedule"); s != "" {
+		cfg.InstancePruneSchedule = s
+	}
+	if n := viper.GetInt("agent_max_instances"); n > 0 {
+		cfg.MaxInstancesPerAgent = n
+	}
+	return cfg
+}

internal/config/agents_test.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadAgentsConfigDefaults(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+
+	cfg := LoadAgentsConfig()
+
+	assert.Equal(t, 10*time.Minute, cfg.InstanceStaleAfter)
+	assert.Equal(t, 720*time.Hour, cfg.InstanceRetention)
+	assert.Equal(t, 24*time.Hour, cfg.OneShotInstanceRetention)
+	assert.True(t, cfg.InstancePruneEnabled)
+	assert.Equal(t, "0 17 * * * *", cfg.InstancePruneSchedule)
+	assert.Equal(t, 500, cfg.MaxInstancesPerAgent)
+	assert.Equal(t, DefaultAgentsConfig(), cfg)
+}
+
+func TestLoadAgentsConfigOverrides(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	viper.Set("agent_instance_stale_after", "5m")
+	viper.Set("agent_instance_retention", "48h")
+	viper.Set("agent_instance_oneshot_retention", "2h")
+	viper.Set("agent_instance_prune_enabled", false)
+	viper.Set("agent_instance_prune_schedule", "0 0 * * * *")
+	viper.Set("agent_max_instances", 20)
+
+	cfg := LoadAgentsConfig()
+
+	assert.Equal(t, 5*time.Minute, cfg.InstanceStaleAfter)
+	assert.Equal(t, 48*time.Hour, cfg.InstanceRetention)
+	assert.Equal(t, 2*time.Hour, cfg.OneShotInstanceRetention)
+	assert.False(t, cfg.InstancePruneEnabled)
+	assert.Equal(t, "0 0 * * * *", cfg.InstancePruneSchedule)
+	assert.Equal(t, 20, cfg.MaxInstancesPerAgent)
+}
+
+func TestLoadAgentsConfigFromEnv(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	viper.SetEnvPrefix("ccf")
+	viper.AutomaticEnv()
+	t.Setenv("CCF_AGENT_INSTANCE_PRUNE_ENABLED", "false")
+	t.Setenv("CCF_AGENT_MAX_INSTANCES", "3")
+	t.Setenv("CCF_AGENT_INSTANCE_STALE_AFTER", "90s")
+
+	cfg := LoadAgentsConfig()
+
+	assert.False(t, cfg.InstancePruneEnabled)
+	assert.Equal(t, 3, cfg.MaxInstancesPerAgent)
+	assert.Equal(t, 90*time.Second, cfg.InstanceStaleAfter)
+}
+
+func TestLoadAgentsConfigIgnoresNonPositiveValues(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+	viper.Set("agent_instance_stale_after", "0s")
+	viper.Set("agent_instance_retention", "-1h")
+	viper.Set("agent_instance_oneshot_retention", "0")
+	viper.Set("agent_instance_prune_schedule", "")
+	viper.Set("agent_max_instances", 0)
+
+	cfg := LoadAgentsConfig()
+
+	assert.Equal(t, DefaultAgentsConfig(), cfg)
+}

internal/config/config.go

@@ -47,6 +47,7 @@ type Config struct {
 	StrictDisablePublicAgentEndpoints bool
 	Authz                             *AuthzConfig
 	Playback                          *PlaybackConfig
+	Agents                            *AgentsConfig
 	Artifact                          *ArtifactConfig
 	EvidenceSubjects                  *EvidenceSubjectConfig
 }
@@ -284,6 +285,7 @@ func NewConfig(logger *zap.SugaredLogger) *Config {
 		StrictDisablePublicAgentEndpoints: strictDisablePublicAgentEndpoints,
 		Authz:                             authzConfig,
 		Playback:                          LoadPlaybackConfig(),
+		Agents:                            LoadAgentsConfig(),
 		Artifact:                          LoadArtifactConfig(),
 	}
 

internal/service/migrator.go

@@ -184,6 +184,7 @@ func MigrateUpWithConfig(db *gorm.DB, cfg *config.Config) error {
 		&relational.Agent{},
 		&relational.AgentServiceAccountKey{},
 		&relational.AgentAuthEvent{},
+		&relational.AgentConfigRevision{},
 		&relational.UserNotificationSubscription{},
 		&relational.SystemNotificationDestination{},
 		&Heartbeat{},
@@ -1159,6 +1160,7 @@ func MigrateDown(db *gorm.DB) error {
 		&poamrel.PoamItemMilestone{},
 		&poamrel.PoamItem{},
 
+		&relational.AgentConfigRevision{},
 		&relational.AgentAuthEvent{},
 		&relational.AgentServiceAccountKey{},
 		&relational.Agent{},

internal/service/relational/agent_config.go

@@ -0,0 +1,37 @@
+package relational
+
+import (
+	"errors"
+	"time"
+
+	"github.com/google/uuid"
+	"gorm.io/datatypes"
+	"gorm.io/gorm"
+)
+
+// ErrAgentConfigRevisionAppendOnly is returned when code tries to update or delete a
+// configuration revision.
+var ErrAgentConfigRevisionAppendOnly = errors.New("agent config revisions are append-only")
+
+// AgentConfigRevision is one immutable revision of an agent's configuration overlay (D11).
+// The desired revision of an agent is MAX(revision). The row id feeds the agent-facing
+// opaque ETag (R7), so a DB reset never produces a false 304.
+type AgentConfigRevision struct {
+	UUIDModel
+	CreatedAt   time.Time      `json:"createdAt"`
+	AgentID     uuid.UUID      `json:"agentId" gorm:"type:uuid;not null;uniqueIndex:idx_agent_config_rev,priority:1"`
+	Revision    int64          `json:"revision" gorm:"not null;uniqueIndex:idx_agent_config_rev,priority:2"`
+	Overlay     datatypes.JSON `json:"overlay" gorm:"type:jsonb;not null"`
+	Comment     *string        `json:"comment,omitempty" gorm:"type:text"`
+	CreatedBy   string         `json:"createdBy" gorm:"type:text;not null"`    // user subject id (email)
+	CreatedByID *uuid.UUID     `json:"createdById,omitempty" gorm:"type:uuid"` // user_uuid claim when present
+	RevertOf    *int64         `json:"revertOf,omitempty"`
+}
+
+func (AgentConfigRevision) TableName() string { return "ccf_agent_config_revisions" }
+
+// BeforeUpdate keeps revisions append-only.
+func (*AgentConfigRevision) BeforeUpdate(*gorm.DB) error { return ErrAgentConfigRevisionAppendOnly }
+
+// BeforeDelete keeps revisions append-only.
+func (*AgentConfigRevision) BeforeDelete(*gorm.DB) error { return ErrAgentConfigRevisionAppendOnly }