From ce4e45db1dcde7accf6f5bc3d76fb89be885c1ad Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 11 Sep 2026 18:45:47 +0200 Subject: [PATCH] fix(hash): pin the service config-hash byte layout to its historical form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ServiceHash digests json.Marshal of types.ServiceConfig, which couples every recorded config-hash to the DECLARATION ORDER of compose-go struct fields: encoding/json emits struct fields in that order and flattens embedded structs at their embedding position. Any compose-go refactoring that moves a field — such as the upcoming container-spec layering, which regroups the whole struct — would change the bytes, and with them the hash, of configurations that did not change at all: every container recreated on the first `up` after an upgrade. The hash now re-emits the marshaled object with its ROOT keys in a frozen list reproducing the historical order (generated by reflection over the last pre-layering compose-go), values byte-verbatim. For today, the output is byte-identical to the direct marshal — proven by a continuity test — so every existing container stamp stays valid: no migration, no recreation, full backward compatibility. From the first struct reorder on, the frozen list alone carries that continuity, locked by golden-value tests; a root attribute added later is appended in sorted order and, thanks to omitempty, only moves the hash of configurations that use it — exactly like a field addition always did. A reflection test fails when compose-go grows a root attribute missing from the list, so extending the hash surface stays a reviewed decision. Nested objects keep their own struct marshal: a reorder inside one of them would still move hashes — the golden tests exist to turn that into a caught, reviewed event rather than a silent side effect. Network and volume hashes are unchanged (their structs are not being reordered) and gain the same golden locks. Co-Authored-By: Claude Fable 5 Signed-off-by: Nicolas De Loof --- pkg/compose/hash.go | 177 +++++++++++++++++++++++++++++++++++++-- pkg/compose/hash_test.go | 157 ++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 6 deletions(-) diff --git a/pkg/compose/hash.go b/pkg/compose/hash.go index 1be19405faa..c79bca8df69 100644 --- a/pkg/compose/hash.go +++ b/pkg/compose/hash.go @@ -17,12 +17,133 @@ package compose import ( + "bytes" "encoding/json" + "sort" "github.com/compose-spec/compose-go/v2/types" "github.com/opencontainers/go-digest" ) +// serviceHashKeyOrder freezes the top-level JSON key order of the service +// config-hash. Hashing json.Marshal of the struct directly would couple every +// recorded hash to the DECLARATION ORDER of compose-go's fields — +// encoding/json emits struct fields in that order, and flattens embedded +// structs at their embedding position — so any compose-go refactoring moving +// a field (or grouping fields into embedded specs) would change the bytes, +// and with them the hash, of configurations that did not change at all: +// every container recreated on the first `up` after an upgrade. +// +// This list pins the byte layout to the historical form instead, generated +// by reflection over compose-go v2.15.1-0.20260908103050 (the last layout +// every released hash was computed from), so existing container stamps stay +// valid verbatim: no migration, no recreation. A root attribute missing from +// the list (added to compose-go later) is emitted after the listed ones, in +// sorted order — deterministic, and thanks to omitempty only configurations +// using the new attribute see their hash move, exactly like a field addition +// always did. Nested objects keep the struct marshal of their own types; a +// reorder inside one of them would still move hashes — the golden tests +// exist to turn that into a reviewed decision instead of a side effect. +var serviceHashKeyOrder = []string{ + "profiles", + "annotations", + "attach", + "build", + "develop", + "blkio_config", + "cap_add", + "cap_drop", + "cgroup_parent", + "cgroup", + "cpu_count", + "cpu_percent", + "cpu_period", + "cpu_quota", + "cpu_rt_period", + "cpu_rt_runtime", + "cpus", + "cpuset", + "cpu_shares", + "command", + "configs", + "container_name", + "credential_spec", + "depends_on", + "deploy", + "device_cgroup_rules", + "devices", + "dns", + "dns_opt", + "dns_search", + "dockerfile", + "domainname", + "entrypoint", + "provider", + "environment", + "env_file", + "expose", + "extends", + "external_links", + "extra_hosts", + "group_add", + "gpus", + "hostname", + "healthcheck", + "image", + "init", + "ipc", + "isolation", + "labels", + "label_file", + "links", + "logging", + "log_driver", + "log_opt", + "mem_limit", + "mem_reservation", + "memswap_limit", + "mem_swappiness", + "mac_address", + "models", + "net", + "network_mode", + "networks", + "oom_kill_disable", + "oom_score_adj", + "pid", + "pids_limit", + "platform", + "ports", + "privileged", + "pull_policy", + "read_only", + "restart", + "runtime", + "scale", + "secrets", + "security_opt", + "shm_size", + "stdin_open", + "stop_grace_period", + "stop_signal", + "storage_opt", + "sysctls", + "tmpfs", + "tty", + "ulimits", + "use_api_socket", + "user", + "userns_mode", + "uts", + "volume_driver", + "volumes", + "volumes_from", + "working_dir", + "pre_start", + "post_start", + "pre_stop", +} + // ServiceHash computes the configuration hash for a service. func ServiceHash(o types.ServiceConfig) (string, error) { // remove the Build config when generating the service hash @@ -37,20 +158,64 @@ func ServiceHash(o types.ServiceConfig) (string, error) { o.DependsOn = nil o.Profiles = nil - bytes, err := json.Marshal(o) + raw, err := json.Marshal(o) + if err != nil { + return "", err + } + pinned, err := pinRootKeyOrder(raw, serviceHashKeyOrder) if err != nil { return "", err } - return digest.SHA256.FromBytes(bytes).Encoded(), nil + return digest.SHA256.FromBytes(pinned).Encoded(), nil +} + +// pinRootKeyOrder re-emits a JSON object with its top-level keys in the +// given order (values kept byte-verbatim), keys absent from the list +// appended in sorted order. For an object whose keys all follow the list, +// the output is byte-identical to the input. +func pinRootKeyOrder(raw []byte, order []string) ([]byte, error) { + var root map[string]json.RawMessage + if err := json.Unmarshal(raw, &root); err != nil { + return nil, err + } + var buf bytes.Buffer + buf.WriteByte('{') + first := true + write := func(key string, val json.RawMessage) { + if !first { + buf.WriteByte(',') + } + first = false + name, _ := json.Marshal(key) + buf.Write(name) + buf.WriteByte(':') + buf.Write(val) + } + for _, key := range order { + if val, ok := root[key]; ok { + write(key, val) + delete(root, key) + } + } + rest := make([]string, 0, len(root)) + for key := range root { + rest = append(rest, key) + } + sort.Strings(rest) + for _, key := range rest { + write(key, root[key]) + } + buf.WriteByte('}') + return buf.Bytes(), nil } // NetworkHash computes the configuration hash for a network. func NetworkHash(o *types.NetworkConfig) (string, error) { - bytes, err := json.Marshal(o) + raw, err := json.Marshal(o) if err != nil { return "", err } - return digest.SHA256.FromBytes(bytes).Encoded(), nil + return digest.SHA256.FromBytes(raw).Encoded(), nil } // VolumeHash computes the configuration hash for a volume. @@ -58,9 +223,9 @@ func VolumeHash(o types.VolumeConfig) (string, error) { if o.Driver == "" { // (TODO: jhrotko) This probably should be fixed in compose-go o.Driver = "local" } - bytes, err := json.Marshal(o) + raw, err := json.Marshal(o) if err != nil { return "", err } - return digest.SHA256.FromBytes(bytes).Encoded(), nil + return digest.SHA256.FromBytes(raw).Encoded(), nil } diff --git a/pkg/compose/hash_test.go b/pkg/compose/hash_test.go index 73b7f387735..8d9c3a3de82 100644 --- a/pkg/compose/hash_test.go +++ b/pkg/compose/hash_test.go @@ -17,9 +17,13 @@ package compose import ( + "encoding/json" + "reflect" + "strings" "testing" "github.com/compose-spec/compose-go/v2/types" + "github.com/opencontainers/go-digest" "gotest.tools/v3/assert" ) @@ -41,3 +45,156 @@ func serviceConfig(replicas int) types.ServiceConfig { Image: "bar", } } + +// TestServiceHashContinuity proves the pinned serializer reproduces the +// historical bytes: while compose-go's struct order still matches the frozen +// list — true on this branch's compose-go — the pinned hash and the plain +// struct-marshal hash are byte-identical. The compose-go upgrade that first +// reorders the struct (the container-spec layering) deletes this test in the +// same commit: from that point the frozen list carries continuity alone, +// locked by TestHashGoldenValues. +func TestServiceHashContinuity(t *testing.T) { + svc := richServiceFixture() + pinned, err := ServiceHash(svc) + assert.NilError(t, err) + + o := svc + o.Build = nil + o.PullPolicy = "" + o.Scale = nil + if o.Deploy != nil { + deploy := *o.Deploy + deploy.Replicas = nil + o.Deploy = &deploy + } + o.DependsOn = nil + o.Profiles = nil + raw, err := json.Marshal(o) + assert.NilError(t, err) + legacy := digest.SHA256.FromBytes(raw).Encoded() + assert.Equal(t, pinned, legacy) + t.Logf("GOLDEN service=%s", pinned) +} + +func richServiceFixture() types.ServiceConfig { + replicas := 3 + return types.ServiceConfig{ + Name: "web", + Image: "nginx:latest", + Command: types.ShellCommand{"nginx", "-g", "daemon off;"}, + User: "nobody", + Tty: true, + StdinOpen: true, + Restart: types.RestartPolicyAlways, + Environment: types.MappingWithEquals{"A": strPtr("1"), "B": nil}, + Labels: types.Labels{"com.example": "v"}, + Annotations: types.Mapping{"note": "x"}, + CapAdd: []string{"NET_ADMIN"}, + ExtraHosts: types.HostsList{"alpha": []string{"10.0.0.1"}}, + Deploy: &types.DeployConfig{Replicas: &replicas}, + Ports: []types.ServicePortConfig{ + {Target: 80, Published: "8080", Protocol: "tcp"}, + }, + HealthCheck: &types.HealthCheckConfig{ + Test: types.HealthCheckTest{"CMD", "true"}, + }, + Volumes: []types.ServiceVolumeConfig{ + {Type: types.VolumeTypeVolume, Source: "data", Target: "/data"}, + }, + Networks: map[string]*types.ServiceNetworkConfig{"default": nil}, + } +} + +func strPtr(s string) *string { return &s } + +func TestHashGoldenValues(t *testing.T) { + svc, err := ServiceHash(richServiceFixture()) + assert.NilError(t, err) + assert.Equal(t, svc, "75bc312132c71fac4971d123202631b6978ac9d796336621af744d46611b42a2") + + nw, err := NetworkHash(&types.NetworkConfig{Name: "proj_default", Driver: "bridge"}) + assert.NilError(t, err) + assert.Equal(t, nw, "eeef29d8955b1d4e9382986e213c80789065d989d7a6d035163e0180159aaac0") + + vol, err := VolumeHash(types.VolumeConfig{Name: "proj_data"}) + assert.NilError(t, err) + assert.Equal(t, vol, "dd3953f0ff20e0f9044086b0483690f2cc2b4653e57aaaa5fa8ea2735da61000") +} + +// pinRootKeyOrder is the identity for an object whose keys already follow +// the frozen order — the property that keeps every released hash valid. +func TestPinRootKeyOrderIdentity(t *testing.T) { + in := []byte(`{"profiles":["p"],"command":["c"],"image":"img","user":"u"}`) + out, err := pinRootKeyOrder(in, serviceHashKeyOrder) + assert.NilError(t, err) + assert.Equal(t, string(out), string(in)) +} + +// The pinned form is a function of the configuration VALUES, not of the +// struct layout it travels in: two types declaring the same JSON fields in a +// different order digest identically. +func TestPinRootKeyOrderIgnoresFieldOrder(t *testing.T) { + type a struct { + Image string `json:"image"` + Ports []string `json:"ports,omitempty"` + User string `json:"user,omitempty"` + } + type b struct { + User string `json:"user,omitempty"` + Image string `json:"image"` + Ports []string `json:"ports,omitempty"` + } + ra, _ := json.Marshal(a{Image: "nginx", Ports: []string{"80:80"}, User: "nobody"}) + rb, _ := json.Marshal(b{Image: "nginx", Ports: []string{"80:80"}, User: "nobody"}) + pa, err := pinRootKeyOrder(ra, serviceHashKeyOrder) + assert.NilError(t, err) + pb, err := pinRootKeyOrder(rb, serviceHashKeyOrder) + assert.NilError(t, err) + assert.Equal(t, string(pa), string(pb)) +} + +// Keys unknown to the frozen list are appended in sorted order, each emitted +// exactly once: nothing an attribute addition brings can be silently dropped. +func TestPinRootKeyOrderAppendsUnknownSorted(t *testing.T) { + in := []byte(`{"zz_new":"2","image":"img","aa_new":"1"}`) + out, err := pinRootKeyOrder(in, serviceHashKeyOrder) + assert.NilError(t, err) + assert.Equal(t, string(out), `{"image":"img","aa_new":"1","zz_new":"2"}`) +} + +// Every root JSON key of compose-go's ServiceConfig must be in the frozen +// list: a compose-go upgrade adding an attribute fails here, so extending the +// hash surface is a reviewed decision — add the new key at the END of +// serviceHashKeyOrder (its hash only moves for configurations using it). +func TestServiceHashKeyOrderCoversStruct(t *testing.T) { + known := map[string]bool{} + for _, k := range serviceHashKeyOrder { + known[k] = true + } + var walk func(t reflect.Type) + walk = func(rt reflect.Type) { + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + tag := f.Tag.Get("json") + name, _, _ := strings.Cut(tag, ",") + if name == "-" { + continue + } + if f.Anonymous && name == "" { + // encoding/json dereferences embedded pointers; match it + ft := f.Type + if ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + walk(ft) + continue + } + if name == "" { + name = f.Name + } + assert.Assert(t, known[name], + "ServiceConfig root attribute %q is not in serviceHashKeyOrder: append it at the end of the list (a reviewed decision — the hash of configurations using it will move)", name) + } + } + walk(reflect.TypeOf(types.ServiceConfig{})) +}