diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4162ff22..4842354b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Improvements * [#3818](https://github.com/sei-protocol/sei-chain/pull/3818) feat(evmrpc): extend HTTP admission control (`max_request_body_bytes`, `max_concurrent_request_bytes`, `ws_admission_timeout`) to the WebSocket plane (:8546). WS oversize frames close with WebSocket close code 1009; budget-wait timeouts return JSON-RPC error `-32005` before the connection closes. `evmrpc_requests_rejected_total` gains a `protocol` label (`http` / `ws`). * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) feat(query): origin-aware pagination limits for ABCI queries. Untrusted callers on the ABCI/gRPC query path get configurable `max-limit`, `max-offset`, and flat `max-iterations` (defaults: 1000 / 10000 / 11000); requests above the caps are rejected upfront, and an exhausted iteration budget returns a partial page with `next_key` instead of failing. Trusted origins (new `[query] trusted-cidrs`) and the `[query] disable-limits` kill switch bypass the caps; the consensus/EVM precompile path is unaffected. +* [#4145](https://github.com/sei-protocol/sei-chain/pull/4145) fix(flatkv): FlatKV keeps 72 old PebbleDB checkpoints instead of the 1 it inherited from memIAVL's `state-commit.sc-keep-recent`. Its snapshot *interval* is still taken from `sc-snapshot-interval`; only the retention count is now independent. This raises the guaranteed reach of `migrate-evm-status`, `dump-flatkv`, a cross-backend digest and a FlatKV rollback from 10,000 blocks (about 74 minutes) to 720,000 blocks (about 89 hours), and costs about 20 GiB of extra disk on a mainnet-sized node, because checkpoints hardlink their SSTs and so only pin what compaction has since obsoleted. No `app.toml` change is needed, and an explicit `state-commit.flatkv.snapshot-keep-recent` left over from an old template still has no effect. * [#3990](https://github.com/sei-protocol/sei-chain/pull/3990) Freeze mode is limited to full nodes and disables transaction and evidence submission, mempool gossip, and state sync from startup while preserving query RPC and mempool-backed reads. Frozen and Autobahn nodes no longer advertise the unused mempool P2P channel. ### Upgrade guide diff --git a/app/testdata/state-commit.golden b/app/testdata/state-commit.golden index 22881423c1..1d1241fd1b 100644 --- a/app/testdata/state-commit.golden +++ b/app/testdata/state-commit.golden @@ -14,7 +14,7 @@ FlatKVConfig.DataDir = string("") FlatKVConfig.Fsync = bool(false) FlatKVConfig.AsyncWriteBuffer = int(0) FlatKVConfig.SnapshotInterval = uint32(10000) -FlatKVConfig.SnapshotKeepRecent = uint32(1) +FlatKVConfig.SnapshotKeepRecent = uint32(72) FlatKVConfig.ExternalPruning = bool(false) FlatKVConfig.EnablePebbleMetrics = bool(true) FlatKVConfig.EnableReadWriteMetrics = bool(false) diff --git a/sei-cosmos/server/config/config.go b/sei-cosmos/server/config/config.go index ee732b85c9..9f30ffe80b 100644 --- a/sei-cosmos/server/config/config.go +++ b/sei-cosmos/server/config/config.go @@ -473,9 +473,10 @@ func GetConfig(v *viper.Viper) (Config, error) { // FlatKV knobs are not rendered in the default app.toml template. GetConfig // is a faithful parse of app.toml/flags: it only reads the explicit // state-commit.flatkv.* keys (if an operator adds them by hand) on top of the - // in-code defaults. The FlatKV-follows-memIAVL mirror (and snapshot cadence - // normalization) is applied later by composite.alignFlatKVSnapshotWithMemIAVL - // at store construction, so we deliberately do not mirror the sc-* keys here. + // in-code defaults. The FlatKV-follows-memIAVL interval mirror (and its + // cadence normalization) is applied later by + // composite.alignFlatKVSnapshotIntervalWithMemIAVL at store construction, so + // we deliberately do not mirror the sc-* keys here. flatKVConfig := config.DefaultStateCommitConfig().FlatKVConfig if v.IsSet("state-commit.flatkv.fsync") { flatKVConfig.Fsync = v.GetBool("state-commit.flatkv.fsync") diff --git a/sei-cosmos/server/config/config_test.go b/sei-cosmos/server/config/config_test.go index 38a1abc0b9..4b588eb735 100644 --- a/sei-cosmos/server/config/config_test.go +++ b/sei-cosmos/server/config/config_test.go @@ -554,8 +554,9 @@ func TestGetConfigParsesRawSnapshotKeepRecent(t *testing.T) { require.NoError(t, err) // GetConfig is a faithful parse of app.toml/flags: the raw 0 is preserved for // memIAVL here and only floored later at store construction. FlatKV does not - // mirror the sc-* keys in GetConfig (that is composite.alignFlatKVSnapshotWithMemIAVL's - // job), so it keeps its in-code default. + // mirror the sc-* keys in GetConfig (the interval mirror is + // composite.alignFlatKVSnapshotIntervalWithMemIAVL's job, and the retention + // count is never mirrored), so it keeps its in-code default. require.Equal(t, uint32(0), cfg.StateCommit.MemIAVLConfig.SnapshotKeepRecent) require.Equal(t, seidbconfig.DefaultStateCommitConfig().FlatKVConfig.SnapshotKeepRecent, cfg.StateCommit.FlatKVConfig.SnapshotKeepRecent) } @@ -582,9 +583,10 @@ func TestGetConfigHonorsExplicitFlatKVOverrides(t *testing.T) { } // TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent locks in the regression fix: -// GetConfig does not mirror the sc-* keys onto FlatKV (that is -// composite.alignFlatKVSnapshotWithMemIAVL's job at store construction), and an -// absent sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV +// GetConfig does not mirror the sc-* keys onto FlatKV (the interval mirror is +// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job at store construction, +// and the retention count is never mirrored), and an absent +// sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV // defaults rather than reading back 0 (which disables FlatKV snapshots and drops // all old snapshots). func TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent(t *testing.T) { diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index e0ec8c8138..7ef8455d8d 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -72,7 +72,7 @@ StateCommit.FlatKVConfig.DataDir = string("") StateCommit.FlatKVConfig.Fsync = bool(false) StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0) StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000) -StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1) +StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(72) StateCommit.FlatKVConfig.ExternalPruning = bool(false) StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true) StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false) diff --git a/sei-db/config/ss_config_test.go b/sei-db/config/ss_config_test.go index 543f50ba0c..81251ab10a 100644 --- a/sei-db/config/ss_config_test.go +++ b/sei-db/config/ss_config_test.go @@ -66,8 +66,9 @@ func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) { require.Zero(t, ssConfig.SnapshotMinTimeInterval) } -// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it -// identically or the two backends drift onto different snapshot heights. +// SS mirrors memIAVL's whole cadence and FlatKV mirrors its interval, and every +// mirror must resolve that interval identically or the backends drift onto +// different snapshot heights. func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) { for _, tc := range []struct { name string diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 98300856de..429ac1cdf9 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -149,7 +149,7 @@ func NewCompositeCommitStore( return nil, fmt.Errorf("invalid state commit config: %w", err) } - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) var memIAVL *memiavl.CommitStore if cfg.WriteMode != types.FlatKVOnly { @@ -194,38 +194,38 @@ func NewCompositeCommitStore( }, nil } -// alignFlatKVSnapshotWithMemIAVL keeps the two backends' snapshot cadence in -// sync. FlatKV has no independently-exposed snapshot knobs in app.toml, so it -// derives its snapshot-interval / keep-recent from memIAVL's sc-* keys. This is -// the single place both backends are constructed from the same config, so it is -// where the alignment is enforced. +// alignFlatKVSnapshotIntervalWithMemIAVL makes FlatKV take its snapshot interval +// from memIAVL's sc-snapshot-interval. This is the single place both backends are +// constructed from the same config, so it is where the alignment is enforced. // -// This derivation is intentionally unconditional across write modes, including -// FlatKVOnly — where NewCompositeCommitStore never constructs a memIAVL store. -// The sc-* keys are the only operator-visible snapshot-cadence knobs now that -// the flatkv.* keys are hidden from the app.toml template, so they must govern -// FlatKV's cadence in every mode; otherwise FlatKVOnly would have no -// template-visible way to tune it. It is harmless when memIAVL is absent: the -// sc-* defaults match FlatKV's own in-code defaults, and only cfg.FlatKVConfig -// is read when building the FlatKVOnly store. +// The interval must match because a composite operation needs a version *both* +// backends hold a snapshot for: a rollback rewinds memIAVL and then FlatKV, and a +// cross-backend digest has to open each at the same height. Two backends +// checkpointing on different heights have no such version in common. // -// FlatKV mirrors memIAVL's *effective* cadence: a zero memIAVL value is first -// resolved to the same default Options.FillDefaults would apply at OpenDB -// (interval 0 -> DefaultSnapshotInterval, keep-recent 0 -> DefaultSnapshotKeepRecent), -// then assigned to FlatKV unconditionally. Resolving-then-assigning (rather than -// skipping on a zero and letting FlatKV keep its own in-code default) keeps the -// two backends in true lockstep without relying on FlatKV's default happening to -// equal memIAVL's healed default. That reliance is fragile — the defaults are -// only kept equal by hand — and it breaks for an upgrading node whose old -// app.toml still carries an explicit state-commit.flatkv.snapshot-keep-recent -// (rendered by the old template) alongside sc-keep-recent = 0: skipping would -// leave FlatKV pinned to the stale explicit value while memIAVL healed to a -// different default. Note that mirroring a raw 0 is never correct here (0 means -// "disable auto-snapshots" for FlatKV), which is why the zero is resolved first. -func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) { - interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) +// Retention count is deliberately not mirrored. It is a per-backend disk decision +// rather than a cadence, and the two backends' costs differ by a factor of +// roughly 200: measured at mainnet state size, one further retained snapshot +// costs 56,782 MiB on memIAVL, whose snapshots are independent full copies, and +// about 286 MiB on FlatKV, whose checkpoints hardlink their SSTs. A single shared +// count cannot serve both — the depth FlatKV wants for forensic reach into the +// migration window would ask memIAVL for more than the volume holds. FlatKV +// therefore keeps config.DefaultSnapshotKeepRecent, which is sized for that reach. +// +// The mirror is unconditional across write modes, including FlatKVOnly, where +// NewCompositeCommitStore never constructs a memIAVL store. sc-snapshot-interval +// is the only operator-visible cadence knob, since the flatkv.* keys are hidden +// from the app.toml template and the production reader does not consult them, so +// it has to govern FlatKV's interval in every mode. It is harmless when memIAVL is +// absent, because only cfg.FlatKVConfig is read when building that store. +// +// A zero is resolved before it is assigned, to the same default +// Options.FillDefaults would apply at OpenDB. Mirroring a raw 0 is never correct: +// 0 disables auto-snapshots for FlatKV, which lets the WAL grow without bound and +// makes every restart replay from snapshot-0. +func alignFlatKVSnapshotIntervalWithMemIAVL(cfg *config.StateCommitConfig) { + interval, _ := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig) cfg.FlatKVConfig.SnapshotInterval = interval - cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent } // Initialize records the set of child store names that should exist on diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index 39a9360c87..d03f4f787c 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -16,6 +16,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" @@ -1037,12 +1038,13 @@ func evmMigratedConfig() config.StateCommitConfig { cfg.MemIAVLConfig.SnapshotInterval = 1 cfg.MemIAVLConfig.SnapshotMinTimeInterval = 0 cfg.MemIAVLConfig.AsyncCommitBuffer = 0 - // With SnapshotInterval=1 every commit produces a snapshot, and FlatKV - // mirrors this cadence via alignFlatKVSnapshotWithMemIAVL. The default - // keep-recent of 1 would prune all but the two newest snapshots, so a + // With SnapshotInterval=1 every commit produces a snapshot, and memIAVL's + // default keep-recent of 1 would prune all but the two newest, so a // rollback/reconcile to an older version (e.g. v3 after committing v5) // could no longer find a base snapshot at-or-below the target. Retain all // snapshots for the short duration of a test so those paths stay valid. + // FlatKV needs no equivalent here: it mirrors the interval but keeps its + // own retention count, which is deep enough already. cfg.MemIAVLConfig.SnapshotKeepRecent = 100 return cfg } @@ -2571,31 +2573,16 @@ func TestLoadVersionReadOnlyDuringMigrateEVMTransition(t *testing.T) { require.Equal(t, []byte(evmVal), got) } -func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) { - t.Run("FlatKV derives interval and keep-recent from a non-zero memIAVL", func(t *testing.T) { +func TestAlignFlatKVSnapshotIntervalWithMemIAVL(t *testing.T) { + t.Run("FlatKV derives its interval from a non-zero memIAVL", func(t *testing.T) { cfg := config.DefaultStateCommitConfig() cfg.MemIAVLConfig.SnapshotInterval = 5000 - cfg.MemIAVLConfig.SnapshotKeepRecent = 3 - // Start FlatKV from divergent values to prove they get overwritten. + // Start FlatKV from a divergent value to prove it gets overwritten. cfg.FlatKVConfig.SnapshotInterval = 111 - cfg.FlatKVConfig.SnapshotKeepRecent = 222 - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) require.Equal(t, uint32(5000), cfg.FlatKVConfig.SnapshotInterval) - require.Equal(t, uint32(3), cfg.FlatKVConfig.SnapshotKeepRecent) - }) - - t.Run("a zero memIAVL keep-recent resolves to the healed default", func(t *testing.T) { - cfg := config.DefaultStateCommitConfig() - cfg.MemIAVLConfig.SnapshotKeepRecent = 0 - // FlatKV must not mirror the raw 0 (which would prune everything but the - // latest). Instead it mirrors the value FillDefaults will heal memIAVL to, - // keeping the two in lockstep. memIAVL's own 0 is left for FillDefaults. - alignFlatKVSnapshotWithMemIAVL(&cfg) - - require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent) - require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent) }) t.Run("a zero memIAVL interval resolves to the healed default", func(t *testing.T) { @@ -2603,28 +2590,51 @@ func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) { cfg.MemIAVLConfig.SnapshotInterval = 0 // A raw 0 would disable FlatKV auto-snapshots; instead FlatKV mirrors the // value FillDefaults will heal memIAVL's interval to. - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval) require.NotZero(t, cfg.FlatKVConfig.SnapshotInterval) }) - t.Run("an explicit FlatKV override loses to memIAVL's healed default", func(t *testing.T) { - // Upgrade scenario: an old app.toml still pins an explicit FlatKV - // keep-recent/interval (the previous template rendered flatkv.* keys) - // while sc-* is 0. FlatKV must follow memIAVL's effective (healed) cadence - // rather than staying pinned to the stale explicit value, otherwise the - // two backends diverge (memIAVL heals 0 -> default, FlatKV keeps the old - // explicit value). + t.Run("retention count is not mirrored", func(t *testing.T) { + // The two backends share a cadence and not a disk budget. A retained + // memIAVL snapshot is an independent full copy where a FlatKV checkpoint + // hardlinks its SSTs, so one shared count cannot serve both. + cfg := config.DefaultStateCommitConfig() + cfg.MemIAVLConfig.SnapshotKeepRecent = 3 + cfg.FlatKVConfig.SnapshotKeepRecent = 222 + + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) + + require.Equal(t, uint32(222), cfg.FlatKVConfig.SnapshotKeepRecent) + require.Equal(t, uint32(3), cfg.MemIAVLConfig.SnapshotKeepRecent) + }) + + t.Run("a zero memIAVL keep-recent does not reach FlatKV", func(t *testing.T) { + // A zero memIAVL keep-recent is healed to memiavl.DefaultSnapshotKeepRecent + // for memIAVL's own use. That healed value must not reach FlatKV, whose + // own default stands. cfg := config.DefaultStateCommitConfig() cfg.MemIAVLConfig.SnapshotKeepRecent = 0 - cfg.MemIAVLConfig.SnapshotInterval = 0 - cfg.FlatKVConfig.SnapshotKeepRecent = 2 - cfg.FlatKVConfig.SnapshotInterval = 7777 - alignFlatKVSnapshotWithMemIAVL(&cfg) + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) - require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent) - require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval) + require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent) + require.Equal(t, flatkvconfig.DefaultSnapshotKeepRecent, cfg.FlatKVConfig.SnapshotKeepRecent) + require.NotEqual(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent, + "FlatKV must not inherit memIAVL's retention count") }) } + +// The default exists to bound what can be asked about a past height, so a reach +// shorter than the migration window it was sized for is the regression to catch. +// 72 checkpoints at a 10000-block interval is 720,000 blocks, about 89 hours at +// mainnet's measured 2.247 blocks/s, against an 85-hour drain at K=1024. +func TestFlatKVDefaultRetentionSpansTheMigrationWindow(t *testing.T) { + cfg := config.DefaultStateCommitConfig() + alignFlatKVSnapshotIntervalWithMemIAVL(&cfg) + + reach := uint64(cfg.FlatKVConfig.SnapshotKeepRecent) * uint64(cfg.FlatKVConfig.SnapshotInterval) + require.GreaterOrEqual(t, reach, uint64(690_000), + "default FlatKV retention must reach back across an 85-hour drain at 2.247 blocks/s") +} diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 5ee296fcd0..a9ce9f3248 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -9,8 +9,24 @@ import ( ) const ( - DefaultSnapshotInterval uint32 = 10000 - DefaultSnapshotKeepRecent uint32 = 1 + DefaultSnapshotInterval uint32 = 10000 + + // DefaultSnapshotKeepRecent keeps 72 old checkpoints besides the latest one, + // which at DefaultSnapshotInterval is a guaranteed reach of 720,000 blocks — + // about 89 hours at mainnet's block rate, so it spans the EVM migration + // window at the rate that window is planned for. + // + // It is this deep because a FlatKV checkpoint is nearly free. Checkpoints + // hardlink their SSTs, so one only costs the bytes compaction has since made + // obsolete: measured at mainnet state size, 261 MiB of pinned SSTs plus about + // 25 MiB of retained state WAL. 72 of them is roughly 20 GiB. The cost is + // linear in depth, because each older checkpoint pins exactly the files + // obsoleted during its own interval and those sets are disjoint. + // + // Reach matters because it bounds what can be answered about a past height at + // all. Below it, migrate-evm-status, dump-flatkv and a cross-backend digest + // cannot open a version, and a rollback has no base snapshot to rewind to. + DefaultSnapshotKeepRecent uint32 = 72 ) // Config defines configuration for the FlatKV (EVM) commit store. @@ -40,7 +56,12 @@ type Config struct { // SnapshotKeepRecent defines how many old snapshots to keep besides the // latest one. 0 means keep only the current snapshot (no old snapshots). // Ignored entirely when ExternalPruning is set. - // Default: 1 + // + // It is not mirrored from memIAVL's sc-keep-recent, and the production store + // reads no app.toml key for it, so a node runs the in-code default. See + // composite.alignFlatKVSnapshotIntervalWithMemIAVL for why the two backends + // share an interval but not a retention count. + // Default: 72 SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"` // ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its