From 1c02b9a4ee1c98da9910abc189c2e0bc70a5851f Mon Sep 17 00:00:00 2001 From: blindchaser Date: Sat, 12 Sep 2026 21:41:23 -0400 Subject: [PATCH] fix(flatkv): keep 10 old checkpoints instead of mirroring memIAVL's count A composite read needs a version both backends still retain. At mainnet state size a memIAVL rewrite takes about six hours against the 10000-block snapshot interval, so memIAVL skips generations and publishes roughly every 50,000 blocks, while a single retained FlatKV checkpoint reaches back only 10,000 to 20,000. FlatKV therefore prunes each version before memIAVL publishes it, and no common version ever exists: measured on the mainnet shadow pair, memIAVL published 229810000 at a point where FlatKV already held only 229840000 and 229850000, so the version was unreachable before it existed. That is what blocks a cross-backend digest and leaves a composite rollback with no shared base snapshot. Ten old checkpoints give a guaranteed reach of 100,000 blocks, about 12 hours, which covers that publication gap with margin. Depth is affordable because a checkpoint hardlinks its SSTs and so pins only what compaction has since obsoleted: 261 MiB of pinned SSTs plus about 25 MiB of retained state WAL each at mainnet state size, roughly 2.8 GiB for ten. The cost is linear in depth, since each older checkpoint pins exactly the files obsoleted during its own interval and those sets are disjoint. No app.toml change is needed. The `[state-commit.flatkv]` section renders no keys, so nothing pins the old value and every node inherits the new default on restart. The configuration characterization records are updated. Co-authored-by: Cursor --- CHANGELOG.md | 1 + app/testdata/state-commit.golden | 2 +- .../config/testdata/server_config.golden | 2 +- sei-db/state_db/sc/flatkv/config/config.go | 26 ++++++++++++++++--- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e4162ff22..24e2662662 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#3818](https://github.com/sei-protocol/sei-chain/pull/3818) feat(evmrpc): extend HTTP admission control (`max_request_body_bytes`, `max_concurrent_request_bytes`, `ws_admission_timeout`) to the WebSocket plane (:8546). WS oversize frames close with WebSocket close code 1009; budget-wait timeouts return JSON-RPC error `-32005` before the connection closes. `evmrpc_requests_rejected_total` gains a `protocol` label (`http` / `ws`). * [#3984](https://github.com/sei-protocol/sei-chain/pull/3984) feat(query): origin-aware pagination limits for ABCI queries. Untrusted callers on the ABCI/gRPC query path get configurable `max-limit`, `max-offset`, and flat `max-iterations` (defaults: 1000 / 10000 / 11000); requests above the caps are rejected upfront, and an exhausted iteration budget returns a partial page with `next_key` instead of failing. Trusted origins (new `[query] trusted-cidrs`) and the `[query] disable-limits` kill switch bypass the caps; the consensus/EVM precompile path is unaffected. * [#3990](https://github.com/sei-protocol/sei-chain/pull/3990) Freeze mode is limited to full nodes and disables transaction and evidence submission, mempool gossip, and state sync from startup while preserving query RPC and mempool-backed reads. Frozen and Autobahn nodes no longer advertise the unused mempool P2P channel. +* [#4158](https://github.com/sei-protocol/sei-chain/pull/4158) fix(flatkv): FlatKV keeps 10 old PebbleDB checkpoints instead of the 1 it inherited from memIAVL's `state-commit.sc-keep-recent`. Its snapshot *interval* is unchanged; only the retention count moves. This raises the guaranteed reach of `migrate-evm-status`, `dump-flatkv`, a cross-backend digest and a FlatKV rollback from 10,000 blocks (about 74 minutes) to 100,000 blocks (about 12 hours), which covers the roughly 50,000-block gap between memIAVL snapshot publications at mainnet state size — below that gap the two backends never retain a common version. It costs about 2.8 GiB of extra disk on a mainnet-sized node, because checkpoints hardlink their SSTs and so pin only what compaction has since obsoleted. No `app.toml` change is needed. ### Upgrade guide * **IBC core removal.** Removes the retired IBC core source, protobufs, light clients, CLI, and simulation support. Retired IBC stores remain mounted but are omitted from `export-genesis`; preserve the state database or use v6.6 freeze nodes for historical IBC data. diff --git a/app/testdata/state-commit.golden b/app/testdata/state-commit.golden index 22881423c1..c43d60f8ca 100644 --- a/app/testdata/state-commit.golden +++ b/app/testdata/state-commit.golden @@ -14,7 +14,7 @@ FlatKVConfig.DataDir = string("") FlatKVConfig.Fsync = bool(false) FlatKVConfig.AsyncWriteBuffer = int(0) FlatKVConfig.SnapshotInterval = uint32(10000) -FlatKVConfig.SnapshotKeepRecent = uint32(1) +FlatKVConfig.SnapshotKeepRecent = uint32(10) FlatKVConfig.ExternalPruning = bool(false) FlatKVConfig.EnablePebbleMetrics = bool(true) FlatKVConfig.EnableReadWriteMetrics = bool(false) diff --git a/sei-cosmos/server/config/testdata/server_config.golden b/sei-cosmos/server/config/testdata/server_config.golden index e0ec8c8138..49f89895b2 100644 --- a/sei-cosmos/server/config/testdata/server_config.golden +++ b/sei-cosmos/server/config/testdata/server_config.golden @@ -72,7 +72,7 @@ StateCommit.FlatKVConfig.DataDir = string("") StateCommit.FlatKVConfig.Fsync = bool(false) StateCommit.FlatKVConfig.AsyncWriteBuffer = int(0) StateCommit.FlatKVConfig.SnapshotInterval = uint32(10000) -StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(1) +StateCommit.FlatKVConfig.SnapshotKeepRecent = uint32(10) StateCommit.FlatKVConfig.ExternalPruning = bool(false) StateCommit.FlatKVConfig.EnablePebbleMetrics = bool(true) StateCommit.FlatKVConfig.EnableReadWriteMetrics = bool(false) diff --git a/sei-db/state_db/sc/flatkv/config/config.go b/sei-db/state_db/sc/flatkv/config/config.go index 5ee296fcd0..4fab2da09f 100644 --- a/sei-db/state_db/sc/flatkv/config/config.go +++ b/sei-db/state_db/sc/flatkv/config/config.go @@ -9,8 +9,25 @@ import ( ) const ( - DefaultSnapshotInterval uint32 = 10000 - DefaultSnapshotKeepRecent uint32 = 1 + DefaultSnapshotInterval uint32 = 10000 + // DefaultSnapshotKeepRecent is how many old checkpoints (besides the latest) to keep, + // which at the default interval is a guaranteed reach of 100,000 blocks — about 12 hours + // at mainnet's block rate. + // + // It is sized against memIAVL's publication rate rather than against FlatKV's own disk + // use. A composite read needs a version both backends still hold, and at mainnet state + // size a memIAVL rewrite takes about six hours against this 10000-block interval, so + // memIAVL skips generations and publishes roughly every 50,000 blocks. Keeping a single + // old checkpoint reaches back 10,000 to 20,000 blocks, so FlatKV prunes each version + // before memIAVL publishes it and no common version ever exists — which is what blocks a + // cross-backend digest and leaves a composite rollback with no shared base. + // + // Depth is affordable here because a checkpoint hardlinks its SSTs, so one costs only the + // bytes compaction has since made obsolete: measured at mainnet state size, 261 MiB of + // pinned SSTs plus about 25 MiB of retained state WAL, or roughly 2.8 GiB for ten. The + // cost is linear in depth, because each older checkpoint pins exactly the files obsoleted + // during its own interval and those sets are disjoint. + DefaultSnapshotKeepRecent uint32 = 10 ) // Config defines configuration for the FlatKV (EVM) commit store. @@ -40,7 +57,10 @@ type Config struct { // SnapshotKeepRecent defines how many old snapshots to keep besides the // latest one. 0 means keep only the current snapshot (no old snapshots). // Ignored entirely when ExternalPruning is set. - // Default: 1 + // + // It is not mirrored from memIAVL's sc-keep-recent, and no app.toml key is rendered for + // it, so a production node runs the DefaultConfig value. + // Default: 10 SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"` // ExternalPruning hands retention to the StorageGarbageCollector: the store stops pruning its