From 82485a385c6072690621cf63f899cb492c342c35 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 2 Mar 2026 10:10:29 -0500 Subject: [PATCH 1/6] feat: readonly mode --- .../bench/wrappers/db_implementations.go | 2 +- .../state_db/bench/wrappers/flatkv_wrapper.go | 2 +- sei-db/state_db/sc/composite/store.go | 10 +- sei-db/state_db/sc/flatkv/api.go | 6 +- .../sc/flatkv/lthash_correctness_test.go | 4 +- sei-db/state_db/sc/flatkv/snapshot.go | 12 +- sei-db/state_db/sc/flatkv/snapshot_test.go | 74 ++--- sei-db/state_db/sc/flatkv/store.go | 160 +++++++++-- sei-db/state_db/sc/flatkv/store_lifecycle.go | 8 +- sei-db/state_db/sc/flatkv/store_test.go | 267 ++++++++++++++++-- sei-db/state_db/sc/flatkv/store_write.go | 8 +- sei-db/state_db/sc/flatkv/store_write_test.go | 18 +- 12 files changed, 470 insertions(+), 101 deletions(-) diff --git a/sei-db/state_db/bench/wrappers/db_implementations.go b/sei-db/state_db/bench/wrappers/db_implementations.go index 9e120cf5ec..60ae2a19ef 100644 --- a/sei-db/state_db/bench/wrappers/db_implementations.go +++ b/sei-db/state_db/bench/wrappers/db_implementations.go @@ -50,7 +50,7 @@ func newFlatKVCommitStore(ctx context.Context, dbDir string) (DBWrapper, error) cfg.Fsync = false fmt.Printf("Opening flatKV from directory %s\n", dbDir) cs := flatkv.NewCommitStore(ctx, dbDir, logger.NewNopLogger(), cfg) - _, err := cs.LoadVersion(0) + _, err := cs.LoadVersion(0, false) if err != nil { if closeErr := cs.Close(); closeErr != nil { fmt.Printf("failed to close commit store during error recovery: %v\n", closeErr) diff --git a/sei-db/state_db/bench/wrappers/flatkv_wrapper.go b/sei-db/state_db/bench/wrappers/flatkv_wrapper.go index 748923569c..567936f593 100644 --- a/sei-db/state_db/bench/wrappers/flatkv_wrapper.go +++ b/sei-db/state_db/bench/wrappers/flatkv_wrapper.go @@ -41,7 +41,7 @@ func (f *flatKVWrapper) Commit() (int64, error) { } func (f *flatKVWrapper) LoadVersion(version int64) error { - _, err := f.base.LoadVersion(version) + _, err := f.base.LoadVersion(version, false) return err } diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 4d750d4437..fec15b655b 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -104,7 +104,13 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) homeDir: cs.homeDir, config: cs.config, } - // TODO: Support loading FlatKV at target version for read only + if cs.evmCommitter != nil { + evmStore, err := cs.evmCommitter.LoadVersion(targetVersion, true) + if err != nil { + return nil, fmt.Errorf("failed to load readonly FlatKV version: %w", err) + } + newStore.evmCommitter = evmStore + } return newStore, nil } @@ -113,7 +119,7 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) // This is the single entry point for evmCommitter.LoadVersion — CMS calls // CompositeCommitStore.LoadVersion(), which internally loads both backends. if cs.evmCommitter != nil { - _, err := cs.evmCommitter.LoadVersion(targetVersion) + _, err := cs.evmCommitter.LoadVersion(targetVersion, false) if err != nil { return nil, fmt.Errorf("failed to load FlatKV version: %w", err) } diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index dce5fc4c01..39b8bfceab 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -24,7 +24,11 @@ type Store interface { // LoadVersion opens the database at the specified version. // targetVersion == 0 opens the latest; targetVersion > 0 seeks the best // snapshot <= target and replays WAL to reach it exactly. - LoadVersion(targetVersion int64) (Store, error) + // + // When readOnly is true a new, isolated CommitStore is returned that + // shares no mutable state with the receiver. The caller must Close it + // when done; Close removes the temporary working directory. + LoadVersion(targetVersion int64, readOnly bool) (Store, error) // ApplyChangeSets buffers EVM changesets (x/evm memiavl keys) and updates LtHash. // Non-EVM modules are ignored. Call Commit to persist. diff --git a/sei-db/state_db/sc/flatkv/lthash_correctness_test.go b/sei-db/state_db/sc/flatkv/lthash_correctness_test.go index 597019e447..7c3c9f96fe 100644 --- a/sei-db/state_db/sc/flatkv/lthash_correctness_test.go +++ b/sei-db/state_db/sc/flatkv/lthash_correctness_test.go @@ -641,7 +641,7 @@ func TestLtHashPersistenceAfterReopen(t *testing.T) { // Phase 1: create state and close s1 := NewCommitStore(t.Context(), dir, nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) for i := 1; i <= 10; i++ { @@ -659,7 +659,7 @@ func TestLtHashPersistenceAfterReopen(t *testing.T) { // Phase 2: reopen and verify s2 := NewCommitStore(t.Context(), dir, nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index bfbcc76021..c19a9144b7 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -187,8 +187,12 @@ func removeTmpDirs(dir string) error { } for _, e := range entries { name := e.Name() - if e.IsDir() && (strings.HasSuffix(name, tmpSuffix) || strings.HasSuffix(name, removingSuffix)) { + if !e.IsDir() { + continue + } + if strings.HasSuffix(name, tmpSuffix) || strings.HasSuffix(name, removingSuffix) { _ = os.RemoveAll(filepath.Join(dir, name)) + continue } } return nil @@ -423,6 +427,9 @@ func (s *CommitStore) migrateFlatLayout(flatkvDir string) (string, error) { // (e.g. flatkv/snapshot-00000000000000000100) and the current symlink is updated. // The dir parameter is ignored; snapshots are always stored alongside the live data. func (s *CommitStore) WriteSnapshot(_ string) error { + if s.readOnly { + return errReadOnly + } version := s.committedVersion if version <= 0 { return fmt.Errorf("cannot snapshot uncommitted store (version %d)", version) @@ -530,6 +537,9 @@ func (s *CommitStore) pruneSnapshots(dir string, currentVersion int64) { // completes, the next restart will simply re-run catchup against the // already-truncated WAL, converging to targetVersion. func (s *CommitStore) Rollback(targetVersion int64) error { + if s.readOnly { + return errReadOnly + } s.log.Info("FlatKV Rollback", "targetVersion", targetVersion) dir := s.flatkvDir() diff --git a/sei-db/state_db/sc/flatkv/snapshot_test.go b/sei-db/state_db/sc/flatkv/snapshot_test.go index d6b43e4c24..321dee24ac 100644 --- a/sei-db/state_db/sc/flatkv/snapshot_test.go +++ b/sei-db/state_db/sc/flatkv/snapshot_test.go @@ -33,7 +33,7 @@ func commitStorageEntry(t *testing.T, s *CommitStore, addr Address, slot Slot, v func TestSnapshotCreatesDir(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -60,7 +60,7 @@ func TestSnapshotCreatesDir(t *testing.T) { func TestSnapshotIdempotent(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -80,7 +80,7 @@ func TestOpenFromSnapshot(t *testing.T) { // Phase 1: create store, commit v1 and v2, snapshot at v2, commit v3 s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s1, Address{0x10}, Slot{0x01}, []byte{0x01}) @@ -97,7 +97,7 @@ func TestOpenFromSnapshot(t *testing.T) { // Phase 2: reopen - should catchup from v2 snapshot + WAL entry for v3 s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -119,7 +119,7 @@ func TestCatchupUpdatesLtHash(t *testing.T) { dir := t.TempDir() s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) // Commit 5 versions, snapshot at v2 @@ -137,7 +137,7 @@ func TestCatchupUpdatesLtHash(t *testing.T) { // Reopen: catchup from v2 snapshot through v3,v4,v5 via WAL s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -151,7 +151,7 @@ func TestRollbackRewindsState(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) // Commit v1..v5, snapshot at v3 @@ -188,7 +188,7 @@ func TestRollbackToSnapshotExact(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0x40}, Slot{0x01}, []byte{0x01}) @@ -209,7 +209,7 @@ func TestRollbackToSnapshotExact(t *testing.T) { func TestPartialSnapshotCleanup(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0x50}, Slot{0x01}, []byte{0x01}) @@ -266,7 +266,7 @@ func TestMigrationFromFlatLayout(t *testing.T) { // Open the store - should trigger migration s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s.LoadVersion(0) + _, err = s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -297,7 +297,7 @@ func TestOpenVersionValidation(t *testing.T) { // Phase 1: create store, commit some data s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s1, Address{0x60}, Slot{0x01}, []byte{0x11}) @@ -320,7 +320,7 @@ func TestOpenVersionValidation(t *testing.T) { // Phase 3: reopen - should detect skew and catchup s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -389,7 +389,7 @@ func TestLoadVersionWithTarget(t *testing.T) { dir := t.TempDir() s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s1, Address{0x70}, Slot{0x01}, []byte{0x01}) @@ -402,7 +402,7 @@ func TestLoadVersionWithTarget(t *testing.T) { // Reopen at specific version 3 s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(3) + _, err = s2.LoadVersion(3, false) require.NoError(t, err) defer s2.Close() @@ -421,7 +421,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // Phase 1: build baseline at v2 and snapshot it. s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s1, addr, slot, []byte{0x01}) // v1 @@ -442,7 +442,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // Phase 3: reopen exactly at v2. If later commits had mutated the snapshot // baseline in place, we'd incorrectly read 0x04 here. s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(2) + _, err = s2.LoadVersion(2, false) require.NoError(t, err) gotV2, ok := s2.Get(key) require.True(t, ok) @@ -451,7 +451,7 @@ func TestSnapshotThenCatchupThenVerifyCorrectness(t *testing.T) { // Phase 4: reopen latest again to ensure catchup/replay still reaches v4. s3 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s3.LoadVersion(0) + _, err = s3.LoadVersion(0, false) require.NoError(t, err) defer s3.Close() @@ -471,7 +471,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { key := evm.BuildMemIAVLEVMKey(evm.EVMKeyStorage, StorageKey(addr, slot)) s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, addr, slot, []byte{0x01}) @@ -486,7 +486,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 1: load exactly v2 s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s1.LoadVersion(2) + _, err = s1.LoadVersion(2, false) require.NoError(t, err) require.Equal(t, int64(2), s1.Version()) require.Equal(t, hashAtV2, s1.RootHash()) @@ -497,7 +497,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 2: load latest (catches up through v3, v4) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) require.Equal(t, int64(4), s2.Version()) require.Equal(t, hashAtV4, s2.RootHash()) @@ -508,7 +508,7 @@ func TestLoadVersionMixedSequence(t *testing.T) { // Round 3: load v2 AGAIN — snapshot must still be clean. s3 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s3.LoadVersion(2) + _, err = s3.LoadVersion(2, false) require.NoError(t, err, "LoadVersion(2) must succeed after LoadVersion(0) dirtied working dir") require.Equal(t, int64(2), s3.Version()) require.Equal(t, hashAtV2, s3.RootHash()) @@ -524,7 +524,7 @@ func TestRollbackTargetBeforeWALStart(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) // Build: v1..v5, snapshot at v2 @@ -558,7 +558,7 @@ func TestRollbackTargetBeforeWALStart(t *testing.T) { // Simulate restart: should stay at v2. require.NoError(t, s.Close()) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -756,7 +756,7 @@ func TestCreateWorkingDirReclones(t *testing.T) { func TestPruneSnapshotsKeepsRecent(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotKeepRecent: 1}) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) for i := 0; i < 5; i++ { @@ -780,7 +780,7 @@ func TestPruneSnapshotsKeepsRecent(t *testing.T) { func TestPruneSnapshotsKeepAll(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotKeepRecent: 100}) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -816,7 +816,7 @@ func TestOrphanSnapshotRecovery(t *testing.T) { require.True(t, os.IsNotExist(err), "no current symlink should exist") s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s.LoadVersion(0) + _, err = s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -878,7 +878,7 @@ func TestTraverseSnapshotsEarlyStop(t *testing.T) { func TestVerifyWALTailSuccess(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -892,7 +892,7 @@ func TestVerifyWALTailSuccess(t *testing.T) { func TestVerifyWALTailMismatch(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -914,7 +914,7 @@ func TestTryTruncateWAL(t *testing.T) { // the manual snapshot at v5 is created; this makes v5 the earliest // snapshot and gives tryTruncateWAL a positive truncation offset. s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotKeepRecent: 0}) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -939,7 +939,7 @@ func TestTryTruncateWAL(t *testing.T) { func TestTryTruncateWALNoSnapshot(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -960,7 +960,7 @@ func TestTryTruncateWALNoSnapshot(t *testing.T) { func TestRollbackRemovesPostTargetSnapshots(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) for i := 0; i < 3; i++ { @@ -1058,7 +1058,7 @@ func TestSeekSnapshotExact(t *testing.T) { func TestMultipleSnapshotsAndReopen(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotKeepRecent: 10}) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) var hashes [][]byte @@ -1072,7 +1072,7 @@ func TestMultipleSnapshotsAndReopen(t *testing.T) { for i, expectedHash := range hashes { ver := int64(i + 1) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotKeepRecent: 10}) - _, err := s2.LoadVersion(ver) + _, err := s2.LoadVersion(ver, false) require.NoError(t, err) require.Equal(t, ver, s2.Version()) require.Equal(t, expectedHash, s2.RootHash(), "hash mismatch at version %d", ver) @@ -1087,7 +1087,7 @@ func TestMultipleSnapshotsAndReopen(t *testing.T) { func TestWriteSnapshotUpdatesSnapshotBase(t *testing.T) { dir := t.TempDir() s := NewCommitStore(context.Background(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0xF0}, Slot{0x01}, []byte{0x01}) @@ -1113,7 +1113,7 @@ func TestWriteSnapshotUpdatesSnapshotBase(t *testing.T) { // so committedVersion should be 5 (from working dir metadata), not 2 // (from the snapshot). Catchup should replay 0 entries. s2 := NewCommitStore(context.Background(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -1124,7 +1124,7 @@ func TestWriteSnapshotUpdatesSnapshotBase(t *testing.T) { func TestSnapshotPreservesAllKeyTypes(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) addr := Address{0xAB} @@ -1145,7 +1145,7 @@ func TestSnapshotPreservesAllKeyTypes(t *testing.T) { require.NoError(t, s.Close()) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 5dabbbba1c..dfbef364de 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -2,10 +2,12 @@ package flatkv import ( "context" + "errors" "fmt" "io" "os" "path/filepath" + "strconv" "time" "github.com/sei-protocol/sei-chain/sei-db/common/logger" @@ -37,6 +39,9 @@ const ( tmpSuffix = "-tmp" removingSuffix = "-removing" + // readOnlyDirPrefix is the directory name prefix for readonly working dirs. + readOnlyDirPrefix = "readonly-" + // Metadata DB keys MetaGlobalVersion = "_meta/version" // Global committed version watermark (8 bytes) MetaGlobalLtHash = "_meta/hash" // Global LtHash (2048 bytes) @@ -100,6 +105,13 @@ type CommitStore struct { // Used to track time spent in various phases of execution. phaseTimer *metrics.PhaseTimer + + // readOnly is set after a readonly LoadVersion completes. All public + // write methods check this flag and return errReadOnly. + readOnly bool + // readOnlyWorkDir is the absolute path of the temporary working + // directory used by a readonly store. Cleaned up by Close(). + readOnlyWorkDir string } var _ Store = (*CommitStore)(nil) @@ -138,13 +150,24 @@ func (s *CommitStore) flatkvDir() string { return s.dbDir } +var errReadOnly = errors.New("flatkv: store is read-only") + // LoadVersion loads the specified version of the database. // // - targetVersion == 0: open latest (follow current symlink + catchup to end of WAL). // - targetVersion > 0: seek the best snapshot <= targetVersion, open it, then // catchup via WAL to reach targetVersion exactly. -func (s *CommitStore) LoadVersion(targetVersion int64) (_ Store, retErr error) { - s.log.Info("FlatKV LoadVersion", "targetVersion", targetVersion) +// +// When readOnly is true a new, self-contained CommitStore is returned. +// It opens its own temporary working directory and changelog handle to +// catch up to the requested version. The caller must Close it when done; +// Close removes the temporary working directory. +func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (_ Store, retErr error) { + s.log.Info("FlatKV LoadVersion", "targetVersion", targetVersion, "readOnly", readOnly) + + if readOnly { + return s.loadVersionReadOnly(targetVersion) + } _ = s.closeDBsOnly() @@ -196,6 +219,91 @@ func (s *CommitStore) LoadVersion(targetVersion int64) (_ Store, retErr error) { return s, nil } +// loadVersionReadOnly creates an isolated, read-only CommitStore at the +// requested version. It opens its own changelog handle for catchup, then +// closes it so the returned store has zero coupling to the parent. +func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr error) { + ro := NewCommitStore(s.ctx, s.dbDir, s.log, s.config) + + ro.readOnlyWorkDir = filepath.Join(ro.flatkvDir(), + readOnlyDirPrefix+strconv.FormatInt(time.Now().UnixNano(), 10)) + + defer func() { + if retErr != nil { + _ = ro.Close() + } + }() + + if err := ro.openReadOnly(targetVersion); err != nil { + return nil, fmt.Errorf("readonly open: %w", err) + } + + return ro, nil +} + +// openReadOnly opens PebbleDBs in a temporary directory, replays the WAL +// to targetVersion, closes the WAL, and marks the store as read-only. +// +// Invariants: +// - readOnlyWorkDir must be set before this call. +// - This method never modifies the global "current" symlink. +func (s *CommitStore) openReadOnly(targetVersion int64) error { + s.clearPendingWrites() + + dir := s.flatkvDir() + + // Find the best snapshot <= targetVersion without touching the symlink. + var snapDir string + if targetVersion > 0 { + baseVer, err := seekSnapshot(dir, targetVersion) + if err != nil { + return fmt.Errorf("seek snapshot for readonly: %w", err) + } + snapDir = filepath.Join(dir, snapshotName(baseVer)) + } else { + var err error + snapDir, _, err = currentSnapshotDir(dir) + if err != nil { + return fmt.Errorf("resolve current snapshot for readonly: %w", err) + } + } + + if err := createWorkingDir(snapDir, s.readOnlyWorkDir); err != nil { + return fmt.Errorf("create readonly working dir: %w", err) + } + + if err := s.openDBs(s.readOnlyWorkDir, dir); err != nil { + return err + } + + if err := s.loadGlobalMetadata(); err != nil { + return err + } + + if err := s.catchup(targetVersion); err != nil { + return fmt.Errorf("readonly catchup: %w", err) + } + + if targetVersion > 0 && s.committedVersion != targetVersion { + return fmt.Errorf("readonly version mismatch: requested %d, reached %d", + targetVersion, s.committedVersion) + } + + if s.changelog != nil { + closeErr := s.changelog.Close() + s.changelog = nil + if closeErr != nil { + return fmt.Errorf("close readonly changelog: %w", closeErr) + } + } + + s.readOnly = true + + s.log.Info("FlatKV readonly store opened", "version", s.committedVersion, + "dir", s.readOnlyWorkDir) + return nil +} + // openTo opens all DBs and catches up via WAL to the given version. // - 0 -> replay to end of WAL (latest). // - >0 -> replay up to (and including) that version. @@ -258,7 +366,7 @@ func (s *CommitStore) open() (retErr error) { return fmt.Errorf("create working dir: %w", err) } - if err := s.openAllDBs(workDir, dir); err != nil { + if err := s.openDBs(workDir, dir); err != nil { return err } @@ -290,20 +398,20 @@ func (s *CommitStore) acquireFileLock(dir string) error { return nil } -// openAllDBs opens the 5 PebbleDBs from the snapshot directory, the changelog -// WAL from the flatkv root, and loads per-DB local metadata. On failure all -// already-opened handles are closed. -func (s *CommitStore) openAllDBs(snapDir, flatkvRoot string) (retErr error) { +// openDBs opens the 5 PebbleDBs from dbDir and loads per-DB local metadata. +// If changelogRoot is non-empty, it also opens the changelog WAL from that +// directory. On failure all already-opened handles are closed. +func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { type namedPath struct { name string path string } dbPaths := []namedPath{ - {accountDBDir, filepath.Join(snapDir, accountDBDir)}, - {codeDBDir, filepath.Join(snapDir, codeDBDir)}, - {storageDBDir, filepath.Join(snapDir, storageDBDir)}, - {legacyDBDir, filepath.Join(snapDir, legacyDBDir)}, - {metadataDir, filepath.Join(snapDir, metadataDir)}, + {accountDBDir, filepath.Join(dbDir, accountDBDir)}, + {codeDBDir, filepath.Join(dbDir, codeDBDir)}, + {storageDBDir, filepath.Join(dbDir, storageDBDir)}, + {legacyDBDir, filepath.Join(dbDir, legacyDBDir)}, + {metadataDir, filepath.Join(dbDir, metadataDir)}, } for _, np := range dbPaths { @@ -323,7 +431,9 @@ func (s *CommitStore) openAllDBs(snapDir, flatkvRoot string) (retErr error) { s.codeDB = nil s.storageDB = nil s.legacyDB = nil - s.changelog = nil + if changelogRoot != "" { + s.changelog = nil + } s.localMeta = make(map[string]*LocalMeta) } }() @@ -354,18 +464,19 @@ func (s *CommitStore) openAllDBs(snapDir, flatkvRoot string) (retErr error) { return err } - changelogPath := filepath.Join(flatkvRoot, changelogDir) - s.changelog, err = wal.NewChangelogWAL(s.log, changelogPath, wal.Config{ - WriteBufferSize: 0, - KeepRecent: 0, - PruneInterval: 0, - }) - if err != nil { - return fmt.Errorf("failed to open changelog: %w", err) + if changelogRoot != "" { + changelogPath := filepath.Join(changelogRoot, changelogDir) + s.changelog, err = wal.NewChangelogWAL(s.log, changelogPath, wal.Config{ + WriteBufferSize: 0, + KeepRecent: 0, + PruneInterval: 0, + }) + if err != nil { + return fmt.Errorf("failed to open changelog: %w", err) + } + toClose = append(toClose, s.changelog) } - toClose = append(toClose, s.changelog) - // Load per-DB local metadata (or initialize if not present) dataDBs := map[string]seidbtypes.KeyValueDB{ accountDBDir: s.accountDB, codeDBDir: s.codeDB, @@ -438,6 +549,9 @@ func (s *CommitStore) RootHash() []byte { } func (s *CommitStore) Importer(version int64) (types.Importer, error) { + if s.readOnly { + return nil, errReadOnly + } // rootmulti.Restore closes the store before creating an importer. // Reopen the DBs so the importer can write data. if s.isClosed() { diff --git a/sei-db/state_db/sc/flatkv/store_lifecycle.go b/sei-db/state_db/sc/flatkv/store_lifecycle.go index 4c73f50136..0ebb647930 100644 --- a/sei-db/state_db/sc/flatkv/store_lifecycle.go +++ b/sei-db/state_db/sc/flatkv/store_lifecycle.go @@ -3,6 +3,7 @@ package flatkv import ( "errors" "fmt" + "os" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" ) @@ -22,8 +23,8 @@ func (s *CommitStore) closeDBsOnly() error { if err := s.changelog.Close(); err != nil { errs = append(errs, fmt.Errorf("changelog close: %w", err)) } - s.changelog = nil } + s.changelog = nil if s.metadataDB != nil { if err := s.metadataDB.Close(); err != nil { @@ -67,6 +68,7 @@ func (s *CommitStore) closeDBsOnly() error { } // Close closes all database instances and releases the file lock. +// For readonly stores it also removes the temporary working directory. func (s *CommitStore) Close() error { err := s.closeDBsOnly() @@ -77,6 +79,10 @@ func (s *CommitStore) Close() error { s.fileLock = nil } + if s.readOnlyWorkDir != "" { + _ = os.RemoveAll(s.readOnlyWorkDir) + } + if err != nil { return err } diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index dadba42c1b..f8dc04fa36 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -68,7 +68,7 @@ func setupTestStore(t *testing.T) *CommitStore { t.Helper() dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) return s } @@ -78,7 +78,7 @@ func setupTestStoreWithConfig(t *testing.T, cfg Config) *CommitStore { t.Helper() dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, cfg) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) return s } @@ -98,7 +98,7 @@ func commitAndCheck(t *testing.T, s *CommitStore) int64 { func TestStoreOpenClose(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) require.NoError(t, s.Close()) @@ -107,7 +107,7 @@ func TestStoreOpenClose(t *testing.T) { func TestStoreClose(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) // Close should succeed @@ -302,7 +302,7 @@ func TestStorePersistence(t *testing.T) { // Write and close s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) cs := makeChangeSet(key, value, false) @@ -312,7 +312,7 @@ func TestStorePersistence(t *testing.T) { // Reopen and verify s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -429,17 +429,17 @@ func TestFileLockPreventsDoubleOpen(t *testing.T) { dir := t.TempDir() s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.Error(t, err, "second open on same dir should fail due to file lock") require.Contains(t, err.Error(), "file lock") require.NoError(t, s1.Close()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err, "should succeed after first store releases lock") require.NoError(t, s2.Close()) } @@ -451,7 +451,7 @@ func TestFileLockPreventsDoubleOpen(t *testing.T) { func TestClearChangelog(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -476,7 +476,7 @@ func TestClearChangelog(t *testing.T) { func TestCloseDBsOnlyIdempotent(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) require.NoError(t, s.closeDBsOnly()) @@ -494,7 +494,7 @@ func TestLoadVersionTargetBeyondWALFails(t *testing.T) { dir := t.TempDir() s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s1, Address{0x01}, Slot{0x01}, []byte{0x01}) @@ -503,7 +503,7 @@ func TestLoadVersionTargetBeyondWALFails(t *testing.T) { require.NoError(t, s1.Close()) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(100) + _, err = s2.LoadVersion(100, false) require.Error(t, err, "loading version beyond WAL should fail") } @@ -515,7 +515,7 @@ func TestReopenReusesWorkingDir(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0x01}, Slot{0x01}, []byte{0x01}) @@ -528,7 +528,7 @@ func TestReopenReusesWorkingDir(t *testing.T) { require.NoError(t, err, "SNAPSHOT_BASE should exist after close") s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -542,7 +542,7 @@ func TestReopenReusesWorkingDir(t *testing.T) { func TestWalOffsetForVersionFastPath(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -564,7 +564,7 @@ func TestWalOffsetForVersionFastPath(t *testing.T) { func TestWalOffsetForVersionBeforeWAL(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -580,7 +580,7 @@ func TestWalOffsetForVersionBeforeWAL(t *testing.T) { func TestWalOffsetForVersionNotFound(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -598,7 +598,7 @@ func TestWalOffsetForVersionNotFound(t *testing.T) { func TestCatchupFromSpecificVersion(t *testing.T) { dir := t.TempDir() s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) for i := 0; i < 10; i++ { @@ -610,7 +610,7 @@ func TestCatchupFromSpecificVersion(t *testing.T) { require.NoError(t, s1.Close()) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -659,7 +659,7 @@ func TestPersistenceAllKeyTypes(t *testing.T) { slot := Slot{0xBB} s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s1.LoadVersion(0) + _, err := s1.LoadVersion(0, false) require.NoError(t, err) storageKey := evm.BuildMemIAVLEVMKey(evm.EVMKeyStorage, StorageKey(addr, slot)) @@ -678,7 +678,7 @@ func TestPersistenceAllKeyTypes(t *testing.T) { require.NoError(t, s1.Close()) s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err = s2.LoadVersion(0) + _, err = s2.LoadVersion(0, false) require.NoError(t, err) defer s2.Close() @@ -697,3 +697,226 @@ func TestPersistenceAllKeyTypes(t *testing.T) { require.True(t, ok) require.Equal(t, []byte{0x60, 0x80}, v) } + +// ============================================================================= +// ReadOnly LoadVersion Tests +// ============================================================================= + +func TestReadOnlyBasicLoadAndRead(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xAA} + slot := Slot{0xBB} + key := memiavlStorageKey(addr, slot) + value := []byte{0xCC} + + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, value, false)})) + commitAndCheck(t, s) + + ro, err := s.LoadVersion(0, true) + require.NoError(t, err) + defer ro.Close() + + require.Equal(t, int64(1), ro.Version()) + got, found := ro.Get(key) + require.True(t, found) + require.Equal(t, value, got) + require.NotNil(t, ro.RootHash()) + require.Len(t, ro.RootHash(), 32) +} + +func TestReadOnlyLoadFromUnopenedStore(t *testing.T) { + dir := t.TempDir() + writer := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := writer.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xCC} + slot := Slot{0xDD} + key := memiavlStorageKey(addr, slot) + value := []byte{0xEE} + + require.NoError(t, writer.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, value, false)})) + commitAndCheck(t, writer) + require.NoError(t, writer.Close()) + + fresh := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + ro, err := fresh.LoadVersion(0, true) + require.NoError(t, err) + defer ro.Close() + + require.Equal(t, int64(1), ro.Version()) + got, found := ro.Get(key) + require.True(t, found) + require.Equal(t, value, got) +} + +func TestReadOnlyAtSpecificVersion(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotInterval: 2, SnapshotKeepRecent: 10}) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0x11} + slot := Slot{0x22} + key := memiavlStorageKey(addr, slot) + + for i := byte(1); i <= 5; i++ { + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{ + makeChangeSet(key, []byte{i}, false), + })) + commitAndCheck(t, s) + } + + ro, err := s.LoadVersion(3, true) + require.NoError(t, err) + defer ro.Close() + + require.Equal(t, int64(3), ro.Version()) + got, found := ro.Get(key) + require.True(t, found) + require.Equal(t, []byte{3}, got) +} + +func TestReadOnlyWriteGuards(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xAA} + slot := Slot{0xBB} + key := memiavlStorageKey(addr, slot) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{1}, false)})) + commitAndCheck(t, s) + + ro, err := s.LoadVersion(0, true) + require.NoError(t, err) + defer ro.Close() + + require.ErrorIs(t, ro.ApplyChangeSets(nil), errReadOnly) + _, err = ro.Commit() + require.ErrorIs(t, err, errReadOnly) + require.ErrorIs(t, ro.WriteSnapshot(""), errReadOnly) + require.ErrorIs(t, ro.Rollback(1), errReadOnly) +} + +func TestReadOnlyParentWritesDuringReadOnly(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xAA} + slot := Slot{0xBB} + key := memiavlStorageKey(addr, slot) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{1}, false)})) + commitAndCheck(t, s) + + ro, err := s.LoadVersion(0, true) + require.NoError(t, err) + defer ro.Close() + + // Parent continues to commit while readonly is open. + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{2}, false)})) + commitAndCheck(t, s) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{3}, false)})) + commitAndCheck(t, s) + + require.Equal(t, int64(3), s.Version()) + + // Readonly should still see version-1 data. + require.Equal(t, int64(1), ro.Version()) + got, found := ro.Get(key) + require.True(t, found) + require.Equal(t, []byte{1}, got) +} + +func TestReadOnlyConcurrentInstances(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{SnapshotInterval: 2, SnapshotKeepRecent: 10}) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0x11} + slot := Slot{0x22} + key := memiavlStorageKey(addr, slot) + + for i := byte(1); i <= 4; i++ { + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{ + makeChangeSet(key, []byte{i}, false), + })) + commitAndCheck(t, s) + } + + ro1, err := s.LoadVersion(0, true) + require.NoError(t, err) + defer ro1.Close() + + ro2, err := s.LoadVersion(0, true) + require.NoError(t, err) + defer ro2.Close() + + require.Equal(t, int64(4), ro1.Version()) + require.Equal(t, int64(4), ro2.Version()) + + g1, ok1 := ro1.Get(key) + g2, ok2 := ro2.Get(key) + require.True(t, ok1) + require.True(t, ok2) + require.Equal(t, []byte{4}, g1) + require.Equal(t, []byte{4}, g2) +} + +func TestReadOnlyFailureDoesNotAffectParent(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xAA} + slot := Slot{0xBB} + key := memiavlStorageKey(addr, slot) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{1}, false)})) + commitAndCheck(t, s) + + // Attempt to load a version that doesn't exist. + _, err = s.LoadVersion(999, true) + require.Error(t, err) + + // Parent should still work normally. + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{2}, false)})) + v, err := s.Commit() + require.NoError(t, err) + require.Equal(t, int64(2), v) + + got, found := s.Get(key) + require.True(t, found) + require.Equal(t, []byte{2}, got) +} + +func TestReadOnlyCloseRemovesTempDir(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + _, err := s.LoadVersion(0, false) + require.NoError(t, err) + + addr := Address{0xAA} + slot := Slot{0xBB} + key := memiavlStorageKey(addr, slot) + require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{1}, false)})) + commitAndCheck(t, s) + + roStore, err := s.LoadVersion(0, true) + require.NoError(t, err) + + roCommit := roStore.(*CommitStore) + tmpDir := roCommit.readOnlyWorkDir + require.DirExists(t, tmpDir) + + require.NoError(t, roStore.Close()) + require.NoDirExists(t, tmpDir) +} diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index cf1350f9be..9df15df883 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -20,8 +20,11 @@ import ( // - codeDB: key=addr, value=bytecode // - legacyDB: key=full original key (with prefix), value=raw value func (s *CommitStore) ApplyChangeSets(cs []*proto.NamedChangeSet) error { - s.phaseTimer.SetPhase("apply_change_sets_prepare") + if s.readOnly { + return errReadOnly + } + s.phaseTimer.SetPhase("apply_change_sets_prepare") // Save original changesets for changelog s.pendingChangeSets = append(s.pendingChangeSets, cs...) @@ -231,6 +234,9 @@ func (s *CommitStore) ApplyChangeSets(cs []*proto.NamedChangeSet) error { // Protocol: WAL → per-DB batch (with LocalMeta) → flush → update metaDB. // On crash, catchup replays WAL to recover incomplete commits. func (s *CommitStore) Commit() (int64, error) { + if s.readOnly { + return 0, errReadOnly + } // Auto-increment version version := s.committedVersion + 1 diff --git a/sei-db/state_db/sc/flatkv/store_write_test.go b/sei-db/state_db/sc/flatkv/store_write_test.go index 04915daf51..de8192dabb 100644 --- a/sei-db/state_db/sc/flatkv/store_write_test.go +++ b/sei-db/state_db/sc/flatkv/store_write_test.go @@ -524,7 +524,7 @@ func TestStoreFsyncConfig(t *testing.T) { t.Run("DefaultConfig", func(t *testing.T) { dir := t.TempDir() store := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := store.LoadVersion(0) + _, err := store.LoadVersion(0, false) require.NoError(t, err) defer store.Close() @@ -538,7 +538,7 @@ func TestStoreFsyncConfig(t *testing.T) { store := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, Config{ Fsync: false, }) - _, err := store.LoadVersion(0) + _, err := store.LoadVersion(0, false) require.NoError(t, err) defer store.Close() @@ -572,7 +572,7 @@ func TestAutoSnapshotTriggeredByInterval(t *testing.T) { SnapshotKeepRecent: 2, } s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, cfg) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -596,7 +596,7 @@ func TestAutoSnapshotNotTriggeredBeforeInterval(t *testing.T) { SnapshotKeepRecent: 2, } s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, cfg) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -623,7 +623,7 @@ func TestAutoSnapshotDisabledWhenIntervalZero(t *testing.T) { dir := t.TempDir() cfg := Config{SnapshotInterval: 0} s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, cfg) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -715,7 +715,7 @@ func TestLtHashDeterministicAcrossReopen(t *testing.T) { writeAndGetHash := func() []byte { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0x01}, Slot{0x01}, []byte{0xAA}) @@ -839,7 +839,7 @@ func TestStoreFsyncEnabled(t *testing.T) { dir := t.TempDir() cfg := Config{Fsync: true} s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, cfg) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -860,7 +860,7 @@ func TestStoreFsyncEnabled(t *testing.T) { func TestLastSnapshotTimeUpdated(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) defer s.Close() @@ -880,7 +880,7 @@ func TestLastSnapshotTimeUpdated(t *testing.T) { func TestWALRecordsChangesets(t *testing.T) { dir := t.TempDir() s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) - _, err := s.LoadVersion(0) + _, err := s.LoadVersion(0, false) require.NoError(t, err) commitStorageEntry(t, s, Address{0x01}, Slot{0x01}, []byte{0xAA}) From 4ac54a427f392e5241746b2ead351accf16a6406 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Thu, 5 Mar 2026 18:02:36 -0500 Subject: [PATCH 2/6] clean --- sei-db/state_db/sc/composite/store.go | 1 + sei-db/state_db/sc/flatkv/snapshot.go | 38 ++++++++++++++++++++++++- sei-db/state_db/sc/flatkv/store.go | 32 ++++++++------------- sei-db/state_db/sc/flatkv/store_test.go | 5 ++++ 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index fec15b655b..ca7e52ad8b 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -107,6 +107,7 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) if cs.evmCommitter != nil { evmStore, err := cs.evmCommitter.LoadVersion(targetVersion, true) if err != nil { + _ = cosmosCommitter.Close() return nil, fmt.Errorf("failed to load readonly FlatKV version: %w", err) } newStore.evmCommitter = evmStore diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index c19a9144b7..23526c20e9 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -111,6 +111,43 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return found, nil } +// resolveSnapshotDirReadOnly finds the snapshot directory for a readonly open +// without creating directories, symlinks, or performing migrations. +// +// When targetVersion > 0 it seeks the highest snapshot <= targetVersion. +// When targetVersion == 0 it tries the current symlink first, then falls back +// to the highest existing snapshot directory. This handles crash-recovery +// states (orphaned snapshots with no current symlink) that the writable +// resolveSnapshotDir would repair. +func resolveSnapshotDirReadOnly(root string, targetVersion int64) (string, error) { + if targetVersion > 0 { + baseVer, err := seekSnapshot(root, targetVersion) + if err != nil { + return "", fmt.Errorf("seek snapshot: %w", err) + } + return filepath.Join(root, snapshotName(baseVer)), nil + } + + snapDir, _, err := currentSnapshotDir(root) + if err == nil { + return snapDir, nil + } + if !os.IsNotExist(err) { + return "", fmt.Errorf("read current symlink: %w", err) + } + + var latestSnap int64 = -1 + _ = traverseSnapshots(root, false, func(v int64) (bool, error) { + latestSnap = v + return true, nil + }) + if latestSnap >= 0 { + return filepath.Join(root, snapshotName(latestSnap)), nil + } + + return "", fmt.Errorf("no snapshot found and DB not initialized") +} + // traverseSnapshots iterates snapshot directories in the given order. // ascending=true -> lowest version first // ascending=false -> highest version first @@ -192,7 +229,6 @@ func removeTmpDirs(dir string) error { } if strings.HasSuffix(name, tmpSuffix) || strings.HasSuffix(name, removingSuffix) { _ = os.RemoveAll(filepath.Join(dir, name)) - continue } } return nil diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index dfbef364de..084037c9a7 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -7,7 +7,6 @@ import ( "io" "os" "path/filepath" - "strconv" "time" "github.com/sei-protocol/sei-chain/sei-db/common/logger" @@ -166,6 +165,9 @@ func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (_ Store, s.log.Info("FlatKV LoadVersion", "targetVersion", targetVersion, "readOnly", readOnly) if readOnly { + if s.readOnly { + return nil, errReadOnly + } return s.loadVersionReadOnly(targetVersion) } @@ -225,8 +227,11 @@ func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (_ Store, func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr error) { ro := NewCommitStore(s.ctx, s.dbDir, s.log, s.config) - ro.readOnlyWorkDir = filepath.Join(ro.flatkvDir(), - readOnlyDirPrefix+strconv.FormatInt(time.Now().UnixNano(), 10)) + workDir, err := os.MkdirTemp("", readOnlyDirPrefix) + if err != nil { + return nil, fmt.Errorf("create readonly temp dir: %w", err) + } + ro.readOnlyWorkDir = workDir defer func() { if retErr != nil { @@ -252,20 +257,9 @@ func (s *CommitStore) openReadOnly(targetVersion int64) error { dir := s.flatkvDir() - // Find the best snapshot <= targetVersion without touching the symlink. - var snapDir string - if targetVersion > 0 { - baseVer, err := seekSnapshot(dir, targetVersion) - if err != nil { - return fmt.Errorf("seek snapshot for readonly: %w", err) - } - snapDir = filepath.Join(dir, snapshotName(baseVer)) - } else { - var err error - snapDir, _, err = currentSnapshotDir(dir) - if err != nil { - return fmt.Errorf("resolve current snapshot for readonly: %w", err) - } + snapDir, err := resolveSnapshotDirReadOnly(dir, targetVersion) + if err != nil { + return fmt.Errorf("resolve snapshot for readonly: %w", err) } if err := createWorkingDir(snapDir, s.readOnlyWorkDir); err != nil { @@ -431,9 +425,7 @@ func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { s.codeDB = nil s.storageDB = nil s.legacyDB = nil - if changelogRoot != "" { - s.changelog = nil - } + s.changelog = nil s.localMeta = make(map[string]*LocalMeta) } }() diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index f8dc04fa36..57e73c406f 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -802,6 +802,11 @@ func TestReadOnlyWriteGuards(t *testing.T) { require.ErrorIs(t, err, errReadOnly) require.ErrorIs(t, ro.WriteSnapshot(""), errReadOnly) require.ErrorIs(t, ro.Rollback(1), errReadOnly) + _, err = ro.(*CommitStore).Importer(1) + require.ErrorIs(t, err, errReadOnly) + + _, err = ro.LoadVersion(0, true) + require.ErrorIs(t, err, errReadOnly) } func TestReadOnlyParentWritesDuringReadOnly(t *testing.T) { From c72f10db2dc2dffba7a5c2d2b60bc3b9582cc02e Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 6 Mar 2026 17:20:07 -0500 Subject: [PATCH 3/6] concise comments --- sei-db/state_db/sc/composite/store.go | 10 +--- sei-db/state_db/sc/flatkv/api.go | 10 +--- sei-db/state_db/sc/flatkv/snapshot.go | 42 +------------- sei-db/state_db/sc/flatkv/store.go | 58 +++++++++----------- sei-db/state_db/sc/flatkv/store_lifecycle.go | 1 - sei-db/state_db/sc/flatkv/store_test.go | 4 -- sei-db/state_db/sc/flatkv/store_write.go | 1 - 7 files changed, 31 insertions(+), 95 deletions(-) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index ca7e52ad8b..21a7351abf 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -81,10 +81,8 @@ func (cs *CompositeCommitStore) SetInitialVersion(initialVersion int64) error { return cs.cosmosCommitter.SetInitialVersion(initialVersion) } -// LoadVersion loads the specified version of the database. -// Being used for two scenarios: -// ReadOnly: Either for state sync or for historical proof -// Writable: Opened during initialization for root multistore +// LoadVersion opens the database at the given version (0 = latest). +// When readOnly is true an isolated composite store is returned. func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) (types.Committer, error) { cosmosSC, err := cs.cosmosCommitter.LoadVersion(targetVersion, readOnly) if err != nil { @@ -96,7 +94,6 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) return nil, fmt.Errorf("unexpected committer type from cosmos LoadVersion") } - // Read only mode should return a new SC if readOnly { newStore := &CompositeCommitStore{ logger: cs.logger, @@ -116,9 +113,6 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) } cs.cosmosCommitter = cosmosCommitter - // Load evmCommitter if initialized (nil when WriteMode is CosmosOnlyWrite). - // This is the single entry point for evmCommitter.LoadVersion — CMS calls - // CompositeCommitStore.LoadVersion(), which internally loads both backends. if cs.evmCommitter != nil { _, err := cs.evmCommitter.LoadVersion(targetVersion, false) if err != nil { diff --git a/sei-db/state_db/sc/flatkv/api.go b/sei-db/state_db/sc/flatkv/api.go index 39b8bfceab..f764812667 100644 --- a/sei-db/state_db/sc/flatkv/api.go +++ b/sei-db/state_db/sc/flatkv/api.go @@ -21,13 +21,9 @@ type Options struct { // Read path: Get/Has/Iterator read committed state only. // Key format: x/evm memiavl keys (mapped internally to account/code/storage DBs). type Store interface { - // LoadVersion opens the database at the specified version. - // targetVersion == 0 opens the latest; targetVersion > 0 seeks the best - // snapshot <= target and replays WAL to reach it exactly. - // - // When readOnly is true a new, isolated CommitStore is returned that - // shares no mutable state with the receiver. The caller must Close it - // when done; Close removes the temporary working directory. + // LoadVersion opens the database at the given version (0 = latest). + // When readOnly is true an isolated, read-only store is returned; + // the caller must Close it when done. LoadVersion(targetVersion int64, readOnly bool) (Store, error) // ApplyChangeSets buffers EVM changesets (x/evm memiavl keys) and updates LtHash. diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 23526c20e9..9c9ce87c51 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -111,43 +111,6 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return found, nil } -// resolveSnapshotDirReadOnly finds the snapshot directory for a readonly open -// without creating directories, symlinks, or performing migrations. -// -// When targetVersion > 0 it seeks the highest snapshot <= targetVersion. -// When targetVersion == 0 it tries the current symlink first, then falls back -// to the highest existing snapshot directory. This handles crash-recovery -// states (orphaned snapshots with no current symlink) that the writable -// resolveSnapshotDir would repair. -func resolveSnapshotDirReadOnly(root string, targetVersion int64) (string, error) { - if targetVersion > 0 { - baseVer, err := seekSnapshot(root, targetVersion) - if err != nil { - return "", fmt.Errorf("seek snapshot: %w", err) - } - return filepath.Join(root, snapshotName(baseVer)), nil - } - - snapDir, _, err := currentSnapshotDir(root) - if err == nil { - return snapDir, nil - } - if !os.IsNotExist(err) { - return "", fmt.Errorf("read current symlink: %w", err) - } - - var latestSnap int64 = -1 - _ = traverseSnapshots(root, false, func(v int64) (bool, error) { - latestSnap = v - return true, nil - }) - if latestSnap >= 0 { - return filepath.Join(root, snapshotName(latestSnap)), nil - } - - return "", fmt.Errorf("no snapshot found and DB not initialized") -} - // traverseSnapshots iterates snapshot directories in the given order. // ascending=true -> lowest version first // ascending=false -> highest version first @@ -224,10 +187,7 @@ func removeTmpDirs(dir string) error { } for _, e := range entries { name := e.Name() - if !e.IsDir() { - continue - } - if strings.HasSuffix(name, tmpSuffix) || strings.HasSuffix(name, removingSuffix) { + if e.IsDir() && (strings.HasSuffix(name, tmpSuffix) || strings.HasSuffix(name, removingSuffix)) { _ = os.RemoveAll(filepath.Join(dir, name)) } } diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 084037c9a7..f678234c74 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -38,7 +38,6 @@ const ( tmpSuffix = "-tmp" removingSuffix = "-removing" - // readOnlyDirPrefix is the directory name prefix for readonly working dirs. readOnlyDirPrefix = "readonly-" // Metadata DB keys @@ -102,15 +101,10 @@ type CommitStore struct { // File lock prevents multiple processes from opening the same DB. fileLock filelock.TryLockerSafe - // Used to track time spent in various phases of execution. phaseTimer *metrics.PhaseTimer - // readOnly is set after a readonly LoadVersion completes. All public - // write methods check this flag and return errReadOnly. - readOnly bool - // readOnlyWorkDir is the absolute path of the temporary working - // directory used by a readonly store. Cleaned up by Close(). - readOnlyWorkDir string + readOnly bool // Set by readonly LoadVersion; guards all write methods. + readOnlyWorkDir string // Temp working dir for readonly store; removed by Close. } var _ Store = (*CommitStore)(nil) @@ -151,16 +145,9 @@ func (s *CommitStore) flatkvDir() string { var errReadOnly = errors.New("flatkv: store is read-only") -// LoadVersion loads the specified version of the database. -// -// - targetVersion == 0: open latest (follow current symlink + catchup to end of WAL). -// - targetVersion > 0: seek the best snapshot <= targetVersion, open it, then -// catchup via WAL to reach targetVersion exactly. -// -// When readOnly is true a new, self-contained CommitStore is returned. -// It opens its own temporary working directory and changelog handle to -// catch up to the requested version. The caller must Close it when done; -// Close removes the temporary working directory. +// LoadVersion opens the database at the given version (0 = latest). +// When readOnly is true an isolated, read-only CommitStore is returned; +// the caller must Close it when done. func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (_ Store, retErr error) { s.log.Info("FlatKV LoadVersion", "targetVersion", targetVersion, "readOnly", readOnly) @@ -222,12 +209,11 @@ func (s *CommitStore) LoadVersion(targetVersion int64, readOnly bool) (_ Store, } // loadVersionReadOnly creates an isolated, read-only CommitStore at the -// requested version. It opens its own changelog handle for catchup, then -// closes it so the returned store has zero coupling to the parent. +// requested version. func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr error) { ro := NewCommitStore(s.ctx, s.dbDir, s.log, s.config) - workDir, err := os.MkdirTemp("", readOnlyDirPrefix) + workDir, err := os.MkdirTemp(ro.flatkvDir(), readOnlyDirPrefix) if err != nil { return nil, fmt.Errorf("create readonly temp dir: %w", err) } @@ -246,20 +232,27 @@ func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr return ro, nil } -// openReadOnly opens PebbleDBs in a temporary directory, replays the WAL -// to targetVersion, closes the WAL, and marks the store as read-only. -// -// Invariants: -// - readOnlyWorkDir must be set before this call. -// - This method never modifies the global "current" symlink. +// openReadOnly opens PebbleDBs in readOnlyWorkDir, replays the WAL to +// targetVersion, then closes the WAL and marks the store as read-only. +// It never modifies the global "current" symlink. func (s *CommitStore) openReadOnly(targetVersion int64) error { s.clearPendingWrites() dir := s.flatkvDir() - snapDir, err := resolveSnapshotDirReadOnly(dir, targetVersion) - if err != nil { - return fmt.Errorf("resolve snapshot for readonly: %w", err) + var snapDir string + if targetVersion > 0 { + baseVer, err := seekSnapshot(dir, targetVersion) + if err != nil { + return fmt.Errorf("seek snapshot for readonly: %w", err) + } + snapDir = filepath.Join(dir, snapshotName(baseVer)) + } else { + var err error + snapDir, _, err = currentSnapshotDir(dir) + if err != nil { + return fmt.Errorf("resolve current snapshot for readonly: %w", err) + } } if err := createWorkingDir(snapDir, s.readOnlyWorkDir); err != nil { @@ -392,9 +385,8 @@ func (s *CommitStore) acquireFileLock(dir string) error { return nil } -// openDBs opens the 5 PebbleDBs from dbDir and loads per-DB local metadata. -// If changelogRoot is non-empty, it also opens the changelog WAL from that -// directory. On failure all already-opened handles are closed. +// openDBs opens all PebbleDBs from dbDir and optionally the changelog WAL +// from changelogRoot. On failure all already-opened handles are closed. func (s *CommitStore) openDBs(dbDir, changelogRoot string) (retErr error) { type namedPath struct { name string diff --git a/sei-db/state_db/sc/flatkv/store_lifecycle.go b/sei-db/state_db/sc/flatkv/store_lifecycle.go index 0ebb647930..347bf659d6 100644 --- a/sei-db/state_db/sc/flatkv/store_lifecycle.go +++ b/sei-db/state_db/sc/flatkv/store_lifecycle.go @@ -68,7 +68,6 @@ func (s *CommitStore) closeDBsOnly() error { } // Close closes all database instances and releases the file lock. -// For readonly stores it also removes the temporary working directory. func (s *CommitStore) Close() error { err := s.closeDBsOnly() diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index 57e73c406f..b19ad59dfa 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -825,7 +825,6 @@ func TestReadOnlyParentWritesDuringReadOnly(t *testing.T) { require.NoError(t, err) defer ro.Close() - // Parent continues to commit while readonly is open. require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{2}, false)})) commitAndCheck(t, s) require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{3}, false)})) @@ -833,7 +832,6 @@ func TestReadOnlyParentWritesDuringReadOnly(t *testing.T) { require.Equal(t, int64(3), s.Version()) - // Readonly should still see version-1 data. require.Equal(t, int64(1), ro.Version()) got, found := ro.Get(key) require.True(t, found) @@ -888,11 +886,9 @@ func TestReadOnlyFailureDoesNotAffectParent(t *testing.T) { require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{1}, false)})) commitAndCheck(t, s) - // Attempt to load a version that doesn't exist. _, err = s.LoadVersion(999, true) require.Error(t, err) - // Parent should still work normally. require.NoError(t, s.ApplyChangeSets([]*proto.NamedChangeSet{makeChangeSet(key, []byte{2}, false)})) v, err := s.Commit() require.NoError(t, err) diff --git a/sei-db/state_db/sc/flatkv/store_write.go b/sei-db/state_db/sc/flatkv/store_write.go index 9df15df883..7bf6a2d93b 100644 --- a/sei-db/state_db/sc/flatkv/store_write.go +++ b/sei-db/state_db/sc/flatkv/store_write.go @@ -25,7 +25,6 @@ func (s *CommitStore) ApplyChangeSets(cs []*proto.NamedChangeSet) error { } s.phaseTimer.SetPhase("apply_change_sets_prepare") - // Save original changesets for changelog s.pendingChangeSets = append(s.pendingChangeSets, cs...) // Collect LtHash pairs per DB (using internal key format) From 8add61662d802f74461804e82a7f37bbfcd886a9 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 9 Mar 2026 10:33:59 -0400 Subject: [PATCH 4/6] log error --- sei-db/state_db/sc/composite/store.go | 4 +++- sei-db/state_db/sc/flatkv/store.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 21a7351abf..4fc2d43334 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -104,7 +104,9 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) if cs.evmCommitter != nil { evmStore, err := cs.evmCommitter.LoadVersion(targetVersion, true) if err != nil { - _ = cosmosCommitter.Close() + if closeErr := cosmosCommitter.Close(); closeErr != nil { + cs.logger.Error("failed to close cosmos committer during FlatKV error cleanup", "err", closeErr) + } return nil, fmt.Errorf("failed to load readonly FlatKV version: %w", err) } newStore.evmCommitter = evmStore diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index f678234c74..6410a44efa 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -221,7 +221,9 @@ func (s *CommitStore) loadVersionReadOnly(targetVersion int64) (_ Store, retErr defer func() { if retErr != nil { - _ = ro.Close() + if closeErr := ro.Close(); closeErr != nil { + s.log.Error("failed to close readonly store during error cleanup", "err", closeErr) + } } }() From 6d931d1ae5eb0a1a9377a628ad8107c505a8402f Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 10 Mar 2026 15:50:42 -0400 Subject: [PATCH 5/6] fix: soft-fail FlatKV on readonly LoadVersion to unblock Cosmos queries --- sei-db/state_db/sc/composite/store.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 4fc2d43334..43cb9d881c 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -104,12 +104,11 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) if cs.evmCommitter != nil { evmStore, err := cs.evmCommitter.LoadVersion(targetVersion, true) if err != nil { - if closeErr := cosmosCommitter.Close(); closeErr != nil { - cs.logger.Error("failed to close cosmos committer during FlatKV error cleanup", "err", closeErr) - } - return nil, fmt.Errorf("failed to load readonly FlatKV version: %w", err) + cs.logger.Error("FlatKV unavailable for readonly load, EVM data will not be served", + "version", targetVersion, "err", err) + } else { + newStore.evmCommitter = evmStore } - newStore.evmCommitter = evmStore } return newStore, nil } From 2e19408678a84a76e078e771331a144c9adffa39 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Tue, 10 Mar 2026 18:19:37 -0400 Subject: [PATCH 6/6] add a lock-guarded startup API (CleanupCrashArtifacts) that removes readonly-* working directories left by a previous process crash --- sei-cosmos/storev2/rootmulti/store.go | 3 + .../bench/wrappers/db_implementations.go | 3 + sei-db/state_db/sc/composite/store.go | 16 ++++- sei-db/state_db/sc/composite/store_test.go | 70 +++++++++++++++++++ sei-db/state_db/sc/flatkv/store_lifecycle.go | 31 ++++++++ sei-db/state_db/sc/flatkv/store_test.go | 37 ++++++++++ 6 files changed, 159 insertions(+), 1 deletion(-) diff --git a/sei-cosmos/storev2/rootmulti/store.go b/sei-cosmos/storev2/rootmulti/store.go index 10d378b630..b132b7723b 100644 --- a/sei-cosmos/storev2/rootmulti/store.go +++ b/sei-cosmos/storev2/rootmulti/store.go @@ -93,6 +93,9 @@ func NewStore( } ctx := context.Background() scStore := composite.NewCompositeCommitStore(ctx, scDir, logger, scConfig) + if err := scStore.CleanupCrashArtifacts(); err != nil { + panic(err) + } store := &Store{ logger: logger, scStore: scStore, diff --git a/sei-db/state_db/bench/wrappers/db_implementations.go b/sei-db/state_db/bench/wrappers/db_implementations.go index 60ae2a19ef..bd99044b46 100644 --- a/sei-db/state_db/bench/wrappers/db_implementations.go +++ b/sei-db/state_db/bench/wrappers/db_implementations.go @@ -67,6 +67,9 @@ func newCompositeCommitStore(ctx context.Context, dbDir string, writeMode config cfg.MemIAVLConfig.SnapshotInterval = 100 cs := composite.NewCompositeCommitStore(ctx, dbDir, logger.NewNopLogger(), cfg) + if err := cs.CleanupCrashArtifacts(); err != nil { + return nil, fmt.Errorf("failed to cleanup crash artifacts: %w", err) + } cs.Initialize([]string{EVMStoreName}) loaded, err := cs.LoadVersion(0, false) diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 43cb9d881c..aee2c29991 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -76,6 +76,20 @@ func (cs *CompositeCommitStore) Initialize(initialStores []string) { cs.cosmosCommitter.Initialize(initialStores) } +// CleanupCrashArtifacts removes temporary/orphaned files left by a +// previous process crash (e.g. FlatKV readonly-* working directories). +// Must be called once at process startup, before any read-only clones +// are created. Any writer lock acquired during cleanup is retained for +// the subsequent LoadVersion(..., false) call. +func (cs *CompositeCommitStore) CleanupCrashArtifacts() error { + if fkv, ok := cs.evmCommitter.(*flatkv.CommitStore); ok { + if err := fkv.CleanupOrphanedReadOnlyDirs(); err != nil { + return err + } + } + return nil +} + // SetInitialVersion sets the initial version for the store func (cs *CompositeCommitStore) SetInitialVersion(initialVersion int64) error { return cs.cosmosCommitter.SetInitialVersion(initialVersion) @@ -104,7 +118,7 @@ func (cs *CompositeCommitStore) LoadVersion(targetVersion int64, readOnly bool) if cs.evmCommitter != nil { evmStore, err := cs.evmCommitter.LoadVersion(targetVersion, true) if err != nil { - cs.logger.Error("FlatKV unavailable for readonly load, EVM data will not be served", + cs.logger.Info("FlatKV unavailable for readonly load, EVM data will not be served", "version", targetVersion, "err", err) } else { newStore.evmCommitter = evmStore diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index e4ed33ebe7..395290a708 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -1,16 +1,43 @@ package composite import ( + "fmt" "testing" "github.com/stretchr/testify/require" "github.com/sei-protocol/sei-chain/sei-db/common/logger" + "github.com/sei-protocol/sei-chain/sei-db/common/metrics" "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" iavl "github.com/sei-protocol/sei-chain/sei-iavl" ) +// failingEVMStore is a mock flatkv.Store whose LoadVersion always fails. +type failingEVMStore struct{} + +var _ flatkv.Store = (*failingEVMStore)(nil) + +func (f *failingEVMStore) LoadVersion(int64, bool) (flatkv.Store, error) { + return nil, fmt.Errorf("flatkv unavailable") +} +func (f *failingEVMStore) ApplyChangeSets([]*proto.NamedChangeSet) error { return nil } +func (f *failingEVMStore) Commit() (int64, error) { return 0, nil } +func (f *failingEVMStore) Get([]byte) ([]byte, bool) { return nil, false } +func (f *failingEVMStore) Has([]byte) bool { return false } +func (f *failingEVMStore) Iterator(_, _ []byte) flatkv.Iterator { return nil } +func (f *failingEVMStore) IteratorByPrefix([]byte) flatkv.Iterator { return nil } +func (f *failingEVMStore) RootHash() []byte { return nil } +func (f *failingEVMStore) Version() int64 { return 0 } +func (f *failingEVMStore) WriteSnapshot(string) error { return nil } +func (f *failingEVMStore) Rollback(int64) error { return nil } +func (f *failingEVMStore) Exporter(int64) (types.Exporter, error) { return nil, nil } +func (f *failingEVMStore) Importer(int64) (types.Importer, error) { return nil, nil } +func (f *failingEVMStore) GetPhaseTimer() *metrics.PhaseTimer { return nil } +func (f *failingEVMStore) Close() error { return nil } + func TestCompositeStoreBasicOperations(t *testing.T) { dir := t.TempDir() cfg := config.DefaultStateCommitConfig() @@ -223,3 +250,46 @@ func TestGetVersions(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(3), latestVersion) } + +func TestReadOnlyLoadVersionSoftFailsWhenFlatKVUnavailable(t *testing.T) { + dir := t.TempDir() + cfg := config.DefaultStateCommitConfig() + + cs := NewCompositeCommitStore(t.Context(), dir, logger.NewNopLogger(), cfg) + cs.Initialize([]string{"test"}) + + _, err := cs.LoadVersion(0, false) + require.NoError(t, err) + + err = cs.ApplyChangeSets([]*proto.NamedChangeSet{ + { + Name: "test", + Changeset: iavl.ChangeSet{ + Pairs: []*iavl.KVPair{ + {Key: []byte("key1"), Value: []byte("value1")}, + }, + }, + }, + }) + require.NoError(t, err) + _, err = cs.Commit() + require.NoError(t, err) + + // Inject a failing EVM committer to simulate FlatKV being unavailable + // for historical versions (different retention, late enablement, etc). + cs.evmCommitter = &failingEVMStore{} + + readOnly, err := cs.LoadVersion(0, true) + require.NoError(t, err, "readonly LoadVersion should succeed even when FlatKV fails") + defer func() { _ = readOnly.Close() }() + + compositeRO, ok := readOnly.(*CompositeCommitStore) + require.True(t, ok) + require.Nil(t, compositeRO.evmCommitter, "evmCommitter should be nil when FlatKV failed") + + // Cosmos data should still be accessible + store := compositeRO.GetChildStoreByName("test") + require.NotNil(t, store) + val := store.Get([]byte("key1")) + require.Equal(t, []byte("value1"), val) +} diff --git a/sei-db/state_db/sc/flatkv/store_lifecycle.go b/sei-db/state_db/sc/flatkv/store_lifecycle.go index 347bf659d6..95e93196bd 100644 --- a/sei-db/state_db/sc/flatkv/store_lifecycle.go +++ b/sei-db/state_db/sc/flatkv/store_lifecycle.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "os" + "path/filepath" + "strings" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" ) @@ -90,6 +92,35 @@ func (s *CommitStore) Close() error { return nil } +// CleanupOrphanedReadOnlyDirs acquires the writer lock and removes readonly-* +// working directories left behind by a previous process crash. It is a +// startup-only API and must be called before any read-only instances are +// created in the current process. The acquired writer lock is retained for +// subsequent LoadVersion(..., false) calls. +func (s *CommitStore) CleanupOrphanedReadOnlyDirs() error { + dir := s.flatkvDir() + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("create flatkv dir: %w", err) + } + if s.fileLock == nil { + if err := s.acquireFileLock(dir); err != nil { + return err + } + } + + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), readOnlyDirPrefix) { + s.log.Info("removing orphaned readonly dir", "dir", e.Name()) + _ = os.RemoveAll(filepath.Join(dir, e.Name())) + } + } + return nil +} + // Exporter creates an exporter for the given version. // NOTE: Not yet implemented. Will be added with state-sync support. // The future implementation will export each DB separately with internal key format. diff --git a/sei-db/state_db/sc/flatkv/store_test.go b/sei-db/state_db/sc/flatkv/store_test.go index b19ad59dfa..3b51ceb15c 100644 --- a/sei-db/state_db/sc/flatkv/store_test.go +++ b/sei-db/state_db/sc/flatkv/store_test.go @@ -921,3 +921,40 @@ func TestReadOnlyCloseRemovesTempDir(t *testing.T) { require.NoError(t, roStore.Close()) require.NoDirExists(t, tmpDir) } + +func TestCleanupOrphanedReadOnlyDirs(t *testing.T) { + dir := t.TempDir() + s := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + defer func() { require.NoError(t, s.Close()) }() + + fkvDir := s.flatkvDir() + require.NoError(t, os.MkdirAll(fkvDir, 0o755)) + + // Simulate orphaned dirs left by a crash. + orphan1 := filepath.Join(fkvDir, "readonly-old1") + orphan2 := filepath.Join(fkvDir, "readonly-old2") + require.NoError(t, os.Mkdir(orphan1, 0o755)) + require.NoError(t, os.Mkdir(orphan2, 0o755)) + + require.NoError(t, s.CleanupOrphanedReadOnlyDirs()) + + require.NoDirExists(t, orphan1) + require.NoDirExists(t, orphan2) + + _, err := s.LoadVersion(0, false) + require.NoError(t, err) +} + +func TestCleanupOrphanedReadOnlyDirsHoldsWriterLock(t *testing.T) { + dir := t.TempDir() + s1 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + defer func() { require.NoError(t, s1.Close()) }() + require.NoError(t, s1.CleanupOrphanedReadOnlyDirs()) + + s2 := NewCommitStore(t.Context(), filepath.Join(dir, flatkvRootDir), nil, DefaultConfig()) + defer func() { require.NoError(t, s2.Close()) }() + + err := s2.CleanupOrphanedReadOnlyDirs() + require.Error(t, err) + require.ErrorContains(t, err, "acquire file lock") +}