diff --git a/app/app.go b/app/app.go index 441d031c7f..4af12a9bf6 100644 --- a/app/app.go +++ b/app/app.go @@ -35,7 +35,6 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/codec" "github.com/sei-protocol/sei-chain/sei-cosmos/codec/types" cryptotypes "github.com/sei-protocol/sei-chain/sei-cosmos/crypto/types" - "github.com/sei-protocol/sei-chain/sei-cosmos/server" "github.com/sei-protocol/sei-chain/sei-cosmos/server/api" "github.com/sei-protocol/sei-chain/sei-cosmos/server/config" servertypes "github.com/sei-protocol/sei-chain/sei-cosmos/server/types" @@ -118,7 +117,6 @@ import ( gigautils "github.com/sei-protocol/sei-chain/giga/executor/utils" "github.com/sei-protocol/sei-chain/precompiles" putils "github.com/sei-protocol/sei-chain/precompiles/utils" - ssconfig "github.com/sei-protocol/sei-chain/sei-db/config" "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer" ibctransferkeeper "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/keeper" ibctransfertypes "github.com/sei-protocol/sei-chain/sei-ibc-go/modules/apps/transfer/types" @@ -678,10 +676,10 @@ func New( wasmOpts..., ) - receiptStorePath := filepath.Join(homePath, "data", "receipt.db") - receiptConfig := ssconfig.DefaultReceiptStoreConfig() - receiptConfig.DBDirectory = receiptStorePath - receiptConfig.KeepRecent = cast.ToInt(appOpts.Get(server.FlagMinRetainBlocks)) + receiptConfig, err := readReceiptStoreConfig(homePath, appOpts) + if err != nil { + panic(fmt.Sprintf("error reading receipt store config: %s", err)) + } if app.receiptStore == nil { receiptStore, err := receipt.NewReceiptStore(logger, receiptConfig, keys[evmtypes.StoreKey]) if err != nil { diff --git a/app/receipt_store_config.go b/app/receipt_store_config.go new file mode 100644 index 0000000000..8d2acfacf2 --- /dev/null +++ b/app/receipt_store_config.go @@ -0,0 +1,26 @@ +package app + +import ( + "path/filepath" + + seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" +) + +const ( + receiptStoreBackendKey = "receipt-store.rs-backend" + receiptStoreDBDirectoryKey = "receipt-store.db-directory" + receiptStoreAsyncWriteBufferKey = "receipt-store.async-write-buffer" + receiptStoreKeepRecentKey = "receipt-store.keep-recent" + receiptStorePruneIntervalSecondsKey = "receipt-store.prune-interval-seconds" +) + +func readReceiptStoreConfig(homePath string, appOpts seidbconfig.AppOptions) (seidbconfig.ReceiptStoreConfig, error) { + receiptConfig, err := seidbconfig.ReadReceiptConfig(appOpts) + if err != nil { + return receiptConfig, err + } + if receiptConfig.DBDirectory == "" { + receiptConfig.DBDirectory = filepath.Join(homePath, "data", "receipt.db") + } + return receiptConfig, nil +} diff --git a/app/seidb_test.go b/app/seidb_test.go index 94c7db5257..fa91e1436e 100644 --- a/app/seidb_test.go +++ b/app/seidb_test.go @@ -1,10 +1,14 @@ package app import ( + "path/filepath" "testing" + "github.com/sei-protocol/sei-chain/sei-cosmos/server" "github.com/sei-protocol/sei-chain/sei-db/config" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type TestSeiDBAppOpts struct { @@ -13,6 +17,7 @@ type TestSeiDBAppOpts struct { func (t TestSeiDBAppOpts) Get(s string) interface{} { defaultSCConfig := config.DefaultStateCommitConfig() defaultSSConfig := config.DefaultStateStoreConfig() + defaultReceiptConfig := config.DefaultReceiptStoreConfig() switch s { case FlagSCEnable: return defaultSCConfig.Enable @@ -46,6 +51,8 @@ func (t TestSeiDBAppOpts) Get(s string) interface{} { return defaultSSConfig.PruneIntervalSeconds case FlagSSImportNumWorkers: return defaultSSConfig.ImportNumWorkers + case receiptStoreBackendKey: + return defaultReceiptConfig.Backend case FlagEVMSSDirectory: return defaultSSConfig.EVMDBDirectory case FlagEVMSSWriteMode: @@ -61,8 +68,11 @@ func TestNewDefaultConfig(t *testing.T) { appOpts := TestSeiDBAppOpts{} scConfig := parseSCConfigs(appOpts) ssConfig := parseSSConfigs(appOpts) + receiptConfig, err := config.ReadReceiptConfig(appOpts) + assert.NoError(t, err) assert.Equal(t, scConfig, config.DefaultStateCommitConfig()) assert.Equal(t, ssConfig, config.DefaultStateStoreConfig()) + assert.Equal(t, receiptConfig, config.DefaultReceiptStoreConfig()) } type mapAppOpts map[string]interface{} @@ -85,3 +95,74 @@ func TestParseSCConfigs_HistoricalProofFlags(t *testing.T) { assert.Equal(t, 12.5, scConfig.HistoricalProofRateLimit) assert.Equal(t, 3, scConfig.HistoricalProofBurst) } + +func TestParseReceiptConfigs_DefaultsToPebbleWhenUnset(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{}) + assert.NoError(t, err) + assert.Equal(t, config.DefaultReceiptStoreConfig(), receiptConfig) +} + +func TestParseReceiptConfigs_UsesConfiguredBackend(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreBackendKey: "parquet", + }) + assert.NoError(t, err) + assert.Equal(t, "parquet", receiptConfig.Backend) + assert.Equal(t, config.DefaultReceiptStoreConfig().AsyncWriteBuffer, receiptConfig.AsyncWriteBuffer) + assert.Equal(t, config.DefaultReceiptStoreConfig().KeepRecent, receiptConfig.KeepRecent) +} + +func TestParseReceiptConfigs_UsesConfiguredValues(t *testing.T) { + receiptConfig, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreDBDirectoryKey: "/tmp/custom-receipt-db", + receiptStoreBackendKey: "parquet", + receiptStoreAsyncWriteBufferKey: 7, + receiptStoreKeepRecentKey: 42, + receiptStorePruneIntervalSecondsKey: 9, + }) + assert.NoError(t, err) + assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) + assert.Equal(t, "parquet", receiptConfig.Backend) + assert.Equal(t, 7, receiptConfig.AsyncWriteBuffer) + assert.Equal(t, 42, receiptConfig.KeepRecent) + assert.Equal(t, 9, receiptConfig.PruneIntervalSeconds) +} + +func TestParseReceiptConfigs_RejectsInvalidBackend(t *testing.T) { + _, err := config.ReadReceiptConfig(mapAppOpts{ + receiptStoreBackendKey: "rocksdb", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported receipt-store backend") + assert.Contains(t, err.Error(), "rocksdb") +} + +func TestReadReceiptStoreConfigUsesConfiguredValues(t *testing.T) { + homePath := t.TempDir() + receiptConfig, err := readReceiptStoreConfig(homePath, mapAppOpts{ + receiptStoreDBDirectoryKey: "/tmp/custom-receipt-db", + receiptStoreKeepRecentKey: 5, + server.FlagMinRetainBlocks: 100, + }) + require.NoError(t, err) + assert.Equal(t, "/tmp/custom-receipt-db", receiptConfig.DBDirectory) + assert.Equal(t, 5, receiptConfig.KeepRecent) +} + +func TestReadReceiptStoreConfigUsesDefaultDirectoryWhenUnset(t *testing.T) { + homePath := t.TempDir() + receiptConfig, err := readReceiptStoreConfig(homePath, mapAppOpts{}) + require.NoError(t, err) + assert.Equal(t, filepath.Join(homePath, "data", "receipt.db"), receiptConfig.DBDirectory) +} + +// TestFullAppPathWithParquetReceiptStore exercises the full app.New path with rs-backend = "parquet" +// and asserts the parquet receipt store is actually instantiated (not pebble). +func TestFullAppPathWithParquetReceiptStore(t *testing.T) { + app := SetupWithScReceiptFromOpts(t, false, false, TestAppOpts{ + UseSc: true, + ReceiptBackend: "parquet", + }) + require.NotNil(t, app.receiptStore, "receipt store should be created") + assert.Equal(t, "parquet", receipt.BackendTypeName(app.receiptStore), "receipt store backend should be parquet") +} diff --git a/app/test_helpers.go b/app/test_helpers.go index c9d5320548..8785e8c568 100644 --- a/app/test_helpers.go +++ b/app/test_helpers.go @@ -76,9 +76,10 @@ func (t TestTx) GetGasEstimate() uint64 { } type TestAppOpts struct { - UseSc bool - EnableGiga bool - EnableGigaOCC bool + UseSc bool + EnableGiga bool + EnableGigaOCC bool + ReceiptBackend string // e.g. "parquet" to use parquet receipt store; empty = default (pebble) } func (t TestAppOpts) Get(s string) interface{} { @@ -99,6 +100,9 @@ func (t TestAppOpts) Get(s string) interface{} { if s == gigaconfig.FlagOCCEnabled { return t.EnableGigaOCC } + if s == receiptStoreBackendKey && t.ReceiptBackend != "" { + return t.ReceiptBackend + } // Disable EVM HTTP and WebSocket servers in tests to avoid port conflicts // when multiple tests run in parallel (all would try to bind to port 8545) if s == "evm.http_enabled" || s == "evm.ws_enabled" { @@ -472,6 +476,55 @@ func SetupWithDB(tb testing.TB, db dbm.DB, isCheckTx bool, enableEVMCustomPrecom return res } +// SetupWithScReceiptFromOpts is like SetupWithSc but does not inject a receipt store via AppOption. +// The receipt store is created inside New() from testAppOpts (e.g. testAppOpts.ReceiptBackend = "parquet"). +// Use this to test the full app path with rs-backend from config. +func SetupWithScReceiptFromOpts(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) { + db := dbm.NewMemDB() + encodingConfig := MakeEncodingConfig() + cdc := encodingConfig.Marshaler + + res = New( + log.NewNopLogger(), + db, + nil, + true, + map[int64]bool{}, + t.TempDir(), + 1, + enableEVMCustomPrecompiles, + config.TestConfig(), + encodingConfig, + wasm.EnableAllProposals, + testAppOpts, + EmptyWasmOpts, + nil, // no options: receipt store is created from testAppOpts inside New() + baseAppOptions..., + ) + if !isCheckTx { + genesisState := NewDefaultGenesisState(cdc) + stateBytes, err := json.MarshalIndent(genesisState, "", " ") + if err != nil { + panic(err) + } + + defer func() { _ = recover() }() + + _, err = res.InitChain( + context.Background(), &abci.RequestInitChain{ + Validators: []abci.ValidatorUpdate{}, + ConsensusParams: DefaultConsensusParams, + AppStateBytes: stateBytes, + }, + ) + if err != nil { + panic(err) + } + } + + return res +} + func SetupWithSc(t *testing.T, isCheckTx bool, enableEVMCustomPrecompiles bool, testAppOpts TestAppOpts, baseAppOptions ...func(*bam.BaseApp)) (res *App) { db := dbm.NewMemDB() encodingConfig := MakeEncodingConfig() diff --git a/cmd/seid/cmd/app_config.go b/cmd/seid/cmd/app_config.go index 2ed7d4c3ae..22305f00cf 100644 --- a/cmd/seid/cmd/app_config.go +++ b/cmd/seid/cmd/app_config.go @@ -22,23 +22,25 @@ type WASMConfig struct { type CustomAppConfig struct { srvconfig.Config - StateCommit seidbconfig.StateCommitConfig `mapstructure:"state-commit"` - StateStore seidbconfig.StateStoreConfig `mapstructure:"state-store"` - WASM WASMConfig `mapstructure:"wasm"` - EVM evmrpcconfig.Config `mapstructure:"evm"` - GigaExecutor gigaconfig.Config `mapstructure:"giga_executor"` - ETHReplay replay.Config `mapstructure:"eth_replay"` - ETHBlockTest blocktest.Config `mapstructure:"eth_block_test"` - EvmQuery querier.Config `mapstructure:"evm_query"` - LightInvariance seiapp.LightInvarianceConfig `mapstructure:"light_invariance"` + StateCommit seidbconfig.StateCommitConfig `mapstructure:"state-commit"` + StateStore seidbconfig.StateStoreConfig `mapstructure:"state-store"` + ReceiptStore seidbconfig.ReceiptStoreConfig `mapstructure:"receipt-store"` + WASM WASMConfig `mapstructure:"wasm"` + EVM evmrpcconfig.Config `mapstructure:"evm"` + GigaExecutor gigaconfig.Config `mapstructure:"giga_executor"` + ETHReplay replay.Config `mapstructure:"eth_replay"` + ETHBlockTest blocktest.Config `mapstructure:"eth_block_test"` + EvmQuery querier.Config `mapstructure:"evm_query"` + LightInvariance seiapp.LightInvarianceConfig `mapstructure:"light_invariance"` } // NewCustomAppConfig creates a CustomAppConfig with the given base config and EVM config func NewCustomAppConfig(baseConfig *srvconfig.Config, evmConfig evmrpcconfig.Config) CustomAppConfig { return CustomAppConfig{ - Config: *baseConfig, - StateCommit: seidbconfig.DefaultStateCommitConfig(), - StateStore: seidbconfig.DefaultStateStoreConfig(), + Config: *baseConfig, + StateCommit: seidbconfig.DefaultStateCommitConfig(), + StateStore: seidbconfig.DefaultStateStoreConfig(), + ReceiptStore: seidbconfig.DefaultReceiptStoreConfig(), WASM: WASMConfig{ QueryGasLimit: 300000, LruSize: 1, diff --git a/cmd/seid/cmd/init_test.go b/cmd/seid/cmd/init_test.go index 5c4f50abd4..06fd629bc1 100644 --- a/cmd/seid/cmd/init_test.go +++ b/cmd/seid/cmd/init_test.go @@ -1,11 +1,13 @@ package cmd import ( + "bytes" "encoding/json" "fmt" "os" "path/filepath" "testing" + "text/template" "github.com/sei-protocol/sei-chain/app" "github.com/sei-protocol/sei-chain/app/genesis" @@ -149,6 +151,7 @@ func TestInitModeConfiguration(t *testing.T) { // Build custom template with all sections customAppTemplate := srvconfig.ManualConfigTemplate + seidbconfig.StateCommitConfigTemplate + seidbconfig.StateStoreConfigTemplate + + seidbconfig.ReceiptStoreConfigTemplate + srvconfig.AutoManagedConfigTemplate // Simplified - just need the pruning config srvconfig.SetConfigTemplate(customAppTemplate) @@ -167,10 +170,40 @@ func TestInitModeConfiguration(t *testing.T) { if tt.validateApp != nil { tt.validateApp(t, configDir) } + + v := viper.New() + v.SetConfigFile(appTomlPath) + err = v.ReadInConfig() + require.NoError(t, err) + require.Equal(t, "pebbledb", v.GetString("receipt-store.rs-backend")) + require.Equal(t, "", v.GetString("receipt-store.db-directory")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().AsyncWriteBuffer, v.GetInt("receipt-store.async-write-buffer")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().KeepRecent, v.GetInt("receipt-store.keep-recent")) + require.Equal(t, seidbconfig.DefaultReceiptStoreConfig().PruneIntervalSeconds, v.GetInt("receipt-store.prune-interval-seconds")) }) } } +func TestInitAppConfigIncludesReceiptStoreDefaults(t *testing.T) { + customAppTemplate, customAppConfig := initAppConfig() + + tmpl, err := template.New("app").Parse(customAppTemplate) + require.NoError(t, err) + + var buf bytes.Buffer + err = tmpl.Execute(&buf, customAppConfig) + require.NoError(t, err) + + output := buf.String() + require.Contains(t, output, "[receipt-store]") + require.Contains(t, output, `rs-backend = "pebbledb"`) + require.Contains(t, output, `db-directory = ""`) + require.Contains(t, output, "async-write-buffer =") + require.Contains(t, output, "keep-recent =") + require.Contains(t, output, "prune-interval-seconds =") + require.NotContains(t, output, "use-default-comparer") +} + // TestInitModeFlag verifies the mode flag validation func TestInitModeFlag(t *testing.T) { validModes := []string{"validator", "full", "seed", "archive"} diff --git a/cmd/seid/cmd/root.go b/cmd/seid/cmd/root.go index 235f98f0f1..00625f9e11 100644 --- a/cmd/seid/cmd/root.go +++ b/cmd/seid/cmd/root.go @@ -430,6 +430,7 @@ func initAppConfig() (string, interface{}) { customAppTemplate := serverconfig.ManualConfigTemplate + seidbconfig.StateCommitConfigTemplate + seidbconfig.StateStoreConfigTemplate + + seidbconfig.ReceiptStoreConfigTemplate + evmrpcconfig.ConfigTemplate + gigaconfig.ConfigTemplate + serverconfig.AutoManagedConfigTemplate + ` diff --git a/sei-db/config/receipt_config.go b/sei-db/config/receipt_config.go index 87b540251e..0d2887ed91 100644 --- a/sei-db/config/receipt_config.go +++ b/sei-db/config/receipt_config.go @@ -1,5 +1,27 @@ package config +import ( + "fmt" + "strings" + + "github.com/spf13/cast" +) + +// AppOptions is a minimal interface for reading config (e.g. from Viper). +// Implemented by sei-cosmos server/types.AppOptions; defined here to avoid import cycles. +type AppOptions interface { + Get(string) interface{} +} + +const ( + flagRSDBDirectory = "receipt-store.db-directory" + flagRSBackend = "receipt-store.rs-backend" + flagRSMisnamedBackend = "receipt-store.backend" + flagRSAsyncWriteBuffer = "receipt-store.async-write-buffer" + flagRSKeepRecent = "receipt-store.keep-recent" + flagRSPruneIntervalSeconds = "receipt-store.prune-interval-seconds" +) + // ReceiptStoreConfig defines configuration for the receipt store database. type ReceiptStoreConfig struct { // DBDirectory defines the directory to store the receipt store db files @@ -7,12 +29,13 @@ type ReceiptStoreConfig struct { // default to empty DBDirectory string `mapstructure:"db-directory"` - // Backend defines the backend database used for receipt-store - // Supported backends: pebbledb, rocksdb + // Backend defines the backend database used for receipt-store. + // Supported backends: pebbledb (aka pebble), parquet // defaults to pebbledb - Backend string `mapstructure:"backend"` + Backend string `mapstructure:"rs-backend"` // AsyncWriteBuffer defines the async queue length for commits to be applied to receipt store + // Applies only to the pebbledb backend. // Set <= 0 for synchronous writes. // defaults to 100 AsyncWriteBuffer int `mapstructure:"async-write-buffer"` @@ -25,12 +48,6 @@ type ReceiptStoreConfig struct { // PruneIntervalSeconds defines the interval in seconds to trigger pruning // default to every 600 seconds PruneIntervalSeconds int `mapstructure:"prune-interval-seconds"` - - // UseDefaultComparer uses Pebble's default lexicographic byte comparer instead of - // the custom MVCCComparer. This is NOT backwards compatible with existing databases - // that were created with MVCCComparer - only use this for NEW databases. - // defaults to false (use MVCCComparer for backwards compatibility) - UseDefaultComparer bool `mapstructure:"use-default-comparer"` } // DefaultReceiptStoreConfig returns the default ReceiptStoreConfig @@ -40,6 +57,55 @@ func DefaultReceiptStoreConfig() ReceiptStoreConfig { AsyncWriteBuffer: DefaultSSAsyncBuffer, KeepRecent: DefaultSSKeepRecent, PruneIntervalSeconds: DefaultSSPruneInterval, - UseDefaultComparer: false, } } + +// ReadReceiptConfig reads receipt store config from app options (e.g. TOML / Viper). +func ReadReceiptConfig(opts AppOptions) (ReceiptStoreConfig, error) { + cfg := DefaultReceiptStoreConfig() + if v := opts.Get(flagRSMisnamedBackend); v != nil { + return cfg, fmt.Errorf("unsupported receipt-store config key %q; use %q instead", flagRSMisnamedBackend, flagRSBackend) + } + if v := opts.Get(flagRSDBDirectory); v != nil { + dbDirectory, err := cast.ToStringE(v) + if err != nil { + return cfg, err + } + cfg.DBDirectory = strings.TrimSpace(dbDirectory) + } + if v := opts.Get(flagRSBackend); v != nil { + backend, err := cast.ToStringE(v) + if err != nil { + return cfg, err + } + backend = strings.ToLower(strings.TrimSpace(backend)) + switch backend { + case "pebbledb", "pebble", "parquet": + cfg.Backend = backend + default: + return cfg, fmt.Errorf("unsupported receipt-store backend %q; supported: pebbledb, parquet", backend) + } + } + if v := opts.Get(flagRSAsyncWriteBuffer); v != nil { + asyncWriteBuffer, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.AsyncWriteBuffer = asyncWriteBuffer + } + if v := opts.Get(flagRSKeepRecent); v != nil { + keepRecent, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.KeepRecent = keepRecent + } + if v := opts.Get(flagRSPruneIntervalSeconds); v != nil { + pruneIntervalSeconds, err := cast.ToIntE(v) + if err != nil { + return cfg, err + } + cfg.PruneIntervalSeconds = pruneIntervalSeconds + } + return cfg, nil +} diff --git a/sei-db/config/receipt_config_test.go b/sei-db/config/receipt_config_test.go new file mode 100644 index 0000000000..68af52c524 --- /dev/null +++ b/sei-db/config/receipt_config_test.go @@ -0,0 +1,23 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type mapAppOpts map[string]interface{} + +func (m mapAppOpts) Get(key string) interface{} { + return m[key] +} + +func TestReadReceiptConfigRejectsMisnamedBackendKey(t *testing.T) { + _, err := ReadReceiptConfig(mapAppOpts{ + "receipt-store.backend": "parquet", + }) + + require.Error(t, err) + require.ErrorContains(t, err, "receipt-store.backend") + require.ErrorContains(t, err, "receipt-store.rs-backend") +} diff --git a/sei-db/config/toml.go b/sei-db/config/toml.go index 8d972cf97f..7588e7b434 100644 --- a/sei-db/config/toml.go +++ b/sei-db/config/toml.go @@ -135,6 +135,24 @@ const ReceiptStoreConfigTemplate = ` # Supported backends: pebble (aka pebbledb), parquet # defaults to pebbledb rs-backend = "{{ .ReceiptStore.Backend }}" + +# Defines the receipt store directory. If unset, defaults to /data/receipt.db +db-directory = "{{ .ReceiptStore.DBDirectory }}" + +# AsyncWriteBuffer defines the async queue length for commits to be applied to receipt store. +# Applies only when rs-backend = "pebbledb"; parquet ignores this setting. +# Set <= 0 for synchronous writes. +# defaults to 100 +async-write-buffer = {{ .ReceiptStore.AsyncWriteBuffer }} + +# KeepRecent defines the number of versions to keep in receipt store +# Setting it to 0 means keep everything. +# Default to keep the last 100,000 blocks +keep-recent = {{ .ReceiptStore.KeepRecent }} + +# PruneIntervalSeconds defines the interval in seconds to trigger pruning. +# defaults to 600 seconds +prune-interval-seconds = {{ .ReceiptStore.PruneIntervalSeconds }} ` // DefaultConfigTemplate combines both templates for backward compatibility diff --git a/sei-db/config/toml_test.go b/sei-db/config/toml_test.go index d35307766f..09a175850c 100644 --- a/sei-db/config/toml_test.go +++ b/sei-db/config/toml_test.go @@ -89,6 +89,36 @@ func TestStateStoreConfigTemplate(t *testing.T) { require.Contains(t, output, "ss-import-num-workers =", "Missing ss-import-num-workers") } +// TestReceiptStoreConfigTemplate verifies that all field paths in the receipt-store TOML template +// are valid and can be resolved against the actual config struct. +func TestReceiptStoreConfigTemplate(t *testing.T) { + type TemplateConfig struct { + ReceiptStore ReceiptStoreConfig + } + + cfg := TemplateConfig{ + ReceiptStore: DefaultReceiptStoreConfig(), + } + + tmpl, err := template.New("receipt").Parse(ReceiptStoreConfigTemplate) + require.NoError(t, err, "Failed to parse ReceiptStoreConfigTemplate") + + var buf bytes.Buffer + err = tmpl.Execute(&buf, cfg) + require.NoError(t, err, "Failed to execute ReceiptStoreConfigTemplate - field path mismatch detected") + + output := buf.String() + + require.Contains(t, output, "[receipt-store]", "Missing receipt-store section") + require.Contains(t, output, `rs-backend = "pebbledb"`, "Missing or incorrect rs-backend") + require.Contains(t, output, `db-directory = ""`, "Missing or incorrect db-directory") + require.Contains(t, output, "async-write-buffer =", "Missing async-write-buffer") + require.Contains(t, output, "keep-recent =", "Missing keep-recent") + require.Contains(t, output, "prune-interval-seconds =", "Missing prune-interval-seconds") + require.Contains(t, output, `Applies only when rs-backend = "pebbledb"`, "Missing pebble-only async-write-buffer note") + require.NotContains(t, output, "use-default-comparer", "use-default-comparer should not be in receipt-store template") +} + // TestDefaultConfigTemplate verifies the combined template works correctly func TestDefaultConfigTemplate(t *testing.T) { type TemplateConfig struct { @@ -116,6 +146,9 @@ func TestDefaultConfigTemplate(t *testing.T) { require.Contains(t, output, "[state-commit]") require.Contains(t, output, "[state-store]") require.Contains(t, output, "[receipt-store]") + require.Contains(t, output, "async-write-buffer =") + require.Contains(t, output, "keep-recent =") + require.Contains(t, output, "prune-interval-seconds =") } // TestWriteModeValues verifies WriteMode enum values match template output diff --git a/sei-db/ledger_db/parquet/store_config_test.go b/sei-db/ledger_db/parquet/store_config_test.go index 2fba4afb66..b7d1d8b07f 100644 --- a/sei-db/ledger_db/parquet/store_config_test.go +++ b/sei-db/ledger_db/parquet/store_config_test.go @@ -65,6 +65,19 @@ func TestNewStoreUsesDefaultIntervalsWhenUnset(t *testing.T) { require.Equal(t, defaultMaxBlocksPerFile, store.CacheRotateInterval()) } +func TestNewStorePreservesKeepRecentAndPruneIntervalSettings(t *testing.T) { + store, err := NewStore(dbLogger.NewNopLogger(), StoreConfig{ + DBDirectory: t.TempDir(), + KeepRecent: 123, + PruneIntervalSeconds: 9, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + + require.Equal(t, int64(123), store.config.KeepRecent) + require.Equal(t, int64(9), store.config.PruneIntervalSeconds) +} + func TestPruneOldFilesKeepsTrackingOnDeleteFailure(t *testing.T) { store, err := NewStore(dbLogger.NewNopLogger(), StoreConfig{ DBDirectory: t.TempDir(), diff --git a/sei-db/ledger_db/receipt/parquet_store_test.go b/sei-db/ledger_db/receipt/parquet_store_test.go index 78e3f8a669..6ba6142f4c 100644 --- a/sei-db/ledger_db/receipt/parquet_store_test.go +++ b/sei-db/ledger_db/receipt/parquet_store_test.go @@ -183,12 +183,38 @@ func TestParquetReceiptStoreWALReplay(t *testing.T) { require.Equal(t, receipt.TxHashHex, got.TxHashHex) } +func TestParquetReceiptStoreUsesConfiguredDirectory(t *testing.T) { + ctx, storeKey := newTestContext() + cfg := dbconfig.DefaultReceiptStoreConfig() + cfg.Backend = "parquet" + cfg.DBDirectory = t.TempDir() + + store, err := NewReceiptStore(dbLogger.NewNopLogger(), cfg, storeKey) + require.NoError(t, err) + + txHash := common.HexToHash("0x31") + receipt := makeTestReceipt(txHash, 11, 0, common.HexToAddress("0x401"), nil) + require.NoError(t, store.SetReceipts(ctx.WithBlockHeight(11), []ReceiptRecord{ + {TxHash: txHash, Receipt: receipt}, + })) + require.NoError(t, store.Close()) + + receiptFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "receipts_*.parquet")) + require.NoError(t, err) + require.NotEmpty(t, receiptFiles, "receipt parquet files should be written under the configured db directory") + + logFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "logs_*.parquet")) + require.NoError(t, err) + require.NotEmpty(t, logFiles, "log parquet files should be written under the configured db directory") +} + func TestParquetFilePruning(t *testing.T) { ctx, storeKey := newTestContext() cfg := dbconfig.DefaultReceiptStoreConfig() cfg.Backend = "parquet" cfg.DBDirectory = t.TempDir() cfg.KeepRecent = 600 // Keep 600 blocks, so files with blocks < (latest - 600) get pruned + cfg.PruneIntervalSeconds = 1 store, err := NewReceiptStore(dbLogger.NewNopLogger(), cfg, storeKey) require.NoError(t, err) @@ -209,20 +235,36 @@ func TestParquetFilePruning(t *testing.T) { // Close to flush all files require.NoError(t, store.Close()) - // Verify we have at least 2 receipt and log files + // Verify we have enough receipt and log files for pruning to remove at least one pair. receiptFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "receipts_*.parquet")) require.NoError(t, err) - require.GreaterOrEqual(t, len(receiptFiles), 2, "should have at least 2 receipt files") + require.GreaterOrEqual(t, len(receiptFiles), 3, "should have at least 3 receipt files") logFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "logs_*.parquet")) require.NoError(t, err) - require.GreaterOrEqual(t, len(logFiles), 2, "should have at least 2 log files") + require.GreaterOrEqual(t, len(logFiles), 3, "should have at least 3 log files") + + initialReceiptFileCount := len(receiptFiles) + initialLogFileCount := len(logFiles) + + const ( + pruneTimeout = 5_000_000_000 + prunePollInterval = 100_000_000 + ) - // Reopen store - pruning will run in background + // Reopen store - pruning runs immediately on startup and then periodically afterward. store, err = NewReceiptStore(dbLogger.NewNopLogger(), cfg, storeKey) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() }) + require.Eventually(t, func() bool { + currentReceiptFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "receipts_*.parquet")) + require.NoError(t, err) + currentLogFiles, err := filepath.Glob(filepath.Join(cfg.DBDirectory, "logs_*.parquet")) + require.NoError(t, err) + return len(currentReceiptFiles) < initialReceiptFileCount && len(currentLogFiles) < initialLogFileCount + }, pruneTimeout, prunePollInterval, "configured keep-recent and prune-interval should trigger parquet file pruning") + // Verify we can still query recent data txHash := common.BigToHash(common.Big1.SetUint64(1400)) _, err = store.GetReceiptFromStore(ctx, txHash) diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 8776a21a92..250ac3ee30 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -83,6 +83,25 @@ func NewReceiptStore(log dbLogger.Logger, config dbconfig.ReceiptStoreConfig, st return newCachedReceiptStore(backend), nil } +// BackendTypeName returns the backend implementation name ("parquet" or "pebble") for testing. +// Returns "" if store is nil or the backend type is unknown. +func BackendTypeName(store ReceiptStore) string { + if store == nil { + return "" + } + if c, ok := store.(*cachedReceiptStore); ok { + store = c.backend + } + switch store.(type) { + case *parquetReceiptStore: + return receiptBackendParquet + case *receiptStore: + return receiptBackendPebble + default: + return "unknown" + } +} + func newReceiptBackend(log dbLogger.Logger, config dbconfig.ReceiptStoreConfig, storeKey sdk.StoreKey) (ReceiptStore, error) { if log == nil { log = dbLogger.NewNopLogger() @@ -98,6 +117,7 @@ func newReceiptBackend(log dbLogger.Logger, config dbconfig.ReceiptStoreConfig, case receiptBackendPebble: ssConfig := dbconfig.DefaultStateStoreConfig() ssConfig.DBDirectory = config.DBDirectory + ssConfig.AsyncWriteBuffer = config.AsyncWriteBuffer ssConfig.KeepRecent = config.KeepRecent if config.PruneIntervalSeconds > 0 { ssConfig.PruneIntervalSeconds = config.PruneIntervalSeconds