feat: add recovery rewind commands

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 
+* Add recovery commands for removing CometBFT block/state and transaction-index suffixes above a target height.
 * [#9](https://github.com/Chaintable/cronos/pull/9) feat(blocksync): add an opt-in continuous sync mode for non-validator nodes.
 * [#2095](https://github.com/crypto-org-chain/cronos/pull/2095) feat: add hardcoded unblockable address list that is filtered out of both the mempool and consensus blocklists.
 

cmd/cronosd/cmd/rewind_cometbft.go

@@ -0,0 +1,128 @@
+package cmd
+
+import (
+	"fmt"
+	"strconv"
+
+	cmtcfg "github.com/cometbft/cometbft/config"
+	cmtstate "github.com/cometbft/cometbft/state"
+	cmtstore "github.com/cometbft/cometbft/store"
+	"github.com/cosmos/cosmos-sdk/server"
+	"github.com/spf13/cobra"
+)
+
+const rewindProgressInterval int64 = 1000
+
+// RewindCometBFTCmd removes the CometBFT block and consensus-state suffix above
+// a target height. Application and index databases are intentionally handled by
+// separate recovery steps.
+func RewindCometBFTCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "rewind-cometbft [height]",
+		Short: "Rewind CometBFT blocks and consensus state to a specific height",
+		Long: `Rewind CometBFT blocks and consensus state to a specific height.
+
+This command always deletes blocks above the target. It does not modify the
+application, transaction-index, or EVM-index databases. Stop cronosd before
+running it and rewind those databases as separate recovery steps.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			target, err := strconv.ParseInt(args[0], 10, 64)
+			if err != nil {
+				return fmt.Errorf("parse target height: %w", err)
+			}
+			if target <= 0 {
+				return fmt.Errorf("target height must be positive: %d", target)
+			}
+
+			ctx := server.GetServerContextFromCmd(cmd)
+			cfg := ctx.Config
+
+			blockDB, err := cmtcfg.DefaultDBProvider(&cmtcfg.DBContext{ID: "blockstore", Config: cfg})
+			if err != nil {
+				return fmt.Errorf("open blockstore: %w", err)
+			}
+			defer blockDB.Close()
+			blockStore := cmtstore.NewBlockStore(blockDB)
+
+			stateDB, err := cmtcfg.DefaultDBProvider(&cmtcfg.DBContext{ID: "state", Config: cfg})
+			if err != nil {
+				return fmt.Errorf("open state store: %w", err)
+			}
+			defer stateDB.Close()
+			stateStore := cmtstate.NewStore(stateDB, cmtstate.StoreOptions{
+				DiscardABCIResponses: cfg.Storage.DiscardABCIResponses,
+			})
+
+			initialState, err := stateStore.Load()
+			if err != nil {
+				return fmt.Errorf("load CometBFT state: %w", err)
+			}
+			cmd.Printf("rewinding CometBFT state=%d blockstore=%d target=%d\n",
+				initialState.LastBlockHeight, blockStore.Height(), target)
+
+			height, hash, removed, err := rewindCometBFT(blockStore, stateStore, target, func(height, removed int64) {
+				if removed%rewindProgressInterval == 0 {
+					cmd.Printf("rewind progress height=%d removed=%d\n", height, removed)
+				}
+			})
+			if err != nil {
+				return err
+			}
+
+			cmd.Printf("rewound CometBFT to height %d app_hash=%X removed=%d\n", height, hash, removed)
+			return nil
+		},
+	}
+}
+
+func rewindCometBFT(
+	blockStore *cmtstore.BlockStore,
+	stateStore cmtstate.Store,
+	target int64,
+	progress func(height, removed int64),
+) (int64, []byte, int64, error) {
+	if target <= 0 {
+		return 0, nil, 0, fmt.Errorf("target height must be positive: %d", target)
+	}
+	if base := blockStore.Base(); base > 0 && target < base {
+		return 0, nil, 0, fmt.Errorf("target %d is below blockstore base %d", target, base)
+	}
+
+	var removed int64
+	for {
+		currentState, err := stateStore.Load()
+		if err != nil {
+			return 0, nil, removed, fmt.Errorf("load CometBFT state: %w", err)
+		}
+		if currentState.IsEmpty() {
+			return 0, nil, removed, fmt.Errorf("no CometBFT state found")
+		}
+
+		stateHeight := currentState.LastBlockHeight
+		blockHeight := blockStore.Height()
+		if stateHeight < target || blockHeight < target {
+			return 0, nil, removed, fmt.Errorf(
+				"target %d is above available state=%d or blockstore=%d", target, stateHeight, blockHeight,
+			)
+		}
+		if stateHeight == target && blockHeight == target {
+			return target, currentState.AppHash, removed, nil
+		}
+
+		height, _, err := cmtstate.Rollback(blockStore, stateStore, true)
+		if err != nil {
+			return 0, nil, removed, fmt.Errorf(
+				"rewind CometBFT from state=%d blockstore=%d: %w", stateHeight, blockHeight, err,
+			)
+		}
+		if height < target {
+			return 0, nil, removed, fmt.Errorf("rewind crossed target %d and reached %d", target, height)
+		}
+
+		removed++
+		if progress != nil {
+			progress(height, removed)
+		}
+	}
+}

cmd/cronosd/cmd/rewind_cometbft_test.go

@@ -0,0 +1,106 @@
+package cmd
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	dbm "github.com/cometbft/cometbft-db"
+	cmtcrypto "github.com/cometbft/cometbft/crypto"
+	"github.com/cometbft/cometbft/crypto/tmhash"
+	cmtstateproto "github.com/cometbft/cometbft/proto/tendermint/state"
+	cmtversion "github.com/cometbft/cometbft/proto/tendermint/version"
+	cmtstate "github.com/cometbft/cometbft/state"
+	cmtstore "github.com/cometbft/cometbft/store"
+	cmttypes "github.com/cometbft/cometbft/types"
+	"github.com/cometbft/cometbft/version"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRewindCometBFT(t *testing.T) {
+	blockStore := cmtstore.NewBlockStore(dbm.NewMemDB())
+	stateStore := cmtstate.NewStore(dbm.NewMemDB(), cmtstate.StoreOptions{})
+	validatorSet, _ := cmttypes.RandValidatorSet(1, 10)
+	params := cmttypes.DefaultConsensusParams()
+
+	var lastBlockID cmttypes.BlockID
+	for height := int64(1); height <= 5; height++ {
+		block := &cmttypes.Block{
+			Header: cmttypes.Header{
+				Version:            cmtversion.Consensus{Block: version.BlockProtocol, App: 1},
+				ChainID:            "rewind-test",
+				Height:             height,
+				Time:               time.Unix(height, 0).UTC(),
+				LastBlockID:        lastBlockID,
+				LastCommitHash:     cmtcrypto.CRandBytes(tmhash.Size),
+				DataHash:           cmtcrypto.CRandBytes(tmhash.Size),
+				ValidatorsHash:     validatorSet.Hash(),
+				NextValidatorsHash: validatorSet.Hash(),
+				ConsensusHash:      params.Hash(),
+				AppHash:            testAppHash(height - 1),
+				LastResultsHash:    tmhash.Sum([]byte(fmt.Sprintf("results-%d", height-1))),
+				EvidenceHash:       cmtcrypto.CRandBytes(tmhash.Size),
+				ProposerAddress:    cmtcrypto.CRandBytes(cmtcrypto.AddressSize),
+			},
+			LastCommit: &cmttypes.Commit{Height: height - 1},
+		}
+		parts, err := block.MakePartSet(cmttypes.BlockPartSizeBytes)
+		require.NoError(t, err)
+		blockStore.SaveBlock(block, parts, &cmttypes.Commit{Height: height})
+		lastBlockID = blockStore.LoadBlockMeta(height).BlockID
+
+		state := cmtstate.State{
+			Version: cmtstateproto.Version{
+				Consensus: block.Version,
+				Software:  version.TMCoreSemVer,
+			},
+			ChainID:                          block.ChainID,
+			InitialHeight:                    1,
+			LastBlockHeight:                  height,
+			LastBlockID:                      lastBlockID,
+			LastBlockTime:                    block.Time,
+			NextValidators:                   validatorSet,
+			Validators:                       validatorSet,
+			LastValidators:                   validatorSet,
+			LastHeightValidatorsChanged:      2,
+			ConsensusParams:                  *params,
+			LastHeightConsensusParamsChanged: 2,
+			LastResultsHash:                  tmhash.Sum([]byte(fmt.Sprintf("results-%d", height))),
+			AppHash:                          testAppHash(height),
+		}
+		if height == 1 {
+			require.NoError(t, stateStore.Bootstrap(state))
+		} else {
+			require.NoError(t, stateStore.Save(state))
+		}
+	}
+
+	var progressCalls int
+	height, hash, removed, err := rewindCometBFT(blockStore, stateStore, 2, func(_, _ int64) {
+		progressCalls++
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, 2, height)
+	require.Equal(t, testAppHash(2), hash)
+	require.EqualValues(t, 3, removed)
+	require.Equal(t, 3, progressCalls)
+	require.EqualValues(t, 2, blockStore.Height())
+	require.Nil(t, blockStore.LoadBlockMeta(3))
+
+	state, err := stateStore.Load()
+	require.NoError(t, err)
+	require.EqualValues(t, 2, state.LastBlockHeight)
+	require.Equal(t, testAppHash(2), state.AppHash)
+}
+
+func TestRewindCometBFTRejectsMissingState(t *testing.T) {
+	blockStore := cmtstore.NewBlockStore(dbm.NewMemDB())
+	stateStore := cmtstate.NewStore(dbm.NewMemDB(), cmtstate.StoreOptions{})
+
+	_, _, _, err := rewindCometBFT(blockStore, stateStore, 2, nil)
+	require.ErrorContains(t, err, "no CometBFT state found")
+}
+
+func testAppHash(height int64) []byte {
+	return tmhash.Sum([]byte(fmt.Sprintf("app-%d", height)))
+}