feat(x/evmengine): support exec engine snap sync (#506)

Previously, when the execution engine was syncing, it would keep
retrying until evm syncing is done. However, evm syncing is stuck while
downloading chain data because the requested head falls behind by more
than 128 blocks, making it impossible to retrieve the missing data from
other peers.

To support snap sync, `newPayloadV3` and `forkChoiceUpdatedV3` requests
are now made with the latest head while the execution engine is syncing.
During syncing, `prepareProposal`, `processProposal`, and `postFinalize`
are skipped.

issue: #505

Author: 0xHansLee

client/x/evmengine/keeper/abci.go

@@ -29,6 +29,11 @@ const prepareTimeout = time.Second * 10
 func (k *Keeper) PrepareProposal(ctx sdk.Context, req *abci.RequestPrepareProposal) (
 	*abci.ResponsePrepareProposal, error,
 ) {
+	if k.IsExecEngSyncing() {
+		log.Warn(ctx, "Skip PrepareProposal while execution engine is syncing", nil)
+		return nil, ErrExecEngSyncing
+	}
+
 	// Only allow 10s to prepare a proposal. Propose empty block otherwise.
 	timeoutCtx, timeoutCancel := context.WithTimeout(ctx.Context(), prepareTimeout)
 	defer timeoutCancel()
@@ -189,6 +194,11 @@ func (k *Keeper) PostFinalize(ctx sdk.Context) error {
 		return nil // Not enabled.
 	}
 
+	if k.IsExecEngSyncing() {
+		log.Warn(ctx, "Skip PostFinalize while execution engine is syncing", nil)
+		return nil
+	}
+
 	// Extract context values
 	height := ctx.BlockHeight()
 	timestamp := ctx.BlockTime()

client/x/evmengine/keeper/keeper.go

@@ -28,6 +28,8 @@ import (
 	"github.com/piplabs/story/lib/k1util"
 )
 
+var ErrExecEngSyncing = errors.New("execution engine is syncing")
+
 type Keeper struct {
 	cdc             codec.BinaryCodec
 	storeService    store.KVStoreService
@@ -56,6 +58,8 @@ type Keeper struct {
 		Height    uint64
 		UpdatedAt time.Time
 	}
+
+	isExecEngSyncing bool
 }
 
 func NewKeeper(
@@ -128,6 +132,15 @@ func (k *Keeper) SetValidatorAddress(addr common.Address) {
 	k.validatorAddr = addr
 }
 
+// IsExecEngSyncing returns if execution engine is syncing or not.
+func (k *Keeper) IsExecEngSyncing() bool {
+	return k.isExecEngSyncing
+}
+
+func (k *Keeper) SetIsExecEngSyncing(isSyncing bool) {
+	k.isExecEngSyncing = isSyncing
+}
+
 // RegisterProposalService registers the proposal service on the provided router.
 // This implements abci.ProcessProposal verification of new proposals.
 func (k *Keeper) RegisterProposalService(server grpc1.Server) {

client/x/evmengine/keeper/msg_server.go

@@ -100,8 +100,11 @@ func (s msgServer) ExecutionPayload(ctx context.Context, msg *types.MsgExecution
 
 			return false, err // Don't retry, error out.
 		} else if isSyncing(status) {
-			log.Warn(ctx, "Processing finalized payload; evm syncing", nil)
-		} /* else isValid(status) */
+			s.SetIsExecEngSyncing(true)
+			log.Warn(ctx, "Push finalized payload while evm syncing", nil)
+		} else if s.IsExecEngSyncing() {
+			s.SetIsExecEngSyncing(false)
+		}
 
 		return true, nil // We are done, don't retry
 	})
@@ -124,9 +127,7 @@ func (s msgServer) ExecutionPayload(ctx context.Context, msg *types.MsgExecution
 
 			return false, nil // Retry
 		} else if isSyncing(fcr.PayloadStatus) {
-			log.Warn(ctx, "Processing finalized payload halted while evm syncing (will retry)", nil, "payload_height", payload.Number)
-
-			return false, nil // Retry
+			log.Warn(ctx, "Processing finalized payload while evm syncing", nil, "payload_height", payload.Number)
 		} else if invalid, err := isInvalid(fcr.PayloadStatus); invalid {
 			// This should never happen. This node will stall now.
 			log.Error(ctx, "Processing finalized payload failed; forkchoice update invalid [BUG]", err,

client/x/evmengine/keeper/proposal_server.go

@@ -19,6 +19,11 @@ type proposalServer struct {
 
 // ExecutionPayload handles a new execution payload proposed in a block.
 func (s proposalServer) ExecutionPayload(ctx context.Context, msg *types.MsgExecutionPayload) (*types.ExecutionPayloadResponse, error) {
+	if s.IsExecEngSyncing() {
+		log.Warn(ctx, "Skip ProcessProposal while execution engine is syncing", nil)
+		return nil, ErrExecEngSyncing
+	}
+
 	payload, err := s.parseAndVerifyProposedPayload(ctx, msg)
 	if err != nil {
 		return nil, errors.Wrap(err, "unmarshal payload")
@@ -41,8 +46,10 @@ func (s proposalServer) ExecutionPayload(ctx context.Context, msg *types.MsgExec
 			return false, errors.Wrap(err, "invalid payload, rejecting proposal") // Don't retry
 		} else if isSyncing(status) {
 			// If this is initial sync, we need to continue and set a target head to sync to, so don't retry.
-			log.Warn(ctx, "Can't properly verifying proposal: evm syncing", err,
+			log.Warn(ctx, "Can't properly verifying proposal: evm syncing", nil,
 				"payload_height", payload.Number)
+
+			return false, ErrExecEngSyncing
 		} /* else isValid(status) */
 
 		return true, nil // We are done, don't retry.