feat(sequencer): catchup from base

CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add disaster recovery for sequencer
+  - Catch up possible DA-only blocks when restarting. [#3057](https://github.com/evstack/ev-node/pull/3057)
+  - Verify DA and P2P state on restart (prevent double-signing). [#3061](https://github.com/evstack/ev-node/pull/3061)
 - Node pruning support. [#2984](https://github.com/evstack/ev-node/pull/2984)
   - Two different sort of pruning implemented:
     _Classic pruning_ (`all`): prunes given `HEAD-n` blocks from the databases, including store metadatas.

apps/evm/server/force_inclusion_test.go

@@ -73,6 +73,10 @@ func (m *mockDA) HasForcedInclusionNamespace() bool {
 	return true
 }
 
+func (m *mockDA) GetLatestDAHeight(_ context.Context) (uint64, error) {
+	return 0, nil
+}
+
 func TestForceInclusionServer_handleSendRawTransaction_Success(t *testing.T) {
 	testHeight := uint64(100)
 

block/internal/da/client.go

@@ -299,6 +299,23 @@ func (c *client) Retrieve(ctx context.Context, height uint64, namespace []byte)
 	}
 }
 
+// GetLatestDAHeight returns the latest height available on the DA layer by
+// querying the network head.
+func (c *client) GetLatestDAHeight(ctx context.Context) (uint64, error) {
+	headCtx, cancel := context.WithTimeout(ctx, c.defaultTimeout)
+	defer cancel()
+
+	header, err := c.headerAPI.NetworkHead(headCtx)
+	if err != nil {
+		return 0, fmt.Errorf("failed to get DA network head: %w", err)
+	}
+	if header == nil {
+		return 0, fmt.Errorf("DA network head returned nil header")
+	}
+
+	return header.Height, nil
+}
+
 // RetrieveForcedInclusion retrieves blobs from the forced inclusion namespace at the specified height.
 func (c *client) RetrieveForcedInclusion(ctx context.Context, height uint64) datypes.ResultRetrieve {
 	if !c.hasForcedNamespace {

block/internal/da/interface.go

@@ -17,6 +17,9 @@ type Client interface {
 	// Get retrieves blobs by their IDs. Used for visualization and fetching specific blobs.
 	Get(ctx context.Context, ids []datypes.ID, namespace []byte) ([]datypes.Blob, error)
 
+	// GetLatestDAHeight returns the latest height available on the DA layer..
+	GetLatestDAHeight(ctx context.Context) (uint64, error)
+
 	// Namespace accessors.
 	GetHeaderNamespace() []byte
 	GetDataNamespace() []byte

block/internal/da/tracing.go

@@ -123,6 +123,20 @@ func (t *tracedClient) Validate(ctx context.Context, ids []datypes.ID, proofs []
 	return res, nil
 }
 
+func (t *tracedClient) GetLatestDAHeight(ctx context.Context) (uint64, error) {
+	ctx, span := t.tracer.Start(ctx, "DA.GetLatestDAHeight")
+	defer span.End()
+
+	height, err := t.inner.GetLatestDAHeight(ctx)
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		return 0, err
+	}
+	span.SetAttributes(attribute.Int64("da.latest_height", int64(height)))
+	return height, nil
+}
+
 func (t *tracedClient) GetHeaderNamespace() []byte { return t.inner.GetHeaderNamespace() }
 func (t *tracedClient) GetDataNamespace() []byte   { return t.inner.GetDataNamespace() }
 func (t *tracedClient) GetForcedInclusionNamespace() []byte {

block/internal/da/tracing_test.go

@@ -54,10 +54,11 @@ func (m *mockFullClient) Validate(ctx context.Context, ids []datypes.ID, proofs
 	}
 	return nil, nil
 }
-func (m *mockFullClient) GetHeaderNamespace() []byte          { return []byte{0x01} }
-func (m *mockFullClient) GetDataNamespace() []byte            { return []byte{0x02} }
-func (m *mockFullClient) GetForcedInclusionNamespace() []byte { return []byte{0x03} }
-func (m *mockFullClient) HasForcedInclusionNamespace() bool   { return true }
+func (m *mockFullClient) GetLatestDAHeight(_ context.Context) (uint64, error) { return 0, nil }
+func (m *mockFullClient) GetHeaderNamespace() []byte                          { return []byte{0x01} }
+func (m *mockFullClient) GetDataNamespace() []byte                            { return []byte{0x02} }
+func (m *mockFullClient) GetForcedInclusionNamespace() []byte                 { return []byte{0x03} }
+func (m *mockFullClient) HasForcedInclusionNamespace() bool                   { return true }
 
 // setup a tracer provider + span recorder
 func setupDATrace(t *testing.T, inner FullClient) (FullClient, *tracetest.SpanRecorder) {

block/internal/syncing/syncer.go

@@ -760,9 +760,49 @@ func (s *Syncer) TrySyncNextBlock(ctx context.Context, event *common.DAHeightEve
 
 	// Update DA height if needed
 	// This height is only updated when a height is processed from DA as P2P
-	// events do not contain DA height information
+	// events do not contain DA height information.
+	//
+	// When a sequencer restarts after extended downtime, it produces "catch-up"
+	// blocks containing forced inclusion transactions from missed DA epochs and
+	// submits them to DA at the current (much higher) DA height. This creates a
+	// gap between the state's DAHeight (tracking forced inclusion epoch progress)
+	// and event.DaHeight (the DA submission height).
+	//
+	// If we jump state.DAHeight directly to event.DaHeight, subsequent calls to
+	// VerifyForcedInclusionTxs would check the wrong epoch (the submission epoch
+	// instead of the next forced-inclusion epoch), causing valid catch-up blocks
+	// to be incorrectly flagged as malicious.
+	//
+	// To handle this, when the gap exceeds one DA epoch, we advance DAHeight by
+	// exactly one epoch per block. This lets the forced inclusion verifier check
+	// the correct epoch for each catch-up block. Once the sequencer finishes
+	// catching up and the gap closes, DAHeight converges to event.DaHeight.
 	if event.DaHeight > newState.DAHeight {
-		newState.DAHeight = event.DaHeight
+		epochSize := s.genesis.DAEpochForcedInclusion
+		gap := event.DaHeight - newState.DAHeight
+
+		if epochSize > 0 && gap > epochSize {
+			// Large gap detected — likely catch-up blocks from a restarted sequencer.
+			// Advance DAHeight by one epoch to keep forced inclusion verification
+			// aligned with the epoch the sequencer is replaying.
+			_, epochEnd, _ := types.CalculateEpochBoundaries(
+				newState.DAHeight, s.genesis.DAStartHeight, epochSize,
+			)
+			nextEpochStart := epochEnd + 1
+			if nextEpochStart > event.DaHeight {
+				// Shouldn't happen, but clamp to event.DaHeight as a safety net.
+				nextEpochStart = event.DaHeight
+			}
+			s.logger.Debug().
+				Uint64("current_da_height", newState.DAHeight).
+				Uint64("event_da_height", event.DaHeight).
+				Uint64("advancing_to", nextEpochStart).
+				Uint64("gap", gap).
+				Msg("large DA height gap detected (sequencer catch-up), advancing DA height by one epoch")
+			newState.DAHeight = nextEpochStart
+		} else {
+			newState.DAHeight = event.DaHeight
+		}
 	}
 
 	batch, err := s.store.NewBatch(ctx)