From 5509d38f00016df16fe07244ea6a147a73f3a9f4 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:32:55 +0100 Subject: [PATCH 01/41] basic prunning --- block/internal/executing/executor.go | 17 ++++++ pkg/config/config.go | 11 ++++ pkg/config/defaults.go | 3 ++ pkg/store/keys.go | 6 +++ pkg/store/store.go | 79 ++++++++++++++++++++++++++++ pkg/store/store_test.go | 51 ++++++++++++++++++ pkg/store/types.go | 27 +++++++++- 7 files changed, 193 insertions(+), 1 deletion(-) diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index bf1b44b6cb..341c5ec289 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -546,6 +546,23 @@ func (e *Executor) ProduceBlock(ctx context.Context) error { // Update in-memory state after successful commit e.setLastState(newState) + // Run height-based pruning of stored block data if enabled. This is a + // best-effort background maintenance step and should not cause block + // production to fail, but it does run in the critical path and may add + // some latency when large ranges are pruned. + if e.config.Node.PruningEnabled && e.config.Node.PruningKeepRecent > 0 && e.config.Node.PruningInterval > 0 { + if newHeight%e.config.Node.PruningInterval == 0 { + // Compute the prune floor: all heights <= targetHeight are candidates + // for pruning of header/data/signature/index entries. + if newHeight > e.config.Node.PruningKeepRecent { + targetHeight := newHeight - e.config.Node.PruningKeepRecent + if err := e.store.PruneBlocks(e.ctx, targetHeight); err != nil { + e.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + } + } + } + // broadcast header and data to P2P network g, broadcastCtx := errgroup.WithContext(e.ctx) g.Go(func() error { diff --git a/pkg/config/config.go b/pkg/config/config.go index e03a277ce8..36354a9090 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -261,6 +261,13 @@ type NodeConfig struct { // Readiness / health configuration ReadinessWindowSeconds uint64 `mapstructure:"readiness_window_seconds" yaml:"readiness_window_seconds" comment:"Time window in seconds used to calculate ReadinessMaxBlocksBehind based on block time. Default: 15 seconds."` ReadinessMaxBlocksBehind uint64 `mapstructure:"readiness_max_blocks_behind" yaml:"readiness_max_blocks_behind" comment:"How many blocks behind best-known head the node can be and still be considered ready. 0 means must be exactly at head."` + + // Pruning configuration + // When enabled, the node will periodically prune old block data (headers, data, + // signatures, and hash index) from the local store while keeping recent history. + PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` + PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain. Older blocks will have their header/data/signature removed from the local store. 0 means keep all blocks."` + PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` } // LogConfig contains all logging configuration parameters @@ -436,6 +443,10 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagReadinessWindowSeconds, def.Node.ReadinessWindowSeconds, "time window in seconds for calculating readiness threshold based on block time (default: 15s)") cmd.Flags().Uint64(FlagReadinessMaxBlocksBehind, def.Node.ReadinessMaxBlocksBehind, "how many blocks behind best-known head the node can be and still be considered ready (0 = must be at head)") cmd.Flags().Duration(FlagScrapeInterval, def.Node.ScrapeInterval.Duration, "interval at which the reaper polls the execution layer for new transactions") + // Pruning configuration flags + cmd.Flags().Bool(FlagPrefixEvnode+"node.pruning_enabled", def.Node.PruningEnabled, "enable height-based pruning of stored block data (headers, data, signatures, index)") + cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (0 = keep all)") + cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_interval", def.Node.PruningInterval, "run pruning every N blocks (must be >= 1 when pruning is enabled)") // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 0de2f4bc27..ee2cbfbeec 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -69,6 +69,9 @@ func DefaultConfig() Config { ReadinessWindowSeconds: defaultReadinessWindowSeconds, ReadinessMaxBlocksBehind: calculateReadinessMaxBlocksBehind(defaultBlockTime.Duration, defaultReadinessWindowSeconds), ScrapeInterval: DurationWrapper{1 * time.Second}, + PruningEnabled: false, + PruningKeepRecent: 0, + PruningInterval: 0, }, DA: DAConfig{ Address: "http://localhost:7980", diff --git a/pkg/store/keys.go b/pkg/store/keys.go index dd989c0e82..ff96fd8955 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -25,6 +25,12 @@ const ( // LastSubmittedHeaderHeightKey is the key used for persisting the last submitted header height in store. LastSubmittedHeaderHeightKey = "last-submitted-header-height" + // LastPrunedBlockHeightKey is the metadata key used for persisting the last + // pruned block height in the store. All block data (header, data, + // signature, and hash index) for heights <= this value are considered + // pruned and may be missing from the store. + LastPrunedBlockHeightKey = "last-pruned-block-height" + headerPrefix = "h" dataPrefix = "d" signaturePrefix = "c" diff --git a/pkg/store/store.go b/pkg/store/store.go index eafa47ae75..a050cbfe83 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -332,6 +332,85 @@ func (s *DefaultStore) Rollback(ctx context.Context, height uint64, aggregator b return nil } +// PruneBlocks removes block data (header, data, signature, and hash index) +// up to and including the given height from the store. It does not modify +// the current chain height or any state snapshots. +// +// This method is intended for long-term storage reduction and is safe to +// call repeatedly with the same or increasing heights. +func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { + batch, err := s.db.Batch(ctx) + if err != nil { + return fmt.Errorf("failed to create a new batch for pruning: %w", err) + } + + // Track the last successfully pruned height so we can resume across restarts. + var lastPruned uint64 + meta, err := s.GetMetadata(ctx, LastPrunedBlockHeightKey) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to get last pruned height: %w", err) + } + } else if len(meta) == heightLength { + lastPruned, err = decodeHeight(meta) + if err != nil { + return fmt.Errorf("failed to decode last pruned height: %w", err) + } + } + + // Nothing new to prune. + if height <= lastPruned { + return nil + } + + // Delete block data for heights in (lastPruned, height]. + for h := lastPruned + 1; h <= height; h++ { + // Get header blob to compute the hash index key. If header is already + // missing (e.g. due to previous partial pruning), just skip this height. + headerBlob, err := s.db.Get(ctx, ds.NewKey(getHeaderKey(h))) + if err != nil { + if errors.Is(err, ds.ErrNotFound) { + continue + } + return fmt.Errorf("failed to get header at height %d during pruning: %w", h, err) + } + + if err := batch.Delete(ctx, ds.NewKey(getHeaderKey(h))); err != nil { + return fmt.Errorf("failed to delete header at height %d during pruning: %w", h, err) + } + + if err := batch.Delete(ctx, ds.NewKey(getDataKey(h))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete data at height %d during pruning: %w", h, err) + } + } + + if err := batch.Delete(ctx, ds.NewKey(getSignatureKey(h))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete signature at height %d during pruning: %w", h, err) + } + } + + headerHash := sha256.Sum256(headerBlob) + if err := batch.Delete(ctx, ds.NewKey(getIndexKey(headerHash[:]))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete index for height %d during pruning: %w", h, err) + } + } + } + + // Persist the updated last pruned height. + if err := batch.Put(ctx, ds.NewKey(getMetaKey(LastPrunedBlockHeightKey)), encodeHeight(height)); err != nil { + return fmt.Errorf("failed to update last pruned height: %w", err) + } + + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit pruning batch: %w", err) + } + + return nil +} + const heightLength = 8 func encodeHeight(height uint64) []byte { diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 6a09db465e..0d47f73058 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -789,6 +789,57 @@ func TestRollback(t *testing.T) { require.Equal(rollbackToHeight, state.LastBlockHeight) } +func TestPruneBlocks_RemovesOldBlockDataOnly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + ds, err := NewTestInMemoryKVStore() + require.NoError(t, err) + + s := New(ds).(*DefaultStore) + + // create and store a few blocks with headers, data, signatures and state + batch, err := s.NewBatch(ctx) + require.NoError(t, err) + + var lastState types.State + for h := uint64(1); h <= 5; h++ { + header := &types.SignedHeader{Header: types.Header{BaseHeader: types.BaseHeader{Height: h}}} + data := &types.Data{} + sig := types.Signature([]byte{byte(h)}) + + require.NoError(t, batch.SaveBlockData(header, data, &sig)) + + // fake state snapshot per height + lastState = types.State{LastBlockHeight: h} + require.NoError(t, batch.UpdateState(lastState)) + } + require.NoError(t, batch.SetHeight(5)) + require.NoError(t, batch.Commit()) + + // prune everything up to height 3 + require.NoError(t, s.PruneBlocks(ctx, 3)) + + // old block data should be gone + for h := uint64(1); h <= 3; h++ { + _, _, err := s.GetBlockData(ctx, h) + assert.Error(t, err, "expected block data at height %d to be pruned", h) + } + + // recent block data should remain + for h := uint64(4); h <= 5; h++ { + _, _, err := s.GetBlockData(ctx, h) + assert.NoError(t, err, "expected block data at height %d to be kept", h) + } + + // state snapshots are not pruned by PruneBlocks + for h := uint64(1); h <= 5; h++ { + st, err := s.GetStateAtHeight(ctx, h) + assert.NoError(t, err, "expected state at height %d to remain", h) + assert.Equal(t, h, st.LastBlockHeight) + } +} + // TestRollbackToSameHeight verifies that rollback to same height is a no-op func TestRollbackToSameHeight(t *testing.T) { t.Parallel() diff --git a/pkg/store/types.go b/pkg/store/types.go index 13b62e2f1e..1162d0075e 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -30,10 +30,24 @@ type Batch interface { } // Store is minimal interface for storing and retrieving blocks, commits and state. +// +// It is composed from three concerns: +// - Reader: read access to blocks, state, and metadata +// - Rollback: consensus rollback logic (used for chain reorgs / recovery) +// - Pruner: long-term height-based pruning of historical block data type Store interface { - Rollback Reader Metadata + Rollback + Pruner + + // SetMetadata saves arbitrary value in the store. + // + // This method enables evolve to safely persist any information. + SetMetadata(ctx context.Context, key string, value []byte) error + + // DeleteMetadata removes a metadata key from the store. + DeleteMetadata(ctx context.Context, key string) error // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error @@ -91,3 +105,14 @@ type Rollback interface { // Aggregator is used to determine if the rollback is performed on the aggregator node. Rollback(ctx context.Context, height uint64, aggregator bool) error } + +// Pruner provides long-term, height-based pruning of historical block data. +// +// Implementations SHOULD be idempotent and safe to call multiple times for +// the same or increasing target heights. +type Pruner interface { + // PruneBlocks removes block data (header, data, signature, and hash index) + // up to and including the given height from the store, without modifying + // state snapshots or the current chain height. + PruneBlocks(ctx context.Context, height uint64) error +} From 95a387684f4ee31fb60ea89248bfd8092c7e8c59 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:10:28 +0100 Subject: [PATCH 02/41] prune metadata from ev-node store --- pkg/store/store.go | 12 ++++++++++++ pkg/store/store_test.go | 30 +++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/store/store.go b/pkg/store/store.go index a050cbfe83..8f045762d7 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -391,6 +391,18 @@ func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { } } + // Delete per-height DA metadata associated with this height, if any. + if err := batch.Delete(ctx, ds.NewKey(getMetaKey(GetHeightToDAHeightHeaderKey(h)))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete header DA height metadata at height %d during pruning: %w", h, err) + } + } + if err := batch.Delete(ctx, ds.NewKey(getMetaKey(GetHeightToDAHeightDataKey(h)))); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete data DA height metadata at height %d during pruning: %w", h, err) + } + } + headerHash := sha256.Sum256(headerBlob) if err := batch.Delete(ctx, ds.NewKey(getIndexKey(headerHash[:]))); err != nil { if !errors.Is(err, ds.ErrNotFound) { diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 0d47f73058..f1b9a131cc 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -798,7 +798,7 @@ func TestPruneBlocks_RemovesOldBlockDataOnly(t *testing.T) { s := New(ds).(*DefaultStore) - // create and store a few blocks with headers, data, signatures and state + // create and store a few blocks with headers, data, signatures, state, and per-height DA metadata batch, err := s.NewBatch(ctx) require.NoError(t, err) @@ -813,6 +813,14 @@ func TestPruneBlocks_RemovesOldBlockDataOnly(t *testing.T) { // fake state snapshot per height lastState = types.State{LastBlockHeight: h} require.NoError(t, batch.UpdateState(lastState)) + + // store fake DA metadata per height + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + bz := make([]byte, 8) + binary.LittleEndian.PutUint64(bz, h+100) // arbitrary DA height + require.NoError(t, s.SetMetadata(ctx, hDaKey, bz)) + require.NoError(t, s.SetMetadata(ctx, dDaKey, bz)) } require.NoError(t, batch.SetHeight(5)) require.NoError(t, batch.Commit()) @@ -838,6 +846,26 @@ func TestPruneBlocks_RemovesOldBlockDataOnly(t *testing.T) { assert.NoError(t, err, "expected state at height %d to remain", h) assert.Equal(t, h, st.LastBlockHeight) } + + // per-height DA metadata for pruned heights should be gone + for h := uint64(1); h <= 3; h++ { + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + _, err := s.GetMetadata(ctx, hDaKey) + assert.Error(t, err, "expected header DA metadata at height %d to be pruned", h) + _, err = s.GetMetadata(ctx, dDaKey) + assert.Error(t, err, "expected data DA metadata at height %d to be pruned", h) + } + + // per-height DA metadata for unpruned heights should remain + for h := uint64(4); h <= 5; h++ { + hDaKey := GetHeightToDAHeightHeaderKey(h) + dDaKey := GetHeightToDAHeightDataKey(h) + _, err := s.GetMetadata(ctx, hDaKey) + assert.NoError(t, err, "expected header DA metadata at height %d to remain", h) + _, err = s.GetMetadata(ctx, dDaKey) + assert.NoError(t, err, "expected data DA metadata at height %d to remain", h) + } } // TestRollbackToSameHeight verifies that rollback to same height is a no-op From b8ca7fb8e971f35f2bb0aa84076c892ed3185684 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Fri, 23 Jan 2026 17:27:04 +0100 Subject: [PATCH 03/41] wiring prunning config to go-header --- pkg/sync/sync_service.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index 8567e79764..c9e26fd31e 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -189,11 +189,21 @@ func (syncService *SyncService[H]) Start(ctx context.Context) error { } // create syncer, must be before initFromP2PWithRetry which calls startSyncer. + syncOpts := []goheadersync.Option{goheadersync.WithBlockTime(syncService.conf.Node.BlockTime.Duration)} + // Map ev-node pruning configuration to go-header's pruning window: we approximate + // "keep N recent heights" as "retain headers for N * blockTime". + if syncService.conf.Node.PruningEnabled && syncService.conf.Node.PruningKeepRecent > 0 { + pruningWindow := syncService.conf.Node.BlockTime.Duration * time.Duration(syncService.conf.Node.PruningKeepRecent) + // Only set a pruning window if the computed duration is positive. + if pruningWindow > 0 { + syncOpts = append(syncOpts, goheadersync.WithPruningWindow(pruningWindow)) + } + } if syncService.syncer, err = newSyncer( syncService.ex, syncService.store, syncService.sub, - []goheadersync.Option{goheadersync.WithBlockTime(syncService.conf.Node.BlockTime.Duration)}, + syncOpts, ); err != nil { return fmt.Errorf("failed to create syncer: %w", err) } @@ -467,7 +477,10 @@ func newSyncer[H header.Header[H]]( opts = append(opts, goheadersync.WithMetrics(), +<<<<<<< HEAD goheadersync.WithPruningWindow(ninetyNineYears), // pruning window not relevant, because of the store wrapper. +======= +>>>>>>> 3b9a0f70 (wiring prunning config to go-header) goheadersync.WithTrustingPeriod(ninetyNineYears), ) return goheadersync.NewSyncer(ex, store, sub, opts...) From d09b8033a51314e27559e7a8f7be5b78749d64a3 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:42:55 +0100 Subject: [PATCH 04/41] prune evm exec store --- block/internal/executing/executor.go | 11 ++++ core/execution/execution.go | 13 ++++ execution/evm/execution.go | 17 +++++ execution/evm/store.go | 53 +++++++++++++++ execution/evm/store_test.go | 99 ++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+) create mode 100644 execution/evm/store_test.go diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 341c5ec289..426afdcf06 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -559,6 +559,17 @@ func (e *Executor) ProduceBlock(ctx context.Context) error { if err := e.store.PruneBlocks(e.ctx, targetHeight); err != nil { e.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") } + + // If the execution client exposes execution-metadata pruning, + // prune ExecMeta using the same target height. This keeps EVM + // execution metadata aligned with ev-node's block store pruning + // while remaining a no-op for execution environments that don't + // implement ExecMetaPruner (e.g. ABCI-based executors). + if pruner, ok := e.exec.(coreexecutor.ExecMetaPruner); ok { + if err := pruner.PruneExecMeta(e.ctx, targetHeight); err != nil { + e.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") + } + } } } } diff --git a/core/execution/execution.go b/core/execution/execution.go index 276da369e9..1f85e2068a 100644 --- a/core/execution/execution.go +++ b/core/execution/execution.go @@ -161,3 +161,16 @@ type Rollbackable interface { // Rollback resets the execution layer head to the specified height. Rollback(ctx context.Context, targetHeight uint64) error } + +// ExecMetaPruner is an optional interface that execution clients can implement +// to support height-based pruning of their execution metadata. This is used by +// EVM-based execution clients to keep ExecMeta consistent with ev-node's +// pruning window while remaining a no-op for execution environments that +// don't persist per-height metadata in ev-node's datastore. +type ExecMetaPruner interface { + // PruneExecMeta should delete execution metadata for all heights up to and + // including the given height. Implementations should be idempotent and track + // their own progress so that repeated calls with the same or decreasing + // heights are cheap no-ops. + PruneExecMeta(ctx context.Context, height uint64) error +} diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 7a0a9dfb5a..5263539429 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -65,6 +65,12 @@ var _ execution.HeightProvider = (*EngineClient)(nil) // Ensure EngineClient implements the execution.Rollbackable interface var _ execution.Rollbackable = (*EngineClient)(nil) +// Ensure EngineClient implements optional pruning interface when used with +// ev-node's height-based pruning. This enables coordinated pruning of EVM +// ExecMeta alongside ev-node's own block data pruning, while remaining a +// no-op for non-EVM execution environments. +var _ execution.ExecMetaPruner = (*EngineClient)(nil) + // validatePayloadStatus checks the payload status and returns appropriate errors. // It implements the Engine API specification's status handling: // - VALID: Operation succeeded, return nil @@ -265,6 +271,17 @@ func NewEngineExecutionClient( }, nil } +// PruneExecMeta implements execution.ExecMetaPruner by delegating to the +// underlying EVMStore. It is safe to call this multiple times with the same +// or increasing heights; the store tracks its own last-pruned height. +func (c *EngineClient) PruneExecMeta(ctx context.Context, height uint64) error { + if c.store == nil { + return nil + } + + return c.store.PruneExecMeta(ctx, height) +} + // SetLogger allows callers to attach a structured logger. func (c *EngineClient) SetLogger(l zerolog.Logger) { c.logger = l diff --git a/execution/evm/store.go b/execution/evm/store.go index af731ee7c6..51a3056b66 100644 --- a/execution/evm/store.go +++ b/execution/evm/store.go @@ -15,6 +15,11 @@ import ( // Store prefix for execution/evm data - keeps it isolated from other ev-node data const evmStorePrefix = "evm/" +// lastPrunedExecMetaKey is the datastore key used to track the highest +// execution height for which ExecMeta has been pruned. All ExecMeta entries +// for heights <= this value are considered pruned. +const lastPrunedExecMetaKey = evmStorePrefix + "last-pruned-execmeta-height" + // ExecMeta stages const ( ExecStageStarted = "started" @@ -140,6 +145,54 @@ func (s *EVMStore) SaveExecMeta(ctx context.Context, meta *ExecMeta) error { return nil } +// PruneExecMeta removes ExecMeta entries up to and including the given height. +// It is safe to call this multiple times with the same or increasing heights; +// previously pruned ranges will be skipped based on the last-pruned marker. +func (s *EVMStore) PruneExecMeta(ctx context.Context, height uint64) error { + // Load last pruned height, if any. + var lastPruned uint64 + data, err := s.db.Get(ctx, ds.NewKey(lastPrunedExecMetaKey)) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to get last pruned execmeta height: %w", err) + } + } else if len(data) == 8 { + lastPruned = binary.BigEndian.Uint64(data) + } + + // Nothing new to prune. + if height <= lastPruned { + return nil + } + + batch, err := s.db.Batch(ctx) + if err != nil { + return fmt.Errorf("failed to create batch for execmeta pruning: %w", err) + } + + for h := lastPruned + 1; h <= height; h++ { + key := execMetaKey(h) + if err := batch.Delete(ctx, key); err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete exec meta at height %d: %w", h, err) + } + } + } + + // Persist updated last pruned height. + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, height) + if err := batch.Put(ctx, ds.NewKey(lastPrunedExecMetaKey), buf); err != nil { + return fmt.Errorf("failed to update last pruned execmeta height: %w", err) + } + + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit execmeta pruning batch: %w", err) + } + + return nil +} + // Sync ensures all pending writes are flushed to disk. func (s *EVMStore) Sync(ctx context.Context) error { return s.db.Sync(ctx, ds.NewKey(evmStorePrefix)) diff --git a/execution/evm/store_test.go b/execution/evm/store_test.go new file mode 100644 index 0000000000..64389701f9 --- /dev/null +++ b/execution/evm/store_test.go @@ -0,0 +1,99 @@ +package evm + +import ( + "context" + "encoding/binary" + "testing" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" +) + +// newTestDatastore creates an in-memory datastore for testing. +func newTestDatastore(t *testing.T) ds.Batching { + t.Helper() + // Wrap the in-memory MapDatastore to satisfy the Batching interface. + return dssync.MutexWrap(ds.NewMapDatastore()) +} + +func TestPruneExecMeta_PrunesUpToTargetHeight(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := newTestDatastore(t) + store := NewEVMStore(db) + + // Seed ExecMeta entries at heights 1..5 + for h := uint64(1); h <= 5; h++ { + meta := &ExecMeta{Height: h} + require.NoError(t, store.SaveExecMeta(ctx, meta)) + } + + // Sanity: all heights should be present + for h := uint64(1); h <= 5; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, h, meta.Height) + } + + // Prune up to height 3 + require.NoError(t, store.PruneExecMeta(ctx, 3)) + + // Heights 1..3 should be gone + for h := uint64(1); h <= 3; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.Nil(t, meta) + } + + // Heights 4..5 should remain + for h := uint64(4); h <= 5; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.NotNil(t, meta) + } + + // Re-pruning with the same height should be a no-op + require.NoError(t, store.PruneExecMeta(ctx, 3)) +} + +func TestPruneExecMeta_TracksLastPrunedHeight(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := newTestDatastore(t) + store := NewEVMStore(db) + + // Seed ExecMeta entries at heights 1..5 + for h := uint64(1); h <= 5; h++ { + meta := &ExecMeta{Height: h} + require.NoError(t, store.SaveExecMeta(ctx, meta)) + } + + // First prune up to 2 + require.NoError(t, store.PruneExecMeta(ctx, 2)) + + // Then prune up to 4; heights 3..4 should be deleted in this run + require.NoError(t, store.PruneExecMeta(ctx, 4)) + + // Verify all heights 1..4 are gone, 5 remains + for h := uint64(1); h <= 4; h++ { + meta, err := store.GetExecMeta(ctx, h) + require.NoError(t, err) + require.Nil(t, meta) + } + + meta, err := store.GetExecMeta(ctx, 5) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, uint64(5), meta.Height) + + // Ensure last-pruned marker is set to 4 + raw, err := db.Get(ctx, ds.NewKey(lastPrunedExecMetaKey)) + require.NoError(t, err) + require.Len(t, raw, 8) + last := binary.BigEndian.Uint64(raw) + require.Equal(t, uint64(4), last) +} From c329f769a82c62177458119b680d37e7b2f6b904 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:45:24 +0100 Subject: [PATCH 05/41] add parameters validation --- pkg/config/config.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 36354a9090..7a14aaae0a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -383,6 +383,21 @@ func (c *Config) Validate() error { if err := c.Raft.Validate(); err != nil { return err } + + // Validate pruning configuration + if c.Node.PruningEnabled { + // When pruning is enabled, pruning_interval must be >= 1 + if c.Node.PruningInterval == 0 { + return fmt.Errorf("pruning_interval must be >= 1 when pruning is enabled") + } + + // When pruning is enabled, keeping 0 blocks is contradictory; use pruning_enabled=false + // for archive mode instead. + if c.Node.PruningKeepRecent == 0 { + return fmt.Errorf("pruning_keep_recent must be > 0 when pruning is enabled; use pruning_enabled=false to keep all blocks") + } + } + return nil } From 9ae2f6a0bd0aba19bdc8d77ddcdb5a2284c737b8 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:47:32 +0100 Subject: [PATCH 06/41] make error handling consistent --- pkg/store/store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/store/store.go b/pkg/store/store.go index 8f045762d7..d981400f07 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -376,7 +376,9 @@ func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { } if err := batch.Delete(ctx, ds.NewKey(getHeaderKey(h))); err != nil { - return fmt.Errorf("failed to delete header at height %d during pruning: %w", h, err) + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete header at height %d during pruning: %w", h, err) + } } if err := batch.Delete(ctx, ds.NewKey(getDataKey(h))); err != nil { From 907f28caea0e90d53648912e2cab64b96d50c9e7 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:19:26 +0100 Subject: [PATCH 07/41] add method to tracedstore to respect interface --- pkg/store/tracing.go | 16 ++++++++++++++++ pkg/store/tracing_test.go | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/pkg/store/tracing.go b/pkg/store/tracing.go index 33a21f93ad..42d686d612 100644 --- a/pkg/store/tracing.go +++ b/pkg/store/tracing.go @@ -247,6 +247,22 @@ func (t *tracedStore) Rollback(ctx context.Context, height uint64, aggregator bo return nil } +func (t *tracedStore) PruneBlocks(ctx context.Context, height uint64) error { + ctx, span := t.tracer.Start(ctx, "Store.PruneBlocks", + trace.WithAttributes(attribute.Int64("height", int64(height))), + ) + defer span.End() + + err := t.inner.PruneBlocks(ctx, height) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + return nil +} + func (t *tracedStore) Close() error { return t.inner.Close() } diff --git a/pkg/store/tracing_test.go b/pkg/store/tracing_test.go index 0c0b967605..5477a66985 100644 --- a/pkg/store/tracing_test.go +++ b/pkg/store/tracing_test.go @@ -124,6 +124,12 @@ func (m *tracingMockStore) Rollback(ctx context.Context, height uint64, aggregat return nil } +func (m *tracingMockStore) PruneBlocks(ctx context.Context, height uint64) error { + // For tracing tests we don't need pruning behavior; just satisfy the Store + // interface. Specific pruning behavior is tested separately in store_test.go. + return nil +} + func (m *tracingMockStore) Close() error { return nil } From b022f10d8d64e6d2bc5b0c99e1ffb858a1ae75e8 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:42:23 +0100 Subject: [PATCH 08/41] add prune block to mockstore --- test/mocks/store.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/mocks/store.go b/test/mocks/store.go index 353fa0a1bb..efd1939940 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -39,6 +39,14 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter { return &MockStore_Expecter{mock: &_m.Mock} } +// PruneBlocks provides a mock implementation for the Store's pruning method. +// Tests using MockStore currently do not exercise pruning behavior, so this +// method simply satisfies the interface and can be extended with expectations +// later if needed. +func (_mock *MockStore) PruneBlocks(ctx context.Context, height uint64) error { + return nil +} + // Close provides a mock function for the type MockStore func (_mock *MockStore) Close() error { ret := _mock.Called() From 99d586b1db8e82648c445aa15b6369ae6e4608cc Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:49:29 +0100 Subject: [PATCH 09/41] fix helper for flag for consistency --- pkg/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a14aaae0a..f42bfb666b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -266,7 +266,7 @@ type NodeConfig struct { // When enabled, the node will periodically prune old block data (headers, data, // signatures, and hash index) from the local store while keeping recent history. PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` - PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain. Older blocks will have their header/data/signature removed from the local store. 0 means keep all blocks."` + PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set pruning_enabled=false to keep all blocks (archive mode)."` PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` } @@ -460,7 +460,7 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Duration(FlagScrapeInterval, def.Node.ScrapeInterval.Duration, "interval at which the reaper polls the execution layer for new transactions") // Pruning configuration flags cmd.Flags().Bool(FlagPrefixEvnode+"node.pruning_enabled", def.Node.PruningEnabled, "enable height-based pruning of stored block data (headers, data, signatures, index)") - cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (0 = keep all)") + cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (must be > 0; disable pruning to keep all blocks)") cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_interval", def.Node.PruningInterval, "run pruning every N blocks (must be >= 1 when pruning is enabled)") // Data Availability configuration flags From f798b18d4e87c3729b67a8613d776a35a39dcc88 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:16:17 +0100 Subject: [PATCH 10/41] add replace statement for local packages --- apps/evm/go.mod | 1 + apps/evm/go.sum | 2 -- apps/grpc/go.mod | 1 + apps/grpc/go.sum | 2 -- apps/testapp/go.mod | 5 ++++- apps/testapp/go.sum | 2 -- execution/evm/go.mod | 5 +++++ execution/evm/go.sum | 4 ---- go.mod | 3 +++ go.sum | 2 -- 10 files changed, 14 insertions(+), 13 deletions(-) diff --git a/apps/evm/go.mod b/apps/evm/go.mod index b4a4d2a3b4..4e373eb123 100644 --- a/apps/evm/go.mod +++ b/apps/evm/go.mod @@ -4,6 +4,7 @@ go 1.25.6 replace ( github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core github.com/evstack/ev-node/execution/evm => ../../execution/evm ) diff --git a/apps/evm/go.sum b/apps/evm/go.sum index 5ea0339d44..58c363be55 100644 --- a/apps/evm/go.sum +++ b/apps/evm/go.sum @@ -411,8 +411,6 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/apps/grpc/go.mod b/apps/grpc/go.mod index 9a14cf9cfb..31ecc287fd 100644 --- a/apps/grpc/go.mod +++ b/apps/grpc/go.mod @@ -4,6 +4,7 @@ go 1.25.6 replace ( github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core github.com/evstack/ev-node/execution/grpc => ../../execution/grpc ) diff --git a/apps/grpc/go.sum b/apps/grpc/go.sum index fb175ac5cd..d1fd875f09 100644 --- a/apps/grpc/go.sum +++ b/apps/grpc/go.sum @@ -367,8 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/apps/testapp/go.mod b/apps/testapp/go.mod index d68a967d63..c67b20897d 100644 --- a/apps/testapp/go.mod +++ b/apps/testapp/go.mod @@ -2,7 +2,10 @@ module github.com/evstack/ev-node/apps/testapp go 1.25.6 -replace github.com/evstack/ev-node => ../../ +replace ( + github.com/evstack/ev-node => ../../. + github.com/evstack/ev-node/core => ../../core +) require ( github.com/evstack/ev-node v1.0.0-rc.3 diff --git a/apps/testapp/go.sum b/apps/testapp/go.sum index fb175ac5cd..d1fd875f09 100644 --- a/apps/testapp/go.sum +++ b/apps/testapp/go.sum @@ -367,8 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= diff --git a/execution/evm/go.mod b/execution/evm/go.mod index ecd188489f..7d46e86573 100644 --- a/execution/evm/go.mod +++ b/execution/evm/go.mod @@ -103,3 +103,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) + +replace ( + github.com/evstack/ev-node => ../../ + github.com/evstack/ev-node/core => ../../core +) diff --git a/execution/evm/go.sum b/execution/evm/go.sum index f00f1139ab..3294e40da9 100644 --- a/execution/evm/go.sum +++ b/execution/evm/go.sum @@ -78,10 +78,6 @@ github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9i github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= -github.com/evstack/ev-node v1.0.0-rc.3 h1:hphJBI0b1TgGN9wajB1twouMVMjhyHXXrS9QaG1XwvQ= -github.com/evstack/ev-node v1.0.0-rc.3/go.mod h1:5Cf3SauhgIV+seQKBJavv3f8ZZw+YTnH5DRJcI4Ooj0= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= diff --git a/go.mod b/go.mod index d07ea7a7e8..8f10b14e53 100644 --- a/go.mod +++ b/go.mod @@ -195,3 +195,6 @@ replace ( google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 ) + +// use local core module during development/CI +replace github.com/evstack/ev-node/core => ./core diff --git a/go.sum b/go.sum index fb175ac5cd..d1fd875f09 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= -github.com/evstack/ev-node/core v1.0.0-rc.1 h1:Dic2PMUMAYUl5JW6DkDj6HXDEWYzorVJQuuUJOV0FjE= -github.com/evstack/ev-node/core v1.0.0-rc.1/go.mod h1:n2w/LhYQTPsi48m6lMj16YiIqsaQw6gxwjyJvR+B3sY= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= From 70e4901c12b75466407b5d8aa74b7d592ea27be3 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:41:18 +0100 Subject: [PATCH 11/41] flags --- pkg/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1834e1b405..9189aa57e5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -112,7 +112,7 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization) // Count the number of flags we're explicitly checking - expectedFlagCount := 63 // Update this number if you add more flag checks above + expectedFlagCount := 66 // Update this number if you add more flag checks above // Get the actual number of flags (both regular and persistent) actualFlagCount := 0 From 9315c6c022b6dba9bf5e25a103f93678e92faa89 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Mon, 2 Feb 2026 18:03:17 +0100 Subject: [PATCH 12/41] add safetey mechanism for pruning only da included blocks --- block/internal/executing/executor.go | 49 +++++++++++++++++++--------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 426afdcf06..89b9c4f1f4 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -3,6 +3,7 @@ package executing import ( "bytes" "context" + "encoding/binary" "errors" "fmt" "reflect" @@ -551,23 +552,41 @@ func (e *Executor) ProduceBlock(ctx context.Context) error { // production to fail, but it does run in the critical path and may add // some latency when large ranges are pruned. if e.config.Node.PruningEnabled && e.config.Node.PruningKeepRecent > 0 && e.config.Node.PruningInterval > 0 { - if newHeight%e.config.Node.PruningInterval == 0 { - // Compute the prune floor: all heights <= targetHeight are candidates - // for pruning of header/data/signature/index entries. - if newHeight > e.config.Node.PruningKeepRecent { - targetHeight := newHeight - e.config.Node.PruningKeepRecent - if err := e.store.PruneBlocks(e.ctx, targetHeight); err != nil { - e.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + interval := e.config.Node.PruningInterval + // Only attempt pruning when we're exactly at an interval boundary. + if newHeight%interval == 0 && newHeight > e.config.Node.PruningKeepRecent { + targetHeight := newHeight - e.config.Node.PruningKeepRecent + + // Determine the DA-included floor for pruning, so we never prune + // beyond what has been confirmed in DA. + var daIncludedHeight uint64 + meta, err := e.store.GetMetadata(e.ctx, store.DAIncludedHeightKey) + if err == nil && len(meta) == 8 { + daIncludedHeight = binary.LittleEndian.Uint64(meta) + } + + // If nothing is known to be DA-included yet, skip pruning. + if daIncludedHeight == 0 { + // Nothing known to be DA-included yet; skip pruning. + } else { + if targetHeight > daIncludedHeight { + targetHeight = daIncludedHeight } - // If the execution client exposes execution-metadata pruning, - // prune ExecMeta using the same target height. This keeps EVM - // execution metadata aligned with ev-node's block store pruning - // while remaining a no-op for execution environments that don't - // implement ExecMetaPruner (e.g. ABCI-based executors). - if pruner, ok := e.exec.(coreexecutor.ExecMetaPruner); ok { - if err := pruner.PruneExecMeta(e.ctx, targetHeight); err != nil { - e.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") + if targetHeight > 0 { + if err := e.store.PruneBlocks(e.ctx, targetHeight); err != nil { + return fmt.Errorf("failed to prune old block data: %w", err) + } + + // If the execution client exposes execution-metadata pruning, + // prune ExecMeta using the same target height. This keeps + // execution-layer metadata aligned with + // ev-node's block store pruning while remaining a no-op for + // execution environments that don't implement ExecMetaPruner yet. + if pruner, ok := e.exec.(coreexecutor.ExecMetaPruner); ok { + if err := pruner.PruneExecMeta(e.ctx, targetHeight); err != nil { + return fmt.Errorf("failed to prune execution metadata: %w", err) + } } } } From 23af6b75384a0873138bbe6373bc09381c41117d Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Mon, 2 Feb 2026 18:12:43 +0100 Subject: [PATCH 13/41] fix rebase --- pkg/sync/sync_service.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index c9e26fd31e..d4a5f0e818 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -477,10 +477,7 @@ func newSyncer[H header.Header[H]]( opts = append(opts, goheadersync.WithMetrics(), -<<<<<<< HEAD goheadersync.WithPruningWindow(ninetyNineYears), // pruning window not relevant, because of the store wrapper. -======= ->>>>>>> 3b9a0f70 (wiring prunning config to go-header) goheadersync.WithTrustingPeriod(ninetyNineYears), ) return goheadersync.NewSyncer(ex, store, sub, opts...) From f250b646bcb4f44237ae91b15b6b353b4b597ee2 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:39:34 +0100 Subject: [PATCH 14/41] remove useless check --- execution/evm/execution.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 5263539429..370887cfd7 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -275,10 +275,6 @@ func NewEngineExecutionClient( // underlying EVMStore. It is safe to call this multiple times with the same // or increasing heights; the store tracks its own last-pruned height. func (c *EngineClient) PruneExecMeta(ctx context.Context, height uint64) error { - if c.store == nil { - return nil - } - return c.store.PruneExecMeta(ctx, height) } From 135c925d66c513b4c161c7e49c7655e9b17c2c0f Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:40:19 +0100 Subject: [PATCH 15/41] use lastprunedheight in Tail() to optimize --- pkg/store/store_adapter.go | 57 +++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 08743c6eb9..2a751749a3 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -52,6 +52,12 @@ type EntityWithDAHint[H any] interface { DAHint() uint64 } +// lastPrunedHeightGetter is an optional interface that store getters can +// implement to expose the last pruned block height. +type lastPrunedHeightGetter interface { + LastPrunedHeight(ctx context.Context) (uint64, bool) +} + // heightSub provides a mechanism for waiting on a specific height to be stored. // This is critical for go-header syncer which expects GetByHeight to block until // the requested height is available. @@ -400,8 +406,19 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { height = h } - // Try genesisInitialHeight first (most common case - no pruning) - item, err := a.getter.GetByHeight(ctx, a.genesisInitialHeight) + // Determine the first candidate tail height. By default, this is the + // genesis initial height, but if pruning metadata is available we can + // skip directly past fully-pruned ranges. + startHeight := a.genesisInitialHeight + if getter, ok := any(a.getter).(lastPrunedHeightGetter); ok { + if lastPruned, ok := getter.LastPrunedHeight(ctx); ok { + if lastPruned < ^uint64(0) { + startHeight = lastPruned + 1 + } + } + } + + item, err := a.getter.GetByHeight(ctx, startHeight) if err == nil { return item, nil } @@ -411,8 +428,8 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { return pendingItem, nil } - // Walk up from genesisInitialHeight to find the first available item (pruning case) - for h := a.genesisInitialHeight + 1; h <= height; h++ { + // Walk up from startHeight to find the first available item + for h := startHeight + 1; h <= height; h++ { item, err = a.getter.GetByHeight(ctx, h) if err == nil { return item, nil @@ -804,6 +821,22 @@ func (g *HeaderStoreGetter) HasAt(ctx context.Context, height uint64) bool { return err == nil } +// LastPrunedHeight implements lastPrunedHeightGetter for HeaderStoreGetter by +// reading the pruning metadata from the underlying store. +func (g *HeaderStoreGetter) LastPrunedHeight(ctx context.Context) (uint64, bool) { + meta, err := g.store.GetMetadata(ctx, LastPrunedBlockHeightKey) + if err != nil || len(meta) != heightLength { + return 0, false + } + + height, err := decodeHeight(meta) + if err != nil { + return 0, false + } + + return height, true +} + // DataStoreGetter implements StoreGetter for *types.Data. type DataStoreGetter struct { store Store @@ -874,6 +907,22 @@ func (g *DataStoreGetter) HasAt(ctx context.Context, height uint64) bool { return err == nil } +// LastPrunedHeight implements lastPrunedHeightGetter for DataStoreGetter by +// reading the pruning metadata from the underlying store. +func (g *DataStoreGetter) LastPrunedHeight(ctx context.Context) (uint64, bool) { + meta, err := g.store.GetMetadata(ctx, LastPrunedBlockHeightKey) + if err != nil || len(meta) != heightLength { + return 0, false + } + + height, err := decodeHeight(meta) + if err != nil { + return 0, false + } + + return height, true +} + // Type aliases for convenience type HeaderStoreAdapter = StoreAdapter[*types.P2PSignedHeader] type DataStoreAdapter = StoreAdapter[*types.P2PData] From c1168c9affbe74728b6714ce30600e3a38b5176c Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 3 Feb 2026 19:52:32 +0100 Subject: [PATCH 16/41] move pruning from executor to dainclusionloop --- block/internal/executing/executor.go | 47 -------------------------- block/internal/submitting/submitter.go | 33 ++++++++++++++++++ 2 files changed, 33 insertions(+), 47 deletions(-) diff --git a/block/internal/executing/executor.go b/block/internal/executing/executor.go index 89b9c4f1f4..bf1b44b6cb 100644 --- a/block/internal/executing/executor.go +++ b/block/internal/executing/executor.go @@ -3,7 +3,6 @@ package executing import ( "bytes" "context" - "encoding/binary" "errors" "fmt" "reflect" @@ -547,52 +546,6 @@ func (e *Executor) ProduceBlock(ctx context.Context) error { // Update in-memory state after successful commit e.setLastState(newState) - // Run height-based pruning of stored block data if enabled. This is a - // best-effort background maintenance step and should not cause block - // production to fail, but it does run in the critical path and may add - // some latency when large ranges are pruned. - if e.config.Node.PruningEnabled && e.config.Node.PruningKeepRecent > 0 && e.config.Node.PruningInterval > 0 { - interval := e.config.Node.PruningInterval - // Only attempt pruning when we're exactly at an interval boundary. - if newHeight%interval == 0 && newHeight > e.config.Node.PruningKeepRecent { - targetHeight := newHeight - e.config.Node.PruningKeepRecent - - // Determine the DA-included floor for pruning, so we never prune - // beyond what has been confirmed in DA. - var daIncludedHeight uint64 - meta, err := e.store.GetMetadata(e.ctx, store.DAIncludedHeightKey) - if err == nil && len(meta) == 8 { - daIncludedHeight = binary.LittleEndian.Uint64(meta) - } - - // If nothing is known to be DA-included yet, skip pruning. - if daIncludedHeight == 0 { - // Nothing known to be DA-included yet; skip pruning. - } else { - if targetHeight > daIncludedHeight { - targetHeight = daIncludedHeight - } - - if targetHeight > 0 { - if err := e.store.PruneBlocks(e.ctx, targetHeight); err != nil { - return fmt.Errorf("failed to prune old block data: %w", err) - } - - // If the execution client exposes execution-metadata pruning, - // prune ExecMeta using the same target height. This keeps - // execution-layer metadata aligned with - // ev-node's block store pruning while remaining a no-op for - // execution environments that don't implement ExecMetaPruner yet. - if pruner, ok := e.exec.(coreexecutor.ExecMetaPruner); ok { - if err := pruner.PruneExecMeta(e.ctx, targetHeight); err != nil { - return fmt.Errorf("failed to prune execution metadata: %w", err) - } - } - } - } - } - } - // broadcast header and data to P2P network g, broadcastCtx := errgroup.WithContext(e.ctx) g.Go(func() error { diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 1c5b034c19..3b1cdd0d2d 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -360,6 +360,39 @@ func (s *Submitter) processDAInclusionLoop() { s.logger.Error().Err(err).Uint64("height", nextHeight).Msg("failed to persist DA included height") } + // Run height-based pruning if enabled. + if s.config.Node.PruningEnabled && s.config.Node.PruningKeepRecent > 0 && s.config.Node.PruningInterval > 0 { + // Trigger pruning only when we reach the configured interval. + if currentDAIncluded%s.config.Node.PruningInterval == 0 { + // We must make sure not to prune blocks that have not yet been included in DA. + daIncludedHeight := s.GetDAIncludedHeight() + + storeHeight, err := s.store.Height(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get store height for pruning") + break + } + + upperBound := min(storeHeight, daIncludedHeight) + if upperBound <= s.config.Node.PruningKeepRecent { + // Not enough fully included blocks to prune while respecting keep-recent. + break + } + + targetHeight := upperBound - s.config.Node.PruningKeepRecent + + if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + + if pruner, ok := s.exec.(coreexecutor.ExecMetaPruner); ok { + if err := pruner.PruneExecMeta(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") + } + } + } + } + // Delete height cache for that height // This can only be performed after the height has been persisted to store s.cache.DeleteHeight(nextHeight) From 155ed5c8792d68869dda27e731ea8135df2dbe10 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 3 Feb 2026 19:52:48 +0100 Subject: [PATCH 17/41] don't prune go-header store --- pkg/sync/sync_service.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pkg/sync/sync_service.go b/pkg/sync/sync_service.go index d4a5f0e818..8567e79764 100644 --- a/pkg/sync/sync_service.go +++ b/pkg/sync/sync_service.go @@ -189,21 +189,11 @@ func (syncService *SyncService[H]) Start(ctx context.Context) error { } // create syncer, must be before initFromP2PWithRetry which calls startSyncer. - syncOpts := []goheadersync.Option{goheadersync.WithBlockTime(syncService.conf.Node.BlockTime.Duration)} - // Map ev-node pruning configuration to go-header's pruning window: we approximate - // "keep N recent heights" as "retain headers for N * blockTime". - if syncService.conf.Node.PruningEnabled && syncService.conf.Node.PruningKeepRecent > 0 { - pruningWindow := syncService.conf.Node.BlockTime.Duration * time.Duration(syncService.conf.Node.PruningKeepRecent) - // Only set a pruning window if the computed duration is positive. - if pruningWindow > 0 { - syncOpts = append(syncOpts, goheadersync.WithPruningWindow(pruningWindow)) - } - } if syncService.syncer, err = newSyncer( syncService.ex, syncService.store, syncService.sub, - syncOpts, + []goheadersync.Option{goheadersync.WithBlockTime(syncService.conf.Node.BlockTime.Duration)}, ); err != nil { return fmt.Errorf("failed to create syncer: %w", err) } From 52c005e8fa66cec85e142b6d066f8540b42ba7a6 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:08:46 +0100 Subject: [PATCH 18/41] update tail function --- pkg/store/store_adapter.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 2a751749a3..d593576f98 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -410,7 +410,7 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if getter, ok := any(a.getter).(lastPrunedHeightGetter); ok { + if getter, ok := a.getter.(lastPrunedHeightGetter); ok { if lastPruned, ok := getter.LastPrunedHeight(ctx); ok { if lastPruned < ^uint64(0) { startHeight = lastPruned + 1 @@ -821,22 +821,6 @@ func (g *HeaderStoreGetter) HasAt(ctx context.Context, height uint64) bool { return err == nil } -// LastPrunedHeight implements lastPrunedHeightGetter for HeaderStoreGetter by -// reading the pruning metadata from the underlying store. -func (g *HeaderStoreGetter) LastPrunedHeight(ctx context.Context) (uint64, bool) { - meta, err := g.store.GetMetadata(ctx, LastPrunedBlockHeightKey) - if err != nil || len(meta) != heightLength { - return 0, false - } - - height, err := decodeHeight(meta) - if err != nil { - return 0, false - } - - return height, true -} - // DataStoreGetter implements StoreGetter for *types.Data. type DataStoreGetter struct { store Store From 9e046c4e5c8e8949383eaf52691f861e0848f3a4 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 09:13:18 +0100 Subject: [PATCH 19/41] rename execmetapruner to execpruner --- block/internal/submitting/submitter.go | 4 ++-- core/execution/execution.go | 8 ++++---- execution/evm/execution.go | 8 ++++---- execution/evm/store.go | 4 ++-- execution/evm/store_test.go | 12 ++++++------ 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 3b1cdd0d2d..28f6252435 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -385,8 +385,8 @@ func (s *Submitter) processDAInclusionLoop() { s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") } - if pruner, ok := s.exec.(coreexecutor.ExecMetaPruner); ok { - if err := pruner.PruneExecMeta(s.ctx, targetHeight); err != nil { + if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { + if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") } } diff --git a/core/execution/execution.go b/core/execution/execution.go index 1f85e2068a..4eb8482f2c 100644 --- a/core/execution/execution.go +++ b/core/execution/execution.go @@ -162,15 +162,15 @@ type Rollbackable interface { Rollback(ctx context.Context, targetHeight uint64) error } -// ExecMetaPruner is an optional interface that execution clients can implement +// ExecPruner is an optional interface that execution clients can implement // to support height-based pruning of their execution metadata. This is used by // EVM-based execution clients to keep ExecMeta consistent with ev-node's // pruning window while remaining a no-op for execution environments that // don't persist per-height metadata in ev-node's datastore. -type ExecMetaPruner interface { - // PruneExecMeta should delete execution metadata for all heights up to and +type ExecPruner interface { + // PruneExec should delete execution metadata for all heights up to and // including the given height. Implementations should be idempotent and track // their own progress so that repeated calls with the same or decreasing // heights are cheap no-ops. - PruneExecMeta(ctx context.Context, height uint64) error + PruneExec(ctx context.Context, height uint64) error } diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 370887cfd7..1c988b67e3 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -69,7 +69,7 @@ var _ execution.Rollbackable = (*EngineClient)(nil) // ev-node's height-based pruning. This enables coordinated pruning of EVM // ExecMeta alongside ev-node's own block data pruning, while remaining a // no-op for non-EVM execution environments. -var _ execution.ExecMetaPruner = (*EngineClient)(nil) +var _ execution.ExecPruner = (*EngineClient)(nil) // validatePayloadStatus checks the payload status and returns appropriate errors. // It implements the Engine API specification's status handling: @@ -271,11 +271,11 @@ func NewEngineExecutionClient( }, nil } -// PruneExecMeta implements execution.ExecMetaPruner by delegating to the +// PruneExec implements execution.ExecPruner by delegating to the // underlying EVMStore. It is safe to call this multiple times with the same // or increasing heights; the store tracks its own last-pruned height. -func (c *EngineClient) PruneExecMeta(ctx context.Context, height uint64) error { - return c.store.PruneExecMeta(ctx, height) +func (c *EngineClient) PruneExec(ctx context.Context, height uint64) error { + return c.store.PruneExec(ctx, height) } // SetLogger allows callers to attach a structured logger. diff --git a/execution/evm/store.go b/execution/evm/store.go index 51a3056b66..21fe5008ae 100644 --- a/execution/evm/store.go +++ b/execution/evm/store.go @@ -145,10 +145,10 @@ func (s *EVMStore) SaveExecMeta(ctx context.Context, meta *ExecMeta) error { return nil } -// PruneExecMeta removes ExecMeta entries up to and including the given height. +// PruneExec removes ExecMeta entries up to and including the given height. // It is safe to call this multiple times with the same or increasing heights; // previously pruned ranges will be skipped based on the last-pruned marker. -func (s *EVMStore) PruneExecMeta(ctx context.Context, height uint64) error { +func (s *EVMStore) PruneExec(ctx context.Context, height uint64) error { // Load last pruned height, if any. var lastPruned uint64 data, err := s.db.Get(ctx, ds.NewKey(lastPrunedExecMetaKey)) diff --git a/execution/evm/store_test.go b/execution/evm/store_test.go index 64389701f9..d3067fc1a6 100644 --- a/execution/evm/store_test.go +++ b/execution/evm/store_test.go @@ -17,7 +17,7 @@ func newTestDatastore(t *testing.T) ds.Batching { return dssync.MutexWrap(ds.NewMapDatastore()) } -func TestPruneExecMeta_PrunesUpToTargetHeight(t *testing.T) { +func TestPruneExec_PrunesUpToTargetHeight(t *testing.T) { t.Parallel() ctx := context.Background() @@ -39,7 +39,7 @@ func TestPruneExecMeta_PrunesUpToTargetHeight(t *testing.T) { } // Prune up to height 3 - require.NoError(t, store.PruneExecMeta(ctx, 3)) + require.NoError(t, store.PruneExec(ctx, 3)) // Heights 1..3 should be gone for h := uint64(1); h <= 3; h++ { @@ -56,10 +56,10 @@ func TestPruneExecMeta_PrunesUpToTargetHeight(t *testing.T) { } // Re-pruning with the same height should be a no-op - require.NoError(t, store.PruneExecMeta(ctx, 3)) + require.NoError(t, store.PruneExec(ctx, 3)) } -func TestPruneExecMeta_TracksLastPrunedHeight(t *testing.T) { +func TestPruneExec_TracksLastPrunedHeight(t *testing.T) { t.Parallel() ctx := context.Background() @@ -73,10 +73,10 @@ func TestPruneExecMeta_TracksLastPrunedHeight(t *testing.T) { } // First prune up to 2 - require.NoError(t, store.PruneExecMeta(ctx, 2)) + require.NoError(t, store.PruneExec(ctx, 2)) // Then prune up to 4; heights 3..4 should be deleted in this run - require.NoError(t, store.PruneExecMeta(ctx, 4)) + require.NoError(t, store.PruneExec(ctx, 4)) // Verify all heights 1..4 are gone, 5 remains for h := uint64(1); h <= 4; h++ { From 3c1641b5a8e42aec65daf510cd47b87717dfe300 Mon Sep 17 00:00:00 2001 From: Pierrick <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 13:40:24 +0100 Subject: [PATCH 20/41] Update core/execution/execution.go Co-authored-by: julienrbrt --- core/execution/execution.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/execution/execution.go b/core/execution/execution.go index 4eb8482f2c..f3ebe1da91 100644 --- a/core/execution/execution.go +++ b/core/execution/execution.go @@ -163,10 +163,7 @@ type Rollbackable interface { } // ExecPruner is an optional interface that execution clients can implement -// to support height-based pruning of their execution metadata. This is used by -// EVM-based execution clients to keep ExecMeta consistent with ev-node's -// pruning window while remaining a no-op for execution environments that -// don't persist per-height metadata in ev-node's datastore. +// to support height-based pruning of their execution metadata. type ExecPruner interface { // PruneExec should delete execution metadata for all heights up to and // including the given height. Implementations should be idempotent and track From 2566dae489545b6f65fdceaac7ce850f6660272b Mon Sep 17 00:00:00 2001 From: Pierrick <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 13:41:30 +0100 Subject: [PATCH 21/41] Update execution/evm/execution.go Co-authored-by: julienrbrt --- execution/evm/execution.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 1c988b67e3..399f7c91ee 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -66,9 +66,7 @@ var _ execution.HeightProvider = (*EngineClient)(nil) var _ execution.Rollbackable = (*EngineClient)(nil) // Ensure EngineClient implements optional pruning interface when used with -// ev-node's height-based pruning. This enables coordinated pruning of EVM -// ExecMeta alongside ev-node's own block data pruning, while remaining a -// no-op for non-EVM execution environments. +// ev-node's height-based pruning. var _ execution.ExecPruner = (*EngineClient)(nil) // validatePayloadStatus checks the payload status and returns appropriate errors. From 535cf1ce343a5b80ad4bb95d12f644def1faeb81 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:29:40 +0100 Subject: [PATCH 22/41] trigger pruning every ticker --- block/internal/submitting/submitter.go | 65 ++++++++++++++------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 28f6252435..886f1ac0f9 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -360,42 +360,47 @@ func (s *Submitter) processDAInclusionLoop() { s.logger.Error().Err(err).Uint64("height", nextHeight).Msg("failed to persist DA included height") } - // Run height-based pruning if enabled. - if s.config.Node.PruningEnabled && s.config.Node.PruningKeepRecent > 0 && s.config.Node.PruningInterval > 0 { - // Trigger pruning only when we reach the configured interval. - if currentDAIncluded%s.config.Node.PruningInterval == 0 { - // We must make sure not to prune blocks that have not yet been included in DA. - daIncludedHeight := s.GetDAIncludedHeight() - - storeHeight, err := s.store.Height(s.ctx) - if err != nil { - s.logger.Error().Err(err).Msg("failed to get store height for pruning") - break - } + // Delete height cache for that height + // This can only be performed after the height has been persisted to store + s.cache.DeleteHeight(nextHeight) + } - upperBound := min(storeHeight, daIncludedHeight) - if upperBound <= s.config.Node.PruningKeepRecent { - // Not enough fully included blocks to prune while respecting keep-recent. - break - } + // Run height-based pruning if enabled. + if s.config.Node.PruningEnabled && s.config.Node.PruningKeepRecent > 0 && s.config.Node.PruningInterval > 0 { + currentDAIncluded = s.GetDAIncludedHeight() - targetHeight := upperBound - s.config.Node.PruningKeepRecent + var lastPruned uint64 + if bz, err := s.store.GetMetadata(s.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { + lastPruned = binary.LittleEndian.Uint64(bz) + } - if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") - } + storeHeight, err := s.store.Height(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get store height for pruning") + continue + } + if storeHeight <= lastPruned+uint64(s.config.Node.PruningInterval) { + continue + } - if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { - if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") - } - } - } + // Never prune blocks that are not DA included + upperBound := min(storeHeight, currentDAIncluded) + if upperBound <= s.config.Node.PruningKeepRecent { + // Not enough fully included blocks to prune + continue } - // Delete height cache for that height - // This can only be performed after the height has been persisted to store - s.cache.DeleteHeight(nextHeight) + targetHeight := upperBound - s.config.Node.PruningKeepRecent + + if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + + if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { + if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") + } + } } } } From 4ee40d636a593c217dc54e6b02fa6ca12c0d9d95 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 15:25:10 +0100 Subject: [PATCH 23/41] add store to store adapter --- block/internal/submitting/submitter.go | 2 +- pkg/store/store_adapter.go | 41 ++++++++------------------ 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 886f1ac0f9..6b1a2971f9 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -379,7 +379,7 @@ func (s *Submitter) processDAInclusionLoop() { s.logger.Error().Err(err).Msg("failed to get store height for pruning") continue } - if storeHeight <= lastPruned+uint64(s.config.Node.PruningInterval) { + if storeHeight <= lastPruned+s.config.Node.PruningInterval { continue } diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index d593576f98..479d7d6915 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -52,12 +52,6 @@ type EntityWithDAHint[H any] interface { DAHint() uint64 } -// lastPrunedHeightGetter is an optional interface that store getters can -// implement to expose the last pruned block height. -type lastPrunedHeightGetter interface { - LastPrunedHeight(ctx context.Context) (uint64, bool) -} - // heightSub provides a mechanism for waiting on a specific height to be stored. // This is critical for go-header syncer which expects GetByHeight to block until // the requested height is available. @@ -292,6 +286,7 @@ func (c *pendingCache[H]) recalcMaxHeight() { // a block, it writes to the underlying store, and subsequent reads will come from the store. type StoreAdapter[H EntityWithDAHint[H]] struct { getter StoreGetter[H] + store Store genesisInitialHeight uint64 // heightSub tracks the current height and allows waiting for specific heights. @@ -410,10 +405,12 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if getter, ok := a.getter.(lastPrunedHeightGetter); ok { - if lastPruned, ok := getter.LastPrunedHeight(ctx); ok { - if lastPruned < ^uint64(0) { - startHeight = lastPruned + 1 + if a.store != nil { + if meta, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(meta) == heightLength { + if lastPruned, err := decodeHeight(meta); err == nil { + if candidate := lastPruned + 1; candidate > startHeight { + startHeight = candidate + } } } } @@ -891,22 +888,6 @@ func (g *DataStoreGetter) HasAt(ctx context.Context, height uint64) bool { return err == nil } -// LastPrunedHeight implements lastPrunedHeightGetter for DataStoreGetter by -// reading the pruning metadata from the underlying store. -func (g *DataStoreGetter) LastPrunedHeight(ctx context.Context) (uint64, bool) { - meta, err := g.store.GetMetadata(ctx, LastPrunedBlockHeightKey) - if err != nil || len(meta) != heightLength { - return 0, false - } - - height, err := decodeHeight(meta) - if err != nil { - return 0, false - } - - return height, true -} - // Type aliases for convenience type HeaderStoreAdapter = StoreAdapter[*types.P2PSignedHeader] type DataStoreAdapter = StoreAdapter[*types.P2PData] @@ -914,11 +895,15 @@ type DataStoreAdapter = StoreAdapter[*types.P2PData] // NewHeaderStoreAdapter creates a new StoreAdapter for headers. // The genesis is used to determine the initial height for efficient Tail lookups. func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter { - return NewStoreAdapter(NewHeaderStoreGetter(store), gen) + adapter := NewStoreAdapter(NewHeaderStoreGetter(store), gen) + adapter.store = store + return adapter } // NewDataStoreAdapter creates a new StoreAdapter for data. // The genesis is used to determine the initial height for efficient Tail lookups. func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { - return NewStoreAdapter(NewDataStoreGetter(store), gen) + adapter := NewStoreAdapter(NewDataStoreGetter(store), gen) + adapter.store = store + return adapter } From a4434d1ce40ee2bb7f3739e15cb0fe19b3f9a477 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 17:57:28 +0100 Subject: [PATCH 24/41] invalidate cache store data after pruning --- pkg/store/cached_store.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index e59d4e28c2..3fdabc9e5f 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -157,6 +157,19 @@ func (cs *CachedStore) Rollback(ctx context.Context, height uint64, aggregator b return nil } +// PruneBlocks wraps the underlying store's PruneBlocks and invalidates caches. +// After pruning historical block data from disk, any cached entries for pruned +// heights must not be served, so we conservatively clear the entire cache. +func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { + if err := cs.Store.PruneBlocks(ctx, height); err != nil { + return err + } + + // Invalidate cache for pruned heights + cs.InvalidateRange(1, height) + return nil +} + // Close closes the underlying store. func (cs *CachedStore) Close() error { cs.ClearCache() From 86eb1a207a9e6bdef6994407e81dc20a3d849fe3 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Wed, 4 Feb 2026 20:21:19 +0100 Subject: [PATCH 25/41] nit --- pkg/store/store_adapter.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 479d7d6915..7aa08e6a2c 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -305,9 +305,9 @@ type StoreAdapter[H EntityWithDAHint[H]] struct { onDeleteFn func(context.Context, uint64) error } -// NewStoreAdapter creates a new StoreAdapter wrapping the given store getter. +// NewStoreAdapter creates a new StoreAdapter wrapping the given store getter and backing Store. // The genesis is used to determine the initial height for efficient Tail lookups. -func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], gen genesis.Genesis) *StoreAdapter[H] { +func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], store Store, gen genesis.Genesis) *StoreAdapter[H] { // Get actual current height from store (0 if empty) var storeHeight uint64 if h, err := getter.Height(context.Background()); err == nil { @@ -316,6 +316,7 @@ func NewStoreAdapter[H EntityWithDAHint[H]](getter StoreGetter[H], gen genesis.G adapter := &StoreAdapter[H]{ getter: getter, + store: store, genesisInitialHeight: max(gen.InitialHeight, 1), pending: newPendingCache[H](), heightSub: newHeightSub(storeHeight), @@ -387,7 +388,6 @@ func (a *StoreAdapter[H]) Head(ctx context.Context, _ ...header.HeadOption[H]) ( // Tail returns the lowest item in the store. // For ev-node, this is typically the genesis/initial height. // If pruning has occurred, it walks up from initialHeight to find the first available item. -// TODO(@julienrbrt): Optimize this when pruning is enabled. func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { var zero H @@ -405,12 +405,10 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if a.store != nil { - if meta, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(meta) == heightLength { - if lastPruned, err := decodeHeight(meta); err == nil { - if candidate := lastPruned + 1; candidate > startHeight { - startHeight = candidate - } + if meta, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(meta) == heightLength { + if lastPruned, err := decodeHeight(meta); err == nil { + if candidate := lastPruned + 1; candidate > startHeight { + startHeight = candidate } } } @@ -895,15 +893,11 @@ type DataStoreAdapter = StoreAdapter[*types.P2PData] // NewHeaderStoreAdapter creates a new StoreAdapter for headers. // The genesis is used to determine the initial height for efficient Tail lookups. func NewHeaderStoreAdapter(store Store, gen genesis.Genesis) *HeaderStoreAdapter { - adapter := NewStoreAdapter(NewHeaderStoreGetter(store), gen) - adapter.store = store - return adapter + return NewStoreAdapter(NewHeaderStoreGetter(store), store, gen) } // NewDataStoreAdapter creates a new StoreAdapter for data. // The genesis is used to determine the initial height for efficient Tail lookups. func NewDataStoreAdapter(store Store, gen genesis.Genesis) *DataStoreAdapter { - adapter := NewStoreAdapter(NewDataStoreGetter(store), gen) - adapter.store = store - return adapter + return NewStoreAdapter(NewDataStoreGetter(store), store, gen) } From ca26acdd1aefe7b220eacb75e0ef6eeb8553b3d0 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:01:05 +0100 Subject: [PATCH 26/41] move pruning to it's own function --- block/internal/submitting/submitter.go | 66 ++++++++++++++------------ 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 6b1a2971f9..6a40bd818e 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -366,42 +366,48 @@ func (s *Submitter) processDAInclusionLoop() { } // Run height-based pruning if enabled. - if s.config.Node.PruningEnabled && s.config.Node.PruningKeepRecent > 0 && s.config.Node.PruningInterval > 0 { - currentDAIncluded = s.GetDAIncludedHeight() + s.pruneBlocks() + } + } +} - var lastPruned uint64 - if bz, err := s.store.GetMetadata(s.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { - lastPruned = binary.LittleEndian.Uint64(bz) - } +func (s *Submitter) pruneBlocks() { + if !s.config.Node.PruningEnabled || s.config.Node.PruningKeepRecent == 0 || s.config.Node.PruningInterval == 0 { + return + } - storeHeight, err := s.store.Height(s.ctx) - if err != nil { - s.logger.Error().Err(err).Msg("failed to get store height for pruning") - continue - } - if storeHeight <= lastPruned+s.config.Node.PruningInterval { - continue - } + currentDAIncluded := s.GetDAIncludedHeight() - // Never prune blocks that are not DA included - upperBound := min(storeHeight, currentDAIncluded) - if upperBound <= s.config.Node.PruningKeepRecent { - // Not enough fully included blocks to prune - continue - } + var lastPruned uint64 + if bz, err := s.store.GetMetadata(s.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { + lastPruned = binary.LittleEndian.Uint64(bz) + } - targetHeight := upperBound - s.config.Node.PruningKeepRecent + storeHeight, err := s.store.Height(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get store height for pruning") + return + } + if storeHeight <= lastPruned+s.config.Node.PruningInterval { + return + } - if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") - } + // Never prune blocks that are not DA included + upperBound := min(storeHeight, currentDAIncluded) + if upperBound <= s.config.Node.PruningKeepRecent { + // Not enough fully included blocks to prune + return + } - if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { - if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") - } - } - } + targetHeight := upperBound - s.config.Node.PruningKeepRecent + + if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + + if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { + if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { + s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") } } } From b1a6a0999d7a62f4d0b6d0f4121c4a6ad8cfab0c Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:05:45 +0100 Subject: [PATCH 27/41] update config.md file --- docs/learn/config.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/learn/config.md b/docs/learn/config.md index ba900a1630..fea509fa5d 100644 --- a/docs/learn/config.md +++ b/docs/learn/config.md @@ -17,6 +17,7 @@ This document provides a comprehensive reference for all configuration options a - [Maximum Pending Blocks](#maximum-pending-blocks) - [Lazy Mode (Lazy Aggregator)](#lazy-mode-lazy-aggregator) - [Lazy Block Interval](#lazy-block-interval) + - [Pruning (Height-Based Pruning)](#pruning-height-based-pruning) - [Data Availability Configuration (`da`)](#data-availability-configuration-da) - [DA Service Address](#da-service-address) - [DA Authentication Token](#da-authentication-token) @@ -279,6 +280,40 @@ _Example:_ `--rollkit.node.lazy_block_interval 1m` _Default:_ `"30s"` _Constant:_ `FlagLazyBlockTime` +### Pruning (Height-Based Pruning) + +**Description:** +Controls height-based pruning of stored block data (headers, data, signatures, and index) from the local store. When pruning is enabled, the node periodically deletes old blocks while keeping a recent window of history. When disabled, the node keeps all blocks (archive mode). + +**YAML:** + +```yaml +node: + pruning_enabled: true + pruning_keep_recent: 100000 + pruning_interval: 1000 +``` + +**Command-line Flags:** + +- `--evnode.node.pruning_enabled` (boolean) + - _Description:_ Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode). +- `--evnode.node.pruning_keep_recent ` + - _Description:_ Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set `pruning_enabled=false` to keep all blocks. +- `--evnode.node.pruning_interval ` + - _Description:_ Run pruning every N blocks. Must be >= 1 when pruning is enabled. + +_Defaults:_ + +```yaml +node: + pruning_enabled: false + pruning_keep_recent: 0 + pruning_interval: 0 +``` + +_Constants:_ `FlagNodePruningEnabled`, `FlagNodePruningKeepRecent`, `FlagNodePruningInterval` + ## Data Availability Configuration (`da`) Parameters for connecting and interacting with the Data Availability (DA) layer, which Evolve uses to publish block data. From fec4fb443d206dfa256424cb40f1d61a44ddd676 Mon Sep 17 00:00:00 2001 From: pthmas <9058370+pthmas@users.noreply.github.com> Date: Tue, 10 Feb 2026 11:50:25 +0100 Subject: [PATCH 28/41] remove ai comments --- pkg/config/config.go | 4 ---- pkg/store/cached_store.go | 5 ++--- pkg/store/types.go | 16 ---------------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f42bfb666b..b964ac7bf2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -386,13 +386,9 @@ func (c *Config) Validate() error { // Validate pruning configuration if c.Node.PruningEnabled { - // When pruning is enabled, pruning_interval must be >= 1 if c.Node.PruningInterval == 0 { return fmt.Errorf("pruning_interval must be >= 1 when pruning is enabled") } - - // When pruning is enabled, keeping 0 blocks is contradictory; use pruning_enabled=false - // for archive mode instead. if c.Node.PruningKeepRecent == 0 { return fmt.Errorf("pruning_keep_recent must be > 0 when pruning is enabled; use pruning_enabled=false to keep all blocks") } diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index 3fdabc9e5f..8a5cca5b38 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -157,9 +157,8 @@ func (cs *CachedStore) Rollback(ctx context.Context, height uint64, aggregator b return nil } -// PruneBlocks wraps the underlying store's PruneBlocks and invalidates caches. -// After pruning historical block data from disk, any cached entries for pruned -// heights must not be served, so we conservatively clear the entire cache. +// PruneBlocks wraps the underlying store's PruneBlocks and invalidates caches +// up to the heigh that we purne func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { if err := cs.Store.PruneBlocks(ctx, height); err != nil { return err diff --git a/pkg/store/types.go b/pkg/store/types.go index 1162d0075e..a5635feec1 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -30,25 +30,12 @@ type Batch interface { } // Store is minimal interface for storing and retrieving blocks, commits and state. -// -// It is composed from three concerns: -// - Reader: read access to blocks, state, and metadata -// - Rollback: consensus rollback logic (used for chain reorgs / recovery) -// - Pruner: long-term height-based pruning of historical block data type Store interface { Reader Metadata Rollback Pruner - // SetMetadata saves arbitrary value in the store. - // - // This method enables evolve to safely persist any information. - SetMetadata(ctx context.Context, key string, value []byte) error - - // DeleteMetadata removes a metadata key from the store. - DeleteMetadata(ctx context.Context, key string) error - // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error @@ -107,9 +94,6 @@ type Rollback interface { } // Pruner provides long-term, height-based pruning of historical block data. -// -// Implementations SHOULD be idempotent and safe to call multiple times for -// the same or increasing target heights. type Pruner interface { // PruneBlocks removes block data (header, data, signature, and hash index) // up to and including the given height from the store, without modifying From 9ec6221da88ce17d1605f4c397862f05f3932698 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Feb 2026 15:31:09 +0100 Subject: [PATCH 29/41] feat: add recovery history depth pruner (#3064) * Initial plan * Add recovery history retention pruning Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * feat: add configurable recovery history retention Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Refactor recovery pruning into pruner component Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Address pruner review feedback Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Update evm test module dependency Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Address code review feedback Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Refine pruner checks and docs Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Clarify PruneExecMeta comment Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Rename recovery history setting Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * Adjust pruner interval and defaults Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> * fixes * updates * updates * updates * comment --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: julienrbrt <29894366+julienrbrt@users.noreply.github.com> Co-authored-by: Julien Robert --- apps/evm/cmd/rollback.go | 8 +- apps/evm/cmd/run.go | 11 +- block/components.go | 27 ++++ block/components_test.go | 2 + block/internal/pruner/pruner.go | 195 +++++++++++++++++++++++++ block/internal/pruner/pruner_test.go | 52 +++++++ block/internal/submitting/submitter.go | 44 ------ execution/evm/execution.go | 22 ++- execution/evm/test/execution_test.go | 3 + execution/evm/test/go.mod | 2 +- pkg/config/config.go | 10 +- pkg/config/config_test.go | 4 +- pkg/config/defaults.go | 3 + pkg/store/cached_store.go | 6 + pkg/store/store.go | 8 + pkg/store/store_test.go | 21 +++ pkg/store/tracing.go | 16 ++ pkg/store/tracing_test.go | 14 +- pkg/store/types.go | 4 + test/mocks/store.go | 122 +++++++++++++++- 20 files changed, 491 insertions(+), 83 deletions(-) create mode 100644 block/internal/pruner/pruner.go create mode 100644 block/internal/pruner/pruner_test.go diff --git a/apps/evm/cmd/rollback.go b/apps/evm/cmd/rollback.go index 3f11ef8d4f..911c330dd6 100644 --- a/apps/evm/cmd/rollback.go +++ b/apps/evm/cmd/rollback.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" ds "github.com/ipfs/go-datastore" + "github.com/rs/zerolog" "github.com/spf13/cobra" "github.com/evstack/ev-node/execution/evm" @@ -30,6 +31,7 @@ func NewRollbackCmd() *cobra.Command { if err != nil { return err } + logger := rollcmd.SetupLogger(nodeConfig.Log) goCtx := cmd.Context() if goCtx == nil { @@ -69,7 +71,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback execution layer via EngineClient - engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB) + engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB, logger.With().Str("module", "engine_client").Logger()) if err != nil { cmd.Printf("Warning: failed to create engine client, skipping EL rollback: %v\n", err) } else { @@ -99,7 +101,7 @@ func NewRollbackCmd() *cobra.Command { return cmd } -func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching) (*evm.EngineClient, error) { +func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching, logger zerolog.Logger) (*evm.EngineClient, error) { ethURL, err := cmd.Flags().GetString(evm.FlagEvmEthURL) if err != nil { return nil, fmt.Errorf("failed to get '%s' flag: %w", evm.FlagEvmEthURL, err) @@ -128,5 +130,5 @@ func createRollbackEngineClient(cmd *cobra.Command, db ds.Batching) (*evm.Engine return nil, fmt.Errorf("JWT secret file '%s' is empty", jwtSecretFile) } - return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, common.Hash{}, common.Address{}, db, false) + return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, common.Hash{}, common.Address{}, db, false, logger) } diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 6690d02d52..5ae854037e 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -55,7 +55,7 @@ var RunCmd = &cobra.Command{ } tracingEnabled := nodeConfig.Instrumentation.IsTracingEnabled() - executor, err := createExecutionClient(cmd, datastore, tracingEnabled) + executor, err := createExecutionClient(cmd, datastore, tracingEnabled, logger.With().Str("module", "engine_client").Logger()) if err != nil { return err } @@ -67,11 +67,6 @@ var RunCmd = &cobra.Command{ daClient := block.NewDAClient(blobClient, nodeConfig, logger) - // Attach logger to the EVM engine client if available - if ec, ok := executor.(*evm.EngineClient); ok { - ec.SetLogger(logger.With().Str("module", "engine_client").Logger()) - } - headerNamespace := da.NamespaceFromString(nodeConfig.DA.GetNamespace()) dataNamespace := da.NamespaceFromString(nodeConfig.DA.GetDataNamespace()) @@ -192,7 +187,7 @@ func createSequencer( return sequencer, nil } -func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEnabled bool) (execution.Executor, error) { +func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEnabled bool, logger zerolog.Logger) (execution.Executor, error) { // Read execution client parameters from flags ethURL, err := cmd.Flags().GetString(evm.FlagEvmEthURL) if err != nil { @@ -237,7 +232,7 @@ func createExecutionClient(cmd *cobra.Command, db datastore.Batching, tracingEna genesisHash := common.HexToHash(genesisHashStr) feeRecipient := common.HexToAddress(feeRecipientStr) - return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, genesisHash, feeRecipient, db, tracingEnabled) + return evm.NewEngineExecutionClient(ethURL, engineURL, jwtSecret, genesisHash, feeRecipient, db, tracingEnabled, logger) } // addFlags adds flags related to the EVM execution client diff --git a/block/components.go b/block/components.go index d5af466ef6..2956a98d81 100644 --- a/block/components.go +++ b/block/components.go @@ -12,6 +12,7 @@ import ( "github.com/evstack/ev-node/block/internal/common" da "github.com/evstack/ev-node/block/internal/da" "github.com/evstack/ev-node/block/internal/executing" + "github.com/evstack/ev-node/block/internal/pruner" "github.com/evstack/ev-node/block/internal/reaping" "github.com/evstack/ev-node/block/internal/submitting" "github.com/evstack/ev-node/block/internal/syncing" @@ -29,6 +30,7 @@ import ( // Components represents the block-related components type Components struct { Executor *executing.Executor + Pruner *pruner.Pruner Reaper *reaping.Reaper Syncer *syncing.Syncer Submitter *submitting.Submitter @@ -60,6 +62,11 @@ func (bc *Components) Start(ctx context.Context) error { return fmt.Errorf("failed to start executor: %w", err) } } + if bc.Pruner != nil { + if err := bc.Pruner.Start(ctxWithCancel); err != nil { + return fmt.Errorf("failed to start pruner: %w", err) + } + } if bc.Reaper != nil { if err := bc.Reaper.Start(ctxWithCancel); err != nil { return fmt.Errorf("failed to start reaper: %w", err) @@ -96,6 +103,11 @@ func (bc *Components) Stop() error { errs = errors.Join(errs, fmt.Errorf("failed to stop executor: %w", err)) } } + if bc.Pruner != nil { + if err := bc.Pruner.Stop(); err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to stop pruner: %w", err)) + } + } if bc.Reaper != nil { if err := bc.Reaper.Stop(); err != nil { errs = errors.Join(errs, fmt.Errorf("failed to stop reaper: %w", err)) @@ -166,6 +178,12 @@ func NewSyncComponents( syncer.SetBlockSyncer(syncing.WithTracingBlockSyncer(syncer)) } + var execPruner coreexecutor.ExecPruner + if p, ok := exec.(coreexecutor.ExecPruner); ok { + execPruner = p + } + pruner := pruner.New(logger, store, execPruner, config.Node) + // Create submitter for sync nodes (no signer, only DA inclusion processing) var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerDAHintAppender, dataDAHintAppender) if config.Instrumentation.IsTracingEnabled() { @@ -189,6 +207,7 @@ func NewSyncComponents( Syncer: syncer, Submitter: submitter, Cache: cacheManager, + Pruner: pruner, errorCh: errorCh, }, nil } @@ -248,6 +267,12 @@ func NewAggregatorComponents( executor.SetBlockProducer(executing.WithTracingBlockProducer(executor)) } + var execPruner coreexecutor.ExecPruner + if p, ok := exec.(coreexecutor.ExecPruner); ok { + execPruner = p + } + pruner := pruner.New(logger, store, execPruner, config.Node) + reaper, err := reaping.NewReaper( exec, sequencer, @@ -264,6 +289,7 @@ func NewAggregatorComponents( if config.Node.BasedSequencer { // no submissions needed for bases sequencer return &Components{ Executor: executor, + Pruner: pruner, Reaper: reaper, Cache: cacheManager, errorCh: errorCh, @@ -290,6 +316,7 @@ func NewAggregatorComponents( return &Components{ Executor: executor, + Pruner: pruner, Reaper: reaper, Submitter: submitter, Cache: cacheManager, diff --git a/block/components_test.go b/block/components_test.go index 93c08a655a..3d1a1f4a1e 100644 --- a/block/components_test.go +++ b/block/components_test.go @@ -127,6 +127,7 @@ func TestNewSyncComponents_Creation(t *testing.T) { assert.NotNil(t, components.Syncer) assert.NotNil(t, components.Submitter) assert.NotNil(t, components.Cache) + assert.NotNil(t, components.Pruner) assert.NotNil(t, components.errorCh) assert.Nil(t, components.Executor) // Sync nodes don't have executors } @@ -183,6 +184,7 @@ func TestNewAggregatorComponents_Creation(t *testing.T) { assert.NotNil(t, components.Executor) assert.NotNil(t, components.Submitter) assert.NotNil(t, components.Cache) + assert.NotNil(t, components.Pruner) assert.NotNil(t, components.errorCh) assert.Nil(t, components.Syncer) // Aggregator nodes currently don't create syncers in this constructor } diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go new file mode 100644 index 0000000000..14e23bc7a4 --- /dev/null +++ b/block/internal/pruner/pruner.go @@ -0,0 +1,195 @@ +package pruner + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "sync" + "time" + + ds "github.com/ipfs/go-datastore" + "github.com/rs/zerolog" + + coreexecutor "github.com/evstack/ev-node/core/execution" + + "github.com/evstack/ev-node/pkg/config" + "github.com/evstack/ev-node/pkg/store" +) + +const defaultPruneInterval = 15 * time.Minute + +// Pruner periodically removes old state and execution metadata entries. +type Pruner struct { + store store.Store + execPruner coreexecutor.ExecPruner + cfg config.NodeConfig + logger zerolog.Logger + lastPruned uint64 + + // Lifecycle + ctx context.Context + wg sync.WaitGroup + cancel context.CancelFunc +} + +// New creates a new Pruner instance. +func New( + logger zerolog.Logger, + store store.Store, + execPruner coreexecutor.ExecPruner, + cfg config.NodeConfig, +) *Pruner { + return &Pruner{ + store: store, + execPruner: execPruner, + cfg: cfg, + logger: logger.With().Str("component", "prune").Logger(), + } +} + +// Start begins the pruning loop. +func (p *Pruner) Start(ctx context.Context) error { + p.ctx, p.cancel = context.WithCancel(ctx) + + // Start pruner loop + p.wg.Go(p.pruneLoop) + + p.logger.Info().Msg("pruner started") + return nil +} + +// Stop stops the pruning loop. +func (p *Pruner) Stop() error { + if p.cancel != nil { + p.cancel() + } + p.wg.Wait() + + p.logger.Info().Msg("pruner stopped") + return nil +} + +func (p *Pruner) pruneLoop() { + ticker := time.NewTicker(defaultPruneInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := p.pruneRecoveryHistory(p.ctx, p.cfg.RecoveryHistoryDepth); err != nil { + p.logger.Error().Err(err).Msg("failed to prune recovery history") + } + + if err := p.pruneBlocks(); err != nil { + p.logger.Error().Err(err).Msg("failed to prune old blocks") + } + + // TODO: add pruning of old blocks // https://github.com/evstack/ev-node/pull/2984 + case <-p.ctx.Done(): + return + } + } +} + +func (p *Pruner) pruneBlocks() error { + if !p.cfg.PruningEnabled || p.cfg.PruningKeepRecent == 0 || p.cfg.PruningInterval == 0 { + return nil + } + + var currentDAIncluded uint64 + currentDAIncludedBz, err := p.store.GetMetadata(p.ctx, store.DAIncludedHeightKey) + if err == nil && len(currentDAIncludedBz) == 8 { + currentDAIncluded = binary.LittleEndian.Uint64(currentDAIncludedBz) + } else { + // if we cannot get the current DA height, we cannot safely prune, so we skip pruning until we can get it. + return nil + } + + var lastPruned uint64 + if bz, err := p.store.GetMetadata(p.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { + lastPruned = binary.LittleEndian.Uint64(bz) + } + + storeHeight, err := p.store.Height(p.ctx) + if err != nil { + return fmt.Errorf("failed to get store height for pruning: %w", err) + } + if storeHeight <= lastPruned+p.cfg.PruningInterval { + return nil + } + + // Never prune blocks that are not DA included + upperBound := min(storeHeight, currentDAIncluded) + if upperBound <= p.cfg.PruningKeepRecent { + // Not enough fully included blocks to prune + return nil + } + + targetHeight := upperBound - p.cfg.PruningKeepRecent + + if err := p.store.PruneBlocks(p.ctx, targetHeight); err != nil { + p.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + } + + if p.execPruner != nil { + if err := p.execPruner.PruneExec(p.ctx, targetHeight); err != nil && !errors.Is(err, ds.ErrNotFound) { + return err + } + } + + return nil +} + +// pruneRecoveryHistory prunes old state and execution metadata entries based on the configured retention depth. +// It does not prunes old blocks, as those are handled by the pruning logic. +// Pruning old state does not lose history but limit the ability to recover (replay or rollback) to the last HEAD-N blocks, where N is the retention depth. +func (p *Pruner) pruneRecoveryHistory(ctx context.Context, retention uint64) error { + if p.cfg.RecoveryHistoryDepth == 0 { + return nil + } + + height, err := p.store.Height(ctx) + if err != nil { + return err + } + + if height <= retention { + return nil + } + + target := height - retention + if target <= p.lastPruned { + return nil + } + + // maxPruneBatch limits how many heights we prune per cycle to bound work. + // it is callibrated to prune the last N blocks in one cycle, where N is the number of blocks produced in the defaultPruneInterval. + blockTime := p.cfg.BlockTime.Duration + if blockTime == 0 { + blockTime = 1 + } + + maxPruneBatch := max(uint64(defaultPruneInterval/blockTime), (target-p.lastPruned)/5) + + start := p.lastPruned + 1 + end := target + if end-start+1 > maxPruneBatch { + end = start + maxPruneBatch - 1 + } + + for h := start; h <= end; h++ { + if err := p.store.DeleteStateAtHeight(ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { + return err + } + + if p.execPruner != nil { + if err := p.execPruner.PruneExec(ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { + return err + } + } + } + + p.lastPruned = end + return nil +} diff --git a/block/internal/pruner/pruner_test.go b/block/internal/pruner/pruner_test.go new file mode 100644 index 0000000000..2d6b73d406 --- /dev/null +++ b/block/internal/pruner/pruner_test.go @@ -0,0 +1,52 @@ +package pruner + +import ( + "context" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/evstack/ev-node/pkg/config" + "github.com/evstack/ev-node/pkg/store" + "github.com/evstack/ev-node/types" +) + +type execMetaAdapter struct { + existing map[uint64]struct{} +} + +func (e *execMetaAdapter) PruneExec(ctx context.Context, height uint64) error { + delete(e.existing, height) + return nil +} + +func TestPrunerPrunesRecoveryHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + kv := dssync.MutexWrap(ds.NewMapDatastore()) + stateStore := store.New(kv) + + for height := uint64(1); height <= 3; height++ { + batch, err := stateStore.NewBatch(ctx) + require.NoError(t, err) + require.NoError(t, batch.SetHeight(height)) + require.NoError(t, batch.UpdateState(types.State{LastBlockHeight: height})) + require.NoError(t, batch.Commit()) + } + + execAdapter := &execMetaAdapter{existing: map[uint64]struct{}{1: {}, 2: {}, 3: {}}} + + recoveryPruner := New(zerolog.Nop(), stateStore, execAdapter, config.NodeConfig{RecoveryHistoryDepth: 2, BlockTime: config.DurationWrapper{Duration: 10 * time.Second}}) + require.NoError(t, recoveryPruner.pruneRecoveryHistory(ctx, recoveryPruner.cfg.RecoveryHistoryDepth)) + + _, err := stateStore.GetStateAtHeight(ctx, 1) + require.ErrorIs(t, err, ds.ErrNotFound) + + _, exists := execAdapter.existing[1] + require.False(t, exists) +} diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 6a40bd818e..1c5b034c19 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -364,50 +364,6 @@ func (s *Submitter) processDAInclusionLoop() { // This can only be performed after the height has been persisted to store s.cache.DeleteHeight(nextHeight) } - - // Run height-based pruning if enabled. - s.pruneBlocks() - } - } -} - -func (s *Submitter) pruneBlocks() { - if !s.config.Node.PruningEnabled || s.config.Node.PruningKeepRecent == 0 || s.config.Node.PruningInterval == 0 { - return - } - - currentDAIncluded := s.GetDAIncludedHeight() - - var lastPruned uint64 - if bz, err := s.store.GetMetadata(s.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { - lastPruned = binary.LittleEndian.Uint64(bz) - } - - storeHeight, err := s.store.Height(s.ctx) - if err != nil { - s.logger.Error().Err(err).Msg("failed to get store height for pruning") - return - } - if storeHeight <= lastPruned+s.config.Node.PruningInterval { - return - } - - // Never prune blocks that are not DA included - upperBound := min(storeHeight, currentDAIncluded) - if upperBound <= s.config.Node.PruningKeepRecent { - // Not enough fully included blocks to prune - return - } - - targetHeight := upperBound - s.config.Node.PruningKeepRecent - - if err := s.store.PruneBlocks(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") - } - - if pruner, ok := s.exec.(coreexecutor.ExecPruner); ok { - if err := pruner.PruneExec(s.ctx, targetHeight); err != nil { - s.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune execution metadata") } } } diff --git a/execution/evm/execution.go b/execution/evm/execution.go index 399f7c91ee..a25fbd3567 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -202,6 +202,7 @@ func NewEngineExecutionClient( feeRecipient common.Address, db ds.Batching, tracingEnabled bool, + logger zerolog.Logger, ) (*EngineClient, error) { if db == nil { return nil, errors.New("db is required for EVM execution client") @@ -265,22 +266,10 @@ func NewEngineExecutionClient( currentSafeBlockHash: genesisHash, currentFinalizedBlockHash: genesisHash, blockHashCache: make(map[uint64]common.Hash), - logger: zerolog.Nop(), + logger: logger, }, nil } -// PruneExec implements execution.ExecPruner by delegating to the -// underlying EVMStore. It is safe to call this multiple times with the same -// or increasing heights; the store tracks its own last-pruned height. -func (c *EngineClient) PruneExec(ctx context.Context, height uint64) error { - return c.store.PruneExec(ctx, height) -} - -// SetLogger allows callers to attach a structured logger. -func (c *EngineClient) SetLogger(l zerolog.Logger) { - c.logger = l -} - // InitChain initializes the blockchain with the given genesis parameters func (c *EngineClient) InitChain(ctx context.Context, genesisTime time.Time, initialHeight uint64, chainID string) ([]byte, error) { if initialHeight != 1 { @@ -1103,6 +1092,13 @@ func (c *EngineClient) Rollback(ctx context.Context, targetHeight uint64) error return nil } +// PruneExec implements execution.ExecPruner by delegating to the +// underlying EVMStore. It is safe to call this multiple times with the same +// or increasing heights; the store tracks its own last-pruned height. +func (c *EngineClient) PruneExec(ctx context.Context, height uint64) error { + return c.store.PruneExec(ctx, height) +} + // decodeSecret decodes a hex-encoded JWT secret string into a byte slice. func decodeSecret(jwtSecret string) ([]byte, error) { secret, err := hex.DecodeString(strings.TrimPrefix(jwtSecret, "0x")) diff --git a/execution/evm/test/execution_test.go b/execution/evm/test/execution_test.go index aa6f2db174..867b99b77d 100644 --- a/execution/evm/test/execution_test.go +++ b/execution/evm/test/execution_test.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/ethclient" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" + "github.com/rs/zerolog" "github.com/stretchr/testify/require" "github.com/evstack/ev-node/execution/evm" @@ -75,6 +76,7 @@ func TestEngineExecution(t *testing.T) { common.Address{}, store, false, + zerolog.Nop(), ) require.NoError(tt, err) @@ -172,6 +174,7 @@ func TestEngineExecution(t *testing.T) { common.Address{}, store, false, + zerolog.Nop(), ) require.NoError(tt, err) diff --git a/execution/evm/test/go.mod b/execution/evm/test/go.mod index 1eaa13d1d7..744c263775 100644 --- a/execution/evm/test/go.mod +++ b/execution/evm/test/go.mod @@ -8,6 +8,7 @@ require ( github.com/evstack/ev-node/execution/evm v0.0.0-00010101000000-000000000000 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/ipfs/go-datastore v0.9.0 + github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 ) @@ -147,7 +148,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect - github.com/rs/zerolog v1.34.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sasha-s/go-deadlock v0.3.5 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect diff --git a/pkg/config/config.go b/pkg/config/config.go index b964ac7bf2..d48501b63d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,6 +51,8 @@ const ( FlagReadinessMaxBlocksBehind = FlagPrefixEvnode + "node.readiness_max_blocks_behind" // FlagScrapeInterval is a flag for specifying the reaper scrape interval FlagScrapeInterval = FlagPrefixEvnode + "node.scrape_interval" + // FlagRecoveryHistoryDepth is a flag for specifying how much recovery history to keep + FlagRecoveryHistoryDepth = FlagPrefixEvnode + "node.recovery_history_depth" // FlagClearCache is a flag for clearing the cache FlagClearCache = FlagPrefixEvnode + "clear_cache" @@ -265,9 +267,10 @@ type NodeConfig struct { // Pruning configuration // When enabled, the node will periodically prune old block data (headers, data, // signatures, and hash index) from the local store while keeping recent history. - PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` - PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set pruning_enabled=false to keep all blocks (archive mode)."` - PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` + PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` + PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set pruning_enabled=false to keep all blocks (archive mode)."` + PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` + RecoveryHistoryDepth uint64 `mapstructure:"recovery_history_depth" yaml:"recovery_history_depth" comment:"Number of recent heights to keep state and execution metadata indexed for recovery (0 keeps all)."` } // LogConfig contains all logging configuration parameters @@ -458,6 +461,7 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Bool(FlagPrefixEvnode+"node.pruning_enabled", def.Node.PruningEnabled, "enable height-based pruning of stored block data (headers, data, signatures, index)") cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (must be > 0; disable pruning to keep all blocks)") cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_interval", def.Node.PruningInterval, "run pruning every N blocks (must be >= 1 when pruning is enabled)") + cmd.Flags().Uint64(FlagRecoveryHistoryDepth, def.Node.RecoveryHistoryDepth, "number of recent heights to keep state and execution metadata indexed for recovery (0 keeps all)") // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9189aa57e5..d8a3ac6c73 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -33,6 +33,7 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, uint64(0), def.Node.MaxPendingHeadersAndData) assert.Equal(t, false, def.Node.LazyMode) assert.Equal(t, 60*time.Second, def.Node.LazyBlockInterval.Duration) + assert.Equal(t, uint64(0), def.Node.RecoveryHistoryDepth) assert.Equal(t, "file", def.Signer.SignerType) assert.Equal(t, "config", def.Signer.SignerPath) assert.Equal(t, "127.0.0.1:7331", def.RPC.Address) @@ -64,6 +65,7 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagReadinessWindowSeconds, DefaultConfig().Node.ReadinessWindowSeconds) assertFlagValue(t, flags, FlagReadinessMaxBlocksBehind, DefaultConfig().Node.ReadinessMaxBlocksBehind) assertFlagValue(t, flags, FlagScrapeInterval, DefaultConfig().Node.ScrapeInterval) + assertFlagValue(t, flags, FlagRecoveryHistoryDepth, DefaultConfig().Node.RecoveryHistoryDepth) // DA flags assertFlagValue(t, flags, FlagDAAddress, DefaultConfig().DA.Address) @@ -112,7 +114,7 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization) // Count the number of flags we're explicitly checking - expectedFlagCount := 66 // Update this number if you add more flag checks above + expectedFlagCount := 67 // Update this number if you add more flag checks above // Get the actual number of flags (both regular and persistent) actualFlagCount := 0 diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index ee2cbfbeec..68c450e26b 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -16,6 +16,8 @@ const ( ConfigName = ConfigFileName + "." + ConfigExtension // AppConfigDir is the directory name for the app configuration. AppConfigDir = "config" + + defaultRecoveryHistoryDepth = uint64(0) ) // DefaultRootDir returns the default root directory for evolve @@ -66,6 +68,7 @@ func DefaultConfig() Config { LazyMode: false, LazyBlockInterval: DurationWrapper{60 * time.Second}, Light: false, + RecoveryHistoryDepth: defaultRecoveryHistoryDepth, ReadinessWindowSeconds: defaultReadinessWindowSeconds, ReadinessMaxBlocksBehind: calculateReadinessMaxBlocksBehind(defaultBlockTime.Duration, defaultReadinessWindowSeconds), ScrapeInterval: DurationWrapper{1 * time.Second}, diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index 8a5cca5b38..490f44411a 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -169,6 +169,12 @@ func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { return nil } +// DeleteStateAtHeight removes the state entry at the given height from the underlying store. +// This value is not cached, so nothing to invalidate. +func (cs *CachedStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + return cs.DeleteStateAtHeight(ctx, height) +} + // Close closes the underlying store. func (cs *CachedStore) Close() error { cs.ClearCache() diff --git a/pkg/store/store.go b/pkg/store/store.go index d981400f07..908f42dbf7 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -172,6 +172,14 @@ func (s *DefaultStore) GetStateAtHeight(ctx context.Context, height uint64) (typ return state, nil } +// DeleteStateAtHeight removes the state entry at the given height. +func (s *DefaultStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + if err := s.db.Delete(ctx, ds.NewKey(getStateAtHeightKey(height))); err != nil && !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("failed to delete state at height %d: %w", height, err) + } + return nil +} + // SetMetadata saves arbitrary value in the store. // // Metadata is separated from other data by using prefix in KV. diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index f1b9a131cc..08ead91074 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -591,6 +591,27 @@ func TestUpdateStateError(t *testing.T) { require.Contains(err.Error(), mockErrPut.Error()) } +func TestDeleteStateAtHeight(t *testing.T) { + t.Parallel() + require := require.New(t) + + kv, err := NewTestInMemoryKVStore() + require.NoError(err) + + store := New(kv) + + batch, err := store.NewBatch(t.Context()) + require.NoError(err) + require.NoError(batch.SetHeight(1)) + require.NoError(batch.UpdateState(types.State{LastBlockHeight: 1})) + require.NoError(batch.Commit()) + + require.NoError(store.(*DefaultStore).DeleteStateAtHeight(t.Context(), 1)) + + _, err = store.GetStateAtHeight(t.Context(), 1) + require.ErrorIs(err, ds.ErrNotFound) +} + func TestGetStateError(t *testing.T) { t.Parallel() require := require.New(t) diff --git a/pkg/store/tracing.go b/pkg/store/tracing.go index 42d686d612..ea1d5c8842 100644 --- a/pkg/store/tracing.go +++ b/pkg/store/tracing.go @@ -228,6 +228,22 @@ func (t *tracedStore) DeleteMetadata(ctx context.Context, key string) error { return nil } +// DeleteStateAtHeight removes the state entry at the given height from the underlying store. +func (t *tracedStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + ctx, span := t.tracer.Start(ctx, "Store.DeleteStateAtHeight", + trace.WithAttributes(attribute.Int64("height", int64(height))), + ) + defer span.End() + + if err := t.inner.DeleteStateAtHeight(ctx, height); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + return nil +} + func (t *tracedStore) Rollback(ctx context.Context, height uint64, aggregator bool) error { ctx, span := t.tracer.Start(ctx, "Store.Rollback", trace.WithAttributes( diff --git a/pkg/store/tracing_test.go b/pkg/store/tracing_test.go index 5477a66985..1109791c76 100644 --- a/pkg/store/tracing_test.go +++ b/pkg/store/tracing_test.go @@ -30,6 +30,8 @@ type tracingMockStore struct { setMetadataFn func(ctx context.Context, key string, value []byte) error deleteMetadataFn func(ctx context.Context, key string) error rollbackFn func(ctx context.Context, height uint64, aggregator bool) error + pruneBlocksFn func(ctx context.Context, height uint64) error + deleteStateAtHeightFn func(ctx context.Context, height uint64) error newBatchFn func(ctx context.Context) (Batch, error) } @@ -125,8 +127,16 @@ func (m *tracingMockStore) Rollback(ctx context.Context, height uint64, aggregat } func (m *tracingMockStore) PruneBlocks(ctx context.Context, height uint64) error { - // For tracing tests we don't need pruning behavior; just satisfy the Store - // interface. Specific pruning behavior is tested separately in store_test.go. + if m.pruneBlocksFn != nil { + return m.pruneBlocksFn(ctx, height) + } + return nil +} + +func (m *tracingMockStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + if m.deleteStateAtHeightFn != nil { + return m.deleteStateAtHeightFn(ctx, height) + } return nil } diff --git a/pkg/store/types.go b/pkg/store/types.go index a5635feec1..f785850b2c 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -99,4 +99,8 @@ type Pruner interface { // up to and including the given height from the store, without modifying // state snapshots or the current chain height. PruneBlocks(ctx context.Context, height uint64) error + + // DeleteStateAtHeight removes the state at the given height from the store. + // It does not affect the current state or any states at other heights, allowing for targeted pruning of historical state snapshots. + DeleteStateAtHeight(ctx context.Context, height uint64) error } diff --git a/test/mocks/store.go b/test/mocks/store.go index efd1939940..2cde50c543 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -39,14 +39,6 @@ func (_m *MockStore) EXPECT() *MockStore_Expecter { return &MockStore_Expecter{mock: &_m.Mock} } -// PruneBlocks provides a mock implementation for the Store's pruning method. -// Tests using MockStore currently do not exercise pruning behavior, so this -// method simply satisfies the interface and can be extended with expectations -// later if needed. -func (_mock *MockStore) PruneBlocks(ctx context.Context, height uint64) error { - return nil -} - // Close provides a mock function for the type MockStore func (_mock *MockStore) Close() error { ret := _mock.Called() @@ -148,6 +140,63 @@ func (_c *MockStore_DeleteMetadata_Call) RunAndReturn(run func(ctx context.Conte return _c } +// DeleteStateAtHeight provides a mock function for the type MockStore +func (_mock *MockStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + ret := _mock.Called(ctx, height) + + if len(ret) == 0 { + panic("no return value specified for DeleteStateAtHeight") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) error); ok { + r0 = returnFunc(ctx, height) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockStore_DeleteStateAtHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteStateAtHeight' +type MockStore_DeleteStateAtHeight_Call struct { + *mock.Call +} + +// DeleteStateAtHeight is a helper method to define mock.On call +// - ctx context.Context +// - height uint64 +func (_e *MockStore_Expecter) DeleteStateAtHeight(ctx interface{}, height interface{}) *MockStore_DeleteStateAtHeight_Call { + return &MockStore_DeleteStateAtHeight_Call{Call: _e.mock.On("DeleteStateAtHeight", ctx, height)} +} + +func (_c *MockStore_DeleteStateAtHeight_Call) Run(run func(ctx context.Context, height uint64)) *MockStore_DeleteStateAtHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uint64 + if args[1] != nil { + arg1 = args[1].(uint64) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockStore_DeleteStateAtHeight_Call) Return(err error) *MockStore_DeleteStateAtHeight_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockStore_DeleteStateAtHeight_Call) RunAndReturn(run func(ctx context.Context, height uint64) error) *MockStore_DeleteStateAtHeight_Call { + _c.Call.Return(run) + return _c +} + // GetBlockByHash provides a mock function for the type MockStore func (_mock *MockStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { ret := _mock.Called(ctx, hash) @@ -888,6 +937,63 @@ func (_c *MockStore_NewBatch_Call) RunAndReturn(run func(ctx context.Context) (s return _c } +// PruneBlocks provides a mock function for the type MockStore +func (_mock *MockStore) PruneBlocks(ctx context.Context, height uint64) error { + ret := _mock.Called(ctx, height) + + if len(ret) == 0 { + panic("no return value specified for PruneBlocks") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(context.Context, uint64) error); ok { + r0 = returnFunc(ctx, height) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockStore_PruneBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PruneBlocks' +type MockStore_PruneBlocks_Call struct { + *mock.Call +} + +// PruneBlocks is a helper method to define mock.On call +// - ctx context.Context +// - height uint64 +func (_e *MockStore_Expecter) PruneBlocks(ctx interface{}, height interface{}) *MockStore_PruneBlocks_Call { + return &MockStore_PruneBlocks_Call{Call: _e.mock.On("PruneBlocks", ctx, height)} +} + +func (_c *MockStore_PruneBlocks_Call) Run(run func(ctx context.Context, height uint64)) *MockStore_PruneBlocks_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 uint64 + if args[1] != nil { + arg1 = args[1].(uint64) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockStore_PruneBlocks_Call) Return(err error) *MockStore_PruneBlocks_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockStore_PruneBlocks_Call) RunAndReturn(run func(ctx context.Context, height uint64) error) *MockStore_PruneBlocks_Call { + _c.Call.Return(run) + return _c +} + // Rollback provides a mock function for the type MockStore func (_mock *MockStore) Rollback(ctx context.Context, height uint64, aggregator bool) error { ret := _mock.Called(ctx, height, aggregator) From 9e11d5a7453a6d00366ecb590e763e938eb06bdb Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 15:35:38 +0100 Subject: [PATCH 30/41] cl --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf7ce9a519..ebd0413c22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Node pruning support. [#2984](https://github.com/evstack/ev-node/pull/2984) + - Two different sort of pruning implemented: + _Classic pruning_: prunes given `HEAD-n` blocks from the databases, including store metadatas. + _Auto Storage Optimization_: prunes only the store metadatas, keep all blocks. + By using one or the other, you are losing the ability to rollback or replay transactions earlier than `HEAD-n`. + When using _classic pruning_, you aren't able to fetch blocks prior to `HEAD-n`. + ## v1.0.0-rc.4 ### Changes From eedfc18a1c335799ef31024924552f41d231db83 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 15:39:42 +0100 Subject: [PATCH 31/41] typo --- pkg/store/cached_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index 490f44411a..d2e5ccec86 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -172,7 +172,7 @@ func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { // DeleteStateAtHeight removes the state entry at the given height from the underlying store. // This value is not cached, so nothing to invalidate. func (cs *CachedStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { - return cs.DeleteStateAtHeight(ctx, height) + return cs.Store.DeleteStateAtHeight(ctx, height) } // Close closes the underlying store. From 050b58cd512cabee7b0addb6c536cb5f14678e33 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 16:33:57 +0100 Subject: [PATCH 32/41] updates --- block/components.go | 4 +- block/internal/pruner/pruner.go | 90 +++++++++++------------- pkg/config/config.go | 82 ++++++++++++++++------ pkg/config/config_test.go | 10 ++- pkg/config/defaults.go | 11 ++- pkg/store/cached_store.go | 2 +- pkg/store/keys.go | 7 +- pkg/store/store.go | 34 +++++++++ pkg/store/store_adapter.go | 8 +-- pkg/store/tracing.go | 29 +++++++- pkg/store/tracing_test.go | 50 ++++++++----- pkg/store/types.go | 6 +- test/mocks/store.go | 120 ++++++++++++++++++++++++++++++++ 13 files changed, 343 insertions(+), 110 deletions(-) diff --git a/block/components.go b/block/components.go index 2956a98d81..5d5954d514 100644 --- a/block/components.go +++ b/block/components.go @@ -182,7 +182,7 @@ func NewSyncComponents( if p, ok := exec.(coreexecutor.ExecPruner); ok { execPruner = p } - pruner := pruner.New(logger, store, execPruner, config.Node) + pruner := pruner.New(logger, store, execPruner, config.Pruning) // Create submitter for sync nodes (no signer, only DA inclusion processing) var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerDAHintAppender, dataDAHintAppender) @@ -271,7 +271,7 @@ func NewAggregatorComponents( if p, ok := exec.(coreexecutor.ExecPruner); ok { execPruner = p } - pruner := pruner.New(logger, store, execPruner, config.Node) + pruner := pruner.New(logger, store, execPruner, config.Pruning) reaper, err := reaping.NewReaper( exec, diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index 14e23bc7a4..88feb125b1 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -17,15 +17,12 @@ import ( "github.com/evstack/ev-node/pkg/store" ) -const defaultPruneInterval = 15 * time.Minute - // Pruner periodically removes old state and execution metadata entries. type Pruner struct { store store.Store execPruner coreexecutor.ExecPruner - cfg config.NodeConfig + cfg config.PruningConfig logger zerolog.Logger - lastPruned uint64 // Lifecycle ctx context.Context @@ -38,7 +35,7 @@ func New( logger zerolog.Logger, store store.Store, execPruner coreexecutor.ExecPruner, - cfg config.NodeConfig, + cfg config.PruningConfig, ) *Pruner { return &Pruner{ store: store, @@ -50,6 +47,10 @@ func New( // Start begins the pruning loop. func (p *Pruner) Start(ctx context.Context) error { + if !p.cfg.IsPruningEnabled() { + return nil + } + p.ctx, p.cancel = context.WithCancel(ctx) // Start pruner loop @@ -61,9 +62,14 @@ func (p *Pruner) Start(ctx context.Context) error { // Stop stops the pruning loop. func (p *Pruner) Stop() error { + if !p.cfg.IsPruningEnabled() { + return nil + } + if p.cancel != nil { p.cancel() } + p.wg.Wait() p.logger.Info().Msg("pruner stopped") @@ -71,32 +77,30 @@ func (p *Pruner) Stop() error { } func (p *Pruner) pruneLoop() { - ticker := time.NewTicker(defaultPruneInterval) + ticker := time.NewTicker(p.cfg.Interval.Duration) defer ticker.Stop() for { select { case <-ticker.C: - if err := p.pruneRecoveryHistory(p.ctx, p.cfg.RecoveryHistoryDepth); err != nil { - p.logger.Error().Err(err).Msg("failed to prune recovery history") - } - - if err := p.pruneBlocks(); err != nil { - p.logger.Error().Err(err).Msg("failed to prune old blocks") + switch p.cfg.Mode { + case config.PruningModeMetadata: + if err := p.pruneMetadata(); err != nil { + p.logger.Error().Err(err).Msg("failed to prune blocks metadata") + } + case config.PruningModeAll: + if err := p.pruneBlocks(); err != nil { + p.logger.Error().Err(err).Msg("failed to prune blocks") + } } - - // TODO: add pruning of old blocks // https://github.com/evstack/ev-node/pull/2984 case <-p.ctx.Done(): return } } } +// pruneBlocks prunes blocks and their metadatas. func (p *Pruner) pruneBlocks() error { - if !p.cfg.PruningEnabled || p.cfg.PruningKeepRecent == 0 || p.cfg.PruningInterval == 0 { - return nil - } - var currentDAIncluded uint64 currentDAIncludedBz, err := p.store.GetMetadata(p.ctx, store.DAIncludedHeightKey) if err == nil && len(currentDAIncludedBz) == 8 { @@ -106,27 +110,19 @@ func (p *Pruner) pruneBlocks() error { return nil } - var lastPruned uint64 - if bz, err := p.store.GetMetadata(p.ctx, store.LastPrunedBlockHeightKey); err == nil && len(bz) == 8 { - lastPruned = binary.LittleEndian.Uint64(bz) - } - storeHeight, err := p.store.Height(p.ctx) if err != nil { return fmt.Errorf("failed to get store height for pruning: %w", err) } - if storeHeight <= lastPruned+p.cfg.PruningInterval { - return nil - } // Never prune blocks that are not DA included upperBound := min(storeHeight, currentDAIncluded) - if upperBound <= p.cfg.PruningKeepRecent { + if upperBound <= p.cfg.KeepRecent { // Not enough fully included blocks to prune return nil } - targetHeight := upperBound - p.cfg.PruningKeepRecent + targetHeight := upperBound - p.cfg.KeepRecent if err := p.store.PruneBlocks(p.ctx, targetHeight); err != nil { p.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") @@ -141,55 +137,53 @@ func (p *Pruner) pruneBlocks() error { return nil } -// pruneRecoveryHistory prunes old state and execution metadata entries based on the configured retention depth. +// pruneMetadata prunes old state and execution metadata entries based on the configured retention depth. // It does not prunes old blocks, as those are handled by the pruning logic. // Pruning old state does not lose history but limit the ability to recover (replay or rollback) to the last HEAD-N blocks, where N is the retention depth. -func (p *Pruner) pruneRecoveryHistory(ctx context.Context, retention uint64) error { - if p.cfg.RecoveryHistoryDepth == 0 { - return nil - } - - height, err := p.store.Height(ctx) +func (p *Pruner) pruneMetadata() error { + height, err := p.store.Height(p.ctx) if err != nil { return err } - if height <= retention { + if height <= p.cfg.KeepRecent { return nil } - target := height - retention - if target <= p.lastPruned { + lastPrunedState, err := p.store.GetLastPrunedStateHeight(p.ctx) + if err != nil { return nil } - // maxPruneBatch limits how many heights we prune per cycle to bound work. - // it is callibrated to prune the last N blocks in one cycle, where N is the number of blocks produced in the defaultPruneInterval. - blockTime := p.cfg.BlockTime.Duration - if blockTime == 0 { - blockTime = 1 + if lastPrunedBlock, err := p.store.GetLastPrunedBlockHeight(p.ctx); err == nil && lastPrunedBlock > lastPrunedState { + lastPrunedState = lastPrunedBlock + } + + target := height - p.cfg.KeepRecent + if target <= lastPrunedState { + return nil } - maxPruneBatch := max(uint64(defaultPruneInterval/blockTime), (target-p.lastPruned)/5) + // maxPruneBatch limits how many heights we prune per cycle to bound work. + maxPruneBatch := (target - lastPrunedState) / 20 - start := p.lastPruned + 1 + start := lastPrunedState + 1 end := target if end-start+1 > maxPruneBatch { end = start + maxPruneBatch - 1 } for h := start; h <= end; h++ { - if err := p.store.DeleteStateAtHeight(ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { + if err := p.store.DeleteStateAtHeight(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } if p.execPruner != nil { - if err := p.execPruner.PruneExec(ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { + if err := p.execPruner.PruneExec(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } } - p.lastPruned = end return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index d48501b63d..85a6cded92 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -51,8 +51,6 @@ const ( FlagReadinessMaxBlocksBehind = FlagPrefixEvnode + "node.readiness_max_blocks_behind" // FlagScrapeInterval is a flag for specifying the reaper scrape interval FlagScrapeInterval = FlagPrefixEvnode + "node.scrape_interval" - // FlagRecoveryHistoryDepth is a flag for specifying how much recovery history to keep - FlagRecoveryHistoryDepth = FlagPrefixEvnode + "node.recovery_history_depth" // FlagClearCache is a flag for clearing the cache FlagClearCache = FlagPrefixEvnode + "clear_cache" @@ -172,6 +170,11 @@ const ( FlagRaftHeartbeatTimeout = FlagPrefixEvnode + "raft.heartbeat_timeout" // FlagRaftLeaderLeaseTimeout is a flag for specifying leader lease timeout FlagRaftLeaderLeaseTimeout = FlagPrefixEvnode + "raft.leader_lease_timeout" + + // Pruning configuration flags + FlagPruningMode = FlagPrefixEvnode + "pruning.pruning_mode" + FlagPruningKeepRecent = FlagPrefixEvnode + "pruning.pruning_keep_recent" + FlagPruningInterval = FlagPrefixEvnode + "pruning.pruning_interval" ) // Config stores Rollkit configuration. @@ -204,6 +207,9 @@ type Config struct { // Raft consensus configuration Raft RaftConfig `mapstructure:"raft" yaml:"raft"` + + // Pruning configuration + Pruning PruningConfig `mapstructure:"pruning" yaml:"pruning"` } // DAConfig contains all Data Availability configuration parameters @@ -263,14 +269,6 @@ type NodeConfig struct { // Readiness / health configuration ReadinessWindowSeconds uint64 `mapstructure:"readiness_window_seconds" yaml:"readiness_window_seconds" comment:"Time window in seconds used to calculate ReadinessMaxBlocksBehind based on block time. Default: 15 seconds."` ReadinessMaxBlocksBehind uint64 `mapstructure:"readiness_max_blocks_behind" yaml:"readiness_max_blocks_behind" comment:"How many blocks behind best-known head the node can be and still be considered ready. 0 means must be exactly at head."` - - // Pruning configuration - // When enabled, the node will periodically prune old block data (headers, data, - // signatures, and hash index) from the local store while keeping recent history. - PruningEnabled bool `mapstructure:"pruning_enabled" yaml:"pruning_enabled" comment:"Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode)."` - PruningKeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set pruning_enabled=false to keep all blocks (archive mode)."` - PruningInterval uint64 `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N blocks. Must be >= 1 when pruning is enabled."` - RecoveryHistoryDepth uint64 `mapstructure:"recovery_history_depth" yaml:"recovery_history_depth" comment:"Number of recent heights to keep state and execution metadata indexed for recovery (0 keeps all)."` } // LogConfig contains all logging configuration parameters @@ -300,6 +298,49 @@ type RPCConfig struct { EnableDAVisualization bool `mapstructure:"enable_da_visualization" yaml:"enable_da_visualization" comment:"Enable DA visualization endpoints for monitoring blob submissions. Default: false"` } +const ( + PruningModeDisabled = "disabled" + PruningModeAll = "all" + PruningModeMetadata = "metadata" +) + +// PruningConfig contains all pruning configuration parameters +type PruningConfig struct { + Mode string `mapstructure:"pruning_mode" yaml:"pruning_mode" comment:"Pruning mode for stored block data and block metadata. Options: 'all' (prune all but recent blocks and their metatadas), 'metadata' (prune all but recent blocks metadatas), 'disabled' (keep all blocks and blocks metadata). Default: 'disabled'."` + KeepRecent uint64 `mapstructure:"pruning_keep_recent" yaml:"pruning_keep_recent" comment:"Number of most recent blocks/blocks metadata to retain when pruning is enabled. Must be > 0."` + Interval DurationWrapper `mapstructure:"pruning_interval" yaml:"pruning_interval" comment:"Run pruning every N minutes. Examples: \"5m\", \"10m\", \"24h\"."` +} + +// IsPruningEnabled returns true if pruning is enabled (i.e. pruning mode is not 'disabled') +func (c PruningConfig) IsPruningEnabled() bool { + return c.Mode != PruningModeDisabled && len(c.Mode) > 0 +} + +// Validate pruning configuration +func (c PruningConfig) Validate(blockTime time.Duration) error { + if c.Mode != PruningModeDisabled && c.Mode != PruningModeAll && c.Mode != PruningModeMetadata { + return fmt.Errorf("invalid pruning mode: %s; must be one of '%s', '%s', or '%s'", c.Mode, PruningModeDisabled, PruningModeAll, PruningModeMetadata) + } + + if c.Mode == PruningModeDisabled { + return nil + } + + if c.Interval.Duration == 0 { + return fmt.Errorf("pruning_interval must be >= 1s when pruning is enabled") + } + + if c.Interval.Duration < blockTime { + return fmt.Errorf("pruning_interval (%v) must be greater than or equal to block time (%v)", c.Interval.Duration, blockTime) + } + + if c.KeepRecent == 0 { + return fmt.Errorf("pruning_keep_recent must be > 0 when pruning is enabled; use pruning_enabled=false to keep all blocks") + } + + return nil +} + // RaftConfig contains all Raft consensus configuration parameters type RaftConfig struct { Enable bool `mapstructure:"enable" yaml:"enable" comment:"Enable Raft consensus for leader election and state replication"` @@ -383,18 +424,13 @@ func (c *Config) Validate() error { return fmt.Errorf("LazyBlockInterval (%v) must be greater than BlockTime (%v) in lazy mode", c.Node.LazyBlockInterval.Duration, c.Node.BlockTime.Duration) } + if err := c.Raft.Validate(); err != nil { return err } - // Validate pruning configuration - if c.Node.PruningEnabled { - if c.Node.PruningInterval == 0 { - return fmt.Errorf("pruning_interval must be >= 1 when pruning is enabled") - } - if c.Node.PruningKeepRecent == 0 { - return fmt.Errorf("pruning_keep_recent must be > 0 when pruning is enabled; use pruning_enabled=false to keep all blocks") - } + if err := c.Pruning.Validate(c.Node.BlockTime.Duration); err != nil { + return err } return nil @@ -457,11 +493,6 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagReadinessWindowSeconds, def.Node.ReadinessWindowSeconds, "time window in seconds for calculating readiness threshold based on block time (default: 15s)") cmd.Flags().Uint64(FlagReadinessMaxBlocksBehind, def.Node.ReadinessMaxBlocksBehind, "how many blocks behind best-known head the node can be and still be considered ready (0 = must be at head)") cmd.Flags().Duration(FlagScrapeInterval, def.Node.ScrapeInterval.Duration, "interval at which the reaper polls the execution layer for new transactions") - // Pruning configuration flags - cmd.Flags().Bool(FlagPrefixEvnode+"node.pruning_enabled", def.Node.PruningEnabled, "enable height-based pruning of stored block data (headers, data, signatures, index)") - cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_keep_recent", def.Node.PruningKeepRecent, "number of most recent blocks to retain when pruning is enabled (must be > 0; disable pruning to keep all blocks)") - cmd.Flags().Uint64(FlagPrefixEvnode+"node.pruning_interval", def.Node.PruningInterval, "run pruning every N blocks (must be >= 1 when pruning is enabled)") - cmd.Flags().Uint64(FlagRecoveryHistoryDepth, def.Node.RecoveryHistoryDepth, "number of recent heights to keep state and execution metadata indexed for recovery (0 keeps all)") // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") @@ -521,6 +552,11 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Duration(FlagRaftSendTimeout, def.Raft.SendTimeout, "max duration to wait for a message to be sent to a peer") cmd.Flags().Duration(FlagRaftHeartbeatTimeout, def.Raft.HeartbeatTimeout, "time between leader heartbeats to followers") cmd.Flags().Duration(FlagRaftLeaderLeaseTimeout, def.Raft.LeaderLeaseTimeout, "duration of the leader lease") + + // Pruning configuration flags + cmd.Flags().String(FlagPruningMode, def.Pruning.Mode, "pruning mode for stored block data and metadata (disabled, all, metadata)") + cmd.Flags().Uint64(FlagPruningKeepRecent, def.Pruning.KeepRecent, "number of most recent blocks and their metadata to retain when pruning is enabled (must be > 0)") + cmd.Flags().Duration(FlagPruningInterval, def.Pruning.Interval.Duration, "interval at which pruning is performed when pruning is enabled") } // Load loads the node configuration in the following order of precedence: diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d8a3ac6c73..c922509e00 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -33,10 +33,10 @@ func TestDefaultConfig(t *testing.T) { assert.Equal(t, uint64(0), def.Node.MaxPendingHeadersAndData) assert.Equal(t, false, def.Node.LazyMode) assert.Equal(t, 60*time.Second, def.Node.LazyBlockInterval.Duration) - assert.Equal(t, uint64(0), def.Node.RecoveryHistoryDepth) assert.Equal(t, "file", def.Signer.SignerType) assert.Equal(t, "config", def.Signer.SignerPath) assert.Equal(t, "127.0.0.1:7331", def.RPC.Address) + assert.Equal(t, PruningModeDisabled, def.Pruning.Mode) assert.NoError(t, def.Validate()) } @@ -65,7 +65,6 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagReadinessWindowSeconds, DefaultConfig().Node.ReadinessWindowSeconds) assertFlagValue(t, flags, FlagReadinessMaxBlocksBehind, DefaultConfig().Node.ReadinessMaxBlocksBehind) assertFlagValue(t, flags, FlagScrapeInterval, DefaultConfig().Node.ScrapeInterval) - assertFlagValue(t, flags, FlagRecoveryHistoryDepth, DefaultConfig().Node.RecoveryHistoryDepth) // DA flags assertFlagValue(t, flags, FlagDAAddress, DefaultConfig().DA.Address) @@ -113,8 +112,13 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagRPCAddress, DefaultConfig().RPC.Address) assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization) + // Pruning flags + assertFlagValue(t, flags, FlagPruningMode, DefaultConfig().Pruning.Mode) + assertFlagValue(t, flags, FlagPruningKeepRecent, DefaultConfig().Pruning.KeepRecent) + assertFlagValue(t, flags, FlagPruningInterval, DefaultConfig().Pruning.Interval.Duration) + // Count the number of flags we're explicitly checking - expectedFlagCount := 67 // Update this number if you add more flag checks above + expectedFlagCount := 66 // Update this number if you add more flag checks above // Get the actual number of flags (both regular and persistent) actualFlagCount := 0 diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 68c450e26b..6cb4f51382 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -16,8 +16,6 @@ const ( ConfigName = ConfigFileName + "." + ConfigExtension // AppConfigDir is the directory name for the app configuration. AppConfigDir = "config" - - defaultRecoveryHistoryDepth = uint64(0) ) // DefaultRootDir returns the default root directory for evolve @@ -68,13 +66,9 @@ func DefaultConfig() Config { LazyMode: false, LazyBlockInterval: DurationWrapper{60 * time.Second}, Light: false, - RecoveryHistoryDepth: defaultRecoveryHistoryDepth, ReadinessWindowSeconds: defaultReadinessWindowSeconds, ReadinessMaxBlocksBehind: calculateReadinessMaxBlocksBehind(defaultBlockTime.Duration, defaultReadinessWindowSeconds), ScrapeInterval: DurationWrapper{1 * time.Second}, - PruningEnabled: false, - PruningKeepRecent: 0, - PruningInterval: 0, }, DA: DAConfig{ Address: "http://localhost:7980", @@ -109,6 +103,11 @@ func DefaultConfig() Config { LeaderLeaseTimeout: 175 * time.Millisecond, RaftDir: filepath.Join(DefaultRootDir, "raft"), }, + Pruning: PruningConfig{ + Mode: PruningModeDisabled, + KeepRecent: 100_000, + Interval: DurationWrapper{15 * time.Minute}, + }, } } diff --git a/pkg/store/cached_store.go b/pkg/store/cached_store.go index d2e5ccec86..86f81129da 100644 --- a/pkg/store/cached_store.go +++ b/pkg/store/cached_store.go @@ -170,8 +170,8 @@ func (cs *CachedStore) PruneBlocks(ctx context.Context, height uint64) error { } // DeleteStateAtHeight removes the state entry at the given height from the underlying store. -// This value is not cached, so nothing to invalidate. func (cs *CachedStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { + // This value is not cached, so nothing to invalidate. return cs.Store.DeleteStateAtHeight(ctx, height) } diff --git a/pkg/store/keys.go b/pkg/store/keys.go index ff96fd8955..054f999901 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -26,10 +26,11 @@ const ( LastSubmittedHeaderHeightKey = "last-submitted-header-height" // LastPrunedBlockHeightKey is the metadata key used for persisting the last - // pruned block height in the store. All block data (header, data, - // signature, and hash index) for heights <= this value are considered - // pruned and may be missing from the store. + // pruned block height in the store. LastPrunedBlockHeightKey = "last-pruned-block-height" + // LastPrunedStateHeightKey is the metadata key used for persisting the last + // pruned state height in the store. + LastPrunedStateHeightKey = "last-pruned-state-height" headerPrefix = "h" dataPrefix = "d" diff --git a/pkg/store/store.go b/pkg/store/store.go index 908f42dbf7..37885f003e 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -433,6 +433,40 @@ func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { return nil } +// GetLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. +func (s *DefaultStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { + meta, err := s.GetMetadata(ctx, LastPrunedBlockHeightKey) + if err != nil { + if errors.Is(err, ds.ErrNotFound) { + return 0, nil + } + return 0, fmt.Errorf("failed to get last pruned block height: %w", err) + } + + if len(meta) != heightLength { + return 0, fmt.Errorf("invalid last pruned block height length: %d (expected %d)", len(meta), heightLength) + } + + return decodeHeight(meta) +} + +// GetLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. +func (s *DefaultStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { + meta, err := s.GetMetadata(ctx, LastPrunedStateHeightKey) + if err != nil { + if errors.Is(err, ds.ErrNotFound) { + return 0, nil + } + return 0, fmt.Errorf("failed to get last pruned state height: %w", err) + } + + if len(meta) != heightLength { + return 0, fmt.Errorf("invalid last pruned state height length: %d (expected %d)", len(meta), heightLength) + } + + return decodeHeight(meta) +} + const heightLength = 8 func encodeHeight(height uint64) []byte { diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 7aa08e6a2c..46fe270073 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -405,12 +405,8 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if meta, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(meta) == heightLength { - if lastPruned, err := decodeHeight(meta); err == nil { - if candidate := lastPruned + 1; candidate > startHeight { - startHeight = candidate - } - } + if lastPrunedHeight, err := a.store.GetLastPrunedBlockHeight(ctx); err == nil && lastPrunedHeight > startHeight { + startHeight = lastPrunedHeight + 1 } item, err := a.getter.GetByHeight(ctx, startHeight) diff --git a/pkg/store/tracing.go b/pkg/store/tracing.go index ea1d5c8842..7740f293ed 100644 --- a/pkg/store/tracing.go +++ b/pkg/store/tracing.go @@ -228,7 +228,6 @@ func (t *tracedStore) DeleteMetadata(ctx context.Context, key string) error { return nil } -// DeleteStateAtHeight removes the state entry at the given height from the underlying store. func (t *tracedStore) DeleteStateAtHeight(ctx context.Context, height uint64) error { ctx, span := t.tracer.Start(ctx, "Store.DeleteStateAtHeight", trace.WithAttributes(attribute.Int64("height", int64(height))), @@ -279,6 +278,34 @@ func (t *tracedStore) PruneBlocks(ctx context.Context, height uint64) error { return nil } +func (t *tracedStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { + ctx, span := t.tracer.Start(ctx, "Store.GetLastPrunedBlockHeight") + defer span.End() + + h, err := t.inner.GetLastPrunedBlockHeight(ctx) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return 0, err + } + + return h, nil +} + +func (t *tracedStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { + ctx, span := t.tracer.Start(ctx, "Store.GetLastPrunedStateHeight") + defer span.End() + + h, err := t.inner.GetLastPrunedStateHeight(ctx) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return 0, err + } + + return h, nil +} + func (t *tracedStore) Close() error { return t.inner.Close() } diff --git a/pkg/store/tracing_test.go b/pkg/store/tracing_test.go index 1109791c76..c395198dd4 100644 --- a/pkg/store/tracing_test.go +++ b/pkg/store/tracing_test.go @@ -17,22 +17,24 @@ import ( ) type tracingMockStore struct { - heightFn func(ctx context.Context) (uint64, error) - getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) - getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) - getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error) - getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error) - getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error) - getStateFn func(ctx context.Context) (types.State, error) - getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error) - getMetadataFn func(ctx context.Context, key string) ([]byte, error) - getMetadataByPrefixFn func(ctx context.Context, prefix string) ([]MetadataEntry, error) - setMetadataFn func(ctx context.Context, key string, value []byte) error - deleteMetadataFn func(ctx context.Context, key string) error - rollbackFn func(ctx context.Context, height uint64, aggregator bool) error - pruneBlocksFn func(ctx context.Context, height uint64) error - deleteStateAtHeightFn func(ctx context.Context, height uint64) error - newBatchFn func(ctx context.Context) (Batch, error) + heightFn func(ctx context.Context) (uint64, error) + getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) + getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) + getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error) + getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error) + getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error) + getStateFn func(ctx context.Context) (types.State, error) + getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error) + getMetadataFn func(ctx context.Context, key string) ([]byte, error) + getMetadataByPrefixFn func(ctx context.Context, prefix string) ([]MetadataEntry, error) + setMetadataFn func(ctx context.Context, key string, value []byte) error + deleteMetadataFn func(ctx context.Context, key string) error + rollbackFn func(ctx context.Context, height uint64, aggregator bool) error + pruneBlocksFn func(ctx context.Context, height uint64) error + deleteStateAtHeightFn func(ctx context.Context, height uint64) error + getLastPrunedBlockHeightFn func(ctx context.Context) (uint64, error) + getLastPrunedStateHeightFn func(ctx context.Context) (uint64, error) + newBatchFn func(ctx context.Context) (Batch, error) } func (m *tracingMockStore) Height(ctx context.Context) (uint64, error) { @@ -140,6 +142,22 @@ func (m *tracingMockStore) DeleteStateAtHeight(ctx context.Context, height uint6 return nil } +func (m *tracingMockStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { + if m.getLastPrunedBlockHeightFn != nil { + return m.getLastPrunedBlockHeightFn(ctx) + } + + return 0, nil +} + +func (m *tracingMockStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { + if m.getLastPrunedStateHeightFn != nil { + return m.getLastPrunedStateHeightFn(ctx) + } + + return 0, nil +} + func (m *tracingMockStore) Close() error { return nil } diff --git a/pkg/store/types.go b/pkg/store/types.go index f785850b2c..56f6a47a7b 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -95,11 +95,15 @@ type Rollback interface { // Pruner provides long-term, height-based pruning of historical block data. type Pruner interface { + // GetLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. + GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) + // GetLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. + GetLastPrunedStateHeight(ctx context.Context) (uint64, error) + // PruneBlocks removes block data (header, data, signature, and hash index) // up to and including the given height from the store, without modifying // state snapshots or the current chain height. PruneBlocks(ctx context.Context, height uint64) error - // DeleteStateAtHeight removes the state at the given height from the store. // It does not affect the current state or any states at other heights, allowing for targeted pruning of historical state snapshots. DeleteStateAtHeight(ctx context.Context, height uint64) error diff --git a/test/mocks/store.go b/test/mocks/store.go index 2cde50c543..a8104ff47c 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -417,6 +417,126 @@ func (_c *MockStore_GetHeader_Call) RunAndReturn(run func(ctx context.Context, h return _c } +// GetLastPrunedBlockHeight provides a mock function for the type MockStore +func (_mock *MockStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { + ret := _mock.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetLastPrunedBlockHeight") + } + + var r0 uint64 + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { + return returnFunc(ctx) + } + if returnFunc, ok := ret.Get(0).(func(context.Context) uint64); ok { + r0 = returnFunc(ctx) + } else { + r0 = ret.Get(0).(uint64) + } + if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = returnFunc(ctx) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockStore_GetLastPrunedBlockHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLastPrunedBlockHeight' +type MockStore_GetLastPrunedBlockHeight_Call struct { + *mock.Call +} + +// GetLastPrunedBlockHeight is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockStore_Expecter) GetLastPrunedBlockHeight(ctx interface{}) *MockStore_GetLastPrunedBlockHeight_Call { + return &MockStore_GetLastPrunedBlockHeight_Call{Call: _e.mock.On("GetLastPrunedBlockHeight", ctx)} +} + +func (_c *MockStore_GetLastPrunedBlockHeight_Call) Run(run func(ctx context.Context)) *MockStore_GetLastPrunedBlockHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockStore_GetLastPrunedBlockHeight_Call) Return(v uint64, err error) *MockStore_GetLastPrunedBlockHeight_Call { + _c.Call.Return(v, err) + return _c +} + +func (_c *MockStore_GetLastPrunedBlockHeight_Call) RunAndReturn(run func(ctx context.Context) (uint64, error)) *MockStore_GetLastPrunedBlockHeight_Call { + _c.Call.Return(run) + return _c +} + +// GetLastPrunedStateHeight provides a mock function for the type MockStore +func (_mock *MockStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { + ret := _mock.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for GetLastPrunedStateHeight") + } + + var r0 uint64 + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { + return returnFunc(ctx) + } + if returnFunc, ok := ret.Get(0).(func(context.Context) uint64); ok { + r0 = returnFunc(ctx) + } else { + r0 = ret.Get(0).(uint64) + } + if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = returnFunc(ctx) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockStore_GetLastPrunedStateHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLastPrunedStateHeight' +type MockStore_GetLastPrunedStateHeight_Call struct { + *mock.Call +} + +// GetLastPrunedStateHeight is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockStore_Expecter) GetLastPrunedStateHeight(ctx interface{}) *MockStore_GetLastPrunedStateHeight_Call { + return &MockStore_GetLastPrunedStateHeight_Call{Call: _e.mock.On("GetLastPrunedStateHeight", ctx)} +} + +func (_c *MockStore_GetLastPrunedStateHeight_Call) Run(run func(ctx context.Context)) *MockStore_GetLastPrunedStateHeight_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockStore_GetLastPrunedStateHeight_Call) Return(v uint64, err error) *MockStore_GetLastPrunedStateHeight_Call { + _c.Call.Return(v, err) + return _c +} + +func (_c *MockStore_GetLastPrunedStateHeight_Call) RunAndReturn(run func(ctx context.Context) (uint64, error)) *MockStore_GetLastPrunedStateHeight_Call { + _c.Call.Return(run) + return _c +} + // GetMetadata provides a mock function for the type MockStore func (_mock *MockStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { ret := _mock.Called(ctx, key) From ce25d7dc60f3a8d5ff1b79b53c523d0b8099d4b3 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 17:16:16 +0100 Subject: [PATCH 33/41] cleanup --- block/internal/pruner/pruner.go | 52 ++++++++++++-- pkg/store/keys.go | 1 + pkg/store/store.go | 34 --------- pkg/store/store_adapter.go | 9 ++- pkg/store/tracing.go | 28 -------- pkg/store/tracing_test.go | 50 +++++-------- pkg/store/types.go | 5 -- test/mocks/store.go | 120 -------------------------------- 8 files changed, 70 insertions(+), 229 deletions(-) diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index 88feb125b1..358eecb165 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -150,12 +150,12 @@ func (p *Pruner) pruneMetadata() error { return nil } - lastPrunedState, err := p.store.GetLastPrunedStateHeight(p.ctx) + lastPrunedState, err := p.getLastPrunedStateHeight(p.ctx) if err != nil { return nil } - if lastPrunedBlock, err := p.store.GetLastPrunedBlockHeight(p.ctx); err == nil && lastPrunedBlock > lastPrunedState { + if lastPrunedBlock, err := p.getLastPrunedBlockHeight(p.ctx); err == nil && lastPrunedBlock > lastPrunedState { lastPrunedState = lastPrunedBlock } @@ -177,13 +177,53 @@ func (p *Pruner) pruneMetadata() error { if err := p.store.DeleteStateAtHeight(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } + } - if p.execPruner != nil { - if err := p.execPruner.PruneExec(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { - return err - } + if p.execPruner != nil { + if err := p.execPruner.PruneExec(p.ctx, end); err != nil && !errors.Is(err, ds.ErrNotFound) { + return err } } + if err := p.setLastPrunedStateHeight(p.ctx, end); err != nil { + return fmt.Errorf("failed to set last pruned block height: %w", err) + } + return nil } + +// getLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. +func (p *Pruner) getLastPrunedBlockHeight(ctx context.Context) (uint64, error) { + lastPrunedBlockHeightBz, err := p.store.GetMetadata(ctx, store.LastPrunedBlockHeightKey) + if err != nil || len(lastPrunedBlockHeightBz) != 8 { + return 0, fmt.Errorf("failed to get last pruned block height or invalid format: %w", err) + } + + lastPrunedBlockHeight := binary.LittleEndian.Uint64(lastPrunedBlockHeightBz) + if lastPrunedBlockHeight == 0 { + return 0, fmt.Errorf("invalid last pruned block height") + } + + return lastPrunedBlockHeight, nil +} + +// getLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. +func (p *Pruner) getLastPrunedStateHeight(ctx context.Context) (uint64, error) { + lastPrunedStateHeightBz, err := p.store.GetMetadata(ctx, store.LastPrunedStateHeightKey) + if err != nil || len(lastPrunedStateHeightBz) != 8 { + return 0, fmt.Errorf("failed to get last pruned block height or invalid format: %w", err) + } + + lastPrunedStateHeight := binary.LittleEndian.Uint64(lastPrunedStateHeightBz) + if lastPrunedStateHeight == 0 { + return 0, fmt.Errorf("invalid last pruned block height") + } + + return lastPrunedStateHeight, nil +} + +func (p *Pruner) setLastPrunedStateHeight(ctx context.Context, height uint64) error { + bz := make([]byte, 8) + binary.LittleEndian.PutUint64(bz, height) + return p.store.SetMetadata(ctx, store.LastPrunedStateHeightKey, bz) +} diff --git a/pkg/store/keys.go b/pkg/store/keys.go index 054f999901..111b91a733 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -28,6 +28,7 @@ const ( // LastPrunedBlockHeightKey is the metadata key used for persisting the last // pruned block height in the store. LastPrunedBlockHeightKey = "last-pruned-block-height" + // LastPrunedStateHeightKey is the metadata key used for persisting the last // pruned state height in the store. LastPrunedStateHeightKey = "last-pruned-state-height" diff --git a/pkg/store/store.go b/pkg/store/store.go index 37885f003e..908f42dbf7 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -433,40 +433,6 @@ func (s *DefaultStore) PruneBlocks(ctx context.Context, height uint64) error { return nil } -// GetLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. -func (s *DefaultStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { - meta, err := s.GetMetadata(ctx, LastPrunedBlockHeightKey) - if err != nil { - if errors.Is(err, ds.ErrNotFound) { - return 0, nil - } - return 0, fmt.Errorf("failed to get last pruned block height: %w", err) - } - - if len(meta) != heightLength { - return 0, fmt.Errorf("invalid last pruned block height length: %d (expected %d)", len(meta), heightLength) - } - - return decodeHeight(meta) -} - -// GetLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. -func (s *DefaultStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { - meta, err := s.GetMetadata(ctx, LastPrunedStateHeightKey) - if err != nil { - if errors.Is(err, ds.ErrNotFound) { - return 0, nil - } - return 0, fmt.Errorf("failed to get last pruned state height: %w", err) - } - - if len(meta) != heightLength { - return 0, fmt.Errorf("invalid last pruned state height length: %d (expected %d)", len(meta), heightLength) - } - - return decodeHeight(meta) -} - const heightLength = 8 func encodeHeight(height uint64) []byte { diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 46fe270073..5e0db4e5bd 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -405,8 +405,13 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if lastPrunedHeight, err := a.store.GetLastPrunedBlockHeight(ctx); err == nil && lastPrunedHeight > startHeight { - startHeight = lastPrunedHeight + 1 + + if lastPrunedBlockHeightBz, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(lastPrunedBlockHeightBz) == heightLength { + if lastPruned, err := decodeHeight(lastPrunedBlockHeightBz); err == nil { + if candidate := lastPruned + 1; candidate > startHeight { + startHeight = candidate + } + } } item, err := a.getter.GetByHeight(ctx, startHeight) diff --git a/pkg/store/tracing.go b/pkg/store/tracing.go index 7740f293ed..796f30289c 100644 --- a/pkg/store/tracing.go +++ b/pkg/store/tracing.go @@ -278,34 +278,6 @@ func (t *tracedStore) PruneBlocks(ctx context.Context, height uint64) error { return nil } -func (t *tracedStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { - ctx, span := t.tracer.Start(ctx, "Store.GetLastPrunedBlockHeight") - defer span.End() - - h, err := t.inner.GetLastPrunedBlockHeight(ctx) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return 0, err - } - - return h, nil -} - -func (t *tracedStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { - ctx, span := t.tracer.Start(ctx, "Store.GetLastPrunedStateHeight") - defer span.End() - - h, err := t.inner.GetLastPrunedStateHeight(ctx) - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return 0, err - } - - return h, nil -} - func (t *tracedStore) Close() error { return t.inner.Close() } diff --git a/pkg/store/tracing_test.go b/pkg/store/tracing_test.go index c395198dd4..1109791c76 100644 --- a/pkg/store/tracing_test.go +++ b/pkg/store/tracing_test.go @@ -17,24 +17,22 @@ import ( ) type tracingMockStore struct { - heightFn func(ctx context.Context) (uint64, error) - getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) - getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) - getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error) - getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error) - getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error) - getStateFn func(ctx context.Context) (types.State, error) - getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error) - getMetadataFn func(ctx context.Context, key string) ([]byte, error) - getMetadataByPrefixFn func(ctx context.Context, prefix string) ([]MetadataEntry, error) - setMetadataFn func(ctx context.Context, key string, value []byte) error - deleteMetadataFn func(ctx context.Context, key string) error - rollbackFn func(ctx context.Context, height uint64, aggregator bool) error - pruneBlocksFn func(ctx context.Context, height uint64) error - deleteStateAtHeightFn func(ctx context.Context, height uint64) error - getLastPrunedBlockHeightFn func(ctx context.Context) (uint64, error) - getLastPrunedStateHeightFn func(ctx context.Context) (uint64, error) - newBatchFn func(ctx context.Context) (Batch, error) + heightFn func(ctx context.Context) (uint64, error) + getBlockDataFn func(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) + getBlockByHashFn func(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) + getSignatureFn func(ctx context.Context, height uint64) (*types.Signature, error) + getSignatureByHash func(ctx context.Context, hash []byte) (*types.Signature, error) + getHeaderFn func(ctx context.Context, height uint64) (*types.SignedHeader, error) + getStateFn func(ctx context.Context) (types.State, error) + getStateAtHeightFn func(ctx context.Context, height uint64) (types.State, error) + getMetadataFn func(ctx context.Context, key string) ([]byte, error) + getMetadataByPrefixFn func(ctx context.Context, prefix string) ([]MetadataEntry, error) + setMetadataFn func(ctx context.Context, key string, value []byte) error + deleteMetadataFn func(ctx context.Context, key string) error + rollbackFn func(ctx context.Context, height uint64, aggregator bool) error + pruneBlocksFn func(ctx context.Context, height uint64) error + deleteStateAtHeightFn func(ctx context.Context, height uint64) error + newBatchFn func(ctx context.Context) (Batch, error) } func (m *tracingMockStore) Height(ctx context.Context) (uint64, error) { @@ -142,22 +140,6 @@ func (m *tracingMockStore) DeleteStateAtHeight(ctx context.Context, height uint6 return nil } -func (m *tracingMockStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { - if m.getLastPrunedBlockHeightFn != nil { - return m.getLastPrunedBlockHeightFn(ctx) - } - - return 0, nil -} - -func (m *tracingMockStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { - if m.getLastPrunedStateHeightFn != nil { - return m.getLastPrunedStateHeightFn(ctx) - } - - return 0, nil -} - func (m *tracingMockStore) Close() error { return nil } diff --git a/pkg/store/types.go b/pkg/store/types.go index 56f6a47a7b..106c644e74 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -95,11 +95,6 @@ type Rollback interface { // Pruner provides long-term, height-based pruning of historical block data. type Pruner interface { - // GetLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. - GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) - // GetLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. - GetLastPrunedStateHeight(ctx context.Context) (uint64, error) - // PruneBlocks removes block data (header, data, signature, and hash index) // up to and including the given height from the store, without modifying // state snapshots or the current chain height. diff --git a/test/mocks/store.go b/test/mocks/store.go index a8104ff47c..2cde50c543 100644 --- a/test/mocks/store.go +++ b/test/mocks/store.go @@ -417,126 +417,6 @@ func (_c *MockStore_GetHeader_Call) RunAndReturn(run func(ctx context.Context, h return _c } -// GetLastPrunedBlockHeight provides a mock function for the type MockStore -func (_mock *MockStore) GetLastPrunedBlockHeight(ctx context.Context) (uint64, error) { - ret := _mock.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for GetLastPrunedBlockHeight") - } - - var r0 uint64 - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { - return returnFunc(ctx) - } - if returnFunc, ok := ret.Get(0).(func(context.Context) uint64); ok { - r0 = returnFunc(ctx) - } else { - r0 = ret.Get(0).(uint64) - } - if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = returnFunc(ctx) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// MockStore_GetLastPrunedBlockHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLastPrunedBlockHeight' -type MockStore_GetLastPrunedBlockHeight_Call struct { - *mock.Call -} - -// GetLastPrunedBlockHeight is a helper method to define mock.On call -// - ctx context.Context -func (_e *MockStore_Expecter) GetLastPrunedBlockHeight(ctx interface{}) *MockStore_GetLastPrunedBlockHeight_Call { - return &MockStore_GetLastPrunedBlockHeight_Call{Call: _e.mock.On("GetLastPrunedBlockHeight", ctx)} -} - -func (_c *MockStore_GetLastPrunedBlockHeight_Call) Run(run func(ctx context.Context)) *MockStore_GetLastPrunedBlockHeight_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockStore_GetLastPrunedBlockHeight_Call) Return(v uint64, err error) *MockStore_GetLastPrunedBlockHeight_Call { - _c.Call.Return(v, err) - return _c -} - -func (_c *MockStore_GetLastPrunedBlockHeight_Call) RunAndReturn(run func(ctx context.Context) (uint64, error)) *MockStore_GetLastPrunedBlockHeight_Call { - _c.Call.Return(run) - return _c -} - -// GetLastPrunedStateHeight provides a mock function for the type MockStore -func (_mock *MockStore) GetLastPrunedStateHeight(ctx context.Context) (uint64, error) { - ret := _mock.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for GetLastPrunedStateHeight") - } - - var r0 uint64 - var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context) (uint64, error)); ok { - return returnFunc(ctx) - } - if returnFunc, ok := ret.Get(0).(func(context.Context) uint64); ok { - r0 = returnFunc(ctx) - } else { - r0 = ret.Get(0).(uint64) - } - if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = returnFunc(ctx) - } else { - r1 = ret.Error(1) - } - return r0, r1 -} - -// MockStore_GetLastPrunedStateHeight_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLastPrunedStateHeight' -type MockStore_GetLastPrunedStateHeight_Call struct { - *mock.Call -} - -// GetLastPrunedStateHeight is a helper method to define mock.On call -// - ctx context.Context -func (_e *MockStore_Expecter) GetLastPrunedStateHeight(ctx interface{}) *MockStore_GetLastPrunedStateHeight_Call { - return &MockStore_GetLastPrunedStateHeight_Call{Call: _e.mock.On("GetLastPrunedStateHeight", ctx)} -} - -func (_c *MockStore_GetLastPrunedStateHeight_Call) Run(run func(ctx context.Context)) *MockStore_GetLastPrunedStateHeight_Call { - _c.Call.Run(func(args mock.Arguments) { - var arg0 context.Context - if args[0] != nil { - arg0 = args[0].(context.Context) - } - run( - arg0, - ) - }) - return _c -} - -func (_c *MockStore_GetLastPrunedStateHeight_Call) Return(v uint64, err error) *MockStore_GetLastPrunedStateHeight_Call { - _c.Call.Return(v, err) - return _c -} - -func (_c *MockStore_GetLastPrunedStateHeight_Call) RunAndReturn(run func(ctx context.Context) (uint64, error)) *MockStore_GetLastPrunedStateHeight_Call { - _c.Call.Return(run) - return _c -} - // GetMetadata provides a mock function for the type MockStore func (_mock *MockStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { ret := _mock.Called(ctx, key) From d64b3383eb97e42ff3187cf81ab7887e08bff00c Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 17:40:49 +0100 Subject: [PATCH 34/41] updates --- CHANGELOG.md | 4 ++-- block/internal/pruner/pruner.go | 27 ++++++++++++++------------- block/internal/pruner/pruner_test.go | 23 ++++++++++++++++++----- pkg/store/store_adapter.go | 1 - 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd0413c22..b2e1ff5347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Node pruning support. [#2984](https://github.com/evstack/ev-node/pull/2984) - Two different sort of pruning implemented: - _Classic pruning_: prunes given `HEAD-n` blocks from the databases, including store metadatas. - _Auto Storage Optimization_: prunes only the store metadatas, keep all blocks. + _Classic pruning_ (`all`): prunes given `HEAD-n` blocks from the databases, including store metadatas. + _Auto Storage Optimization_ (`metadata`): prunes only the state metadatas, keeps all blocks. By using one or the other, you are losing the ability to rollback or replay transactions earlier than `HEAD-n`. When using _classic pruning_, you aren't able to fetch blocks prior to `HEAD-n`. diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index 358eecb165..ebfeadd144 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -152,7 +152,7 @@ func (p *Pruner) pruneMetadata() error { lastPrunedState, err := p.getLastPrunedStateHeight(p.ctx) if err != nil { - return nil + return fmt.Errorf("failed to get last pruned state height: %w", err) } if lastPrunedBlock, err := p.getLastPrunedBlockHeight(p.ctx); err == nil && lastPrunedBlock > lastPrunedState { @@ -164,28 +164,19 @@ func (p *Pruner) pruneMetadata() error { return nil } - // maxPruneBatch limits how many heights we prune per cycle to bound work. - maxPruneBatch := (target - lastPrunedState) / 20 - - start := lastPrunedState + 1 - end := target - if end-start+1 > maxPruneBatch { - end = start + maxPruneBatch - 1 - } - - for h := start; h <= end; h++ { + for h := lastPrunedState + 1; h <= target; h++ { if err := p.store.DeleteStateAtHeight(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } if p.execPruner != nil { - if err := p.execPruner.PruneExec(p.ctx, end); err != nil && !errors.Is(err, ds.ErrNotFound) { + if err := p.execPruner.PruneExec(p.ctx, target); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } - if err := p.setLastPrunedStateHeight(p.ctx, end); err != nil { + if err := p.setLastPrunedStateHeight(p.ctx, target); err != nil { return fmt.Errorf("failed to set last pruned block height: %w", err) } @@ -195,6 +186,11 @@ func (p *Pruner) pruneMetadata() error { // getLastPrunedBlockHeight returns the height of the last block that was pruned using PruneBlocks. func (p *Pruner) getLastPrunedBlockHeight(ctx context.Context) (uint64, error) { lastPrunedBlockHeightBz, err := p.store.GetMetadata(ctx, store.LastPrunedBlockHeightKey) + if errors.Is(err, ds.ErrNotFound) { + // If not found, it means we haven't pruned any blocks yet, so we return 0. + return 0, nil + } + if err != nil || len(lastPrunedBlockHeightBz) != 8 { return 0, fmt.Errorf("failed to get last pruned block height or invalid format: %w", err) } @@ -210,6 +206,11 @@ func (p *Pruner) getLastPrunedBlockHeight(ctx context.Context) (uint64, error) { // getLastPrunedStateHeight returns the height of the last state that was pruned using DeleteStateAtHeight. func (p *Pruner) getLastPrunedStateHeight(ctx context.Context) (uint64, error) { lastPrunedStateHeightBz, err := p.store.GetMetadata(ctx, store.LastPrunedStateHeightKey) + if errors.Is(err, ds.ErrNotFound) { + // If not found, it means we haven't pruned any state yet, so we return 0. + return 0, nil + } + if err != nil || len(lastPrunedStateHeightBz) != 8 { return 0, fmt.Errorf("failed to get last pruned block height or invalid format: %w", err) } diff --git a/block/internal/pruner/pruner_test.go b/block/internal/pruner/pruner_test.go index 2d6b73d406..f0f1463a10 100644 --- a/block/internal/pruner/pruner_test.go +++ b/block/internal/pruner/pruner_test.go @@ -20,18 +20,23 @@ type execMetaAdapter struct { } func (e *execMetaAdapter) PruneExec(ctx context.Context, height uint64) error { - delete(e.existing, height) + for h := range e.existing { + if h < height { + delete(e.existing, h) + } + } + return nil } -func TestPrunerPrunesRecoveryHistory(t *testing.T) { +func TestPrunerPruneMetadata(t *testing.T) { t.Parallel() ctx := context.Background() kv := dssync.MutexWrap(ds.NewMapDatastore()) stateStore := store.New(kv) - for height := uint64(1); height <= 3; height++ { + for height := uint64(1); height <= 5; height++ { batch, err := stateStore.NewBatch(ctx) require.NoError(t, err) require.NoError(t, batch.SetHeight(height)) @@ -40,13 +45,21 @@ func TestPrunerPrunesRecoveryHistory(t *testing.T) { } execAdapter := &execMetaAdapter{existing: map[uint64]struct{}{1: {}, 2: {}, 3: {}}} + cfg := config.PruningConfig{ + Mode: config.PruningModeMetadata, + Interval: config.DurationWrapper{Duration: 1 * time.Second}, + KeepRecent: 1, + } - recoveryPruner := New(zerolog.Nop(), stateStore, execAdapter, config.NodeConfig{RecoveryHistoryDepth: 2, BlockTime: config.DurationWrapper{Duration: 10 * time.Second}}) - require.NoError(t, recoveryPruner.pruneRecoveryHistory(ctx, recoveryPruner.cfg.RecoveryHistoryDepth)) + pruner := New(zerolog.New(zerolog.NewTestWriter(t)), stateStore, execAdapter, cfg) + require.NoError(t, pruner.pruneMetadata()) _, err := stateStore.GetStateAtHeight(ctx, 1) require.ErrorIs(t, err, ds.ErrNotFound) + _, err = stateStore.GetStateAtHeight(ctx, 5) + require.NoError(t, err) + _, exists := execAdapter.existing[1] require.False(t, exists) } diff --git a/pkg/store/store_adapter.go b/pkg/store/store_adapter.go index 5e0db4e5bd..5b21ce1613 100644 --- a/pkg/store/store_adapter.go +++ b/pkg/store/store_adapter.go @@ -405,7 +405,6 @@ func (a *StoreAdapter[H]) Tail(ctx context.Context) (H, error) { // genesis initial height, but if pruning metadata is available we can // skip directly past fully-pruned ranges. startHeight := a.genesisInitialHeight - if lastPrunedBlockHeightBz, err := a.store.GetMetadata(ctx, LastPrunedBlockHeightKey); err == nil && len(lastPrunedBlockHeightBz) == heightLength { if lastPruned, err := decodeHeight(lastPrunedBlockHeightBz); err == nil { if candidate := lastPruned + 1; candidate > startHeight { From 83b4500ef43cf0c07cd943ad43cd02996af5d805 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 17:52:23 +0100 Subject: [PATCH 35/41] delete unused key --- pkg/store/keys.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/store/keys.go b/pkg/store/keys.go index 111b91a733..5654c15f8a 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -19,9 +19,6 @@ const ( // DAIncludedHeightKey is the key used for persisting the da included height in store. DAIncludedHeightKey = "d" - // LastBatchDataKey is the key used for persisting the last batch data in store. - LastBatchDataKey = "l" - // LastSubmittedHeaderHeightKey is the key used for persisting the last submitted header height in store. LastSubmittedHeaderHeightKey = "last-submitted-header-height" From 22bc5e7105920893ee4d5b252c3bd09058040ab2 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 18:16:33 +0100 Subject: [PATCH 36/41] prevent aggresive pruning --- block/components.go | 4 +- block/internal/pruner/pruner.go | 71 +++++++++++++++++++++++++--- block/internal/pruner/pruner_test.go | 2 +- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/block/components.go b/block/components.go index 5d5954d514..40071b6f20 100644 --- a/block/components.go +++ b/block/components.go @@ -182,7 +182,7 @@ func NewSyncComponents( if p, ok := exec.(coreexecutor.ExecPruner); ok { execPruner = p } - pruner := pruner.New(logger, store, execPruner, config.Pruning) + pruner := pruner.New(logger, store, execPruner, config.Pruning, config.Node.BlockTime.Duration) // Create submitter for sync nodes (no signer, only DA inclusion processing) var daSubmitter submitting.DASubmitterAPI = submitting.NewDASubmitter(daClient, config, genesis, blockOpts, metrics, logger, headerDAHintAppender, dataDAHintAppender) @@ -271,7 +271,7 @@ func NewAggregatorComponents( if p, ok := exec.(coreexecutor.ExecPruner); ok { execPruner = p } - pruner := pruner.New(logger, store, execPruner, config.Pruning) + pruner := pruner.New(logger, store, execPruner, config.Pruning, config.Node.BlockTime.Duration) reaper, err := reaping.NewReaper( exec, diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index ebfeadd144..707cfcb0b8 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -22,6 +22,7 @@ type Pruner struct { store store.Store execPruner coreexecutor.ExecPruner cfg config.PruningConfig + blockTime time.Duration logger zerolog.Logger // Lifecycle @@ -36,11 +37,13 @@ func New( store store.Store, execPruner coreexecutor.ExecPruner, cfg config.PruningConfig, + blockTime time.Duration, ) *Pruner { return &Pruner{ store: store, execPruner: execPruner, cfg: cfg, + blockTime: blockTime, logger: logger.With().Str("component", "prune").Logger(), } } @@ -124,12 +127,30 @@ func (p *Pruner) pruneBlocks() error { targetHeight := upperBound - p.cfg.KeepRecent - if err := p.store.PruneBlocks(p.ctx, targetHeight); err != nil { - p.logger.Error().Err(err).Uint64("target_height", targetHeight).Msg("failed to prune old block data") + // Get the last pruned height to determine batch size + lastPruned, err := p.getLastPrunedBlockHeight(p.ctx) + if err != nil { + return fmt.Errorf("failed to get last pruned block height: %w", err) + } + + catchUpBatchSize, normalBatchSize := p.calculateBatchSizes() + + remainingToPrune := targetHeight - lastPruned + batchSize := normalBatchSize + if remainingToPrune > catchUpBatchSize { + batchSize = catchUpBatchSize + } + + // prune in batches to avoid overwhelming the system + batchEnd := min(lastPruned+batchSize, targetHeight) + + if err := p.store.PruneBlocks(p.ctx, batchEnd); err != nil { + p.logger.Error().Err(err).Uint64("target_height", batchEnd).Msg("failed to prune old block data") + return err } if p.execPruner != nil { - if err := p.execPruner.PruneExec(p.ctx, targetHeight); err != nil && !errors.Is(err, ds.ErrNotFound) { + if err := p.execPruner.PruneExec(p.ctx, batchEnd); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } @@ -137,6 +158,33 @@ func (p *Pruner) pruneBlocks() error { return nil } +// calculateBatchSizes returns appropriate batch sizes for catch-up and normal pruning operations. +// The batch sizes are based on the pruning interval and block time to ensure reasonable progress +// without overwhelming the node. +// Catch-up mode is usually triggered when pruning is enabled for the first time ever, and there is a large backlog of blocks to prune. +func (p *Pruner) calculateBatchSizes() (catchUpBatchSize, normalBatchSize uint64) { + // Calculate batch size based on pruning interval and block time. + // We use 2x the blocks produced during one pruning interval as the catch-up batch size, + // and 4x for normal operation. This ensures we make steady progress during catch-up + // without overwhelming the node. + // Example: With 100ms blocks and 15min interval: 15*60/0.1 = 9000 blocks/interval + // - Catch-up batch: 18,000 blocks + // - Normal batch: 36,000 blocks + blocksPerInterval := uint64(p.cfg.Interval.Duration / p.blockTime) + catchUpBatchSize = blocksPerInterval * 2 + normalBatchSize = blocksPerInterval * 4 + + // Ensure reasonable minimums + if catchUpBatchSize < 1000 { + catchUpBatchSize = 1000 + } + if normalBatchSize < 10000 { + normalBatchSize = 10000 + } + + return catchUpBatchSize, normalBatchSize +} + // pruneMetadata prunes old state and execution metadata entries based on the configured retention depth. // It does not prunes old blocks, as those are handled by the pruning logic. // Pruning old state does not lose history but limit the ability to recover (replay or rollback) to the last HEAD-N blocks, where N is the retention depth. @@ -164,19 +212,30 @@ func (p *Pruner) pruneMetadata() error { return nil } - for h := lastPrunedState + 1; h <= target; h++ { + catchUpBatchSize, normalBatchSize := p.calculateBatchSizes() + + remainingToPrune := target - lastPrunedState + batchSize := normalBatchSize + if remainingToPrune > catchUpBatchSize { + batchSize = catchUpBatchSize + } + + // prune in batches to avoid overwhelming the system + batchEnd := min(lastPrunedState+batchSize, target) + + for h := lastPrunedState + 1; h <= batchEnd; h++ { if err := p.store.DeleteStateAtHeight(p.ctx, h); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } if p.execPruner != nil { - if err := p.execPruner.PruneExec(p.ctx, target); err != nil && !errors.Is(err, ds.ErrNotFound) { + if err := p.execPruner.PruneExec(p.ctx, batchEnd); err != nil && !errors.Is(err, ds.ErrNotFound) { return err } } - if err := p.setLastPrunedStateHeight(p.ctx, target); err != nil { + if err := p.setLastPrunedStateHeight(p.ctx, batchEnd); err != nil { return fmt.Errorf("failed to set last pruned block height: %w", err) } diff --git a/block/internal/pruner/pruner_test.go b/block/internal/pruner/pruner_test.go index f0f1463a10..b57cdff9a9 100644 --- a/block/internal/pruner/pruner_test.go +++ b/block/internal/pruner/pruner_test.go @@ -51,7 +51,7 @@ func TestPrunerPruneMetadata(t *testing.T) { KeepRecent: 1, } - pruner := New(zerolog.New(zerolog.NewTestWriter(t)), stateStore, execAdapter, cfg) + pruner := New(zerolog.New(zerolog.NewTestWriter(t)), stateStore, execAdapter, cfg, 100*time.Millisecond) require.NoError(t, pruner.pruneMetadata()) _, err := stateStore.GetStateAtHeight(ctx, 1) From c1ef25b18449e88a12fa157c9f1f63b6087fcb3d Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 19:22:18 +0100 Subject: [PATCH 37/41] config --- docs/learn/config.md | 72 ++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/docs/learn/config.md b/docs/learn/config.md index fea509fa5d..1cd2dcef02 100644 --- a/docs/learn/config.md +++ b/docs/learn/config.md @@ -17,7 +17,7 @@ This document provides a comprehensive reference for all configuration options a - [Maximum Pending Blocks](#maximum-pending-blocks) - [Lazy Mode (Lazy Aggregator)](#lazy-mode-lazy-aggregator) - [Lazy Block Interval](#lazy-block-interval) - - [Pruning (Height-Based Pruning)](#pruning-height-based-pruning) +- [Pruning Configuration (`pruning`)](#pruning-configuration-pruning) - [Data Availability Configuration (`da`)](#data-availability-configuration-da) - [DA Service Address](#da-service-address) - [DA Authentication Token](#da-authentication-token) @@ -280,39 +280,71 @@ _Example:_ `--rollkit.node.lazy_block_interval 1m` _Default:_ `"30s"` _Constant:_ `FlagLazyBlockTime` -### Pruning (Height-Based Pruning) +### Pruning Configuration (`pruning`) **Description:** -Controls height-based pruning of stored block data (headers, data, signatures, and index) from the local store. When pruning is enabled, the node periodically deletes old blocks while keeping a recent window of history. When disabled, the node keeps all blocks (archive mode). +Controls automatic pruning of stored block data and metadata from the local store. Pruning helps manage disk space by periodically removing old blocks and their associated state, while keeping a recent window of history for validation and queries. + +**Pruning Modes:** + +- **`disabled`** (default): Archive mode - keeps all blocks and metadata indefinitely +- **`metadata`**: Prunes only state metadata (execution state snapshots), keeps all blocks +- **`all`**: Prunes both blocks (headers, data, signatures) and metadata + +**How Pruning Works:** + +When pruning is enabled, the pruner runs at the configured interval and removes data beyond the retention window (`pruning_keep_recent`). The system uses intelligent batching to avoid overwhelming the node: + +- **Batch sizes are automatically calculated** based on your `pruning_interval` and `block_time` +- **Catch-up mode**: When first enabling pruning on an existing node, smaller batches (2× blocks per interval) are used to gradually catch up without impacting performance +- **Normal mode**: Once caught up, larger batches (4× blocks per interval) are used for efficient maintenance +- **Progress tracking**: Pruning progress is saved after each batch, so restarts don't lose progress + +**Batch Size Examples:** + +With default settings (15 minute interval, 1 second blocks): +- Catch-up: ~1,800 blocks per run +- Normal: ~3,600 blocks per run + +With high-throughput chain (15 minute interval, 100ms blocks): +- Catch-up: ~18,000 blocks per run +- Normal: ~36,000 blocks per run **YAML:** ```yaml -node: - pruning_enabled: true +pruning: + pruning_mode: "all" pruning_keep_recent: 100000 - pruning_interval: 1000 + pruning_interval: "15m" ``` **Command-line Flags:** -- `--evnode.node.pruning_enabled` (boolean) - - _Description:_ Enable height-based pruning of stored block data. When disabled, all blocks are kept (archive mode). -- `--evnode.node.pruning_keep_recent ` - - _Description:_ Number of most recent blocks to retain when pruning is enabled. Must be > 0 when pruning is enabled; set `pruning_enabled=false` to keep all blocks. -- `--evnode.node.pruning_interval ` - - _Description:_ Run pruning every N blocks. Must be >= 1 when pruning is enabled. +- `--evnode.pruning.pruning_mode ` + - _Description:_ Pruning mode: 'disabled' (keep all), 'metadata' (prune state only), or 'all' (prune blocks and state) + - _Example:_ `--evnode.pruning.pruning_mode all` + - _Default:_ `"disabled"` -_Defaults:_ +- `--evnode.pruning.pruning_keep_recent ` + - _Description:_ Number of most recent blocks/metadata to retain when pruning is enabled. Must be > 0 when pruning is enabled. + - _Example:_ `--evnode.pruning.pruning_keep_recent 100000` + - _Default:_ `0` -```yaml -node: - pruning_enabled: false - pruning_keep_recent: 0 - pruning_interval: 0 -``` +- `--evnode.pruning.pruning_interval ` + - _Description:_ How often to run the pruning process. Must be >= block_time when pruning is enabled. Larger intervals allow larger batch sizes. + - _Example:_ `--evnode.pruning.pruning_interval 15m` + - _Default:_ `0` (disabled) + +_Constants:_ `FlagPruningMode`, `FlagPruningKeepRecent`, `FlagPruningInterval` + +**Important Notes:** -_Constants:_ `FlagNodePruningEnabled`, `FlagNodePruningKeepRecent`, `FlagNodePruningInterval` +- Pruning only removes blocks that have been confirmed on the DA layer (for mode `all`) +- The first pruning run after enabling may take several cycles to catch up, processing data in smaller batches +- Pruning cannot be undone - ensure your retention window is sufficient for your use case +- For production deployments, consider keeping at least 100,000 recent blocks +- The pruning interval should be balanced with your disk space growth rate ## Data Availability Configuration (`da`) From 21b7933336442e3d5ad41bd1bd54afc51b6a086a Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 12 Feb 2026 19:24:32 +0100 Subject: [PATCH 38/41] feedback --- apps/evm/cmd/rollback.go | 2 +- apps/evm/cmd/run.go | 2 +- execution/evm/execution.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/evm/cmd/rollback.go b/apps/evm/cmd/rollback.go index 911c330dd6..ade2a7da1c 100644 --- a/apps/evm/cmd/rollback.go +++ b/apps/evm/cmd/rollback.go @@ -71,7 +71,7 @@ func NewRollbackCmd() *cobra.Command { } // rollback execution layer via EngineClient - engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB, logger.With().Str("module", "engine_client").Logger()) + engineClient, err := createRollbackEngineClient(cmd, rawEvolveDB, logger) if err != nil { cmd.Printf("Warning: failed to create engine client, skipping EL rollback: %v\n", err) } else { diff --git a/apps/evm/cmd/run.go b/apps/evm/cmd/run.go index 5ae854037e..60730eac52 100644 --- a/apps/evm/cmd/run.go +++ b/apps/evm/cmd/run.go @@ -55,7 +55,7 @@ var RunCmd = &cobra.Command{ } tracingEnabled := nodeConfig.Instrumentation.IsTracingEnabled() - executor, err := createExecutionClient(cmd, datastore, tracingEnabled, logger.With().Str("module", "engine_client").Logger()) + executor, err := createExecutionClient(cmd, datastore, tracingEnabled, logger) if err != nil { return err } diff --git a/execution/evm/execution.go b/execution/evm/execution.go index a25fbd3567..15cddf6417 100644 --- a/execution/evm/execution.go +++ b/execution/evm/execution.go @@ -266,7 +266,7 @@ func NewEngineExecutionClient( currentSafeBlockHash: genesisHash, currentFinalizedBlockHash: genesisHash, blockHashCache: make(map[uint64]common.Hash), - logger: logger, + logger: logger.With().Str("module", "engine_client").Logger(), }, nil } From 8111810bd21bec4a3b85a406ce62fd820c0d4d86 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Fri, 13 Feb 2026 13:12:03 +0100 Subject: [PATCH 39/41] shorter key --- pkg/store/keys.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/store/keys.go b/pkg/store/keys.go index 5654c15f8a..f64c9cadc5 100644 --- a/pkg/store/keys.go +++ b/pkg/store/keys.go @@ -24,11 +24,11 @@ const ( // LastPrunedBlockHeightKey is the metadata key used for persisting the last // pruned block height in the store. - LastPrunedBlockHeightKey = "last-pruned-block-height" + LastPrunedBlockHeightKey = "lst-prnd-b" // LastPrunedStateHeightKey is the metadata key used for persisting the last // pruned state height in the store. - LastPrunedStateHeightKey = "last-pruned-state-height" + LastPrunedStateHeightKey = "lst-prnd-s" headerPrefix = "h" dataPrefix = "d" From fc38d6c4967d1946599ef91bea5d7a4912506109 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Fri, 13 Feb 2026 17:35:56 +0100 Subject: [PATCH 40/41] add logs --- block/internal/pruner/pruner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index 707cfcb0b8..a0b9acab72 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -155,6 +155,7 @@ func (p *Pruner) pruneBlocks() error { } } + p.logger.Debug().Uint64("pruned_up_to_height", batchEnd).Msg("pruned blocks up to height") return nil } @@ -239,6 +240,7 @@ func (p *Pruner) pruneMetadata() error { return fmt.Errorf("failed to set last pruned block height: %w", err) } + p.logger.Debug().Uint64("pruned_to", batchEnd).Msg("pruned state height metadata up to height") return nil } From 44191114812aa69c0cbe241b7ded7b68ac5fa47a Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Fri, 13 Feb 2026 18:20:59 +0100 Subject: [PATCH 41/41] improve batch size --- block/internal/pruner/pruner.go | 48 +++++++++------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/block/internal/pruner/pruner.go b/block/internal/pruner/pruner.go index a0b9acab72..63d291a61b 100644 --- a/block/internal/pruner/pruner.go +++ b/block/internal/pruner/pruner.go @@ -44,13 +44,14 @@ func New( execPruner: execPruner, cfg: cfg, blockTime: blockTime, - logger: logger.With().Str("component", "prune").Logger(), + logger: logger.With().Str("component", "pruner").Logger(), } } // Start begins the pruning loop. func (p *Pruner) Start(ctx context.Context) error { if !p.cfg.IsPruningEnabled() { + p.logger.Info().Msg("pruning is disabled, not starting pruner") return nil } @@ -133,15 +134,8 @@ func (p *Pruner) pruneBlocks() error { return fmt.Errorf("failed to get last pruned block height: %w", err) } - catchUpBatchSize, normalBatchSize := p.calculateBatchSizes() - - remainingToPrune := targetHeight - lastPruned - batchSize := normalBatchSize - if remainingToPrune > catchUpBatchSize { - batchSize = catchUpBatchSize - } - // prune in batches to avoid overwhelming the system + batchSize := p.calculateBatchSize() batchEnd := min(lastPruned+batchSize, targetHeight) if err := p.store.PruneBlocks(p.ctx, batchEnd); err != nil { @@ -159,31 +153,19 @@ func (p *Pruner) pruneBlocks() error { return nil } -// calculateBatchSizes returns appropriate batch sizes for catch-up and normal pruning operations. -// The batch sizes are based on the pruning interval and block time to ensure reasonable progress +// calculateBatchSize returns the appropriate batch size for pruning operations. +// The batch size is based on the pruning interval and block time to ensure reasonable progress // without overwhelming the node. -// Catch-up mode is usually triggered when pruning is enabled for the first time ever, and there is a large backlog of blocks to prune. -func (p *Pruner) calculateBatchSizes() (catchUpBatchSize, normalBatchSize uint64) { +func (p *Pruner) calculateBatchSize() uint64 { // Calculate batch size based on pruning interval and block time. - // We use 2x the blocks produced during one pruning interval as the catch-up batch size, - // and 4x for normal operation. This ensures we make steady progress during catch-up - // without overwhelming the node. + // We use 4x the blocks produced during one pruning interval as the batch size. + // This ensures we catch up at 3x the block production rate when there's a backlog. // Example: With 100ms blocks and 15min interval: 15*60/0.1 = 9000 blocks/interval - // - Catch-up batch: 18,000 blocks - // - Normal batch: 36,000 blocks + // - Batch size: 36,000 blocks (prunes 36k, chain grows 9k = net 27k catch-up per interval) blocksPerInterval := uint64(p.cfg.Interval.Duration / p.blockTime) - catchUpBatchSize = blocksPerInterval * 2 - normalBatchSize = blocksPerInterval * 4 - - // Ensure reasonable minimums - if catchUpBatchSize < 1000 { - catchUpBatchSize = 1000 - } - if normalBatchSize < 10000 { - normalBatchSize = 10000 - } - return catchUpBatchSize, normalBatchSize + // Ensure reasonable minimum + return min(blocksPerInterval*4, 10000) } // pruneMetadata prunes old state and execution metadata entries based on the configured retention depth. @@ -213,13 +195,7 @@ func (p *Pruner) pruneMetadata() error { return nil } - catchUpBatchSize, normalBatchSize := p.calculateBatchSizes() - - remainingToPrune := target - lastPrunedState - batchSize := normalBatchSize - if remainingToPrune > catchUpBatchSize { - batchSize = catchUpBatchSize - } + batchSize := p.calculateBatchSize() // prune in batches to avoid overwhelming the system batchEnd := min(lastPrunedState+batchSize, target)