Trace blocks built from payload attributes

CHANGELOG.sf.md

@@ -1,9 +1,13 @@
-## Unreleased
+## v1.4.2-fh3.1-2
 
 ### Changed
 
 * The node no longer installs a Firehose execution extension (ExEx). It only emitted `FIRE INIT` and the genesis block; blocks are traced by the engine. Both are now emitted at startup before the consensus engine exists, so no block can be traced ahead of `FIRE INIT`. Unless another ExEx is enabled, the node stops writing the ExEx write-ahead log. `FIRE INIT` now reports the Base node version, as `reth base-1.4.2`, instead of the `reth-firehose` crate version.
 
+### Fixed
+
+* Firehose now emits the blocks a node builds from payload attributes alone (`no_tx_pool`). This is how the consensus node derives blocks from L1, for example when the node starts behind the chain and catches up from L1 data. These blocks were inserted already executed and skipped the engine validation that traces live blocks, so they were missing from the Firehose stream, which could not continue past the gap. Blocks received through `engine_newPayload` and blocks replayed by staged sync were not affected.
+
 ## v1.4.2-fh3.1-1
 
 ### Changed

crates/execution/firehose/Cargo.toml

@@ -15,10 +15,12 @@ workspace = true
 reth-evm.workspace = true
 reth-revm.workspace = true
 reth-firehose.workspace = true
+reth-storage-api.workspace = true
 firehose-tracer.workspace = true
 reth-primitives-traits.workspace = true
 
 # base
+base-execution-evm.workspace = true
 base-common-evm = { workspace = true, features = ["reth"] }
 base-common-consensus = { workspace = true, features = ["reth"] }
 

crates/execution/firehose/README.md

@@ -25,10 +25,15 @@ This crate is that wrapper for Base / OP Stack.
 - [`OpFirehoseEvmConfig`] — `ConfigureEvm` wrapper whose `batch_executor` constructs
   `FirehoseBlockExecutor::new_with_hooks(..., OpPreTxAdjust, OpPostTxExtras)` so the
   pipeline fires the same OP hooks the live engine-API path already uses.
+- [`BuiltBlock`] and [`BuiltBlockTracer`] — tracing of a block the payload builder assembles
+  from payload attributes alone, which is how the consensus node derives blocks from L1. Such a
+  block enters the chain already executed and skips the engine validation, so the payload
+  builder calls `BuiltBlock::trace` on it.
 
 ## Paths into the hooks
 
 | Path                          | Where hooks are installed                                 |
 | ----------------------------- | --------------------------------------------------------- |
 | Staged sync (pipeline)        | `OpFirehoseEvmConfig::batch_executor` in this crate       |
 | Engine API (live)             | reth's engine validator (`execute_and_trace_block`), using the hooks `OpFirehoseEvmConfig` selects through `reth_firehose::FirehoseLiveHooks` |
+| Block built from attributes   | `BuiltBlock::trace`, called by the payload builder for `no_tx_pool` builds |

crates/execution/firehose/src/built_block.rs

@@ -0,0 +1,115 @@
+//! Firehose tracing of blocks the payload builder assembles from payload attributes alone.
+//!
+//! The consensus node derives blocks from L1 by asking the execution node to build each one from
+//! its payload attributes, with the transaction pool disabled. Such a block enters the chain
+//! already executed and skips the engine validation that traces every other live block, so the
+//! payload builder traces it through [`BuiltBlock::trace`].
+
+use alloy_consensus::BlockHeader;
+use base_common_consensus::BasePrimitives;
+use base_execution_evm::BaseEvmConfig;
+use reth_evm::{ConfigureEvm, Database, execute::BlockExecutionError};
+use reth_firehose::{ChainHooks, FinalizedBlockRef, FirehoseBlockTracer};
+use reth_primitives_traits::{NodePrimitives, RecoveredBlock};
+use reth_revm::{State, database::StateProviderDatabase};
+use reth_storage_api::{HeaderProvider, StateProviderFactory};
+
+use crate::{OpChainHooks, OpFirehoseEvmConfig};
+
+/// EVM configuration hook that emits an already built block to Firehose.
+pub trait BuiltBlockTracer<N: NodePrimitives> {
+    /// Re-executes `block` on `state`, a fresh state over the block's parent, and emits it with
+    /// `finalized` as its finalized block. Does nothing by default.
+    fn trace_built_block<DB: Database>(
+        &self,
+        _state: &mut State<DB>,
+        _block: &RecoveredBlock<N::Block>,
+        _finalized: Option<FinalizedBlockRef>,
+    ) -> Result<(), BlockExecutionError> {
+        Ok(())
+    }
+}
+
+impl<ChainSpec, N: NodePrimitives, R, EvmFactory> BuiltBlockTracer<N>
+    for BaseEvmConfig<ChainSpec, N, R, EvmFactory>
+{
+}
+
+impl<F> BuiltBlockTracer<BasePrimitives> for OpFirehoseEvmConfig<F>
+where
+    F: ConfigureEvm<Primitives = BasePrimitives>,
+    OpChainHooks: ChainHooks<F>,
+{
+    fn trace_built_block<DB: Database>(
+        &self,
+        state: &mut State<DB>,
+        block: &RecoveredBlock<<BasePrimitives as NodePrimitives>::Block>,
+        finalized: Option<FinalizedBlockRef>,
+    ) -> Result<(), BlockExecutionError> {
+        if !reth_firehose::is_tracer_initialized() {
+            return Ok(());
+        }
+
+        let mut tracer =
+            FirehoseBlockTracer::start::<BasePrimitives>(block.sealed_block(), finalized);
+        match OpChainHooks.execute_one_traced(&self.inner, state, block, &mut tracer) {
+            Ok(_) => {
+                tracer.mark_verified();
+                Ok(())
+            }
+            Err(error) => {
+                tracer.mark_failed(&error);
+                Err(error)
+            }
+        }
+    }
+}
+
+/// A block the payload builder assembled from payload attributes alone.
+#[derive(Debug, Clone, Copy)]
+pub struct BuiltBlock;
+
+impl BuiltBlock {
+    /// Emits `block` to Firehose by re-executing it on its parent state, read from `client`.
+    ///
+    /// The block advertises the node's finalized head, or the highest canonical ancestor below
+    /// it. Does nothing when the Firehose tracer is not initialized.
+    pub fn trace<Evm, N, Client>(
+        evm_config: &Evm,
+        client: &Client,
+        block: &RecoveredBlock<N::Block>,
+    ) -> Result<(), BlockExecutionError>
+    where
+        N: NodePrimitives,
+        Evm: BuiltBlockTracer<N>,
+        Client: StateProviderFactory + HeaderProvider,
+    {
+        if !reth_firehose::is_tracer_initialized() {
+            return Ok(());
+        }
+
+        let parent_hash = block.header().parent_hash();
+        let state_provider =
+            client.state_by_block_hash(parent_hash).map_err(BlockExecutionError::other)?;
+        let mut state = State::builder()
+            .with_database(StateProviderDatabase::new(&state_provider))
+            .with_bundle_update()
+            .build();
+
+        let finalized = reth_firehose::finalized_ref_for_block(
+            block.header().number(),
+            parent_hash,
+            client.finalized_block_num_hash().ok().flatten(),
+            |hash| {
+                client
+                    .header(hash)
+                    .ok()
+                    .flatten()
+                    .map(|header| (header.number(), header.parent_hash()))
+            },
+            |number| client.block_hash(number).ok().flatten(),
+        );
+
+        evm_config.trace_built_block(&mut state, block, finalized)
+    }
+}

crates/execution/firehose/src/lib.rs

@@ -7,3 +7,6 @@ pub use extras::{OpPostTxExtras, OpPreTxAdjust};
 
 mod evm_config;
 pub use evm_config::{OpChainHooks, OpFirehoseEvmConfig};
+
+mod built_block;
+pub use built_block::{BuiltBlock, BuiltBlockTracer};

crates/execution/node/src/node.rs

@@ -1173,7 +1173,8 @@ where
                 HeaderTy<Node::Types>,
                 <Node::Types as NodeTypes>::ChainSpec,
             >,
-        > + 'static,
+        > + base_execution_firehose::BuiltBlockTracer<PrimitivesTy<Node::Types>>
+        + 'static,
     Pool:
         TransactionPool<Transaction: BasePooledTx<Consensus = TxTy<Node::Types>>> + Unpin + 'static,
     Txs: BasePayloadTransactions<Pool>,

crates/execution/payload/Cargo.toml

@@ -36,6 +36,7 @@ base-bundles.workspace = true
 base-metrics.workspace = true
 base-common-evm.workspace = true
 base-execution-evm.workspace = true
+base-execution-firehose.workspace = true
 base-execution-txpool.workspace = true
 base-execution-eip8130.workspace = true
 base-observability-events.workspace = true

crates/execution/payload/src/builder.rs

@@ -17,6 +17,7 @@ use base_common_chains::Upgrades;
 use base_common_consensus::{BaseTransaction, CoinbaseTip, Predeploys};
 use base_common_evm::L1BlockInfo;
 use base_execution_eip8130::IntrinsicGas;
+use base_execution_firehose::{BuiltBlock, BuiltBlockTracer};
 use base_execution_txpool::{
     BasePooledTx, GuardMetrics, ParkableTransactionPool, PredicateContext, ValidityPredicate,
     estimated_da_size::DataAvailabilitySized,
@@ -294,7 +295,7 @@ where
     Evm: ConfigureEvm<
             Primitives = N,
             NextBlockEnvCtx: BuildNextEnv<Attrs, N::BlockHeader, Client::ChainSpec>,
-        >,
+        > + BuiltBlockTracer<N>,
     Txs: BasePayloadTransactions<Pool>,
     Attrs: Attributes<Transaction = N::SignedTx>,
 {
@@ -306,7 +307,21 @@ where
         args: BuildArguments<Self::Attributes, Self::BuiltPayload>,
     ) -> Result<BuildOutcome<Self::BuiltPayload>, PayloadBuilderError> {
         let pool = self.pool.clone();
-        self.build_payload(args, |attrs| self.best_transactions.best_transactions(pool, attrs))
+        let no_tx_pool = args.config.attributes.no_tx_pool();
+        let outcome = self
+            .build_payload(args, |attrs| self.best_transactions.best_transactions(pool, attrs))?;
+
+        // A block built from the payload attributes alone enters the chain without the engine
+        // validation that Firehose traces, so it is traced here.
+        if no_tx_pool
+            && let BuildOutcome::Freeze(payload) = &outcome
+            && let Some(executed) = &payload.executed_block
+        {
+            BuiltBlock::trace(&self.evm_config, &self.client, &executed.recovered_block)
+                .map_err(PayloadBuilderError::other)?;
+        }
+
+        Ok(outcome)
     }
 
     fn on_missing_payload(

crates/execution/runner/tests/firehose_live_tracing.rs

@@ -1,4 +1,4 @@
-//! Regression test for Firehose tracing on the engine-tree live-block path.
+//! Regression test for Firehose tracing of live blocks, on both paths a block can take into a node.
 //!
 //! Blocks that a node has to *execute itself* when they arrive via `engine_newPayload` are routed
 //! into the Firehose tracer by reth's engine validator (`validate_block_with_state` →
@@ -9,14 +9,17 @@
 //!
 //! ## Why two nodes
 //!
-//! A node that *builds* a block (the sequencer flow) inserts it into its tree as already-executed
+//! A node that *builds* a block inserts it into its tree as already-executed
 //! (`InsertExecutedBlock`), so a subsequent `engine_newPayload` for that same block short-circuits
-//! and never re-runs `validate_block_with_state` — the traced path is skipped. The live path is
-//! only exercised by a node that did **not** build the block: a follower receiving payloads from a
-//! sequencer. So this test runs two nodes — a `sequencer` that builds payloads and a `follower`
-//! that executes them via `engine_newPayload` — and asserts the follower emits `FIRE BLOCK` lines.
-//! If the dispatch into `execute_and_trace_block` is missing, no `FIRE BLOCK` lines are produced and
-//! the test fails.
+//! and never re-runs `validate_block_with_state`. That is how a consensus node derives blocks from
+//! L1: it asks the node to build each one from payload attributes alone (`no_tx_pool`). The payload
+//! builder traces such a block itself, through `base_execution_firehose::BuiltBlock::trace`.
+//!
+//! The engine path is only exercised by a node that did **not** build the block. So this test runs
+//! two nodes — a `sequencer` that builds payloads from attributes alone and a `follower` that
+//! executes them via `engine_newPayload` — and asserts each block is emitted once by each node, and
+//! that the two emissions are identical. If either path stops tracing, a block is emitted once
+//! instead of twice and the test fails.
 //!
 //! ## OP Stack hooks
 //!
@@ -27,8 +30,7 @@
 //!
 //! It lives in its own integration-test binary because it installs a process-wide tracer;
 //! cargo/nextest run each integration binary in its own process, keeping the global tracer isolated
-//! from the rest of the suite. The tracer is global, but only the follower's
-//! `validate_block_with_state` execution feeds it — building on the sequencer does not trace.
+//! from the rest of the suite. Both nodes feed that one tracer.
 
 use std::{sync::Arc, time::Duration};
 
@@ -57,8 +59,8 @@ use reth_provider::ChainSpecProvider;
 use tokio::time::sleep;
 
 /// Number of blocks to advance. All of them (block 1 included — the genesis block itself is
-/// emitted separately at node startup, not through this path) exercise the
-/// live `execute_and_trace_block` path on the follower.
+/// emitted separately at node startup, not through this path) are built on the sequencer and
+/// exercise the live `execute_and_trace_block` path on the follower.
 const PRODUCED_BLOCKS: u64 = 3;
 
 /// Base EIP-1559 base-fee params (`eip1559Denominator` / `eip1559Elasticity`) matching the
@@ -69,7 +71,7 @@ const EIP1559_ELASTICITY: u32 = 6;
 const MIN_BASE_FEE: u64 = 1_000_000_000;
 
 #[tokio::test(flavor = "multi_thread")]
-async fn live_payload_validation_emits_firehose_blocks() -> Result<()> {
+async fn built_and_validated_blocks_emit_firehose_blocks() -> Result<()> {
     // Install a buffer-backed global Firehose tracer BEFORE any block is validated, so the live
     // path's `is_tracer_initialized()` gate activates and routes execution through
     // `execute_and_trace_block`. The chain id matches the test genesis; the fork timestamps only
@@ -96,9 +98,9 @@ async fn live_payload_validation_emits_firehose_blocks() -> Result<()> {
     genesis.extra_data = Bytes::from(extra_data);
     let chain_spec = Arc::new(BaseChainSpec::from_genesis(genesis));
 
-    // Two nodes on the same genesis: the sequencer builds payloads; the follower executes them via
-    // `engine_newPayload` (the path under test). Building does not trace; only the follower's
-    // `validate_block_with_state` execution feeds the global tracer.
+    // Two nodes on the same genesis: the sequencer builds payloads from attributes alone, which
+    // the payload builder traces; the follower executes them via `engine_newPayload`, which the
+    // engine traces.
     let sequencer = LocalNode::new(vec![], chain_spec.clone()).await?;
     let follower = LocalNode::new(vec![], chain_spec.clone()).await?;
     sleep(Duration::from_millis(NODE_STARTUP_DELAY_MS)).await;
@@ -160,7 +162,7 @@ async fn live_payload_validation_emits_firehose_blocks() -> Result<()> {
             3,
         )?;
 
-        // Sequencer builds the payload (this does NOT trace — it builds, it does not validate).
+        // Sequencer builds the payload from the attributes alone; the payload builder traces it.
         let payload_id = seq_engine
             .update_forkchoice(parent_hash, parent_hash, Some(attributes))
             .await?
@@ -180,7 +182,7 @@ async fn live_payload_validation_emits_firehose_blocks() -> Result<()> {
 
         // Follower validates the externally-produced payload via `engine_newPayload`. Since the
         // follower did not build it, this drives `validate_block_with_state` →
-        // `execute_and_trace_block` — the traced path under test.
+        // `execute_and_trace_block`.
         let status = fol_engine
             .new_payload(execution_payload, vec![], parent_beacon_block_root, execution_requests)
             .await?;
@@ -198,19 +200,23 @@ async fn live_payload_validation_emits_firehose_blocks() -> Result<()> {
     let traced = capture.traced_block_numbers();
     assert!(
         !traced.is_empty(),
-        "no FIRE BLOCK lines were emitted — the follower's live payload-validation path is not \
-         traced.\nCaptured tracer output:\n{}",
+        "no FIRE BLOCK lines were emitted — neither the built-block path nor the follower's live \
+         payload-validation path is traced.\nCaptured tracer output:\n{}",
         capture.raw_text()
     );
 
-    // Every produced block goes through the live `execute_and_trace_block` path. Require each to
-    // have been traced exactly once.
+    // Every produced block is traced once when the sequencer builds it and once when the follower
+    // validates it, and both paths must produce the same block.
+    let blocks = capture.blocks()?;
     for number in 1..=PRODUCED_BLOCKS {
-        let count = traced.iter().filter(|traced| **traced == number).count();
-        assert_eq!(count, 1, "expected one FIRE BLOCK line for live block #{number}, got {traced:?}");
+        let emitted: Vec<_> = blocks.iter().filter(|block| block.number == number).collect();
+        let [built, validated] = emitted.as_slice() else {
+            panic!("expected two FIRE BLOCK lines for block #{number}, got {traced:?}");
+        };
+        assert_eq!(built, validated, "built and validated traces of block #{number} differ");
     }
 
-    for block in capture.blocks()? {
+    for block in blocks {
         let [deposit, transfer] = block.transaction_traces.as_slice() else {
             panic!(
                 "block #{} should hold the L1-info deposit and one transfer, got {} traces",