fix(zone): only remove withdrawals after successful receipts (#378)

Author: mattsse

crates/tempo-zone/src/metrics.rs

@@ -27,9 +27,12 @@ pub(crate) struct WithdrawalProcessorMetrics {
     /// Number of withdrawals confirmed on L1.
     pub(crate) withdrawals_confirmed_total: Counter,
 
-    /// Number of withdrawals that failed to send or confirm.
+    /// Number of withdrawals that failed to send, confirm, or reverted after inclusion.
     pub(crate) withdrawals_failed_total: Counter,
 
+    /// Number of `processWithdrawal` transactions that were included on L1 but reverted.
+    pub(crate) withdrawals_reverted_total: Counter,
+
     /// Time spent processing a withdrawal queue slot.
     pub(crate) slot_processing_duration_seconds: Histogram,
 }

crates/tempo-zone/src/withdrawals.rs

@@ -31,6 +31,7 @@ use std::{
     time::{Duration, Instant},
 };
 
+use alloy_network::ReceiptResponse;
 use alloy_primitives::{Address, B256};
 use alloy_provider::DynProvider;
 use parking_lot::Mutex;
@@ -45,6 +46,8 @@ use crate::{
 };
 use tempo_alloy::rpc::TempoCallBuilderExt;
 
+const PROCESS_WITHDRAWAL_CONFIRM_TIMEOUT: Duration = Duration::from_secs(30);
+
 /// Shared handle to the withdrawal store.
 #[derive(Clone)]
 pub struct SharedWithdrawalStore(Arc<Mutex<WithdrawalStore>>);
@@ -341,6 +344,7 @@ impl WithdrawalProcessor {
             "Processing withdrawal batch"
         );
         let slot_started_at = Instant::now();
+        let slot_queue_hash = abi::Withdrawal::queue_hash(&withdrawals);
 
         for (i, withdrawal) in withdrawals.iter().enumerate() {
             self.metrics.withdrawals_processed_total.increment(1);
@@ -398,12 +402,32 @@ impl WithdrawalProcessor {
                 Ok(pending) => {
                     let tx_hash = *pending.tx_hash();
                     match pending
-                        .with_required_confirmations(1)
-                        .with_timeout(Some(std::time::Duration::from_secs(30)))
-                        .watch()
+                        .with_timeout(Some(PROCESS_WITHDRAWAL_CONFIRM_TIMEOUT))
+                        .get_receipt()
                         .await
                     {
-                        Ok(_) => {
+                        Ok(receipt) => {
+                            if !receipt.status() {
+                                self.metrics.withdrawals_failed_total.increment(1);
+                                self.metrics.withdrawals_reverted_total.increment(1);
+                                self.record_slot_duration(slot_started_at.elapsed());
+                                self.repair_notify.notify_one();
+
+                                error!(
+                                    slot = head_val,
+                                    index = i,
+                                    %tx_hash,
+                                    to = %withdrawal.to,
+                                    amount = %withdrawal.amount,
+                                    queue_head = head_val,
+                                    queue_tail = tail_val,
+                                    expected_slot_queue_hash = %slot_queue_hash,
+                                    expected_remaining_queue = %remaining_queue,
+                                    "processWithdrawal tx was included but reverted; keeping batch in store and requesting repair"
+                                );
+                                return Ok(());
+                            }
+
                             self.metrics.withdrawals_confirmed_total.increment(1);
                             info!(
                                 slot = head_val,
@@ -422,6 +446,8 @@ impl WithdrawalProcessor {
                                 slot = head_val,
                                 index = i,
                                 %tx_hash,
+                                expected_slot_queue_hash = %slot_queue_hash,
+                                expected_remaining_queue = %remaining_queue,
                                 to = %withdrawal.to,
                                 amount = %withdrawal.amount,
                                 error = %e,
@@ -437,6 +463,8 @@ impl WithdrawalProcessor {
                     error!(
                         slot = head_val,
                         index = i,
+                        expected_slot_queue_hash = %slot_queue_hash,
+                        expected_remaining_queue = %remaining_queue,
                         to = %withdrawal.to,
                         amount = %withdrawal.amount,
                         error = %e,