perf(provider): lazily refresh cached static-file jars

Pass the live segment header from static-file writers into provider index updates so append and truncate bookkeeping can use in-memory block and transaction ranges without reopening the jar.

Only reload and remap a static-file jar when that segment/range is already cached by readers, and keep uncached ranges lazy. Add a regression test covering both the uncached fast path and cached refresh behavior.

Amp-Thread-ID: https://ampcode.com/threads/T-019df204-bb69-724c-a1e4-fb1180d49bec
Co-authored-by: Amp <[email protected]>

Author: amp

crates/static-file/static-file/src/static_file_producer.rs

@@ -152,9 +152,11 @@ where
 
         self.provider.static_file_provider().commit()?;
         for (segment, block_range) in segments {
-            self.provider
-                .static_file_provider()
-                .update_index(segment.segment(), Some(*block_range.end()))?;
+            self.provider.static_file_provider().update_index(
+                segment.segment(),
+                Some(*block_range.end()),
+                None,
+            )?;
         }
 
         let elapsed = start.elapsed(); // TODO(alexey): track in metrics

crates/storage/provider/src/providers/static_file/manager.rs

@@ -1036,10 +1036,15 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
     /// Any entry higher than `segment_max_block` will be deleted from the previous structures.
     ///
     /// If `segment_max_block` is None it means there's no static file for this segment.
+    ///
+    /// If `segment_header` comes from the live writer for the indexed range, its in-memory state is
+    /// used for bookkeeping so the append path does not need to reopen the jar unless readers are
+    /// already caching that range.
     pub fn update_index(
         &self,
         segment: StaticFileSegment,
         segment_max_block: Option<BlockNumber>,
+        segment_header: Option<&SegmentHeader>,
     ) -> ProviderResult<()> {
         debug!(
             target: "providers::static_file",
@@ -1056,11 +1061,21 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
                     indexes.get(segment).map(|index| &index.expected_block_ranges_by_max_block),
                     segment_max_block,
                 );
-
-                let jar = NippyJar::<SegmentHeader>::load(
-                    &self.path.join(segment.filename(&fixed_range)),
-                )
-                .map_err(ProviderError::other)?;
+                let key = (fixed_range.end(), segment);
+                let path = self.path.join(segment.filename(&fixed_range));
+                let should_refresh_cached_jar = self.map.contains_key(&key);
+                let mut loaded_jar = None;
+                let current_header = if let Some(segment_header) =
+                    segment_header.filter(|header| header.expected_block_range() == fixed_range)
+                {
+                    segment_header.clone()
+                } else {
+                    let jar =
+                        NippyJar::<SegmentHeader>::load(&path).map_err(ProviderError::other)?;
+                    let header = jar.user_header().clone();
+                    loaded_jar = Some(jar);
+                    header
+                };
 
                 let index = indexes
                     .entry(segment)
@@ -1104,7 +1119,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
                 // 2. Sync to block 100, this update sets min_block = [0..=100]
                 // 3. Pruner calls get_lowest_static_file_block() -> returns 100 (correct). Without
                 //    this update, it would incorrectly return 0 (stale)
-                if let Some(current_block_range) = jar.user_header().block_range() {
+                if let Some(current_block_range) = current_header.block_range() {
                     if let Some(min_block_range) = index.min_block_range.as_mut() {
                         // delete_jar WILL ALWAYS re-initialize all indexes, so we are always
                         // sure that current_min is always the lowest.
@@ -1118,10 +1133,10 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
 
                 // Updates the tx index by first removing all entries which have a higher
                 // block_start than our current static file.
-                if let Some(tx_range) = jar.user_header().tx_range() {
+                if let Some(tx_range) = current_header.tx_range() {
                     // Current block range has the same block start as `fixed_range``, but block end
                     // might be different if we are still filling this static file.
-                    if let Some(current_block_range) = jar.user_header().block_range() {
+                    if let Some(current_block_range) = current_header.block_range() {
                         let tx_end = tx_range.end();
 
                         // Considering that `update_index` is called when we either append/truncate,
@@ -1153,9 +1168,16 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
                     index.available_block_ranges_by_max_tx.take_if(|index| index.is_empty());
                 }
 
-                // Update the cached provider.
-                debug!(target: "providers::static_file", ?segment, "Inserting updated jar into cache");
-                self.map.insert((fixed_range.end(), segment), LoadedJar::new(jar)?);
+                // Refresh the cached provider only when readers already have this range resident.
+                if should_refresh_cached_jar {
+                    debug!(target: "providers::static_file", ?segment, ?fixed_range, "Refreshing cached jar");
+                    let jar = if let Some(jar) = loaded_jar {
+                        jar
+                    } else {
+                        NippyJar::<SegmentHeader>::load(&path).map_err(ProviderError::other)?
+                    };
+                    self.map.insert(key, LoadedJar::new(jar)?);
+                }
 
                 // Delete any cached provider that no longer has an associated jar.
                 debug!(target: "providers::static_file", ?segment, "Cleaning up jar map");
@@ -3020,11 +3042,13 @@ where
 mod tests {
     use std::collections::BTreeMap;
 
+    use alloy_consensus::Header;
+    use alloy_primitives::BlockHash;
     use reth_chain_state::EthPrimitives;
     use reth_db::test_utils::create_test_static_files_dir;
     use reth_static_file_types::{SegmentRangeInclusive, StaticFileSegment};
 
-    use crate::{providers::StaticFileProvider, StaticFileProviderBuilder};
+    use crate::{providers::StaticFileProvider, StaticFileProviderBuilder, StaticFileWriter};
 
     #[test]
     fn test_find_fixed_range_with_block_index() -> eyre::Result<()> {
@@ -3145,4 +3169,42 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn update_index_only_refreshes_cached_jars() -> eyre::Result<()> {
+        let (static_dir, _) = create_test_static_files_dir();
+        let sf_rw: StaticFileProvider<EthPrimitives> =
+            StaticFileProviderBuilder::read_write(&static_dir).with_blocks_per_file(10).build()?;
+
+        {
+            let mut writer = sf_rw.latest_writer(StaticFileSegment::Headers)?;
+            let mut header = Header::default();
+            for number in 0..=4 {
+                header.number = number;
+                writer.append_header(&header, &BlockHash::default())?;
+            }
+            writer.commit()?;
+        }
+
+        assert!(sf_rw.map.is_empty(), "uncached writes should not populate the jar cache");
+
+        {
+            let _provider =
+                sf_rw.get_segment_provider_for_block(StaticFileSegment::Headers, 0, None)?;
+        }
+        assert!(sf_rw.map.contains_key(&(9, StaticFileSegment::Headers)));
+
+        {
+            let mut writer = sf_rw.latest_writer(StaticFileSegment::Headers)?;
+            let mut header = Header::default();
+            header.number = 5;
+            writer.append_header(&header, &BlockHash::default())?;
+            writer.commit()?;
+        }
+
+        let cached = sf_rw.map.get(&(9, StaticFileSegment::Headers)).expect("cached jar exists");
+        assert_eq!(cached.user_header().block_range(), Some(SegmentRangeInclusive::new(0, 5)));
+
+        Ok(())
+    }
 }

crates/storage/provider/src/providers/static_file/writer.rs

@@ -697,6 +697,7 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
     /// Updates the `self.reader` internal index.
     fn update_index(&self) -> ProviderResult<()> {
         let segment = self.writer.user_header().segment();
+        let segment_header = self.writer.user_header();
 
         // We find the maximum block of the segment by checking this writer's last block.
         //
@@ -723,7 +724,7 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
                 prev_path.exists().then_some(prev_block)
             });
 
-        self.reader().update_index(segment, segment_max_block)
+        self.reader().update_index(segment, segment_max_block, Some(segment_header))
     }
 
     /// Ensures that the writer is positioned at the specified block number.