From 056ca8d767a4c29d5692bd6b2677ff249fe2b563 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 10:52:12 -0500
Subject: [PATCH 01/19] perf(mem_wal): index small batches inline instead of
 one thread per index
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`insert_batches_parallel` spawned one OS thread per index on every call via
`std::thread::scope`. That is sized for flush-time batches; for a handful of
rows the spawn costs more than the indexing itself.

Merge it into `insert_batches`, which now picks the path by size: inline on the
calling thread when there is a single index or the batch is at or below
`PARALLEL_INDEX_MIN_ROWS` (64) rows, and threaded above it. Atomicity is
unchanged — both paths run every task and keep only the first error.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../mem_wal/vector/mem_wal_index_micro.rs     |   4 +-
 rust/lance/src/dataset/mem_wal/index.rs       | 341 ++++++++++--------
 rust/lance/src/dataset/mem_wal/wal.rs         |   2 +-
 rust/lance/src/dataset/mem_wal/write.rs       |   2 +-
 4 files changed, 186 insertions(+), 163 deletions(-)

diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index 8bff64dfcca..bb822775b53 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -10,7 +10,9 @@
 //!    index maintained, opens `mem_wal_writer` with a `ShardWriterConfig`
 //!    sized to hold the largest checkpoint without flushing, and times
 //!    `writer.put(batch)` calls. Index updates therefore go through the
-//!    parallel `IndexStore::insert_batches_parallel` path. At each
+//!    `IndexStore::insert_batches` path, which indexes inline at or below
+//!    `PARALLEL_INDEX_MIN_ROWS` rows and spawns a thread per index above it —
+//!    so the checkpoint sizes here straddle that crossover. At each
 //!    checkpoint, queries are issued against `active_memtable_ref()` via
 //!    `MemTableScanner::nearest`.
 //!
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 8077f06149e..f7e49ca4d0f 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -58,6 +58,19 @@ use pk_key::encode_pk_batch;
 /// [`BTreeMemIndex`]'s byte backend indexes it directly.
 const PK_KEY_COLUMN: &str = "__pk_key__";
 
+/// Row count at or below which [`IndexStore::insert_batches`] indexes inline
+/// rather than spawning a thread per index.
+///
+/// The spawn is one OS thread *per index* — tens of microseconds each, and a table can
+/// carry several BTrees alongside its HNSW and FTS — so for a small batch it costs more
+/// than the indexing it parallelizes. Small batches are not the exceptional case: a
+/// durable put triggers a WAL flush covering only the batch it just inserted, so this
+/// path is routinely called with a single short batch.
+///
+/// The crossover depends on per-row HNSW cost, which varies with dimension and
+/// `ef_construction`; tune against `benches/mem_wal/vector/mem_wal_index_micro.rs`.
+const PARALLEL_INDEX_MIN_ROWS: usize = 64;
+
 /// The memtable's primary-key index, used to answer "newest visible version of
 /// this key" for dedup. Single-column PKs reuse the column's compact typed
 /// [`BTreeMemIndex`] (no second copy); composite PKs key a `BTreeMemIndex` on
@@ -724,43 +737,136 @@ impl IndexStore {
         }
     }
 
-    /// Insert multiple batches into all indexes with cross-batch optimization.
+    /// Insert multiple batches into every index.
+    ///
+    /// Above [`PARALLEL_INDEX_MIN_ROWS`] rows each index runs on its own thread, which
+    /// maximizes parallelism when several indexes are maintained. At or below it they run
+    /// inline on the calling thread: the spawn is one OS thread *per index*, and for a
+    /// handful of rows that costs more than the indexing itself.
+    ///
+    /// Returns a map of index names to their update durations for performance tracking.
     #[instrument(name = "idx_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len()))]
-    pub fn insert_batches(&self, batches: &[StoredBatch]) -> Result<()> {
+    pub fn insert_batches(
+        &self,
+        batches: &[StoredBatch],
+    ) -> Result<std::collections::HashMap<String, std::time::Duration>> {
+        use std::time::Instant;
+
         if batches.is_empty() {
-            return Ok(());
+            return Ok(std::collections::HashMap::new());
         }
 
         let track_pk_overrides = self.should_track_pk_overrides();
-        // BTree indexes: iterate batches (no cross-batch optimization benefit)
-        for index in self.btree_indexes.values() {
+
+        // One task per index, boxed so the inline and the threaded path drive the very
+        // same closures. Each reports whether it saw an already-present PK.
+        type IndexTask<'a> = Box<dyn Fn() -> Result<bool> + Send + Sync + 'a>;
+        let mut tasks: Vec<(&str, IndexTask<'_>)> = Vec::new();
+
+        for (name, index) in &self.btree_indexes {
             let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
-            let mut had_existing = false;
-            for stored in batches {
-                if track_this_index {
-                    had_existing |=
-                        index.insert_and_report_existing(&stored.data, stored.row_offset)?;
-                } else {
-                    index.insert(&stored.data, stored.row_offset)?;
-                }
-            }
-            self.mark_pk_overrides_if_needed(had_existing);
+            tasks.push((
+                name.as_str(),
+                Box::new(move || {
+                    let mut had_existing = false;
+                    for stored in batches {
+                        if track_this_index {
+                            had_existing |= index
+                                .insert_and_report_existing(&stored.data, stored.row_offset)?;
+                        } else {
+                            index.insert(&stored.data, stored.row_offset)?;
+                        }
+                    }
+                    Ok(had_existing)
+                }),
+            ));
         }
 
-        // HNSW indexes: use batched insert
-        for index in self.hnsw_indexes.values() {
-            index.insert_batches(batches)?;
+        for (name, index) in &self.hnsw_indexes {
+            tasks.push((
+                name.as_str(),
+                Box::new(move || index.insert_batches(batches).map(|_| false)),
+            ));
         }
 
-        // FTS indexes: iterate batches (potential future optimization)
-        for index in self.fts_indexes.values() {
-            for stored in batches {
-                index.insert(&stored.data, stored.row_offset)?;
+        for (name, index) in &self.fts_indexes {
+            tasks.push((
+                name.as_str(),
+                Box::new(move || {
+                    for stored in batches {
+                        index.insert(&stored.data, stored.row_offset)?;
+                    }
+                    Ok(false)
+                }),
+            ));
+        }
+
+        // Keep the raw `Duration` so sub-millisecond timings (the steady state for BTree
+        // updates) survive instead of truncating to 0.
+        let total_rows: usize = batches.iter().map(|b| b.num_rows).sum();
+        let results: Vec<(&str, std::time::Duration, Result<bool>)> =
+            if tasks.len() < 2 || total_rows <= PARALLEL_INDEX_MIN_ROWS {
+                tasks
+                    .iter()
+                    .map(|(name, task)| {
+                        let start = Instant::now();
+                        let result = task();
+                        (*name, start.elapsed(), result)
+                    })
+                    .collect()
+            } else {
+                std::thread::scope(|scope| {
+                    let handles: Vec<_> = tasks
+                        .iter()
+                        .map(|(name, task)| {
+                            let handle = scope.spawn(move || {
+                                let start = Instant::now();
+                                let result = task();
+                                (start.elapsed(), result)
+                            });
+                            (*name, handle)
+                        })
+                        .collect();
+
+                    handles
+                        .into_iter()
+                        .map(|(name, handle)| match handle.join() {
+                            Ok((duration, result)) => (name, duration, result),
+                            Err(_) => (
+                                name,
+                                std::time::Duration::ZERO,
+                                Err(Error::internal(format!("Index '{}' thread panicked", name))),
+                            ),
+                        })
+                        .collect()
+                })
+            };
+
+        // Every task ran to completion whether or not a peer failed (the threaded path
+        // joins all handles unconditionally). Keep the first error; there is no rollback,
+        // so a failure here is terminal for the writer.
+        let mut first_error: Option<Error> = None;
+        let mut had_existing_pk = false;
+        let mut duration_map =
+            std::collections::HashMap::<String, std::time::Duration>::with_capacity(results.len());
+
+        for (name, duration, result) in results {
+            duration_map.insert(name.to_string(), duration);
+            match result {
+                Ok(had_existing) => had_existing_pk |= had_existing,
+                Err(e) if first_error.is_none() => first_error = Some(e),
+                Err(_) => {}
             }
         }
 
-        // Single-column PK aliases a `btree_indexes` entry (maintained above);
-        // a composite PK has its own index, maintained here.
+        if let Some(e) = first_error {
+            return Err(e);
+        }
+        self.mark_pk_overrides_if_needed(had_existing_pk);
+
+        // Single-column PK aliases a `btree_indexes` entry — its task above already
+        // maintained it. A composite PK has its own index; maintain it here before the
+        // watermark advances so the visible prefix is fully indexed.
         let mut had_existing = false;
         for stored in batches {
             had_existing |=
@@ -772,142 +878,7 @@ impl IndexStore {
         let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
         self.advance_max_visible_batch_position(max_bp);
 
-        Ok(())
-    }
-
-    /// Insert multiple batches into all indexes in parallel.
-    ///
-    /// Each individual index runs in its own thread, regardless of type.
-    /// This maximizes parallelism when multiple indexes are maintained.
-    ///
-    /// This is used during WAL flush to parallelize index updates with WAL I/O.
-    /// Insert batches into all indexes in parallel.
-    ///
-    /// Returns a map of index names to their update durations for performance tracking.
-    #[allow(clippy::print_stderr)]
-    #[instrument(name = "idx_insert_batches_parallel", level = "debug", skip_all, fields(batch_count = batches.len()))]
-    pub fn insert_batches_parallel(
-        &self,
-        batches: &[StoredBatch],
-    ) -> Result<std::collections::HashMap<String, std::time::Duration>> {
-        use std::time::Instant;
-
-        if batches.is_empty() {
-            return Ok(std::collections::HashMap::new());
-        }
-
-        let track_pk_overrides = self.should_track_pk_overrides();
-        // Use std::thread::scope for parallel CPU-bound work
-        std::thread::scope(|scope| {
-            // Each handle returns (index_name, index_type, duration, Result)
-            let mut handles: Vec<(
-                &str,
-                &str,
-                std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result<bool>)>,
-            )> = Vec::new();
-
-            // Spawn a thread for each BTree index
-            for (name, index) in &self.btree_indexes {
-                let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = (|| {
-                        let mut had_existing = false;
-                        for stored in batches {
-                            if track_this_index {
-                                had_existing |= index
-                                    .insert_and_report_existing(&stored.data, stored.row_offset)?;
-                            } else {
-                                index.insert(&stored.data, stored.row_offset)?;
-                            }
-                        }
-                        Ok(had_existing)
-                    })();
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "btree", handle));
-            }
-
-            // Spawn a thread for each HNSW index
-            for (name, index) in &self.hnsw_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = index.insert_batches(batches).map(|_| false);
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "hnsw", handle));
-            }
-
-            // Spawn a thread for each FTS index
-            for (name, index) in &self.fts_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
-                    let start = Instant::now();
-                    let result = (|| {
-                        for stored in batches {
-                            index.insert(&stored.data, stored.row_offset)?;
-                        }
-                        Ok(false)
-                    })();
-                    (start.elapsed(), result)
-                });
-                handles.push((name.as_str(), "fts", handle));
-            }
-
-            // Collect results, log timing, and check for errors. Keep the raw
-            // `Duration` so sub-millisecond timings (the steady-state case for
-            // BTree updates) are preserved instead of getting truncated to 0.
-            let mut first_error: Option<Error> = None;
-            let mut timings: Vec<(&str, &str, std::time::Duration)> = Vec::new();
-            let mut had_existing_pk = false;
-
-            for (name, idx_type, handle) in handles {
-                match handle.join() {
-                    Ok((duration, Ok(had_existing))) => {
-                        timings.push((name, idx_type, duration));
-                        had_existing_pk |= had_existing;
-                    }
-                    Ok((duration, Err(e))) => {
-                        timings.push((name, idx_type, duration));
-                        if first_error.is_none() {
-                            first_error = Some(e);
-                        }
-                    }
-                    Err(_) => {
-                        if first_error.is_none() {
-                            first_error =
-                                Some(Error::internal(format!("Index '{}' thread panicked", name)));
-                        }
-                    }
-                }
-            }
-
-            if let Some(e) = first_error {
-                return Err(e);
-            }
-            self.mark_pk_overrides_if_needed(had_existing_pk);
-
-            let duration_map: std::collections::HashMap<String, std::time::Duration> = timings
-                .into_iter()
-                .map(|(name, _idx_type, duration)| (name.to_string(), duration))
-                .collect();
-
-            // Single-column PK aliases a `btree_indexes` entry — its thread above
-            // already maintained it (and joined). A composite PK has its own
-            // index; maintain it here before the watermark advances so the
-            // visible prefix is fully indexed.
-            let mut had_existing = false;
-            for stored in batches {
-                had_existing |=
-                    self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?;
-            }
-            self.mark_pk_overrides_if_needed(had_existing);
-
-            // Update global watermark to the max batch position
-            let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
-            self.advance_max_visible_batch_position(max_bp);
-
-            Ok(duration_map)
-        })
+        Ok(duration_map)
     }
 
     /// Get a BTree index by name.
@@ -1003,6 +974,7 @@ mod tests {
     use arrow_array::{Int32Array, StringArray};
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use log::warn;
+    use rstest::rstest;
     use std::sync::Arc;
     use uuid::Uuid;
 
@@ -1047,6 +1019,21 @@ mod tests {
         .unwrap()
     }
 
+    fn create_sized_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch {
+        let ids: Vec<i32> = (0..num_rows as i32).map(|i| start_id + i).collect();
+        let names: Vec<String> = ids.iter().map(|id| format!("name-{id}")).collect();
+        let descriptions: Vec<String> = ids.iter().map(|id| format!("hello world {id}")).collect();
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+                Arc::new(StringArray::from(descriptions)),
+            ],
+        )
+        .unwrap()
+    }
+
     fn fts_index_metadata(index_version: i32) -> IndexMetadata {
         let details =
             pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
@@ -1526,6 +1513,40 @@ mod tests {
         assert_eq!(registry.max_visible_batch_position(), 10);
     }
 
+    /// `insert_batches` picks the inline or the threaded path by row count, so
+    /// exercise both and assert they leave the same index state: every row indexed
+    /// exactly once, in every index, with a timing reported for each.
+    #[rstest]
+    #[case::inline(8)]
+    #[case::threaded(PARALLEL_INDEX_MIN_ROWS + 64)]
+    fn test_insert_batches_indexes_every_row_once(#[case] num_rows: usize) {
+        let schema = create_test_schema();
+        let mut registry = IndexStore::new();
+        registry.add_btree("id_idx".to_string(), 0, "id".to_string());
+        registry.add_fts("desc_idx".to_string(), 2, "description".to_string());
+
+        let batch = create_sized_batch(&schema, 0, num_rows);
+        let durations = registry
+            .insert_batches(&[StoredBatch::new(batch, 0, 2)])
+            .unwrap();
+
+        assert_eq!(durations.len(), 2, "expected one timing per index");
+        assert!(durations.contains_key("id_idx"));
+        assert!(durations.contains_key("desc_idx"));
+
+        let btree = registry.get_btree("id_idx").unwrap();
+        for id in 0..num_rows as i32 {
+            let positions = btree.get(&ScalarValue::Int32(Some(id)));
+            assert_eq!(
+                positions.len(),
+                1,
+                "id={id} should be indexed exactly once, got {positions:?}"
+            );
+        }
+        assert_eq!(registry.get_fts("desc_idx").unwrap().doc_count(), num_rows);
+        assert_eq!(registry.max_visible_batch_position(), 2);
+    }
+
     #[test]
     fn test_get_index_by_name_and_field_id() {
         let mut registry = IndexStore::new();
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index b4caff180e1..61546c18fe3 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -592,7 +592,7 @@ impl WalFlusher {
             let index_future = async {
                 let start = Instant::now();
                 let per_index = tokio::task::spawn_blocking(move || {
-                    idx_registry.insert_batches_parallel(&stored_batches)
+                    idx_registry.insert_batches(&stored_batches)
                 })
                 .await
                 .map_err(|e| Error::internal(format!("Index update task panicked: {}", e)))??;
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 4c1cccd5af8..8a467fa1117 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -926,7 +926,7 @@ async fn replay_memtable_from_wal(
                     stored.push(s.clone());
                 }
             }
-            tokio::task::spawn_blocking(move || indexes.insert_batches_parallel(&stored))
+            tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
                 .await
                 .map_err(|e| {
                     Error::internal(format!("WAL replay index update task panicked: {}", e))

From fe6acb6a10186086257fc9b4eb76cd8783af8c8a Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 10:55:02 -0500
Subject: [PATCH 02/19] fix(mem_wal): fail reads on a poisoned writer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`check_poisoned()` guarded the three write paths but no read path, so a writer
that had been fenced by a peer or had latched a persistence failure kept serving
scans. Those scans can hand out rows that are not durable and that replay will
not reproduce — a divergent snapshot from a shard that is already known to be
dead. Recovery is evict and reopen; until then the shard should fail closed.

Guard `scan()`, `active_memtable_ref()`, and `in_memory_memtable_refs()`,
mirroring SlateDB's `check_closed()` at the top of every read.

`memtable_stats()` is deliberately left unguarded: a poisoned writer is exactly
when an operator — and the eviction path — most needs to read its state.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/write.rs | 43 ++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 8a467fa1117..47319141b78 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -2082,6 +2082,10 @@ impl ShardWriter {
 
     /// Get current MemTable statistics. Returns an error in WAL-only mode
     /// (no MemTable exists).
+    ///
+    /// Deliberately does *not* `check_poisoned`, unlike the read and write
+    /// paths: a poisoned writer is exactly when an operator most needs to see
+    /// its state, and the caller deciding whether to evict reads these stats.
     pub async fn memtable_stats(&self) -> Result<MemTableStats> {
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
@@ -2110,8 +2114,9 @@ impl ShardWriter {
     /// The scanner captures the current `max_visible_batch_position` from the
     /// `IndexStore` at construction time to ensure consistent visibility.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn scan(&self) -> Result<MemTableScanner> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(state.memtable.scan())
@@ -2121,10 +2126,11 @@ impl ShardWriter {
     /// Prefer [`Self::in_memory_memtable_refs`] on the read path — it also
     /// carries frozen-awaiting-flush generations.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn active_memtable_ref(
         &self,
     ) -> Result<crate::dataset::mem_wal::scanner::InMemoryMemTableRef> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(in_memory_ref(&state.memtable))
@@ -2136,10 +2142,11 @@ impl ShardWriter {
     /// path uses this instead of [`Self::active_memtable_ref`] so a
     /// concurrent reader sees no hole while a flush drains.
     ///
-    /// Returns an error in WAL-only mode.
+    /// Returns an error in WAL-only mode, or if the writer is poisoned.
     pub async fn in_memory_memtable_refs(
         &self,
     ) -> Result<crate::dataset::mem_wal::scanner::InMemoryMemTables> {
+        self.wal_flusher.check_poisoned()?;
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         Ok(crate::dataset::mem_wal::scanner::InMemoryMemTables {
@@ -4783,8 +4790,9 @@ mod tests {
     }
 
     // A durable write whose WAL PUT keeps failing poisons the writer with a
-    // typed persistence failure; the next write fails fast with the same reason;
-    // and once storage heals, reopening replays the WAL and writes resume.
+    // typed persistence failure; the next write *and every read* fail fast with
+    // the same reason; and once storage heals, reopening replays the WAL and
+    // writes resume.
     #[tokio::test]
     async fn test_writer_poisons_on_persistence_failure_and_recovers_on_reopen() {
         let (store, base_path, controls) = failing_memory_store().await;
@@ -4823,6 +4831,31 @@ mod tests {
             .await
             .unwrap_err();
         assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // ...and rejects *reads* too. Batch 0 was committed to the BatchStore
+        // before its WAL PUT failed, so a poisoned writer that still served
+        // reads would hand out a row that is not durable and that replay will
+        // not reproduce — a divergent snapshot. Mirrors SlateDB's
+        // `check_closed()` at the top of every read.
+        for reason in [
+            writer.scan().await.err().and_then(|e| e.fence_reason()),
+            writer
+                .active_memtable_ref()
+                .await
+                .err()
+                .and_then(|e| e.fence_reason()),
+            writer
+                .in_memory_memtable_refs()
+                .await
+                .err()
+                .and_then(|e| e.fence_reason()),
+        ] {
+            assert_eq!(reason, Some(FenceReason::PersistenceFailure));
+        }
+
+        // Stats stay readable: this is what an operator (and the eviction path)
+        // inspects to decide what to do about the poisoned shard.
+        writer.memtable_stats().await.unwrap();
         drop(writer);
 
         // Storage heals: reopening replays the WAL and accepts writes again.

From 4f6b7db76b1eb43172902a36d495e98a1396e54e Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 10:56:56 -0500
Subject: [PATCH 03/19] fix(mem_wal): mark replayed batches durable so the next
 flush does not re-append
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

After `replay_memtable_from_wal` rehydrated a memtable, the durability cursor
stayed at "nothing flushed". The next WAL flush therefore re-covered `[0, end)`:
it appended the already-durable rows to the WAL a second time *and* re-inserted
every replayed row into the in-memory indexes.

None of the three indexes is idempotent. HNSW mints fresh node ids for the same
row, so KNN returns it twice and burns two of the `k` slots. FTS increments
`doc_count`, `total_tokens`, and every term's `df` rather than recomputing them,
corrupting BM25 for the whole memtable. BTree is a multiset whose second insert
sets `pk_has_overrides` permanently, disabling HNSW plan selection and WAND
pruning. A full scan kept looking healthy while every index-accelerated query
silently returned duplicates — and it compounded, because the WAL now held those
rows twice, so the next crash replayed both copies.

Stamp the durability cursor at the end of replay: the batches came from the WAL,
and `insert_batches` has just re-derived the indexes over them. The index cursor
already advanced itself; only durability was missing.

Regression test reproduces the original report: reopen + replay + one 2-row put
grew the WAL from 8 rows to 18 instead of to 10.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/write.rs | 118 ++++++++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 47319141b78..df873f2099f 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -934,6 +934,25 @@ async fn replay_memtable_from_wal(
         }
     }
 
+    // Mark the replayed batches durable. They came *from* the WAL, and
+    // `insert_batches` above just re-derived the indexes over them, so both
+    // cursors are satisfied — but only the index one advances itself.
+    //
+    // Without this stamp the durability cursor stays at "nothing flushed", so
+    // the next WAL flush re-covers [0, end): it re-appends already-durable rows
+    // to the WAL *and* re-inserts every replayed row into the indexes. None of
+    // the three in-memory indexes is idempotent (HNSW mints fresh node ids for
+    // the same row, FTS increments doc_count/df instead of recomputing them,
+    // BTree is a multiset), so a full scan keeps looking healthy while every
+    // index-accelerated query silently returns duplicates. It compounds, too:
+    // the WAL now holds those rows twice, so the next crash replays both copies.
+    let replayed_batches = memtable.batch_count();
+    if replayed_batches > 0 {
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(replayed_batches - 1);
+    }
+
     Ok(position)
 }
 
@@ -4931,6 +4950,105 @@ mod tests {
         writer_b.close().await.unwrap();
     }
 
+    /// Replayed batches are already WAL-durable, so the first flush after a
+    /// reopen must not re-append them to the WAL or re-insert them into the
+    /// indexes. Before replay stamped the durability cursor it stayed at
+    /// "nothing flushed", so the next flush re-covered `[0, end)`: it appended
+    /// the already-durable rows a second time *and* re-indexed them. None of
+    /// the in-memory indexes is idempotent, so an indexed PK lookup returned
+    /// the row twice while a full scan still looked healthy.
+    #[tokio::test]
+    async fn test_replay_does_not_reappend_or_reindex() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Writer A: two durable batches (5 + 3 = 8 rows), dropped without close.
+        {
+            let writer_a = ShardWriter::open(
+                store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                memtable_config_with_pk(shard_id),
+                schema.clone(),
+                vec![],
+            )
+            .await
+            .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 0, 5)])
+                .await
+                .unwrap();
+            writer_a
+                .put(vec![create_test_batch(&schema, 100, 3)])
+                .await
+                .unwrap();
+        }
+
+        // Writer B reopens and replays A's two batches.
+        let writer_b = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        let stats = writer_b.memtable_stats().await.unwrap();
+        assert_eq!(stats.batch_count, 2);
+        assert_eq!(
+            stats.max_flushed_batch_position,
+            Some(1),
+            "replayed batches came from the WAL, so the durability cursor must already cover them"
+        );
+
+        // One more durable put. Its flush must cover only the new batch.
+        writer_b
+            .put(vec![create_test_batch(&schema, 200, 2)])
+            .await
+            .unwrap();
+
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut wal_rows = 0;
+        for position in first..next {
+            if let Some(entry) = tailer.read_entry(position).await.unwrap() {
+                wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::<usize>();
+            }
+        }
+        assert_eq!(
+            wal_rows, 10,
+            "WAL must hold 8 replayed + 2 new rows; a re-covering flush re-appends the replayed 8"
+        );
+
+        // The indexed arm must not see a replayed row twice.
+        let mut scanner = writer_b.scan().await.unwrap();
+        scanner.filter("id = 0").unwrap();
+        let hit = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            hit.num_rows(),
+            1,
+            "indexed PK lookup returned the replayed row more than once"
+        );
+
+        let all = writer_b
+            .scan()
+            .await
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(all.num_rows(), 10);
+
+        writer_b.close().await.unwrap();
+    }
+
     /// Replay is a no-op on a fresh shard: the MemTable starts empty.
     #[tokio::test]
     async fn test_memtable_replay_no_op_on_fresh_shard() {

From 0745e28ec1f2b538c1dd1e4a80ce7a7187e190d6 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 10:59:02 -0500
Subject: [PATCH 04/19] fix(mem_wal): reject index configs that disagree with
 the schema at shard open
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An in-memory index configured on a column that cannot support it fails
deterministically on every insert — including inserts replayed from the WAL. So
once a row is durable the shard can never reopen: replay re-reads the same rows,
hits the same error, and `open()` propagates it. That is a permanently down
shard, and it is the failure mode that makes poison-and-replay non-terminating.

Validate once at open, before a single row is accepted: FTS columns must be
Utf8/LargeUtf8/Utf8View, HNSW columns must be FixedSizeList<Float32> with a
non-zero dimension, every index column must exist, and composite primary-key
columns must have an order-preserving key encoding. BTree needs only existence —
its backend falls through to per-row `ScalarValue` extraction and accepts any
type the schema can hold.

Also close two related gaps:

- FTS silently appended an empty batch when its column was missing from the
  batch, so a misconfigured index stayed empty forever while the shard reported
  healthy. It now errors, matching BTree and HNSW.
- HNSW's runtime capacity-exhaustion errors were labelled `invalid_input`, which
  blames the caller for a well-formed batch. They mean the index was sized below
  what the memtable holds, so they are `internal`.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/hnsw/graph.rs  |   7 +-
 .../lance/src/dataset/mem_wal/hnsw/storage.rs |  17 +-
 rust/lance/src/dataset/mem_wal/index.rs       | 237 ++++++++++++++++++
 rust/lance/src/dataset/mem_wal/index/fts.rs   |  21 +-
 rust/lance/src/dataset/mem_wal/write.rs       |  47 ++++
 5 files changed, 310 insertions(+), 19 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
index 4009557702c..570122581b7 100644
--- a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
+++ b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs
@@ -594,9 +594,12 @@ impl HnswGraph {
     }
 
     fn validate_source(&self, vectors: &impl VectorSource, needed_len: usize) -> Result<()> {
+        // Not caller input: the graph was sized below what the memtable holds.
+        // See the matching note in `storage.rs::append_batch`.
         if needed_len > self.nodes.len() {
-            return Err(Error::invalid_input(format!(
-                "graph capacity {} exhausted: need {needed_len}",
+            return Err(Error::internal(format!(
+                "HNSW graph capacity {} exhausted: need {needed_len}; \
+                 the graph is sized below the memtable's row capacity",
                 self.nodes.len()
             )));
         }
diff --git a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
index bbeb57a5fe2..543c1172500 100644
--- a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
+++ b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
@@ -269,24 +269,31 @@ impl ArrowFixedSizeListVectorStore {
             )));
         };
 
+        // Exhaustion is not a caller-input problem — the batch is well-formed and
+        // has already passed the memtable's schema gate. It means the HNSW store
+        // was sized below what the memtable will hold before it flushes, which is
+        // a shard-construction bug. `invalid_input` mislabelled it as the writer's
+        // fault and hid that.
         let start = self.committed_len.load(Ordering::Relaxed);
         let end = start.checked_add(num_rows).ok_or_else(|| {
-            Error::invalid_input(format!(
+            Error::internal(format!(
                 "vector count overflow: start={}, batch_len={}",
                 start, num_rows
             ))
         })?;
         if end > self.capacity {
-            return Err(Error::invalid_input(format!(
-                "capacity {} exhausted: inserting rows [{}..{})",
+            return Err(Error::internal(format!(
+                "HNSW vector store capacity {} exhausted: inserting rows [{}..{}); \
+                 the store is sized below the memtable's row capacity",
                 self.capacity, start, end
             )));
         }
 
         let batch_idx = self.committed_batches.load(Ordering::Relaxed);
         if batch_idx >= self.max_batches {
-            return Err(Error::invalid_input(format!(
-                "max_batches {} exhausted",
+            return Err(Error::internal(format!(
+                "HNSW vector store max_batches {} exhausted; \
+                 the store is sized below the memtable's batch capacity",
                 self.max_batches
             )));
         }
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index f7e49ca4d0f..0df3c2499e2 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -28,6 +28,7 @@ use datafusion::common::ScalarValue;
 
 use super::memtable::batch_store::StoredBatch;
 use arrow_array::RecordBatch;
+use arrow_schema::{DataType, Schema as ArrowSchema};
 use lance_core::datatypes::Schema as LanceSchema;
 use lance_core::{Error, Result};
 use lance_index::pbold;
@@ -93,6 +94,143 @@ enum PkIndex {
 // Index Store
 // ============================================================================
 
+/// Validate every configured in-memory index, and the composite primary key,
+/// against the shard schema. Call once at shard open, before any write can land.
+///
+/// This is what makes poison-and-replay *terminating*. An index insert that
+/// fails deterministically on a row that is already WAL-durable cannot be
+/// recovered from: the writer poisons, the operator reopens, replay re-reads the
+/// same WAL rows, the same insert fails again, and `open()` propagates it — a
+/// shard that never comes back. Every such failure is an index *config*
+/// disagreeing with the schema, never a property of the data, so one pass here
+/// closes the whole class before a single row is accepted.
+///
+/// The data-dependent errors inside the index layer are already unreachable
+/// through `put`: `MemTable::insert_batches_only` does a full `Arc<Schema>`
+/// equality check, so a batch that would trip one is rejected before it reaches
+/// the batch store, let alone the WAL.
+pub fn validate_index_configs(
+    configs: &[MemIndexConfig],
+    schema: &ArrowSchema,
+    pk_columns: &[String],
+) -> Result<()> {
+    for config in configs {
+        let column = config.column();
+        let field = schema.field_with_name(column).map_err(|_| {
+            Error::invalid_input(format!(
+                "index '{}' is configured on column '{}', which is not in the shard schema; \
+                 available columns: [{}]",
+                config.name(),
+                column,
+                schema
+                    .fields()
+                    .iter()
+                    .map(|f| f.name().as_str())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ))
+        })?;
+
+        match config {
+            // BTree falls back to per-row `ScalarValue` extraction, so it
+            // accepts any column type the schema can hold. Existence is the
+            // only precondition.
+            MemIndexConfig::BTree(_) => {}
+            MemIndexConfig::Fts(_) => {
+                if !matches!(
+                    field.data_type(),
+                    DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+                ) {
+                    return Err(Error::invalid_input(format!(
+                        "FTS index '{}' requires a Utf8, LargeUtf8, or Utf8View column; \
+                         column '{}' is {:?}",
+                        config.name(),
+                        column,
+                        field.data_type()
+                    )));
+                }
+            }
+            MemIndexConfig::Hnsw(_) => match field.data_type() {
+                DataType::FixedSizeList(item, dim) => {
+                    if item.data_type() != &DataType::Float32 {
+                        return Err(Error::invalid_input(format!(
+                            "HNSW index '{}' requires a FixedSizeList<Float32> column; \
+                             column '{}' has item type {:?}",
+                            config.name(),
+                            column,
+                            item.data_type()
+                        )));
+                    }
+                    // `HnswMemIndex.dim` is a placeholder until the first batch
+                    // pins it (`hnsw.rs`), so a zero-width vector would only
+                    // surface at insert time — i.e. on already-durable data.
+                    if *dim <= 0 {
+                        return Err(Error::invalid_input(format!(
+                            "HNSW index '{}' requires a vector dimension > 0; column '{}' has \
+                             dimension {dim}",
+                            config.name(),
+                            column,
+                        )));
+                    }
+                }
+                other => {
+                    return Err(Error::invalid_input(format!(
+                        "HNSW index '{}' requires a FixedSizeList<Float32> column; \
+                         column '{}' is {:?}",
+                        config.name(),
+                        column,
+                        other
+                    )));
+                }
+            },
+        }
+    }
+
+    // A single-column PK aliases a BTree entry (any type). Only a *composite* PK
+    // builds an order-preserving encoded key, and only some types encode.
+    if pk_columns.len() > 1 {
+        for column in pk_columns {
+            let field = schema.field_with_name(column).map_err(|_| {
+                Error::invalid_input(format!(
+                    "primary-key column '{column}' is not in the shard schema"
+                ))
+            })?;
+            if !is_encodable_pk_type(field.data_type()) {
+                return Err(Error::invalid_input(format!(
+                    "composite primary-key column '{column}' has type {:?}, which has no \
+                     order-preserving key encoding",
+                    field.data_type()
+                )));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Types `pk_key::encode_value` can encode into an order-preserving composite key.
+fn is_encodable_pk_type(data_type: &DataType) -> bool {
+    matches!(
+        data_type,
+        DataType::Int8
+            | DataType::Int16
+            | DataType::Int32
+            | DataType::Int64
+            | DataType::UInt8
+            | DataType::UInt16
+            | DataType::UInt32
+            | DataType::UInt64
+            | DataType::Date32
+            | DataType::Date64
+            | DataType::Boolean
+            | DataType::Utf8
+            | DataType::LargeUtf8
+            | DataType::Binary
+            | DataType::LargeBinary
+            | DataType::FixedSizeBinary(_)
+    )
+}
+
 /// Configuration for an index in MemWAL.
 ///
 /// Each variant contains all the configuration needed for that index type.
@@ -1479,6 +1617,105 @@ mod tests {
         assert!(registry.get_fts_by_field_id(2).is_some());
     }
 
+    fn vector_schema() -> Arc<ArrowSchema> {
+        Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("description", DataType::Utf8, true),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+            Field::new(
+                "f64_vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), 4),
+                true,
+            ),
+        ]))
+    }
+
+    /// Every index config that would fail *deterministically* on insert must be
+    /// rejected at open instead. Such a config also fails on WAL replay, so once
+    /// a row is durable the shard could never reopen — poison-and-replay would
+    /// not terminate.
+    #[rstest]
+    #[case::btree_ok(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 0, column: "id".into(),
+    }), None)]
+    #[case::btree_missing_column(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 9, column: "nope".into(),
+    }), Some("not in the shard schema"))]
+    #[case::fts_ok(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 1, "description".into(),
+    )), None)]
+    #[case::fts_non_utf8(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 0, "id".into(),
+    )), Some("requires a Utf8, LargeUtf8, or Utf8View column"))]
+    #[case::fts_missing_column(MemIndexConfig::Fts(FtsIndexConfig::new(
+        "idx".into(), 9, "nope".into(),
+    )), Some("not in the shard schema"))]
+    #[case::hnsw_ok(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 2, "vector".into(), DistanceType::L2,
+    ))), None)]
+    #[case::hnsw_not_a_vector(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 0, "id".into(), DistanceType::L2,
+    ))), Some("requires a FixedSizeList<Float32> column"))]
+    #[case::hnsw_wrong_item_type(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 3, "f64_vector".into(), DistanceType::L2,
+    ))), Some("item type Float64"))]
+    #[case::hnsw_missing_column(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new(
+        "idx".into(), 9, "nope".into(), DistanceType::L2,
+    ))), Some("not in the shard schema"))]
+    fn test_validate_index_configs(
+        #[case] config: MemIndexConfig,
+        #[case] expected_error: Option<&str>,
+    ) {
+        let schema = vector_schema();
+        let result = validate_index_configs(&[config], &schema, &[]);
+        match expected_error {
+            None => result.expect("valid config must pass validation"),
+            Some(fragment) => {
+                let message = result
+                    .expect_err("invalid config must be rejected")
+                    .to_string();
+                assert!(
+                    message.contains(fragment),
+                    "error must explain the mismatch; wanted {fragment:?}, got {message:?}"
+                );
+            }
+        }
+    }
+
+    /// A composite PK builds an order-preserving encoded key, so its columns must
+    /// be encodable. A single-column PK aliases a BTree entry, which accepts any
+    /// type — so it must *not* be rejected here.
+    #[test]
+    fn test_validate_composite_pk_column_types() {
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new(
+                "coords",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+
+        validate_index_configs(&[], &schema, &["id".into(), "name".into()])
+            .expect("Int32 + Utf8 composite PK must be encodable");
+
+        let err = validate_index_configs(&[], &schema, &["id".into(), "coords".into()])
+            .expect_err("a FixedSizeList PK column has no order-preserving encoding");
+        assert!(
+            err.to_string().contains("order-preserving key encoding"),
+            "error must name the reason, got {err}"
+        );
+
+        // A single-column PK of the same type is fine: it aliases a BTree.
+        validate_index_configs(&[], &schema, &["coords".into()])
+            .expect("single-column PK aliases a BTree and accepts any type");
+    }
+
     #[test]
     fn test_index_store_max_visible_batch_position() {
         let schema = create_test_schema();
diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 6e0d12a276e..04b07fcbda2 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -1098,23 +1098,20 @@ impl FtsMemIndex {
         let st = self.state.load_full();
         let batch_position = st.tail.next_position();
 
+        // A missing column is a config error, not an empty document: silently
+        // appending an empty batch would leave the index empty forever while the
+        // shard reported healthy. BTree and HNSW both reject this; so do we.
+        // `validate_index_configs` catches it at open, so reaching here means a
+        // batch got past the memtable's schema-equality gate.
         let Some(col_idx) = batch
             .schema()
             .column_with_name(&self.column_name)
             .map(|(idx, _)| idx)
         else {
-            // Column missing: nothing to index, but publish an empty batch so
-            // the tail's visibility counters keep up with the writer.
-            st.tail.append_batch(
-                batch_position,
-                row_offset,
-                batch.num_rows() as u32,
-                vec![0; batch.num_rows()],
-                0,
-                FxHashMap::default(),
-                self.params.has_positions(),
-            );
-            return Ok(());
+            return Err(Error::invalid_input(format!(
+                "FTS index column '{}' is not in the inserted batch schema",
+                self.column_name
+            )));
         };
 
         let column = batch.column(col_idx);
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index df873f2099f..72a24180bbc 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -37,6 +37,7 @@ use uuid::Uuid;
 
 pub use super::index::{
     BTreeIndexConfig, BTreeMemIndex, FtsIndexConfig, HnswIndexConfig, IndexStore, MemIndexConfig,
+    validate_index_configs,
 };
 pub use super::memtable::CacheConfig;
 pub use super::memtable::MemTable;
@@ -1495,6 +1496,13 @@ impl ShardWriter {
         let pk_fields = lance_schema.unenforced_primary_key();
         let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
         let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
+
+        // Reject an index config that disagrees with the schema *before* a
+        // single row is accepted. Such a config fails deterministically on every
+        // insert, including inserts replayed from the WAL — so once a row is
+        // durable the shard can never reopen. Fail the open instead.
+        validate_index_configs(index_configs, schema.as_ref(), &pk_columns)?;
+
         let mut memtable = MemTable::with_capacity(
             schema.clone(),
             manifest.current_generation,
@@ -5049,6 +5057,45 @@ mod tests {
         writer_b.close().await.unwrap();
     }
 
+    /// An index config that disagrees with the schema fails `open()` outright.
+    /// It must not be allowed to accept writes: the insert would fail
+    /// deterministically on every batch, including batches replayed from the
+    /// WAL, so once a row was durable the shard could never reopen. Before this
+    /// check, an FTS index on a non-Utf8 column silently indexed nothing and the
+    /// shard reported healthy.
+    #[tokio::test]
+    async fn test_open_rejects_index_config_that_disagrees_with_schema() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // `id` is Int32, not a string column.
+        let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new(
+            "bad_fts".to_string(),
+            0,
+            "id".to_string(),
+        ));
+
+        let Err(err) = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema,
+            vec![bad_fts],
+        )
+        .await
+        else {
+            panic!("open must reject an FTS index on a non-Utf8 column");
+        };
+
+        let message = err.to_string();
+        assert!(
+            message.contains("bad_fts") && message.contains("Utf8"),
+            "error must name the index and the constraint, got: {message}"
+        );
+    }
+
     /// Replay is a no-op on a fresh shard: the MemTable starts empty.
     #[tokio::test]
     async fn test_memtable_replay_no_op_on_fresh_shard() {

From aea043aa11d59847ea2afc20b5a356780ce5a2b2 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 11:10:36 -0500
Subject: [PATCH 05/19] refactor(mem_wal): delete dead WAL-tracking state from
 MemTable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`mark_wal_flushed` had no production callers. The two collections it maintained
— `wal_batch_mapping` and `flushed_batch_positions` — were allocated per memtable
and read only from tests, as was `MemTable::last_flushed_wal_entry_position`
(production tracks that on `WriterState`, a different struct). `BatchStore::
is_wal_flush_complete` had no callers at all.

What the L0 flush actually gates on is `MemTable::all_flushed_to_wal()`, which
derives from the batch store's durability watermark and stays. The tests were
calling `mark_wal_flushed` only to satisfy that precondition, so they now set the
watermark directly — the same thing production does, instead of a parallel
bookkeeping path that only tests could reach.

The two tests that covered nothing but the deleted mapping are replaced by one
that covers the surviving behaviour: `all_flushed_to_wal()` flips only once the
watermark covers every committed batch.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../mem_wal/vector/mem_wal_index_micro.rs     |  8 +-
 rust/lance/src/dataset/mem_wal/memtable.rs    | 95 ++++---------------
 .../dataset/mem_wal/memtable/batch_store.rs   |  6 --
 .../src/dataset/mem_wal/memtable/flush.rs     | 36 +++++--
 4 files changed, 48 insertions(+), 97 deletions(-)

diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index bb822775b53..88e2151a5fa 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -377,12 +377,14 @@ async fn measure_flush(
     memtable.set_indexes(registry);
 
     let total_batches = cp.div_ceil(batch_size);
-    for (wal_pos, i) in (0_u64..).zip(0..total_batches) {
+    for i in 0..total_batches {
         let start = (i * batch_size) as i64;
         let rows = batch_size.min(cp - i * batch_size);
         let batch = make_batch(start, rows, dim);
         let frag_id = memtable.insert(batch).await?;
-        memtable.mark_wal_flushed(&[frag_id], wal_pos + 1, &[i]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
     }
 
     let temp_dir =
@@ -408,7 +410,7 @@ async fn measure_flush(
     let flusher = MemTableFlusher::new(store, base_path, uri, shard_id, manifest_store);
 
     // total_batches WAL entries were stamped at positions 1..=total_batches
-    // by the mark_wal_flushed loop above (1-based positions).
+    // by the loop above (1-based positions).
     let covered_wal_entry_position = total_batches as u64;
     let t = Instant::now();
     let _result = flusher
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index 77611fd7c47..e30d01720bf 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -7,7 +7,6 @@ pub mod batch_store;
 pub mod flush;
 pub mod scanner;
 
-use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -79,13 +78,6 @@ pub struct MemTable {
     /// Generation number (incremented on flush).
     generation: u64,
 
-    /// WAL batch mapping: batch_position -> (wal_entry_position, position within WAL entry).
-    wal_batch_mapping: HashMap<usize, (u64, usize)>,
-    /// Last WAL entry position that has been flushed.
-    last_flushed_wal_entry_position: u64,
-    /// Set of batch IDs that have been flushed to WAL.
-    flushed_batch_positions: HashSet<usize>,
-
     /// Primary key bloom filter for staleness detection.
     pk_bloom_filter: Sbbf,
     /// Primary key field IDs (for bloom filter updates).
@@ -220,9 +212,6 @@ impl MemTable {
             cache_config,
             cached_dataset: RwLock::new(None),
             generation,
-            wal_batch_mapping: HashMap::new(),
-            last_flushed_wal_entry_position: 0,
-            flushed_batch_positions: HashSet::new(),
             pk_bloom_filter,
             pk_field_ids,
             // Initialize with an empty IndexStore so the visibility cursor has
@@ -561,30 +550,6 @@ impl MemTable {
         Ok(())
     }
 
-    /// Mark batches as flushed to WAL.
-    ///
-    /// Updates the WAL batch mapping for use during MemTable flush.
-    /// Also updates the batch_store's watermark to the highest flushed batch_position.
-    pub fn mark_wal_flushed(
-        &mut self,
-        batch_positions: &[usize],
-        wal_entry_position: u64,
-        positions: &[usize],
-    ) {
-        for (idx, &batch_position) in batch_positions.iter().enumerate() {
-            self.wal_batch_mapping
-                .insert(batch_position, (wal_entry_position, positions[idx]));
-            self.flushed_batch_positions.insert(batch_position);
-        }
-        self.last_flushed_wal_entry_position = wal_entry_position;
-
-        // Update batch_store watermark to the highest batch_position flushed (inclusive)
-        if let Some(&max_batch_position) = batch_positions.iter().max() {
-            self.batch_store
-                .set_max_flushed_batch_position(max_batch_position);
-        }
-    }
-
     /// Get or create a Dataset for reading.
     ///
     /// Uses caching based on the configured eventual consistency strategy:
@@ -726,16 +691,6 @@ impl MemTable {
         self.batch_store.estimated_bytes() + self.pk_bloom_filter.estimated_memory_size()
     }
 
-    /// Get the WAL batch mapping.
-    pub fn wal_batch_mapping(&self) -> &HashMap<usize, (u64, usize)> {
-        &self.wal_batch_mapping
-    }
-
-    /// Get the last flushed WAL entry position.
-    pub fn last_flushed_wal_entry_position(&self) -> u64 {
-        self.last_flushed_wal_entry_position
-    }
-
     /// Get the bloom filter for serialization.
     pub fn bloom_filter(&self) -> &Sbbf {
         &self.pk_bloom_filter
@@ -762,14 +717,6 @@ impl MemTable {
         self.batch_store.pending_wal_flush_count() == 0
     }
 
-    /// Get unflushed batch IDs.
-    pub fn unflushed_batch_positions(&self) -> Vec<usize> {
-        let batch_count = self.batch_count();
-        (0..batch_count)
-            .filter(|id| !self.flushed_batch_positions.contains(id))
-            .collect()
-    }
-
     /// Get cache configuration.
     pub fn cache_config(&self) -> &CacheConfig {
         &self.cache_config
@@ -931,29 +878,11 @@ mod tests {
         assert_eq!(total_rows, 15);
     }
 
+    /// `all_flushed_to_wal()` is the L0 flush's precondition (`flush.rs:171`):
+    /// false while any committed batch is still un-appended, true once the
+    /// durability watermark covers every one of them.
     #[tokio::test]
-    async fn test_memtable_wal_mapping() {
-        let schema = create_test_schema();
-        let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
-
-        let batch_position = memtable
-            .insert(create_test_batch(&schema, 10))
-            .await
-            .unwrap();
-        assert!(!memtable.all_flushed_to_wal());
-
-        memtable.mark_wal_flushed(&[batch_position], 5, &[0]);
-
-        assert!(memtable.all_flushed_to_wal());
-        assert_eq!(
-            memtable.wal_batch_mapping().get(&batch_position),
-            Some(&(5, 0))
-        );
-        assert_eq!(memtable.last_flushed_wal_entry_position(), 5);
-    }
-
-    #[tokio::test]
-    async fn test_memtable_unflushed_batches() {
+    async fn test_all_flushed_to_wal_tracks_the_durability_watermark() {
         let schema = create_test_schema();
         let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
 
@@ -965,12 +894,20 @@ mod tests {
             .insert(create_test_batch(&schema, 5))
             .await
             .unwrap();
+        assert!(!memtable.all_flushed_to_wal());
 
-        assert_eq!(memtable.unflushed_batch_positions(), vec![batch1, batch2]);
-
-        memtable.mark_wal_flushed(&[batch1], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(batch1);
+        assert!(
+            !memtable.all_flushed_to_wal(),
+            "batch2 is still waiting on its WAL append"
+        );
 
-        assert_eq!(memtable.unflushed_batch_positions(), vec![batch2]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(batch2);
+        assert!(memtable.all_flushed_to_wal());
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
index 30e47e1a9bb..ff514e6bcfe 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
@@ -478,12 +478,6 @@ impl BatchStore {
         }
     }
 
-    /// Check if all committed batches have been WAL-flushed.
-    #[inline]
-    pub fn is_wal_flush_complete(&self) -> bool {
-        self.pending_wal_flush_count() == 0
-    }
-
     /// Get the range of batch IDs pending WAL flush: [start, end).
     /// Returns None if nothing pending.
     #[inline]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index de0df7f0b28..75a5f1cfc92 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -1319,7 +1319,9 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
         assert!(memtable.all_flushed_to_wal());
 
         let flusher = MemTableFlusher::new(
@@ -1383,7 +1385,9 @@ mod tests {
             .insert(create_test_batch(&schema, 10))
             .await
             .unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
         let warmer: Arc<dyn GenerationWarmer> = Arc::new(CountingWarmer {
@@ -1444,7 +1448,9 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1547,7 +1553,9 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1647,7 +1655,9 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1741,7 +1751,9 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1873,7 +1885,9 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -2010,7 +2024,9 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -2160,7 +2176,9 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+        memtable
+            .batch_store()
+            .set_max_flushed_batch_position(frag_id);
 
         let flusher = MemTableFlusher::new(
             store.clone(),

From e2a76babc8c897ca19dbdf4167775cf8152d3900 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 11:47:01 -0500
Subject: [PATCH 06/19] fix(mem_wal): make the WAL durability cursor
 writer-global
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The durable watermark was meaningless across a memtable rotation. `WalFlusher`
is built once per writer and its watch channel is never reset, but the targets
put against it were *memtable-local* batch positions — and positions restart at
0 in every new memtable.

So after memtable A flushed N batches (watermark = N), the first put into
memtable B targeted position 1, saw N >= 1, and acked immediately with no WAL
append at all. The first N puts into every post-rotation memtable were falsely
acked as durable: `durable_write: true` silently degraded to non-durable. When
B's append finally landed it sent a *smaller* value, walking the watermark
backwards — so a watcher from A targeting N could miss N entirely and block
until some later memtable climbed back to it. A hang, not just a stale read.

Fix the coordinate system rather than the symptom:

- `BatchStore` carries an immutable `global_offset`, stamped at freeze from the
  outgoing store's `global_end()`. It is a coordinate, not a cursor: it cannot
  restart and cannot move backwards.
- The durability cursor moves onto `WalFlusher` as a writer-global exclusive
  count, and is the only thing the watch channel carries. It advances
  monotonically (`max`), so an out-of-order completion cannot walk it back.
- `BatchStore::local_end(global_cursor)` is the single place the global-to-local
  subtraction is written. It saturates in both directions, and both are reachable:
  a cursor below a store's offset means "nothing here yet" — the ordinary state of
  a freshly rotated memtable — and open-coding the subtraction underflows there,
  which in release wraps to a huge end and makes the entire new memtable visible
  at once.
- The per-memtable `max_flushed_batch_position` (inclusive, `usize::MAX`-sentinel)
  is deleted. `pending_wal_flush_*` and `all_flushed_to_wal` now derive from the
  global cursor.

Two things fell out while wiring it up:

- The put path captured `batch_store` *after* the freeze check that may rotate
  the memtable, pairing the new store with the old store's end position. Capture
  it before, next to the insert that produced those positions.
- `flush_memtable` sampled the cursor before awaiting the WAL-append completion
  it depends on, so the L0 precondition saw a stale value. Sample it after.

Regression test: with a local target, a post-rotation put acks at durable=2 while
its own batch spans [2, 3) — acked, never appended.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../mem_wal/vector/mem_wal_index_micro.rs     |  15 +-
 .../mem_wal_shard_writer_backpressure.rs      |   4 +-
 rust/lance/src/dataset/mem_wal/memtable.rs    |  53 +++--
 .../dataset/mem_wal/memtable/batch_store.rs   | 101 ++++-----
 .../src/dataset/mem_wal/memtable/flush.rs     |  71 +++---
 rust/lance/src/dataset/mem_wal/wal.rs         |  85 ++++---
 rust/lance/src/dataset/mem_wal/write.rs       | 210 ++++++++++++++----
 7 files changed, 343 insertions(+), 196 deletions(-)

diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index 88e2151a5fa..f7b2ac4e709 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -381,10 +381,7 @@ async fn measure_flush(
         let start = (i * batch_size) as i64;
         let rows = batch_size.min(cp - i * batch_size);
         let batch = make_batch(start, rows, dim);
-        let frag_id = memtable.insert(batch).await?;
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let _frag_id = memtable.insert(batch).await?;
     }
 
     let temp_dir =
@@ -412,9 +409,17 @@ async fn measure_flush(
     // total_batches WAL entries were stamped at positions 1..=total_batches
     // by the loop above (1-based positions).
     let covered_wal_entry_position = total_batches as u64;
+    // Every batch was appended above, so the writer-global durable count is all of them.
+    let durable = total_batches;
     let t = Instant::now();
     let _result = flusher
-        .flush_with_indexes(&memtable, epoch, index_configs, covered_wal_entry_position)
+        .flush_with_indexes(
+            &memtable,
+            epoch,
+            index_configs,
+            covered_wal_entry_position,
+            durable,
+        )
         .await?;
     let elapsed = t.elapsed();
 
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
index f4e4fb47395..56915fb205d 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
@@ -650,7 +650,7 @@ fn push_sample(
         "active_memtable_bytes": memtable.as_ref().map(|stats| stats.estimated_size),
         "active_memtable_generation": memtable.as_ref().map(|stats| stats.generation),
         "active_memtable_max_buffered_batch_position": memtable.as_ref().and_then(|stats| stats.max_buffered_batch_position),
-        "active_memtable_max_flushed_batch_position": memtable.as_ref().and_then(|stats| stats.max_flushed_batch_position),
+        "active_memtable_durable_batch_count": memtable.as_ref().map(|stats| stats.durable_batch_count),
         "wal_queue_pending_batches": memtable.as_ref().map(|stats| stats.pending_wal_batch_count),
         "wal_queue_pending_rows": memtable.as_ref().map(|stats| stats.pending_wal_row_count),
         "wal_queue_pending_bytes": memtable.as_ref().map(|stats| stats.pending_wal_estimated_bytes),
@@ -667,7 +667,7 @@ fn memtable_stats_json(memtable: Option<&MemTableStats>) -> serde_json::Value {
             "estimated_size": stats.estimated_size,
             "generation": stats.generation,
             "max_buffered_batch_position": stats.max_buffered_batch_position,
-            "max_flushed_batch_position": stats.max_flushed_batch_position,
+            "durable_batch_count": stats.durable_batch_count,
             "wal_queue_pending_start_batch_position": stats.pending_wal_start_batch_position,
             "wal_queue_pending_end_batch_position": stats.pending_wal_end_batch_position,
             "wal_queue_pending_batches": stats.pending_wal_batch_count,
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index e30d01720bf..27d185fce14 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -179,6 +179,29 @@ impl MemTable {
         pk_field_ids: Vec<i32>,
         cache_config: CacheConfig,
         batch_capacity: usize,
+    ) -> Result<Self> {
+        Self::with_capacity_at(
+            schema,
+            generation,
+            pk_field_ids,
+            cache_config,
+            batch_capacity,
+            0,
+        )
+    }
+
+    /// Create a memtable whose batch 0 sits at `global_offset` in the writer's
+    /// batch sequence. Every memtable after the writer's first is rotated in by
+    /// `freeze_memtable`, which stamps the outgoing memtable's `global_end()`
+    /// here so writer-global cursors stay mappable onto local batch positions.
+    #[allow(clippy::too_many_arguments)]
+    pub fn with_capacity_at(
+        schema: Arc<ArrowSchema>,
+        generation: u64,
+        pk_field_ids: Vec<i32>,
+        cache_config: CacheConfig,
+        batch_capacity: usize,
+        global_offset: usize,
     ) -> Result<Self> {
         let lance_schema = Schema::try_from(schema.as_ref())?;
 
@@ -197,7 +220,7 @@ impl MemTable {
         let dataset_uri = format!("memory://{}", Uuid::new_v4());
 
         // Create lock-free batch store
-        let batch_store = Arc::new(BatchStore::with_capacity(batch_capacity));
+        let batch_store = Arc::new(BatchStore::with_capacity_at(batch_capacity, global_offset));
 
         // Create memtable_flush_completion cell immediately so backpressure can
         // wait on it even before the memtable is frozen. Every memtable will
@@ -712,9 +735,15 @@ impl MemTable {
         self.indexes.take()
     }
 
-    /// Check if all batches have been flushed to WAL.
-    pub fn all_flushed_to_wal(&self) -> bool {
-        self.batch_store.pending_wal_flush_count() == 0
+    /// Whether every committed batch in this memtable is WAL-durable, given the
+    /// writer-global durability cursor. The L0 flush's precondition.
+    pub fn all_flushed_to_wal(&self, durable: usize) -> bool {
+        self.batch_store.pending_wal_flush_count(durable) == 0
+    }
+
+    /// Writer-global coordinate one past this memtable's last committed batch.
+    pub fn global_end(&self) -> usize {
+        self.batch_store.global_end()
     }
 
     /// Get cache configuration.
@@ -878,7 +907,7 @@ mod tests {
         assert_eq!(total_rows, 15);
     }
 
-    /// `all_flushed_to_wal()` is the L0 flush's precondition (`flush.rs:171`):
+    /// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`):
     /// false while any committed batch is still un-appended, true once the
     /// durability watermark covers every one of them.
     #[tokio::test]
@@ -894,20 +923,16 @@ mod tests {
             .insert(create_test_batch(&schema, 5))
             .await
             .unwrap();
-        assert!(!memtable.all_flushed_to_wal());
+        assert!(!memtable.all_flushed_to_wal(0), "nothing is durable yet");
 
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(batch1);
+        let durable = batch1 + 1;
         assert!(
-            !memtable.all_flushed_to_wal(),
+            !memtable.all_flushed_to_wal(durable),
             "batch2 is still waiting on its WAL append"
         );
 
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(batch2);
-        assert!(memtable.all_flushed_to_wal());
+        let durable = batch2 + 1;
+        assert!(memtable.all_flushed_to_wal(durable));
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
index ff514e6bcfe..23f1c9f4767 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
@@ -195,10 +195,14 @@ pub struct BatchStore {
     /// Estimated size in bytes (for flush threshold).
     estimated_bytes: AtomicUsize,
 
-    /// WAL flush watermark: the last batch ID that has been flushed to WAL (inclusive).
-    /// Uses usize::MAX as sentinel for "nothing flushed yet".
-    /// This is per-memtable tracking, not global.
-    max_flushed_batch_position: AtomicUsize,
+    /// Writer-global coordinate of this store's batch 0.
+    ///
+    /// A *coordinate*, not a cursor: stamped once at construction and never
+    /// moved. `global_position = global_offset + local_position`. Batch
+    /// positions restart at 0 in every memtable, so this is the only thing that
+    /// lets a writer-global cursor (the WAL durability count) be mapped onto a
+    /// particular store.
+    global_offset: usize,
 }
 
 // SAFETY: Safe to share across threads because:
@@ -221,6 +225,13 @@ impl BatchStore {
     ///
     /// Panics if capacity is 0.
     pub fn with_capacity(capacity: usize) -> Self {
+        Self::with_capacity_at(capacity, 0)
+    }
+
+    /// Create a store whose batch 0 sits at `global_offset` in the writer's
+    /// batch sequence. Used by `freeze_memtable` for every memtable after the
+    /// first; the first starts at 0.
+    pub fn with_capacity_at(capacity: usize, global_offset: usize) -> Self {
         assert!(capacity > 0, "capacity must be > 0");
 
         // Allocate uninitialized storage
@@ -235,7 +246,7 @@ impl BatchStore {
             capacity,
             total_rows: AtomicUsize::new(0),
             estimated_bytes: AtomicUsize::new(0),
-            max_flushed_batch_position: AtomicUsize::new(usize::MAX), // Nothing flushed yet
+            global_offset,
         }
     }
 
@@ -437,68 +448,54 @@ impl BatchStore {
     // WAL Flush Tracking API
     // =========================================================================
 
-    /// Get the WAL flush watermark (the last batch ID that was flushed, inclusive).
-    /// Returns None if nothing has been flushed yet.
+    /// Writer-global coordinate one past this store's last committed batch.
     #[inline]
-    pub fn max_flushed_batch_position(&self) -> Option<usize> {
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        if watermark == usize::MAX {
-            None
-        } else {
-            Some(watermark)
-        }
+    pub fn global_end(&self) -> usize {
+        self.global_offset + self.committed_len.load(Ordering::Acquire)
+    }
+
+    /// This store's writer-global coordinate for batch 0.
+    #[inline]
+    pub fn global_offset(&self) -> usize {
+        self.global_offset
     }
 
-    /// Update the WAL flush watermark after successful WAL flush.
+    /// The local exclusive end of this store covered by a writer-global cursor.
     ///
-    /// # Arguments
+    /// Saturating in both directions, and both directions are reachable in
+    /// normal operation: a cursor *below* this store's offset means "nothing
+    /// here yet" (the store was rotated in after the cursor last advanced —
+    /// the ordinary state of a fresh memtable), and a cursor beyond its end
+    /// clamps to what is committed.
     ///
-    /// * `batch_position` - The last batch ID that was flushed (inclusive)
+    /// This is the **only** place the global-to-local subtraction is written.
+    /// Open-coding it underflows on every memtable rotation, which in release
+    /// wraps to a huge end and makes the whole new memtable instantly visible.
     #[inline]
-    pub fn set_max_flushed_batch_position(&self, batch_position: usize) {
-        debug_assert!(
-            batch_position != usize::MAX,
-            "batch_position cannot be usize::MAX (reserved as sentinel)"
-        );
-        self.max_flushed_batch_position
-            .store(batch_position, Ordering::Release);
+    pub fn local_end(&self, global_cursor: usize) -> usize {
+        global_cursor
+            .saturating_sub(self.global_offset)
+            .min(self.committed_len.load(Ordering::Acquire))
     }
 
-    /// Get the number of batches pending WAL flush.
+    /// Batches in this store still waiting on their WAL append.
     #[inline]
-    pub fn pending_wal_flush_count(&self) -> usize {
-        let committed = self.committed_len.load(Ordering::Acquire);
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        if watermark == usize::MAX {
-            // Nothing flushed yet, all committed batches are pending
-            committed
-        } else {
-            // Batches [0, watermark] are flushed, so pending = committed - (watermark + 1)
-            committed.saturating_sub(watermark + 1)
-        }
+    pub fn pending_wal_flush_count(&self, durable: usize) -> usize {
+        self.committed_len.load(Ordering::Acquire) - self.local_end(durable)
     }
 
-    /// Get the range of batch IDs pending WAL flush: [start, end).
-    /// Returns None if nothing pending.
+    /// Local range `[start, end)` of batches still waiting on their WAL append,
+    /// or `None` when the store is fully durable.
     #[inline]
-    pub fn pending_wal_flush_range(&self) -> Option<(usize, usize)> {
-        let committed = self.committed_len.load(Ordering::Acquire);
-        let watermark = self.max_flushed_batch_position.load(Ordering::Acquire);
-        let start = if watermark == usize::MAX {
-            0
-        } else {
-            watermark + 1
-        };
-        if committed > start {
-            Some((start, committed))
-        } else {
-            None
-        }
+    pub fn pending_wal_flush_range(&self, durable: usize) -> Option<(usize, usize)> {
+        let start = self.local_end(durable);
+        let end = self.committed_len.load(Ordering::Acquire);
+        (end > start).then_some((start, end))
     }
 
     /// Get a point-in-time summary of batches pending WAL flush.
-    pub fn pending_wal_flush_stats(&self) -> PendingWalFlushStats {
-        let Some((start, end)) = self.pending_wal_flush_range() else {
+    pub fn pending_wal_flush_stats(&self, durable: usize) -> PendingWalFlushStats {
+        let Some((start, end)) = self.pending_wal_flush_range(durable) else {
             return PendingWalFlushStats::default();
         };
 
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index 75a5f1cfc92..8e87f19483a 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -219,6 +219,7 @@ impl MemTableFlusher {
         memtable: &MemTable,
         epoch: u64,
         covered_wal_entry_position: u64,
+        durable: usize,
     ) -> Result<FlushResult> {
         self.manifest_store.check_fenced(epoch).await?;
 
@@ -226,7 +227,7 @@ impl MemTableFlusher {
             return Err(Error::invalid_input("Cannot flush empty MemTable"));
         }
 
-        if !memtable.all_flushed_to_wal() {
+        if !memtable.all_flushed_to_wal(durable) {
             return Err(Error::invalid_input(
                 "MemTable has unflushed fragments - WAL flush required first",
             ));
@@ -452,6 +453,7 @@ impl MemTableFlusher {
         epoch: u64,
         index_configs: &[MemIndexConfig],
         covered_wal_entry_position: u64,
+        durable: usize,
     ) -> Result<FlushResult> {
         self.manifest_store.check_fenced(epoch).await?;
 
@@ -459,7 +461,7 @@ impl MemTableFlusher {
             return Err(Error::invalid_input("Cannot flush empty MemTable"));
         }
 
-        if !memtable.all_flushed_to_wal() {
+        if !memtable.all_flushed_to_wal(durable) {
             return Err(Error::invalid_input(
                 "MemTable has unflushed fragments - WAL flush required first",
             ));
@@ -1258,11 +1260,12 @@ mod tests {
             .await
             .unwrap();
 
-        // Not flushed to WAL yet
-        assert!(!memtable.all_flushed_to_wal());
+        // Nothing is durable yet, so the L0 flush must refuse.
+        let durable = 0;
+        assert!(!memtable.all_flushed_to_wal(durable));
 
         let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
-        let result = flusher.flush(&memtable, epoch, 0).await;
+        let result = flusher.flush(&memtable, epoch, 0, 0).await;
 
         assert!(result.is_err());
         assert!(
@@ -1291,7 +1294,7 @@ mod tests {
         let memtable = MemTable::new(schema, 1, vec![]).unwrap();
 
         let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store);
-        let result = flusher.flush(&memtable, epoch, 0).await;
+        let result = flusher.flush(&memtable, epoch, 0, 0).await;
 
         assert!(result.is_err());
         assert!(result.unwrap_err().to_string().contains("empty MemTable"));
@@ -1319,10 +1322,8 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
-        assert!(memtable.all_flushed_to_wal());
+        let durable = frag_id + 1;
+        assert!(memtable.all_flushed_to_wal(durable));
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1331,7 +1332,7 @@ mod tests {
             shard_id,
             manifest_store.clone(),
         );
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
         assert_eq!(result.generation.generation, 1);
         assert_eq!(result.rows_flushed, 10);
@@ -1385,9 +1386,7 @@ mod tests {
             .insert(create_test_batch(&schema, 10))
             .await
             .unwrap();
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
         let warmer: Arc<dyn GenerationWarmer> = Arc::new(CountingWarmer {
@@ -1404,7 +1403,7 @@ mod tests {
         )
         .with_warmer(Some(warmer));
         // Flush must succeed despite the warmer erroring.
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
         assert_eq!(result.generation.generation, 1);
         assert_eq!(
@@ -1448,9 +1447,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1459,7 +1456,7 @@ mod tests {
             shard_id,
             manifest_store,
         );
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
         assert_eq!(result.rows_flushed, 5, "all physical rows are written");
 
         // Scanning the flushed generation must honor the deletion vector and
@@ -1553,9 +1550,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1565,7 +1560,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &[], 1)
+            .flush_with_indexes(&memtable, epoch, &[], 1, durable)
             .await
             .unwrap();
 
@@ -1655,9 +1650,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1667,7 +1660,7 @@ mod tests {
             manifest_store.clone(),
         );
         // The plain-flush path — what the writer dispatches to with no indexes.
-        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+        let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap();
 
         let gen_path = base_path
             .clone()
@@ -1751,9 +1744,7 @@ mod tests {
         )
         .unwrap();
         let frag_id = memtable.insert(batch).await.unwrap();
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1763,7 +1754,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
         assert_eq!(result.rows_flushed, 5, "all physical rows are written");
@@ -1885,9 +1876,7 @@ mod tests {
             .unwrap();
 
         // Simulate WAL flush
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -1897,7 +1886,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
@@ -2024,9 +2013,7 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -2036,7 +2023,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
@@ -2176,9 +2163,7 @@ mod tests {
         let frag_id = memtable.insert(batch).await.unwrap();
 
         // Simulate WAL flush
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(frag_id);
+        let durable = frag_id + 1;
 
         let flusher = MemTableFlusher::new(
             store.clone(),
@@ -2188,7 +2173,7 @@ mod tests {
             manifest_store.clone(),
         );
         let result = flusher
-            .flush_with_indexes(&memtable, epoch, &index_configs, 1)
+            .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable)
             .await
             .unwrap();
 
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 61546c18fe3..719a2b36a96 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -198,15 +198,10 @@ pub enum WalFlushSource {
 }
 
 impl WalFlushSource {
-    fn pending_count(&self) -> usize {
+    fn kind(&self) -> &'static str {
         match self {
-            Self::BatchStore { batch_store, .. } => batch_store.pending_wal_flush_count(),
-            Self::WalOnly { state } => state
-                .pending
-                .lock()
-                .ok()
-                .map(|p| p.batches.len())
-                .unwrap_or(0),
+            Self::BatchStore { .. } => "BatchStore",
+            Self::WalOnly { .. } => "WalOnly",
         }
     }
 }
@@ -232,7 +227,7 @@ pub struct TriggerWalFlush {
 impl std::fmt::Debug for TriggerWalFlush {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("TriggerWalFlush")
-            .field("pending_batches", &self.source.pending_count())
+            .field("source", &self.source.kind())
             .field("end_batch_position", &self.end_batch_position)
             .finish()
     }
@@ -420,16 +415,33 @@ impl WalFlusher {
     ///
     /// Returns a `BatchDurableWatcher` that can be awaited for durability.
     /// The actual batch data is stored in the BatchStore.
-    pub fn track_batch(&self, batch_position: usize) -> BatchDurableWatcher {
-        // Return a watcher that waits for this batch to become durable
-        // batch_position is 0-indexed, so we wait for watermark > batch_position (i.e., >= batch_position + 1)
+    pub fn track_batch(&self, target_durable_count: usize) -> BatchDurableWatcher {
+        // `target_durable_count` is writer-global and exclusive: "N batches into
+        // this writer's batch sequence are durable". It must NOT be a memtable-
+        // local batch position — positions restart at 0 in every memtable, while
+        // this channel spans the writer's whole life, so a local target would be
+        // satisfied by a *previous* memtable's appends and ack a write that was
+        // never persisted. Callers globalize via `BatchStore::global_offset`.
         BatchDurableWatcher::new(
             self.durable_watermark_rx.clone(),
-            batch_position + 1,
+            target_durable_count,
             Arc::clone(&self.terminal_error),
         )
     }
 
+    /// The writer-global WAL durability cursor: how many batches of this
+    /// writer's batch sequence are durable. Exclusive count; 0 means none.
+    pub fn durable(&self) -> usize {
+        *self.durable_watermark_rx.borrow()
+    }
+
+    /// Advance the durability cursor and wake waiters. Monotonic: a stale or
+    /// out-of-order completion can never walk the cursor backwards.
+    pub(crate) fn advance_durable(&self, global_count: usize) {
+        self.durable_watermark_tx
+            .send_modify(|current| *current = (*current).max(global_count));
+    }
+
     /// Latch a terminal flush failure and wake every durability waiter (the
     /// watermark never advances, so they must observe the error, not block).
     /// Idempotent: only the first failure is retained.
@@ -552,12 +564,10 @@ impl WalFlusher {
         indexes: Option<Arc<IndexStore>>,
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
-        // Get current flush position from per-memtable watermark (inclusive)
-        // start_batch_position is the first batch to flush
-        let start_batch_position = batch_store
-            .max_flushed_batch_position()
-            .map(|w| w + 1)
-            .unwrap_or(0);
+        // Where this store's un-appended suffix begins, derived from the
+        // writer-global durability cursor. `local_end` clamps a cursor that
+        // predates this memtable to 0 and one past its end to `committed_len`.
+        let start_batch_position = batch_store.local_end(self.durable());
 
         // If we've already flushed past this end, nothing to do
         if start_batch_position >= end_batch_position {
@@ -614,11 +624,12 @@ impl WalFlusher {
         let (append_result, wal_io_duration) = append_result?;
         let (index_update_duration, index_update_duration_breakdown) = index_result?;
 
-        // Update per-memtable watermark (inclusive: last batch ID that was flushed)
-        batch_store.set_max_flushed_batch_position(end_batch_position - 1);
-
-        // Notify durability waiters (global channel)
-        let _ = self.durable_watermark_tx.send(end_batch_position);
+        // Advance the writer-global durability cursor and wake waiters. The
+        // range we just appended was `[start, end)` *local to this store*, so it
+        // must be lifted into the writer's coordinate space before it is
+        // published — otherwise a fresh memtable's small local end would be
+        // compared against a channel carrying a previous memtable's larger one.
+        self.advance_durable(batch_store.global_offset() + end_batch_position);
         // Signal WAL flush completion for backpressure waiters
         self.signal_wal_flush_complete();
 
@@ -1576,7 +1587,7 @@ mod tests {
         let buffer = build_test_flusher(store, &base_path, shard_id, 1);
 
         // Track a batch
-        let watcher = buffer.track_batch(0);
+        let watcher = buffer.track_batch(1);
 
         // Watcher should not be durable yet
         assert!(!watcher.is_durable());
@@ -1595,7 +1606,7 @@ mod tests {
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 10)).unwrap();
 
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(1);
 
         // wait() must NOT resolve before the flush happens
         let result =
@@ -1628,13 +1639,13 @@ mod tests {
         batch_store.append(batch2).unwrap();
 
         // Track batch IDs in WAL flusher
-        let mut watcher1 = buffer.track_batch(0);
-        let mut watcher2 = buffer.track_batch(1);
+        let mut watcher1 = buffer.track_batch(1);
+        let mut watcher2 = buffer.track_batch(2);
 
         // Verify initial state
         assert!(!watcher1.is_durable());
         assert!(!watcher2.is_durable());
-        assert!(batch_store.max_flushed_batch_position().is_none());
+        assert_eq!(buffer.durable(), 0);
 
         // Flush all pending batches
         let source = batch_store_source(&batch_store);
@@ -1646,8 +1657,8 @@ mod tests {
         assert_eq!(entry.position, FIRST_WAL_ENTRY_POSITION);
         assert_eq!(entry.writer_epoch, 1);
         assert_eq!(entry.num_batches, 2);
-        // After flushing 2 batches (positions 0 and 1), max flushed position is 1 (inclusive)
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(1));
+        // Two batches appended => the writer-global durable count is 2 (exclusive).
+        assert_eq!(buffer.durable(), 2);
 
         // Watchers should be notified
         watcher1.wait().await.unwrap();
@@ -1684,7 +1695,7 @@ mod tests {
         // Cursor must advance to the highest flushed batch position (2),
         // making all three batches visible to scanners.
         assert_eq!(indexes.max_visible_batch_position(), 2);
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(2));
+        assert_eq!(flusher.durable(), 3);
     }
 
     // Regression guard for the indexed path: with at least one BTree index
@@ -1710,7 +1721,7 @@ mod tests {
         flusher.flush(&source, batch_store.len()).await.unwrap();
 
         assert_eq!(indexes.max_visible_batch_position(), 2);
-        assert_eq!(batch_store.max_flushed_batch_position(), Some(2));
+        assert_eq!(flusher.durable(), 3);
     }
 
     #[tokio::test]
@@ -1726,8 +1737,8 @@ mod tests {
         batch_store.append(create_test_batch(&schema, 5)).unwrap();
 
         // Track batch IDs and flush all pending batches
-        let _watcher1 = buffer.track_batch(0);
-        let _watcher2 = buffer.track_batch(1);
+        let _watcher1 = buffer.track_batch(1);
+        let _watcher2 = buffer.track_batch(2);
         let source = batch_store_source(&batch_store);
         let result = buffer.flush(&source, batch_store.len()).await.unwrap();
         let entry = result.entry.unwrap();
@@ -1905,7 +1916,7 @@ mod tests {
         // A durable put on the predecessor: stage a batch and track it.
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(1);
 
         // Flushing collides with the sentinel and fences. Both the flush result
         // and the watcher must report the fence — and the watcher must resolve
@@ -2094,7 +2105,7 @@ mod tests {
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(0);
+        let mut watcher = flusher.track_batch(1);
 
         let source = batch_store_source(&batch_store);
         let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err();
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 72a24180bbc..2391bd78f77 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -935,25 +935,6 @@ async fn replay_memtable_from_wal(
         }
     }
 
-    // Mark the replayed batches durable. They came *from* the WAL, and
-    // `insert_batches` above just re-derived the indexes over them, so both
-    // cursors are satisfied — but only the index one advances itself.
-    //
-    // Without this stamp the durability cursor stays at "nothing flushed", so
-    // the next WAL flush re-covers [0, end): it re-appends already-durable rows
-    // to the WAL *and* re-inserts every replayed row into the indexes. None of
-    // the three in-memory indexes is idempotent (HNSW mints fresh node ids for
-    // the same row, FTS increments doc_count/df instead of recomputing them,
-    // BTree is a multiset), so a full scan keeps looking healthy while every
-    // index-accelerated query silently returns duplicates. It compounds, too:
-    // the WAL now holds those rows twice, so the next crash replays both copies.
-    let replayed_batches = memtable.batch_count();
-    if replayed_batches > 0 {
-        memtable
-            .batch_store()
-            .set_max_flushed_batch_position(replayed_batches - 1);
-    }
-
     Ok(position)
 }
 
@@ -1081,19 +1062,29 @@ impl SharedWriterState {
     ///
     /// Takes `&mut WriterState` directly since caller already holds the lock.
     fn freeze_memtable(&self, state: &mut WriterState) -> Result<u64> {
-        let pending_wal_range = state.memtable.batch_store().pending_wal_flush_range();
+        let durable = self.wal_flusher.durable();
+        let pending_wal_range = state
+            .memtable
+            .batch_store()
+            .pending_wal_flush_range(durable);
         let last_wal_entry_position = state.last_flushed_wal_entry_position;
 
         let old_batch_store = state.memtable.batch_store();
         let old_indexes = state.memtable.indexes_arc();
 
         let next_generation = state.memtable.generation() + 1;
-        let mut new_memtable = MemTable::with_capacity(
+        // The incoming memtable's batch 0 continues the writer's batch sequence
+        // where the outgoing one ends. Without this coordinate, local positions
+        // (which restart at 0 every rotation) cannot be mapped onto the
+        // writer-global durability cursor.
+        let next_global_offset = old_batch_store.global_end();
+        let mut new_memtable = MemTable::with_capacity_at(
             self.schema.clone(),
             next_generation,
             self.pk_field_ids.clone(),
             CacheConfig::default(),
             self.max_memtable_batches,
+            next_global_offset,
         )?;
 
         // Build an IndexStore when there are user indexes *or* a primary key:
@@ -1165,9 +1156,13 @@ impl SharedWriterState {
         Ok(next_generation)
     }
 
-    /// Track batch for WAL durability.
-    fn track_batch_for_wal(&self, batch_position: usize) -> super::wal::BatchDurableWatcher {
-        self.wal_flusher.track_batch(batch_position)
+    /// Watch for a write to become WAL-durable.
+    ///
+    /// `target_durable_count` is a writer-global, exclusive batch count — not a
+    /// memtable-local position. See the caller for why that distinction is
+    /// load-bearing.
+    fn track_batch_for_wal(&self, target_durable_count: usize) -> super::wal::BatchDurableWatcher {
+        self.wal_flusher.track_batch(target_durable_count)
     }
 
     /// Check if memtable flush is needed and trigger if so.
@@ -1201,7 +1196,7 @@ impl SharedWriterState {
         let indexes = state.memtable.indexes_arc();
 
         // Check if there are any unflushed batches
-        let has_pending = batch_store.pending_wal_flush_count() > 0;
+        let has_pending = batch_store.pending_wal_flush_count(self.wal_flusher.durable()) > 0;
 
         // Check time-based trigger first
         let time_trigger = if let Some(interval) = self.config.max_wal_flush_interval {
@@ -1542,6 +1537,23 @@ impl ShardWriter {
             &mut memtable,
         )
         .await?;
+
+        // Mark the replayed batches durable. They came *from* the WAL, and the
+        // replay above has already re-derived the indexes over them.
+        //
+        // Without this the durability cursor stays at 0, so the next WAL flush
+        // re-covers [0, end): it re-appends the already-durable rows *and*
+        // re-inserts every replayed row into the indexes. None of the three
+        // in-memory indexes is idempotent (HNSW mints fresh node ids for the
+        // same row, FTS increments doc_count/df rather than recomputing them,
+        // BTree is a multiset), so a full scan keeps looking healthy while every
+        // index-accelerated query silently returns duplicates — and it compounds,
+        // because the WAL now holds those rows twice.
+        //
+        // The replayed memtable is the writer's first, so its `global_offset` is
+        // 0 and the durable count is just its batch count.
+        wal_flusher.advance_durable(memtable.batch_count());
+
         wal_flusher
             .wal_appender()
             .seed_next_position(next_wal_position)
@@ -1594,6 +1606,7 @@ impl ShardWriter {
         let memtable_handler = MemTableFlushHandler::new(
             state.clone(),
             flusher,
+            wal_flusher.clone(),
             epoch,
             index_configs.to_vec(),
             stats,
@@ -1892,26 +1905,36 @@ impl ShardWriter {
             // 1. Insert all batches into memtable atomically
             let results = state.memtable.insert_batches_only(batches).await?;
 
-            // Get batch position range
+            // 2. Capture the store the batches actually landed in, *before* step
+            //    4 below can freeze and swap the active memtable. Reading it
+            //    afterwards hands the flush trigger the **new** store paired with
+            //    the **old** store's end position, so the new store's watermark
+            //    jumps past batches that were never appended.
+            let batch_store = state.memtable.batch_store();
+            let indexes = state.memtable.indexes_arc();
+
             let start_pos = results.first().map(|(pos, _, _)| *pos).unwrap_or(0);
             let end_pos = results.last().map(|(pos, _, _)| pos + 1).unwrap_or(0);
             let batch_positions = start_pos..end_pos;
 
-            // 2. Track last batch for WAL durability
-            let durable_watcher = writer_state.track_batch_for_wal(end_pos.saturating_sub(1));
-
-            // 3. Check if WAL flush should be triggered
+            // 3. Track this write for WAL durability. The target is writer-global
+            //    and exclusive. It must not be a memtable-local position: batch
+            //    positions restart at 0 in every memtable, while the durability
+            //    channel spans the writer's whole life. A local target is
+            //    therefore already satisfied by a *previous* memtable's appends,
+            //    so the first N puts into every post-rotation memtable would ack
+            //    as durable without a WAL append ever happening.
+            let durable_watcher =
+                writer_state.track_batch_for_wal(batch_store.global_offset() + end_pos);
+
+            // 4. Check if WAL flush should be triggered
             writer_state.maybe_trigger_wal_flush(&mut state);
 
-            // 4. Check if memtable flush is needed
+            // 5. Check if memtable flush is needed (may freeze and rotate)
             if let Err(e) = writer_state.maybe_trigger_memtable_flush(&mut state) {
                 warn!("Failed to trigger memtable flush: {}", e);
             }
 
-            // Get batch_store and indexes while we have the lock (for durable_write case)
-            let batch_store = state.memtable.batch_store();
-            let indexes = state.memtable.indexes_arc();
-
             (batch_positions, durable_watcher, batch_store, indexes)
         }; // Lock released here
 
@@ -2117,14 +2140,16 @@ impl ShardWriter {
         let state_lock = self.memtable_state_lock()?;
         let state = state_lock.read().await;
         let batch_store = state.memtable.batch_store();
-        let pending_wal = batch_store.pending_wal_flush_stats();
+        let durable = self.wal_flusher.durable();
+        let pending_wal = batch_store.pending_wal_flush_stats(durable);
         Ok(MemTableStats {
             row_count: state.memtable.row_count(),
             batch_count: state.memtable.batch_count(),
             estimated_size: state.memtable.estimated_size(),
             generation: state.memtable.generation(),
             max_buffered_batch_position: batch_store.max_buffered_batch_position(),
-            max_flushed_batch_position: batch_store.max_flushed_batch_position(),
+            durable_batch_count: durable,
+            global_offset: batch_store.global_offset(),
             pending_wal_start_batch_position: pending_wal.start_batch_position,
             pending_wal_end_batch_position: pending_wal.end_batch_position,
             pending_wal_batch_count: pending_wal.batch_count,
@@ -2433,7 +2458,12 @@ pub struct MemTableStats {
     pub estimated_size: usize,
     pub generation: u64,
     pub max_buffered_batch_position: Option<usize>,
-    pub max_flushed_batch_position: Option<usize>,
+    /// Writer-global count of WAL-durable batches. Exclusive: 0 means none.
+    /// Compare against `global_offset + batch_count` to see what this memtable
+    /// still owes the WAL.
+    pub durable_batch_count: usize,
+    /// Writer-global coordinate of this memtable's batch 0.
+    pub global_offset: usize,
     pub pending_wal_start_batch_position: Option<usize>,
     pub pending_wal_end_batch_position: Option<usize>,
     pub pending_wal_batch_count: usize,
@@ -2552,8 +2582,7 @@ impl WalFlushHandler {
         // BatchStore source, so the early-out simplifies to the watermark
         // comparison.
         if let WalFlushSource::BatchStore { batch_store, .. } = &source {
-            let max_flushed = batch_store.max_flushed_batch_position();
-            let flushed_up_to = max_flushed.map(|p| p + 1).unwrap_or(0);
+            let flushed_up_to = batch_store.local_end(self.wal_flusher.durable());
             let is_frozen_flush = if let Some(state_lock) = &self.memtable_state {
                 let state = state_lock.read().await;
                 !Arc::ptr_eq(batch_store, &state.memtable.batch_store())
@@ -2599,6 +2628,9 @@ impl WalFlushHandler {
 struct MemTableFlushHandler {
     state: Arc<RwLock<WriterState>>,
     flusher: Arc<MemTableFlusher>,
+    /// Source of the writer-global durability cursor, which the L0 flush asserts
+    /// covers the whole frozen memtable before it writes a generation.
+    wal_flusher: Arc<WalFlusher>,
     epoch: u64,
     /// Secondary index configs to rebuild on each flushed generation. When
     /// non-empty the handler flushes via [`MemTableFlusher::flush_with_indexes`]
@@ -2613,9 +2645,11 @@ struct MemTableFlushHandler {
 }
 
 impl MemTableFlushHandler {
+    #[allow(clippy::too_many_arguments)]
     fn new(
         state: Arc<RwLock<WriterState>>,
         flusher: Arc<MemTableFlusher>,
+        wal_flusher: Arc<WalFlusher>,
         epoch: u64,
         index_configs: Vec<MemIndexConfig>,
         stats: SharedWriteStats,
@@ -2624,6 +2658,7 @@ impl MemTableFlushHandler {
         Self {
             state,
             flusher,
+            wal_flusher,
             epoch,
             index_configs,
             stats,
@@ -2735,9 +2770,15 @@ impl MemTableFlushHandler {
             // dataset open when there are no indexes to build. The indexed
             // path's future is boxed to keep this async block's nesting
             // under the type-layout recursion limit.
+            // Read the durability cursor *after* the WAL-append completion above,
+            // not before: the append that makes this memtable durable is the very
+            // thing we just waited on, so a cursor sampled earlier would still be
+            // short of it and trip the flush precondition.
+            let durable = self.wal_flusher.durable();
+
             if self.index_configs.is_empty() {
                 self.flusher
-                    .flush(&memtable, self.epoch, covered_wal_entry_position)
+                    .flush(&memtable, self.epoch, covered_wal_entry_position, durable)
                     .await
             } else {
                 Box::pin(self.flusher.flush_with_indexes(
@@ -2745,6 +2786,7 @@ impl MemTableFlushHandler {
                     self.epoch,
                     &self.index_configs,
                     covered_wal_entry_position,
+                    durable,
                 ))
                 .await
             }
@@ -5010,8 +5052,7 @@ mod tests {
         let stats = writer_b.memtable_stats().await.unwrap();
         assert_eq!(stats.batch_count, 2);
         assert_eq!(
-            stats.max_flushed_batch_position,
-            Some(1),
+            stats.durable_batch_count, 2,
             "replayed batches came from the WAL, so the durability cursor must already cover them"
         );
 
@@ -5057,6 +5098,89 @@ mod tests {
         writer_b.close().await.unwrap();
     }
 
+    /// The durability cursor is writer-global, so a put into a *post-rotation*
+    /// memtable must still wait for its own WAL append.
+    ///
+    /// Batch positions restart at 0 in every memtable, but the durability watch
+    /// channel spans the writer's whole life and is never reset. When the put
+    /// path targeted a memtable-local position, the first N puts into every
+    /// memtable after the first were already "satisfied" by the *previous*
+    /// memtable's N appends: they acked instantly, with no WAL append, and
+    /// `durable_write: true` silently degraded to non-durable. Worse, the next
+    /// append then sent a *smaller* value, walking the watermark backwards and
+    /// hanging any watcher still waiting on the old, higher one.
+    ///
+    /// The cursor is now a writer-global exclusive count and every target is
+    /// lifted through the store's `global_offset`, so it only ever moves forward
+    /// and a post-rotation put can only be acked by its own append.
+    #[tokio::test]
+    async fn test_durable_ack_after_rotation_requires_its_own_wal_append() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // A two-batch memtable, so the third put forces a freeze + rotation.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // Fill and rotate the first memtable.
+        for i in 0..3 {
+            writer
+                .put(vec![create_test_batch(&schema, i * 5, 5)])
+                .await
+                .unwrap();
+        }
+        let stats = writer.memtable_stats().await.unwrap();
+        assert!(
+            stats.global_offset > 0,
+            "expected a rotation; the active memtable is still the writer's first"
+        );
+
+        // Every batch acked so far must be genuinely durable, and the cursor must
+        // cover the active memtable's entire prefix rather than lagging inside it.
+        let stats = writer.memtable_stats().await.unwrap();
+        assert!(
+            stats.durable_batch_count >= stats.global_offset + stats.batch_count,
+            "durable_write acked a put the WAL never received: durable={} but the active \
+             memtable spans [{}, {})",
+            stats.durable_batch_count,
+            stats.global_offset,
+            stats.global_offset + stats.batch_count
+        );
+
+        // And the WAL really holds every row we acked (3 puts x 5 rows).
+        writer.close().await.unwrap();
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut wal_rows = 0;
+        for position in first..next {
+            if let Some(entry) = tailer.read_entry(position).await.unwrap() {
+                wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::<usize>();
+            }
+        }
+        assert_eq!(
+            wal_rows, 15,
+            "every acked row must be in the WAL; a post-rotation put that acked without an \
+             append would leave rows missing"
+        );
+    }
+
     /// An index config that disagrees with the schema fails `open()` outright.
     /// It must not be allowed to accept writes: the insert would fail
     /// deterministically on every batch, including batches replayed from the

From 8a1d5885e912c8941d345a1ed71a6507f772b837 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 12:59:52 -0500
Subject: [PATCH 07/19] fix(mem_wal): make the visibility cursor an exclusive
 count
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`max_visible_batch_position` was an inclusive position starting at 0, so a cursor
of 0 meant *both* "nothing is visible" and "batch 0 is visible". There was no
sentinel and no `Option` to tell them apart.

The window is real, not theoretical. `BatchStore::append` publishes
`committed_len` on the put path, under the state lock, before the WAL flush that
indexes the batch is even triggered — and that flush is a ~100ms S3 PUT on
another task. So batch 0 of every memtable sat committed and readable for a full
round-trip before it was indexed or durable, and only through the arms backed by
the batch store: the index-backed arms, reading the same cursor, saw nothing. The
tiers actively disagreed.

Express the cursor as an exclusive count. `0` now means nothing, the `+1`
disappears, `i < count` replaces `i <= max`, and the off-by-one becomes
inexpressible. `checked_sub(1)` conversions at the consumers fall away — every
site got simpler, as did `bounded_in_memory_membership`, whose `batch_count == 0`
case now falls out of the arithmetic instead of being special-cased.

Rename it to `indexed_count` while we are here. It has only ever been an
*indexed* cursor — it is advanced at the end of `insert_batches`, once every
index insert for a batch completes, and never before. Five read sites treated it
as a visibility cursor, which is a separate thing that belongs to the writer.
Deriving visibility from it is the next change; this one just stops the name from
lying.

Two things the conversion turned up:

- `point_lookup` did `indexed_count().min(len - 1)`, reading the cursor as an
  inclusive index. It had no "nothing visible" case at all.
- `test_shard_writer_e2e_correctness` passed *only* because of this bug: it wrote
  with `durable_write: false` and scanned, and the rows appeared because the
  un-advanced cursor of 0 was misread as "batch 0 is visible". A non-durable put
  is not yet read-your-writes — the index apply is welded to the WAL flush — so
  the test now writes durably, which is what it meant to test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .../vector/hnsw/mem_wal_recall_hnsw.rs        |   4 +-
 .../mem_wal/vector/mem_wal_index_micro.rs     |   4 +-
 rust/lance/src/dataset/mem_wal/index.rs       |  70 ++++----
 rust/lance/src/dataset/mem_wal/memtable.rs    |  85 +++++-----
 .../dataset/mem_wal/memtable/batch_store.rs   | 158 ++++++++++--------
 .../mem_wal/memtable/scanner/builder.rs       |  30 ++--
 .../dataset/mem_wal/memtable/scanner/exec.rs  |   4 +-
 .../scanner/exec/brute_force_vector.rs        |  25 ++-
 .../mem_wal/memtable/scanner/exec/btree.rs    |  31 ++--
 .../memtable/scanner/exec/dedup_scan.rs       |  27 ++-
 .../mem_wal/memtable/scanner/exec/fts.rs      |  25 ++-
 .../mem_wal/memtable/scanner/exec/scan.rs     |  37 ++--
 .../mem_wal/memtable/scanner/exec/vector.rs   |  19 +--
 .../src/dataset/mem_wal/scanner/block_list.rs |   9 +-
 .../mem_wal/scanner/exec/pk_block_filter.rs   |   2 +-
 .../src/dataset/mem_wal/scanner/fts_search.rs |   2 +-
 .../dataset/mem_wal/scanner/point_lookup.rs   |   9 +-
 rust/lance/src/dataset/mem_wal/wal.rs         |  11 +-
 rust/lance/src/dataset/mem_wal/write.rs       |  10 +-
 19 files changed, 290 insertions(+), 272 deletions(-)

diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
index 7b9f35b73c9..85dd5e7ac14 100644
--- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
+++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
@@ -416,11 +416,11 @@ async fn run_checkpoint(
         writer.put(vec![batch]).await?;
     }
 
-    let target_batch_pos = total_batches.saturating_sub(1);
+    let target_indexed_count = total_batches;
     let mut spins = 0u64;
     loop {
         let active = writer.active_memtable_ref().await?;
-        if active.index_store.max_visible_batch_position() >= target_batch_pos {
+        if active.index_store.indexed_count() >= target_indexed_count {
             break;
         }
         drop(active);
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index f7b2ac4e709..812535ce053 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -255,7 +255,7 @@ async fn main() -> lance_core::Result<()> {
 
         while next_cp_idx < checkpoints.len() && total_inserted >= checkpoints[next_cp_idx] {
             let cp = checkpoints[next_cp_idx];
-            let target_batch_pos = (cp / batch_size).saturating_sub(1);
+            let target_indexed_count = cp / batch_size;
             // The WAL flush handler only updates the index watermark when a
             // flush is triggered, and the time-based trigger inside the
             // writer runs only when `put()` is called. After the final put
@@ -267,7 +267,7 @@ async fn main() -> lance_core::Result<()> {
             let mut spins = 0u64;
             loop {
                 let active = writer.active_memtable_ref().await?;
-                if active.index_store.max_visible_batch_position() >= target_batch_pos {
+                if active.index_store.indexed_count() >= target_indexed_count {
                     break;
                 }
                 drop(active);
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 0df3c2499e2..8d1030ce87f 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -389,7 +389,7 @@ impl MemIndexConfig {
 /// Indexes are keyed by index name. Each index stores its field_id for
 /// stable column-to-index resolution (column name → field_id → index).
 ///
-/// The store also carries the MemTable's `max_visible_batch_position`
+/// The store also carries the MemTable's `visible_count`
 /// watermark — the highest batch position that is durable in the WAL and
 /// therefore safe for scanners to read. Scanners snapshot this at plan
 /// construction time so every plan keys on a stable MVCC cursor.
@@ -408,7 +408,16 @@ pub struct IndexStore {
     /// Maximum batch position that is durable in the WAL and therefore
     /// visible to scanners. Advanced unconditionally after a WAL append
     /// succeeds; not gated on whether any indexes are configured.
-    max_visible_batch_position: AtomicUsize,
+    /// How many batches of this memtable have been fully indexed. An exclusive
+    /// count: 0 means none.
+    ///
+    /// This has only ever been an *indexed* cursor — it is advanced at the end of
+    /// `insert_batches`, once every index insert for the batch has completed, and
+    /// never before. It was named `visible_count` and treated as a
+    /// visibility cursor by five read sites, which is how rows became readable
+    /// before they were durable. The derivation from "indexed" to "visible" is a
+    /// separate thing, and it belongs to the writer, not here.
+    indexed_count: AtomicUsize,
     /// Conservative flag set once this memtable has observed any primary-key
     /// rewrite while maintaining a search index. Search planners can push top-k
     /// into HNSW/FTS for append-only PK data, but must switch to
@@ -423,7 +432,7 @@ impl Default for IndexStore {
             hnsw_indexes: HashMap::new(),
             fts_indexes: HashMap::new(),
             pk_index: None,
-            max_visible_batch_position: AtomicUsize::new(0),
+            indexed_count: AtomicUsize::new(0),
             pk_has_overrides: AtomicBool::new(false),
         }
     }
@@ -451,10 +460,7 @@ impl std::fmt::Debug for IndexStore {
                     }
                 },
             )
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position.load(Ordering::Acquire),
-            )
+            .field("indexed_count", &self.indexed_count.load(Ordering::Acquire))
             .field(
                 "pk_has_overrides",
                 &self.pk_has_overrides.load(Ordering::Acquire),
@@ -846,26 +852,25 @@ impl IndexStore {
         let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?;
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update global watermark after all indexes have been updated
+        // Update the indexed prefix after every index has been updated.
         if let Some(bp) = batch_position {
-            self.advance_max_visible_batch_position(bp);
+            self.advance_indexed_count(bp + 1);
         }
 
         Ok(())
     }
 
-    /// Advance the visibility watermark to at least `batch_pos`.
+    /// Advance the indexed prefix to at least `count` batches.
     ///
-    /// The watermark only ever moves forward (idempotent max). The vector
-    /// planner relies on the insert paths setting `pk_has_overrides` before
-    /// calling this method, so any snapshot that can see a PK rewrite also
-    /// observes `pk_has_overrides == true`.
-    pub(crate) fn advance_max_visible_batch_position(&self, batch_pos: usize) {
-        let mut current = self.max_visible_batch_position.load(Ordering::Acquire);
-        while batch_pos > current {
-            match self.max_visible_batch_position.compare_exchange_weak(
+    /// Only ever moves forward (idempotent max). The vector planner relies on the
+    /// insert paths setting `pk_has_overrides` before this is called, so any
+    /// snapshot that can see a PK rewrite also observes `pk_has_overrides == true`.
+    pub(crate) fn advance_indexed_count(&self, count: usize) {
+        let mut current = self.indexed_count.load(Ordering::Acquire);
+        while count > current {
+            match self.indexed_count.compare_exchange_weak(
                 current,
-                batch_pos,
+                count,
                 Ordering::Release,
                 Ordering::Acquire,
             ) {
@@ -1012,9 +1017,10 @@ impl IndexStore {
         }
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update global watermark to the max batch position
+        // The indexed prefix now covers every batch up to and including the
+        // highest position in this call, so the count is that position plus one.
         let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
-        self.advance_max_visible_batch_position(max_bp);
+        self.advance_indexed_count(max_bp + 1);
 
         Ok(duration_map)
     }
@@ -1101,8 +1107,8 @@ impl IndexStore {
     /// construction time so every plan runs against a stable cursor.
     ///
     /// Returns 0 before any WAL flush has advanced the watermark.
-    pub fn max_visible_batch_position(&self) -> usize {
-        self.max_visible_batch_position.load(Ordering::Acquire)
+    pub fn indexed_count(&self) -> usize {
+        self.indexed_count.load(Ordering::Acquire)
     }
 }
 
@@ -1717,7 +1723,7 @@ mod tests {
     }
 
     #[test]
-    fn test_index_store_max_visible_batch_position() {
+    fn test_index_store_indexed_count() {
         let schema = create_test_schema();
         let mut registry = IndexStore::new();
 
@@ -1726,7 +1732,7 @@ mod tests {
         registry.add_fts("desc_idx".to_string(), 2, "description".to_string());
 
         // Initial watermark should be 0 (no data indexed yet)
-        assert_eq!(registry.max_visible_batch_position(), 0);
+        assert_eq!(registry.indexed_count(), 0);
 
         // Insert with batch position tracking
         let batch = create_test_batch(&schema, 0);
@@ -1734,20 +1740,20 @@ mod tests {
             .insert_with_batch_position(&batch, 0, Some(5))
             .unwrap();
 
-        // Now watermark should be 5
-        assert_eq!(registry.max_visible_batch_position(), 5);
+        // Indexing batch position 5 means the prefix [0, 6) is indexed.
+        assert_eq!(registry.indexed_count(), 6);
 
         // Insert with higher batch position
         registry
             .insert_with_batch_position(&batch, 3, Some(10))
             .unwrap();
 
-        // Watermark should advance to 10
-        assert_eq!(registry.max_visible_batch_position(), 10);
+        // Advances to cover batch position 10.
+        assert_eq!(registry.indexed_count(), 11);
 
-        // Insert without batch position shouldn't change watermark
+        // Insert without batch position shouldn't change the cursor
         registry.insert(&batch, 6).unwrap();
-        assert_eq!(registry.max_visible_batch_position(), 10);
+        assert_eq!(registry.indexed_count(), 11);
     }
 
     /// `insert_batches` picks the inline or the threaded path by row count, so
@@ -1781,7 +1787,7 @@ mod tests {
             );
         }
         assert_eq!(registry.get_fts("desc_idx").unwrap().doc_count(), num_rows);
-        assert_eq!(registry.max_visible_batch_position(), 2);
+        assert_eq!(registry.indexed_count(), 3);
     }
 
     #[test]
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index 27d185fce14..f0c5299bbdf 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -495,51 +495,39 @@ impl MemTable {
 
     /// Get batches visible up to a specific batch position (inclusive).
     ///
-    /// A batch at position `i` is visible if `i <= max_visible_batch_position`.
+    /// A batch at position `i` is visible if `i <= visible_count`.
     ///
     /// # Arguments
     ///
-    /// * `max_visible_batch_position` - The maximum batch position to include (inclusive)
+    /// * `visible_count` - The maximum batch position to include (inclusive)
     ///
     /// # Returns
     ///
     /// Vector of visible batches.
-    pub async fn get_visible_batches(&self, max_visible_batch_position: usize) -> Vec<RecordBatch> {
-        self.batch_store
-            .visible_record_batches(max_visible_batch_position)
+    pub async fn get_visible_batches(&self, visible_count: usize) -> Vec<RecordBatch> {
+        self.batch_store.visible_record_batches(visible_count)
     }
 
     /// Get batch positions visible up to a specific batch position (inclusive).
     ///
     /// This is useful for filtering index results by visibility.
-    pub async fn get_max_visible_batch_positions(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Vec<usize> {
-        self.batch_store
-            .max_visible_batch_positions(max_visible_batch_position)
+    pub async fn get_visible_batch_positions(&self, visible_count: usize) -> Vec<usize> {
+        self.batch_store.visible_batch_positions(visible_count)
     }
 
     /// Check if a specific batch is visible at a given visibility position.
     ///
     /// Returns true if the batch is visible, false if not visible or doesn't exist.
-    pub async fn is_batch_visible(
-        &self,
-        batch_position: usize,
-        max_visible_batch_position: usize,
-    ) -> bool {
+    pub async fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool {
         self.batch_store
-            .is_batch_visible(batch_position, max_visible_batch_position)
+            .is_batch_visible(batch_position, visible_count)
     }
 
     /// Scan batches visible up to a specific batch position.
     ///
     /// This combines `get_visible_batches` with the scan interface.
-    pub async fn scan_batches_at_position(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Result<Vec<RecordBatch>> {
-        Ok(self.get_visible_batches(max_visible_batch_position).await)
+    pub async fn scan_batches_at_position(&self, visible_count: usize) -> Result<Vec<RecordBatch>> {
+        Ok(self.get_visible_batches(visible_count).await)
     }
 
     /// Update the bloom filter with primary keys from a batch.
@@ -770,9 +758,9 @@ impl MemTable {
     ///
     /// # Arguments
     ///
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     ///
-    /// The scanner captures the current `max_visible_batch_position` from the
+    /// The scanner captures the current `visible_count` from the
     /// `IndexStore` at construction time to ensure consistent visibility.
     ///
     /// # Panics
@@ -954,23 +942,21 @@ mod tests {
             .await
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 are visible
-        let visible = memtable.get_visible_batches(1).await;
+        // A count of N exposes the prefix [0, N).
+        let visible = memtable.get_visible_batches(2).await;
         assert_eq!(visible.len(), 2);
         let total_rows: usize = visible.iter().map(|b| b.num_rows()).sum();
         assert_eq!(total_rows, 15); // 10 + 5
 
-        // max_visible_batch_position=2 means all batches are visible
-        let visible = memtable.get_visible_batches(2).await;
+        let visible = memtable.get_visible_batches(3).await;
         assert_eq!(visible.len(), 3);
 
-        // max_visible_batch_position=0 means only position 0 is visible
-        let visible = memtable.get_visible_batches(0).await;
-        assert_eq!(visible.len(), 1);
+        // A count of 0 exposes nothing — not "batch 0".
+        assert!(memtable.get_visible_batches(0).await.is_empty());
     }
 
     #[tokio::test]
-    async fn test_memtable_get_max_visible_batch_positions() {
+    async fn test_memtable_get_visible_batch_positions() {
         let schema = create_test_schema();
         let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
 
@@ -988,17 +974,15 @@ mod tests {
             .await
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 visible
-        let visible_ids = memtable.get_max_visible_batch_positions(1).await;
+        // A count of N exposes the prefix [0, N).
+        let visible_ids = memtable.get_visible_batch_positions(2).await;
         assert_eq!(visible_ids, vec![0, 1]);
 
-        // max_visible_batch_position=2 means all positions visible
-        let visible_ids = memtable.get_max_visible_batch_positions(2).await;
+        let visible_ids = memtable.get_visible_batch_positions(3).await;
         assert_eq!(visible_ids, vec![0, 1, 2]);
 
-        // max_visible_batch_position=0 means only position 0 visible
-        let visible_ids = memtable.get_max_visible_batch_positions(0).await;
-        assert_eq!(visible_ids, vec![0]);
+        // A count of 0 exposes nothing.
+        assert!(memtable.get_visible_batch_positions(0).await.is_empty());
     }
 
     #[tokio::test]
@@ -1019,14 +1003,14 @@ mod tests {
             .await
             .unwrap(); // position 2
 
-        // batch_position 0 is visible when max_visible_batch_position >= 0
-        assert!(memtable.is_batch_visible(0, 0).await);
+        // A count of 0 means nothing is visible, batch 0 included.
+        assert!(!memtable.is_batch_visible(0, 0).await);
+
+        // Batch i is visible once the count exceeds i.
         assert!(memtable.is_batch_visible(0, 1).await);
         assert!(memtable.is_batch_visible(0, 2).await);
-
-        // batch_position 2 is only visible when max_visible_batch_position >= 2
         assert!(!memtable.is_batch_visible(2, 1).await);
-        assert!(memtable.is_batch_visible(2, 2).await);
+        assert!(!memtable.is_batch_visible(2, 2).await);
         assert!(memtable.is_batch_visible(2, 3).await);
 
         // Non-existent batch
@@ -1047,12 +1031,21 @@ mod tests {
             .await
             .unwrap(); // position 1
 
-        let batches = memtable.scan_batches_at_position(0).await.unwrap();
+        let batches = memtable.scan_batches_at_position(1).await.unwrap();
         assert_eq!(batches.len(), 1);
         assert_eq!(batches[0].num_rows(), 10);
 
-        let batches = memtable.scan_batches_at_position(1).await.unwrap();
+        let batches = memtable.scan_batches_at_position(2).await.unwrap();
         assert_eq!(batches.len(), 2);
+
+        // Nothing indexed yet => nothing scannable.
+        assert!(
+            memtable
+                .scan_batches_at_position(0)
+                .await
+                .unwrap()
+                .is_empty()
+        );
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
index 23f1c9f4767..f16f9bea1fd 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
@@ -634,66 +634,53 @@ impl BatchStore {
     // Visibility API
     // =========================================================================
 
-    /// Get batches visible up to a specific batch position (inclusive).
+    /// Batches in the visible prefix `[0, visible_count)`.
     ///
-    /// A batch at position `i` is visible if `i <= max_visible_batch_position`.
-    pub fn visible_batches(&self, max_visible_batch_position: usize) -> Vec<&StoredBatch> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        let end = (max_visible_batch_position + 1).min(len);
+    /// `visible_count` is an **exclusive count**, not an inclusive position: 0
+    /// means nothing is visible. As an inclusive position, 0 meant *both*
+    /// "nothing visible" and "batch 0 is visible", so a batch that was committed
+    /// to the store but not yet indexed or WAL-durable was readable for a full
+    /// PUT round-trip. The count makes that off-by-one inexpressible.
+    pub fn visible_batches(&self, visible_count: usize) -> Vec<&StoredBatch> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
         (0..end).filter_map(|i| self.get(i)).collect()
     }
 
-    /// Get batch positions visible up to a specific batch position (inclusive).
-    pub fn max_visible_batch_positions(&self, max_visible_batch_position: usize) -> Vec<usize> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        let end = (max_visible_batch_position + 1).min(len);
+    /// Positions of the batches in the visible prefix.
+    pub fn visible_batch_positions(&self, visible_count: usize) -> Vec<usize> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
         (0..end).collect()
     }
 
-    /// The inclusive maximum visible *row* position at `max_visible_batch_position`,
-    /// or `None` when no rows are visible. The visible batches are the committed
-    /// prefix `[0, last_visible_idx]`; each batch carries its cumulative
-    /// `row_offset`, so this is the end of the last visible batch minus one.
-    /// Used to bound MVCC seeks against the maintained PK-position index.
-    pub fn max_visible_row(&self, max_visible_batch_position: usize) -> Option<u64> {
-        let len = self.committed_len.load(Ordering::Acquire);
-        if len == 0 {
-            return None;
-        }
-        let last_visible_idx = max_visible_batch_position.min(len - 1);
-        let last = self.get(last_visible_idx)?;
+    /// The inclusive maximum visible *row* position, or `None` when no rows are
+    /// visible. Each batch carries its cumulative `row_offset`, so this is the
+    /// end of the last visible batch minus one. Bounds MVCC seeks against the
+    /// maintained PK-position index.
+    pub fn max_visible_row(&self, visible_count: usize) -> Option<u64> {
+        let end = visible_count.min(self.committed_len.load(Ordering::Acquire));
+        let last = self.get(end.checked_sub(1)?)?;
         let visible_end = last.row_offset + last.num_rows as u64; // exclusive
         visible_end.checked_sub(1)
     }
 
-    /// Check if a specific batch is visible at a given visibility position.
+    /// Whether a batch falls inside the visible prefix.
     #[inline]
-    pub fn is_batch_visible(
-        &self,
-        batch_position: usize,
-        max_visible_batch_position: usize,
-    ) -> bool {
+    pub fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool {
         let len = self.committed_len.load(Ordering::Acquire);
-        batch_position < len && batch_position <= max_visible_batch_position
+        batch_position < len && batch_position < visible_count
     }
 
-    /// Get visible RecordBatches (clones the data).
-    pub fn visible_record_batches(&self, max_visible_batch_position: usize) -> Vec<RecordBatch> {
-        self.visible_batches(max_visible_batch_position)
+    /// Visible RecordBatches (clones the data).
+    pub fn visible_record_batches(&self, visible_count: usize) -> Vec<RecordBatch> {
+        self.visible_batches(visible_count)
             .into_iter()
             .map(|b| b.data.clone())
             .collect()
     }
 
-    /// Get visible RecordBatches with their row offsets.
-    ///
-    /// Returns tuples of (batch, row_offset) for each visible batch.
-    /// The row_offset is the starting row position for that batch.
-    pub fn visible_batches_with_offsets(
-        &self,
-        max_visible_batch_position: usize,
-    ) -> Vec<(RecordBatch, u64)> {
-        self.visible_batches(max_visible_batch_position)
+    /// Visible RecordBatches paired with the row position each one starts at.
+    pub fn visible_batches_with_offsets(&self, visible_count: usize) -> Vec<(RecordBatch, u64)> {
+        self.visible_batches(visible_count)
             .into_iter()
             .map(|b| (b.data.clone(), b.row_offset))
             .collect()
@@ -926,17 +913,52 @@ mod tests {
         store.append(create_test_batch(10)).unwrap(); // position 3
         store.append(create_test_batch(10)).unwrap(); // position 4
 
-        // max_visible_batch_position=2 means positions 0, 1, 2 are visible
-        let visible = store.max_visible_batch_positions(2);
-        assert_eq!(visible, vec![0, 1, 2]);
+        // A count of N exposes the prefix [0, N).
+        assert_eq!(store.visible_batch_positions(3), vec![0, 1, 2]);
+        assert_eq!(store.visible_batch_positions(5), vec![0, 1, 2, 3, 4]);
+
+        // A count of 0 exposes nothing. Under the old inclusive cursor this
+        // case was indistinguishable from "batch 0 is visible", so every
+        // memtable leaked its first batch before it was indexed or durable.
+        assert!(store.visible_batch_positions(0).is_empty());
+
+        // Beyond the committed range, clamp.
+        assert_eq!(store.visible_batch_positions(99), vec![0, 1, 2, 3, 4]);
+    }
+
+    /// The zero of the visibility cursor must be unambiguous.
+    ///
+    /// `BatchStore::append` publishes `committed_len` on the put path, under the
+    /// state lock, *before* the WAL flush that indexes the batch is even
+    /// triggered — and that flush is a ~100ms S3 PUT on another task. So batch 0
+    /// sits committed and readable for a full round-trip before it is indexed or
+    /// durable. As an inclusive position, a cursor of 0 meant both "nothing is
+    /// visible" and "batch 0 is visible", so every read arm backed by the batch
+    /// store served that batch while the index-backed arms did not — the tiers
+    /// actively disagreed. An exclusive count makes the state inexpressible.
+    #[test]
+    fn test_zero_cursor_hides_the_committed_but_unindexed_prefix() {
+        let store = BatchStore::with_capacity(4);
+        store.append(create_test_batch(10)).unwrap();
+        store.append(create_test_batch(10)).unwrap();
+
+        // Committed, but nothing indexed yet: every visibility query must agree
+        // that there is nothing to read.
+        assert!(store.visible_batches(0).is_empty());
+        assert!(store.visible_batch_positions(0).is_empty());
+        assert!(store.visible_record_batches(0).is_empty());
+        assert!(store.visible_batches_with_offsets(0).is_empty());
+        assert!(!store.is_batch_visible(0, 0));
+        assert_eq!(store.max_visible_row(0), None);
 
-        // max_visible_batch_position=4 means all visible
-        let visible = store.max_visible_batch_positions(4);
-        assert_eq!(visible, vec![0, 1, 2, 3, 4]);
+        // The batches are there — they are simply not yet published.
+        assert_eq!(store.len(), 2);
 
-        // max_visible_batch_position=0 means only position 0 visible
-        let visible = store.max_visible_batch_positions(0);
-        assert_eq!(visible, vec![0]);
+        // Indexing batch 0 publishes exactly batch 0.
+        assert_eq!(store.visible_batches(1).len(), 1);
+        assert!(store.is_batch_visible(0, 1));
+        assert!(!store.is_batch_visible(1, 1));
+        assert_eq!(store.max_visible_row(1), Some(9));
     }
 
     #[test]
@@ -947,14 +969,17 @@ mod tests {
         store.append(create_test_batch(10)).unwrap(); // position 1
         store.append(create_test_batch(10)).unwrap(); // position 2
 
-        // Batch at position 0 is visible when max_visible_batch_position >= 0
-        assert!(store.is_batch_visible(0, 0));
+        // A count of 0 means *nothing* is visible — including batch 0. As an
+        // inclusive position this case was indistinguishable from "batch 0 is
+        // visible", so a batch that was committed to the store but not yet
+        // indexed or WAL-durable was readable for a full PUT round-trip.
+        assert!(!store.is_batch_visible(0, 0));
+
+        // Batch i is visible once the count exceeds i.
         assert!(store.is_batch_visible(0, 1));
         assert!(store.is_batch_visible(0, 2));
-
-        // Batch at position 2 is only visible when max_visible_batch_position >= 2
         assert!(!store.is_batch_visible(2, 1));
-        assert!(store.is_batch_visible(2, 2));
+        assert!(!store.is_batch_visible(2, 2));
         assert!(store.is_batch_visible(2, 3));
 
         // Batch 3 doesn't exist
@@ -963,7 +988,7 @@ mod tests {
 
     #[test]
     fn test_max_visible_row() {
-        // (1) Empty store: no rows are visible at any position.
+        // (1) Empty store: no rows are visible at any count.
         let store = BatchStore::with_capacity(10);
         assert_eq!(store.max_visible_row(0), None);
         assert_eq!(store.max_visible_row(100), None);
@@ -973,23 +998,24 @@ mod tests {
         store.append(create_test_batch(20)).unwrap(); // position 1
         store.append(create_test_batch(30)).unwrap(); // position 2
 
-        // (2) A position within range yields the inclusive end of that prefix.
-        assert_eq!(store.max_visible_row(0), Some(9)); // batch 0: 0..10
-        assert_eq!(store.max_visible_row(1), Some(29)); // batch 1: 10..30
-        assert_eq!(store.max_visible_row(2), Some(59)); // batch 2: 30..60
+        // (2) A count of 0 means nothing is visible — not "batch 0 is visible".
+        assert_eq!(store.max_visible_row(0), None);
+
+        // (3) A count of N yields the inclusive last row of the prefix [0, N).
+        assert_eq!(store.max_visible_row(1), Some(9)); // batch 0: 0..10
+        assert_eq!(store.max_visible_row(2), Some(29)); // + batch 1: 10..30
+        assert_eq!(store.max_visible_row(3), Some(59)); // + batch 2: 30..60
 
-        // (3) A position beyond the committed range clamps to the last batch,
-        // i.e. the inclusive max over all rows.
+        // (4) A count beyond the committed range clamps to the last batch.
         assert_eq!(store.max_visible_row(100), Some(59));
 
-        // (4) An empty leading batch contributes no rows: at its own position
-        // the inclusive end underflows to None, while a later non-empty batch
-        // is reported correctly.
+        // (5) An empty leading batch contributes no rows, so a prefix covering
+        // only it still yields None, while a later non-empty batch is reported.
         let store = BatchStore::with_capacity(10);
         store.append(create_test_batch(0)).unwrap(); // position 0: rows [0,0)
         store.append(create_test_batch(5)).unwrap(); // position 1: rows [0,5)
-        assert_eq!(store.max_visible_row(0), None); // empty prefix → no rows
-        assert_eq!(store.max_visible_row(1), Some(4)); // through batch 1
+        assert_eq!(store.max_visible_row(1), None); // empty prefix → no rows
+        assert_eq!(store.max_visible_row(2), Some(4)); // through batch 1
     }
 
     #[test]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
index e1f0d6e689c..7ab0d00a273 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
@@ -378,7 +378,7 @@ impl ScalarPredicate {
 ///
 /// # Index Visibility Model
 ///
-/// The scanner captures `max_visible_batch_position` from the `IndexStore` at
+/// The scanner captures `visible_count` from the `IndexStore` at
 /// construction time. This frozen visibility ensures queries only see data
 /// that has been indexed, providing consistent results.
 ///
@@ -401,8 +401,8 @@ pub struct MemTableScanner {
     indexes: Arc<IndexStore>,
     schema: SchemaRef,
     /// Frozen visibility captured at scanner construction time.
-    /// This is the `max_visible_batch_position` from the IndexStore.
-    max_visible_batch_position: usize,
+    /// This is the `visible_count` from the IndexStore.
+    visible_count: usize,
     projection: Option<Vec<String>>,
     filter: Option<Expr>,
     limit: Option<usize>,
@@ -427,7 +427,7 @@ pub struct MemTableScanner {
 impl MemTableScanner {
     /// Create a new scanner.
     ///
-    /// Captures `max_visible_batch_position` from the `IndexStore` at construction
+    /// Captures `visible_count` from the `IndexStore` at construction
     /// time to ensure consistent query visibility.
     ///
     /// # Arguments
@@ -439,13 +439,13 @@ impl MemTableScanner {
         // Snapshot the visibility cursor at construction time. The cursor is
         // advanced by `flush_from_batch_store` after the WAL append succeeds,
         // so this snapshot reflects WAL-durable data.
-        let max_visible_batch_position = indexes.max_visible_batch_position();
+        let visible_count = indexes.indexed_count();
 
         Self {
             batch_store,
             indexes,
             schema,
-            max_visible_batch_position,
+            visible_count,
             projection: None,
             filter: None,
             limit: None,
@@ -504,12 +504,12 @@ impl MemTableScanner {
         self
     }
 
-    /// The `max_visible_batch_position` snapshot this scanner latched at
+    /// The `visible_count` snapshot this scanner latched at
     /// construction. A downstream recency filter must key on this same snapshot
     /// (not a fresh read of the IndexStore watermark, which a concurrent append
     /// could have advanced) so it stays consistent with the rows the search saw.
-    pub fn max_visible_batch_position(&self) -> usize {
-        self.max_visible_batch_position
+    pub fn visible_count(&self) -> usize {
+        self.visible_count
     }
 
     /// Include the _rowaddr column in output.
@@ -971,7 +971,7 @@ impl MemTableScanner {
 
         let scan = MemTableScanExec::with_filter(
             self.batch_store.clone(),
-            self.max_visible_batch_position,
+            self.visible_count,
             projection_indices,
             self.output_schema(),
             self.schema.clone(),
@@ -1033,7 +1033,7 @@ impl MemTableScanner {
 
         Ok(Arc::new(MemTableDedupScanExec::new(
             self.batch_store.clone(),
-            self.max_visible_batch_position,
+            self.visible_count,
             projection_indices,
             self.output_schema(),
             pk_indices,
@@ -1056,7 +1056,7 @@ impl MemTableScanner {
             return self.plan_full_scan().await;
         }
 
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
 
         let index_exec = BTreeIndexExec::new(
@@ -1095,7 +1095,7 @@ impl MemTableScanner {
     }
 
     async fn plan_vector_search(&self, query: &VectorQuery) -> Result<Arc<dyn ExecutionPlan>> {
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
         let base_schema = self.base_output_schema();
         let filter_predicate = self.filter_predicate()?;
@@ -1154,7 +1154,7 @@ impl MemTableScanner {
             return self.empty_fts_plan();
         }
 
-        let max_visible = self.max_visible_batch_position;
+        let max_visible = self.visible_count;
         let projection_indices = self.compute_projection_indices()?;
         let filter_predicate = self.filter_predicate()?;
         if let Some(pk_columns) = &self.pk_columns {
@@ -1426,7 +1426,7 @@ mod tests {
         let indexes = Arc::new(index_store);
         let scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
         let result = scanner.try_into_batch().await.unwrap();
-        // max_visible_batch_position is 1, so we see batches 0 and 1 (20 rows)
+        // visible_count is 1, so we see batches 0 and 1 (20 rows)
         assert_eq!(result.num_rows(), 20);
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
index 2a593a0f215..ac54e4c5617 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
@@ -36,7 +36,7 @@ pub use vector::VectorIndexExec;
 pub(super) fn newest_pk_positions(
     batch_store: &BatchStore,
     pk_columns: &[String],
-    max_visible_batch_position: usize,
+    visible_count: usize,
     max_visible_row: u64,
 ) -> DataFusionResult<HashSet<u64>> {
     let mut newest: HashMap<Vec<ScalarValue>, u64> = HashMap::new();
@@ -46,7 +46,7 @@ pub(super) fn newest_pk_positions(
         if n == 0 {
             continue;
         }
-        if batch_position > max_visible_batch_position {
+        if batch_position > visible_count {
             current_row += n as u64;
             continue;
         }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
index 8a239605b50..bc9421f675d 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
@@ -48,7 +48,7 @@ const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2;
 pub struct MemTableBruteForceVectorExec {
     batch_store: Arc<BatchStore>,
     query: VectorQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -69,10 +69,7 @@ impl Debug for MemTableBruteForceVectorExec {
         f.debug_struct("MemTableBruteForceVectorExec")
             .field("column", &self.query.column)
             .field("k", &self.query.k)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .finish()
     }
@@ -85,7 +82,7 @@ impl MemTableBruteForceVectorExec {
     pub fn new(
         batch_store: Arc<BatchStore>,
         query: VectorQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -111,7 +108,7 @@ impl MemTableBruteForceVectorExec {
         Ok(Self {
             batch_store,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -160,7 +157,7 @@ impl MemTableBruteForceVectorExec {
         Ok(Some(mask))
     }
 
-    /// Last row position visible under `max_visible_batch_position`, or `None`
+    /// Last row position visible under `visible_count`, or `None`
     /// if no batches are visible. Identical to `VectorIndexExec`'s helper so
     /// both arms cut at the same MVCC boundary.
     fn compute_max_visible_row(&self) -> Option<u64> {
@@ -169,7 +166,7 @@ impl MemTableBruteForceVectorExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
@@ -225,7 +222,7 @@ impl MemTableBruteForceVectorExec {
                 newest_pk_positions(
                     &self.batch_store,
                     pk_columns,
-                    self.max_visible_batch_position,
+                    self.visible_count,
                     max_visible_row,
                 )
                 .map_err(|e| Error::invalid_input(e.to_string()))?,
@@ -245,7 +242,7 @@ impl MemTableBruteForceVectorExec {
             if n == 0 {
                 continue;
             }
-            if batch_position > self.max_visible_batch_position {
+            if batch_position > self.visible_count {
                 current_row += n as u64;
                 continue;
             }
@@ -615,7 +612,7 @@ mod tests {
             MemTableBruteForceVectorExec::new(
                 store,
                 query,
-                /* max_visible_batch_position = */ usize::MAX,
+                /* visible_count = */ usize::MAX,
                 None,
                 schema,
                 false,
@@ -668,7 +665,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn respects_max_visible_batch_position() {
+    async fn respects_indexed_count() {
         // Two batches of two rows. Freeze at batch 0 — only ids 0,1 are
         // visible candidates; the (closer) ids 2,3 in batch 1 are excluded.
         let schema = make_schema();
@@ -678,7 +675,7 @@ mod tests {
         let query = query_for([0.0, 0.0], 4);
         let exec = Arc::new(
             MemTableBruteForceVectorExec::new(
-                store, query, /* max_visible_batch_position = */ 0, None, schema, false,
+                store, query, /* visible_count = */ 1, None, schema, false,
             )
             .expect("ctor"),
         );
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
index fed61698fab..62e538f9102 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
@@ -31,7 +31,7 @@ pub struct BTreeIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     predicate: ScalarPredicate,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -48,10 +48,7 @@ impl Debug for BTreeIndexExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("BTreeIndexExec")
             .field("predicate", &self.predicate)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .field("with_row_address", &self.with_row_address)
             .field("column", &self.column)
@@ -67,7 +64,7 @@ impl BTreeIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with BTree indexes
     /// * `predicate` - Scalar predicate to apply
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -77,7 +74,7 @@ impl BTreeIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         predicate: ScalarPredicate,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         with_row_id: bool,
@@ -103,7 +100,7 @@ impl BTreeIndexExec {
             batch_store,
             indexes,
             predicate,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -114,7 +111,7 @@ impl BTreeIndexExec {
         })
     }
 
-    /// Compute the maximum visible row position based on max_visible_batch_position.
+    /// Compute the maximum visible row position based on visible_count.
     /// Returns None if no batches are visible.
     fn compute_max_visible_row(&self) -> Option<u64> {
         let mut max_visible_row_exclusive: u64 = 0;
@@ -122,7 +119,7 @@ impl BTreeIndexExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
@@ -434,7 +431,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0, // max_visible_batch_position (batch at position 0)
+            1, // visible_count (batch at position 0)
             None,
             schema,
             false,
@@ -478,7 +475,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0,
+            1,
             None,
             schema,
             false,
@@ -523,7 +520,7 @@ mod tests {
             batch_store.clone(),
             indexes.clone(),
             predicate.clone(),
-            0,
+            1,
             None,
             schema.clone(),
             false,
@@ -543,7 +540,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            1,
+            2,
             None,
             schema,
             false,
@@ -592,7 +589,7 @@ mod tests {
             batch_store,
             indexes,
             predicate,
-            0,
+            1,
             None,
             schema_with_rowid.clone(),
             true,
@@ -656,7 +653,7 @@ mod tests {
                 batch_store.clone(),
                 indexes.clone(),
                 predicate.clone(),
-                0,
+                1,
                 None,
                 schema.clone(),
                 false,
@@ -684,7 +681,7 @@ mod tests {
                 batch_store,
                 indexes,
                 predicate,
-                0,
+                1,
                 None,
                 schema_with_rowid,
                 true,
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
index ba5947e4b12..635825f742f 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
@@ -44,7 +44,7 @@ use crate::dataset::mem_wal::write::BatchStore;
 /// that satisfy the (optional) predicate. See the module doc.
 pub struct MemTableDedupScanExec {
     batch_store: Arc<BatchStore>,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     /// Column indices to project (into the source schema).
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
@@ -62,10 +62,7 @@ pub struct MemTableDedupScanExec {
 impl Debug for MemTableDedupScanExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MemTableDedupScanExec")
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("projection", &self.projection)
             .field("pk_indices", &self.pk_indices)
             .field("with_row_address", &self.with_row_address)
@@ -79,7 +76,7 @@ impl MemTableDedupScanExec {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         pk_indices: Vec<usize>,
@@ -97,7 +94,7 @@ impl MemTableDedupScanExec {
 
         Self {
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             pk_indices,
@@ -188,7 +185,7 @@ impl ExecutionPlan for MemTableDedupScanExec {
         // back-to-front below.
         let mut batches = self
             .batch_store
-            .visible_batches_with_offsets(self.max_visible_batch_position);
+            .visible_batches_with_offsets(self.visible_count);
         batches.reverse();
 
         let projection = self.projection.clone();
@@ -347,7 +344,7 @@ mod tests {
     /// Run the exec and collect (id -> (value, rowaddr)).
     async fn run(
         store: Arc<BatchStore>,
-        max_visible: usize,
+        visible_count: usize,
         filter: Option<Expr>,
     ) -> HashMap<i32, (Option<i32>, u64)> {
         let filter_predicate = filter.map(|expr| {
@@ -358,7 +355,7 @@ mod tests {
         let filter_expr = None;
         let exec = MemTableDedupScanExec::new(
             store,
-            max_visible,
+            visible_count,
             None,
             output_schema(),
             vec![0],
@@ -392,11 +389,11 @@ mod tests {
         // id=10 inserted (100) then updated to NULL, all in one batch.
         store.append(batch(&[(10, Some(100)), (10, None)])).unwrap();
 
-        let no_filter = run(store.clone(), 0, None).await;
+        let no_filter = run(store.clone(), 1, None).await;
         assert_eq!(no_filter.len(), 1);
         assert_eq!(no_filter[&10].0, None, "newest version of id=10 is NULL");
 
-        let not_null = run(store, 0, Some(col("value").is_not_null())).await;
+        let not_null = run(store, 1, Some(col("value").is_not_null())).await;
         assert!(
             !not_null.contains_key(&10),
             "id=10 newest is NULL; the stale value=100 must not leak under value IS NOT NULL"
@@ -413,7 +410,7 @@ mod tests {
         store.append(batch(&[(20, Some(999)), (30, None)])).unwrap();
 
         // No filter: newest per PK = {10:NULL@2, 20:999@3, 30:NULL@4}.
-        let all = run(store.clone(), 1, None).await;
+        let all = run(store.clone(), 2, None).await;
         assert_eq!(all.len(), 3);
         assert_eq!(all[&10], (None, 2));
         assert_eq!(all[&20], (Some(999), 3));
@@ -421,7 +418,7 @@ mod tests {
 
         // value IS NOT NULL: only id=20 (newest 999) survives; 10 and 30 are
         // newest-NULL so they must be absent (no stale leak).
-        let not_null = run(store, 1, Some(col("value").is_not_null())).await;
+        let not_null = run(store, 2, Some(col("value").is_not_null())).await;
         assert_eq!(not_null.len(), 1);
         assert_eq!(not_null[&20], (Some(999), 3));
     }
@@ -434,7 +431,7 @@ mod tests {
         // id=40 inserted NULL then updated to 400 (newest non-NULL).
         store.append(batch(&[(40, None), (40, Some(400))])).unwrap();
 
-        let is_null = run(store, 0, Some(col("value").is_null())).await;
+        let is_null = run(store, 1, Some(col("value").is_null())).await;
         assert!(
             !is_null.contains_key(&40),
             "id=40 newest is 400; the stale NULL must not leak under value IS NULL"
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
index 364261c3276..97bafbef2eb 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
@@ -48,14 +48,14 @@ pub struct FtsIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     query: FtsQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
     /// Pre-computed batch ranges for O(log n) lookup.
     batch_ranges: Vec<BatchRange>,
-    /// Maximum visible row position based on max_visible_batch_position (None if nothing visible).
+    /// Maximum visible row position based on visible_count (None if nothing visible).
     max_visible_row: Option<u64>,
     /// Whether to include _rowid column (row position) in output.
     with_row_id: bool,
@@ -73,10 +73,7 @@ impl Debug for FtsIndexExec {
         f.debug_struct("FtsIndexExec")
             .field("column", &self.query.column)
             .field("query_type", &self.query.query_type)
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("with_row_id", &self.with_row_id)
             .finish()
     }
@@ -90,7 +87,7 @@ impl FtsIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with FTS indexes
     /// * `query` - FTS query parameters
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `base_schema` - Schema before adding score column (and _rowid if with_row_id)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -98,7 +95,7 @@ impl FtsIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         query: FtsQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -149,7 +146,7 @@ impl FtsIndexExec {
                 end: batch_end,
                 batch_id,
             });
-            if batch_id <= max_visible_batch_position {
+            if batch_id < visible_count {
                 max_visible_row_exclusive = batch_end as u64;
             }
             current_row = batch_end;
@@ -166,7 +163,7 @@ impl FtsIndexExec {
             batch_store,
             indexes,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -459,7 +456,7 @@ impl FtsIndexExec {
             Some(newest_pk_positions(
                 &self.batch_store,
                 pk_columns,
-                self.max_visible_batch_position,
+                self.visible_count,
                 max_visible_row,
             )?)
         };
@@ -646,7 +643,7 @@ mod tests {
 
         let query = FtsQuery::match_query("text", "hello");
 
-        let exec = FtsIndexExec::new(batch_store, indexes, query, 0, None, schema, false).unwrap();
+        let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap();
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -687,7 +684,7 @@ mod tests {
             batch_store.clone(),
             indexes.clone(),
             query.clone(),
-            0,
+            1,
             None,
             schema.clone(),
             false,
@@ -702,7 +699,7 @@ mod tests {
         assert_eq!(total_rows, 2); // "hello" in batch1 docs 0 and 2
 
         // Query with max_visible=1 should see both batches
-        let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap();
+        let exec = FtsIndexExec::new(batch_store, indexes, query, 2, None, schema, false).unwrap();
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
index c56e960048d..f43d871a414 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
@@ -31,12 +31,12 @@ pub const ROW_ADDRESS_COLUMN: &str = "_rowaddr";
 /// ExecutionPlan node that scans all visible batches from a MemTable.
 ///
 /// This node implements visibility filtering, returning only batches
-/// where `batch_position <= max_visible_batch_position`.
+/// where `batch_position <= visible_count`.
 ///
 /// Supports filter pushdown for efficient predicate evaluation during scan.
 pub struct MemTableScanExec {
     batch_store: Arc<BatchStore>,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     /// Schema of the source data (before projection), used for filter evaluation.
@@ -56,10 +56,7 @@ pub struct MemTableScanExec {
 impl Debug for MemTableScanExec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MemTableScanExec")
-            .field(
-                "max_visible_batch_position",
-                &self.max_visible_batch_position,
-            )
+            .field("visible_count", &self.visible_count)
             .field("projection", &self.projection)
             .field("with_row_id", &self.with_row_id)
             .field("with_row_address", &self.with_row_address)
@@ -74,20 +71,20 @@ impl MemTableScanExec {
     /// # Arguments
     ///
     /// * `batch_store` - Lock-free batch store containing data
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `with_row_id` - Whether to include _rowid column (row position)
     pub fn new(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         with_row_id: bool,
     ) -> Self {
         Self::with_filter(
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema.clone(),
             output_schema,
@@ -103,7 +100,7 @@ impl MemTableScanExec {
     /// # Arguments
     ///
     /// * `batch_store` - Lock-free batch store containing data
-    /// * `max_visible_batch_position` - Maximum batch position visible (inclusive)
+    /// * `visible_count` - Maximum batch position visible (inclusive)
     /// * `projection` - Optional column indices to project
     /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested)
     /// * `source_schema` - Schema of source data (before projection), used for filter evaluation
@@ -114,7 +111,7 @@ impl MemTableScanExec {
     #[allow(clippy::too_many_arguments)]
     pub fn with_filter(
         batch_store: Arc<BatchStore>,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         output_schema: SchemaRef,
         source_schema: SchemaRef,
@@ -132,7 +129,7 @@ impl MemTableScanExec {
 
         Self {
             batch_store,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             source_schema,
@@ -226,7 +223,7 @@ impl ExecutionPlan for MemTableScanExec {
         // Get visible batches with their row offsets
         let batches_with_offsets = self
             .batch_store
-            .visible_batches_with_offsets(self.max_visible_batch_position);
+            .visible_batches_with_offsets(self.visible_count);
 
         let projection = self.projection.clone();
         let schema = self.output_schema.clone();
@@ -399,7 +396,7 @@ mod tests {
         batch_store.append(batch).unwrap();
 
         // Batch is at position 0, max_visible=0 means position 0 is visible
-        let exec = MemTableScanExec::new(batch_store, 0, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -425,8 +422,8 @@ mod tests {
             .append(create_test_batch(&schema, 20, 10))
             .unwrap();
 
-        // max_visible_batch_position=1 means positions 0 and 1 are visible (2 batches)
-        let exec = MemTableScanExec::new(batch_store.clone(), 1, None, schema.clone(), false);
+        // visible_count=1 means positions 0 and 1 are visible (2 batches)
+        let exec = MemTableScanExec::new(batch_store.clone(), 2, None, schema.clone(), false);
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
         let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
@@ -447,7 +444,7 @@ mod tests {
         // Project only "id" column (index 0)
         let projected_schema =
             Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        let exec = MemTableScanExec::new(batch_store, 0, Some(vec![0]), projected_schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, Some(vec![0]), projected_schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -464,7 +461,7 @@ mod tests {
         let batch_store = Arc::new(BatchStore::with_capacity(100));
 
         // Empty store with max_visible=0 should return no batches
-        let exec = MemTableScanExec::new(batch_store, 0, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
@@ -486,7 +483,7 @@ mod tests {
             .unwrap();
 
         // max_visible=1 means positions 0 and 1 are visible
-        let exec = MemTableScanExec::new(batch_store, 1, None, schema, false);
+        let exec = MemTableScanExec::new(batch_store, 2, None, schema, false);
 
         let stats = exec.partition_statistics(None).unwrap();
         // Statistics are Absent to avoid DataFusion analysis bugs
@@ -513,7 +510,7 @@ mod tests {
             Field::new("_rowid", DataType::UInt64, true),
         ]));
 
-        let exec = MemTableScanExec::new(batch_store, 1, None, schema_with_rowid, true);
+        let exec = MemTableScanExec::new(batch_store, 2, None, schema_with_rowid, true);
 
         let ctx = Arc::new(TaskContext::default());
         let stream = exec.execute(0, ctx).unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
index c3453db68a2..8cbf40a3039 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
@@ -35,7 +35,7 @@ pub struct VectorIndexExec {
     batch_store: Arc<BatchStore>,
     indexes: Arc<IndexStore>,
     query: VectorQuery,
-    max_visible_batch_position: usize,
+    visible_count: usize,
     projection: Option<Vec<usize>>,
     output_schema: SchemaRef,
     properties: Arc<PlanProperties>,
@@ -59,10 +59,7 @@ impl Debug for VectorIndexExec {
         if let Some(metric) = &self.query.distance_type {
             debug.field("distance_type", metric);
         }
-        debug.field(
-            "max_visible_batch_position",
-            &self.max_visible_batch_position,
-        );
+        debug.field("visible_count", &self.visible_count);
         debug.field("with_row_id", &self.with_row_id);
         debug.finish()
     }
@@ -76,7 +73,7 @@ impl VectorIndexExec {
     /// * `batch_store` - Lock-free batch store containing data
     /// * `indexes` - Index registry with HNSW vector indexes
     /// * `query` - Vector query parameters
-    /// * `max_visible_batch_position` - MVCC visibility sequence number
+    /// * `visible_count` - MVCC visibility sequence number
     /// * `projection` - Optional column indices to project
     /// * `base_schema` - Schema after projection (will add _distance column, and _rowid if with_row_id)
     /// * `with_row_id` - Whether to include _rowid column (row position)
@@ -84,7 +81,7 @@ impl VectorIndexExec {
         batch_store: Arc<BatchStore>,
         indexes: Arc<IndexStore>,
         query: VectorQuery,
-        max_visible_batch_position: usize,
+        visible_count: usize,
         projection: Option<Vec<usize>>,
         base_schema: SchemaRef,
         with_row_id: bool,
@@ -120,7 +117,7 @@ impl VectorIndexExec {
             batch_store,
             indexes,
             query,
-            max_visible_batch_position,
+            visible_count,
             projection,
             output_schema,
             properties,
@@ -129,9 +126,9 @@ impl VectorIndexExec {
         })
     }
 
-    /// Compute the maximum visible row position based on max_visible_batch_position.
+    /// Compute the maximum visible row position based on visible_count.
     ///
-    /// Returns the last row position that is visible at the given max_visible_batch_position,
+    /// Returns the last row position that is visible at the given visible_count,
     /// or None if no batches are visible.
     fn compute_max_visible_row(&self) -> Option<u64> {
         let mut max_visible_row_exclusive: u64 = 0;
@@ -139,7 +136,7 @@ impl VectorIndexExec {
 
         for (batch_position, stored_batch) in self.batch_store.iter().enumerate() {
             let batch_end = current_row + stored_batch.num_rows as u64;
-            if batch_position <= self.max_visible_batch_position {
+            if batch_position < self.visible_count {
                 max_visible_row_exclusive = batch_end;
             }
             current_row = batch_end;
diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
index 30a19650005..eefec568983 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
@@ -328,7 +328,7 @@ fn in_memory_membership(
     batch_store: &Arc<BatchStore>,
     index_store: &Arc<IndexStore>,
 ) -> GenMembership {
-    let max_visible_row = batch_store.max_visible_row(index_store.max_visible_batch_position());
+    let max_visible_row = batch_store.max_visible_row(index_store.indexed_count());
     GenMembership::InMemory {
         index_store: index_store.clone(),
         max_visible_row,
@@ -346,9 +346,10 @@ fn bounded_in_memory_membership(
     index_store: &Arc<IndexStore>,
     batch_count: u64,
 ) -> GenMembership {
-    let max_visible_row = batch_count
-        .checked_sub(1)
-        .and_then(|last_batch| batch_store.max_visible_row(last_batch as usize));
+    // `batch_count` is already an exclusive count, and so is what
+    // `max_visible_row` takes, so it passes straight through: no
+    // count-to-inclusive-position conversion, and `0` yields `None` on its own.
+    let max_visible_row = batch_store.max_visible_row(batch_count as usize);
     GenMembership::InMemory {
         index_store: index_store.clone(),
         max_visible_row,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
index 89dbd7adc61..e3345f60557 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
@@ -322,7 +322,7 @@ mod tests {
             let (bp, off, _) = store.append(b.clone()).unwrap();
             index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
         }
-        let max_visible_row = store.max_visible_row(index.max_visible_batch_position());
+        let max_visible_row = store.max_visible_row(index.indexed_count());
         GenMembership::InMemory {
             index_store: Arc::new(index),
             max_visible_row,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
index 60000666f52..60dc128f4d2 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
@@ -98,7 +98,7 @@ fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool {
                 .get_fts_by_column(column)
                 .is_some_and(|index| !index.is_empty())
                 && batch_store
-                    .max_visible_row(index_store.max_visible_batch_position())
+                    .max_visible_row(index_store.indexed_count())
                     .is_some()
         }
         _ => false,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index d80e0e26795..245bb5f04ce 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -891,7 +891,12 @@ fn probe_position(
     if len == 0 {
         return Ok(ProbePos::Miss);
     }
-    let last_visible_idx = index_store.max_visible_batch_position().min(len - 1);
+    // The cursor is an exclusive count, so the last visible batch sits at
+    // `count - 1`. A count of 0 means nothing is visible yet — not "batch 0".
+    let visible_count = index_store.indexed_count().min(len);
+    let Some(last_visible_idx) = visible_count.checked_sub(1) else {
+        return Ok(ProbePos::Miss);
+    };
     let last = batch_store.get(last_visible_idx).ok_or_else(|| {
         lance_core::Error::internal("point-lookup: visible batch index out of range")
     })?;
@@ -1388,7 +1393,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
-        // BTree on the PK so that `max_visible_batch_position` advances as
+        // BTree on the PK so that `visible_count` advances as
         // we insert, otherwise the scanner sees no batches at all.
         index_store.add_btree("id_idx".to_string(), 0, "id".to_string());
 
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 719a2b36a96..d7f01ea0403 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -1669,7 +1669,7 @@ mod tests {
 
     // Regression test for the visibility-cursor bug: with an empty IndexStore
     // (the common case for WAL-managed tables that mirror an index-less base
-    // dataset), a WAL flush must still advance `max_visible_batch_position` so
+    // dataset), a WAL flush must still advance `visible_count` so
     // scanners can see every batch up to the durable position — not just
     // batch 0. Before the fix, the cursor stayed at 0 for the lifetime of the
     // memtable and scanners returned only the first row.
@@ -1687,14 +1687,13 @@ mod tests {
 
         // Empty registry, mimicking a memtable with `index_configs = []`.
         let indexes = Arc::new(IndexStore::new());
-        assert_eq!(indexes.max_visible_batch_position(), 0);
+        assert_eq!(indexes.indexed_count(), 0);
 
         let source = batch_store_source_with_indexes(&batch_store, &indexes);
         flusher.flush(&source, batch_store.len()).await.unwrap();
 
-        // Cursor must advance to the highest flushed batch position (2),
-        // making all three batches visible to scanners.
-        assert_eq!(indexes.max_visible_batch_position(), 2);
+        // All three batches indexed => the indexed prefix is 3 (exclusive count).
+        assert_eq!(indexes.indexed_count(), 3);
         assert_eq!(flusher.durable(), 3);
     }
 
@@ -1720,7 +1719,7 @@ mod tests {
         let source = batch_store_source_with_indexes(&batch_store, &indexes);
         flusher.flush(&source, batch_store.len()).await.unwrap();
 
-        assert_eq!(indexes.max_visible_batch_position(), 2);
+        assert_eq!(indexes.indexed_count(), 3);
         assert_eq!(flusher.durable(), 3);
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 2391bd78f77..790a89cb057 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -2163,7 +2163,7 @@ impl ShardWriter {
     /// The scanner provides read access to all data currently in the MemTable,
     /// with optional filtering, projection, and index support.
     ///
-    /// The scanner captures the current `max_visible_batch_position` from the
+    /// The scanner captures the current `visible_count` from the
     /// `IndexStore` at construction time to ensure consistent visibility.
     ///
     /// Returns an error in WAL-only mode, or if the writer is poisoned.
@@ -7055,8 +7055,14 @@ mod shard_writer_tests {
         // Re-open dataset and create new writer to verify recovery
         let dataset = Dataset::open(&uri).await.expect("Failed to reopen dataset");
         let new_shard_id = Uuid::new_v4();
+        // `durable_write(true)` so the put waits for its flush, which is what
+        // currently publishes the rows. A non-durable put is *not* yet
+        // read-your-writes: the index apply is welded to the WAL flush, so the
+        // rows stay invisible until the next flush. This test used to pass with
+        // `durable_write(false)` only because an un-advanced cursor of 0 was
+        // misread as "batch 0 is visible" — it was asserting the dirty read.
         let new_config = ShardWriterConfig::new(new_shard_id)
-            .with_durable_write(false)
+            .with_durable_write(true)
             .with_sync_indexed_write(true);
 
         let new_writer = dataset

From 462a4f9b1559599e54148447ee54f87319936dd2 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 13:36:32 -0500
Subject: [PATCH 08/19] fix(mem_wal): publish rows only once they are indexed
 and durable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Readers keyed off `indexed_count`, which the index insert advances itself. But
the WAL append and the index apply run under one `tokio::join!`, and `join!` runs
both arms to completion — it does not cancel the index arm when the append fails.
So on a failed append the index arm still advanced the cursor every reader treats
as visibility, publishing rows that are not in the WAL and that replay will not
reproduce. `MemTableScanner::new` asserts the opposite in its own doc comment.

Split the cursor in two. `indexed_count` stays the *apply* cursor: it tracks what
the index layer has ingested, it is what HNSW's contiguity check needs, and it
advances on a failed flush — which is fine, because indexes are derived state and
replay rebuilds them. `visible_count` is new, and is the only thing readers
snapshot: the writer advances it downstream of *both* arms succeeding, so
`visible => durable` holds unconditionally.

Also make an index-apply failure terminal. `insert_batches` joins every index
thread unconditionally, so a failure leaves the others fully applied, and none of
them can be rolled back: HNSW has no delete, FTS has already incremented its
collection statistics, BTree has linked into an append-only skiplist. Both ways
out are corrupt — retry re-covers the range and re-inserts into the indexes that
did succeed; skip it and the failed index is permanently short rows the others
have. So discard instead: poison, and let reopen rebuild the indexes from the
WAL. The rollback primitive already exists and it is `open()`.

Replay publishes explicitly: its batches are durable by construction and it
bypasses the flush, so without it the recovered rows stay invisible.

Regression tests pin both halves. With the index arm publishing, a row whose WAL
append failed becomes readable (visible_count=1 where it must be 0). An index
insert that fails deterministically now poisons the writer instead of leaving it
to limp on with a corrupt index.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/index.rs       |  52 ++++-
 .../mem_wal/memtable/scanner/builder.rs       |   2 +-
 .../src/dataset/mem_wal/scanner/block_list.rs |   2 +-
 .../mem_wal/scanner/exec/pk_block_filter.rs   |   2 +-
 .../src/dataset/mem_wal/scanner/fts_search.rs |   2 +-
 .../dataset/mem_wal/scanner/point_lookup.rs   |   2 +-
 rust/lance/src/dataset/mem_wal/wal.rs         | 184 +++++++++++++++++-
 rust/lance/src/dataset/mem_wal/write.rs       |   9 +
 8 files changed, 242 insertions(+), 13 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 8d1030ce87f..bda064e12df 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -389,9 +389,9 @@ impl MemIndexConfig {
 /// Indexes are keyed by index name. Each index stores its field_id for
 /// stable column-to-index resolution (column name → field_id → index).
 ///
-/// The store also carries the MemTable's `visible_count`
-/// watermark — the highest batch position that is durable in the WAL and
-/// therefore safe for scanners to read. Scanners snapshot this at plan
+/// The store also carries the MemTable's two cursors: `indexed_count` (what the
+/// index layer has ingested) and `visible_count` (what is indexed *and* durable,
+/// and therefore safe for scanners to read). Scanners snapshot the latter at plan
 /// construction time so every plan keys on a stable MVCC cursor.
 pub struct IndexStore {
     /// BTree indexes keyed by index name. `Arc` so the primary-key BTrees can be
@@ -413,11 +413,27 @@ pub struct IndexStore {
     ///
     /// This has only ever been an *indexed* cursor — it is advanced at the end of
     /// `insert_batches`, once every index insert for the batch has completed, and
-    /// never before. It was named `visible_count` and treated as a
+    /// never before. It was named `max_visible_batch_position` and treated as a
     /// visibility cursor by five read sites, which is how rows became readable
-    /// before they were durable. The derivation from "indexed" to "visible" is a
-    /// separate thing, and it belongs to the writer, not here.
+    /// before they were durable. Publishing is a separate step, and it is the
+    /// writer's to make — see `visible_count`.
     indexed_count: AtomicUsize,
+
+    /// How many batches of this memtable are **published to readers**. An
+    /// exclusive count; 0 means none.
+    ///
+    /// Distinct from `indexed_count`, and the distinction is the whole point.
+    /// `indexed_count` is the *apply* cursor: it tracks what the index layer has
+    /// ingested, and the HNSW graph requires it to advance contiguously. It is
+    /// advanced by the index insert itself — including on a flush whose WAL
+    /// append *failed*, since the two run concurrently and the append error only
+    /// surfaces after both have completed.
+    ///
+    /// Readers must never key off that. `visible_count` is advanced by the
+    /// writer only once a batch is both indexed **and** WAL-durable, so
+    /// `visible => durable` holds and a failed append can never publish a row
+    /// that replay will not reproduce.
+    visible_count: AtomicUsize,
     /// Conservative flag set once this memtable has observed any primary-key
     /// rewrite while maintaining a search index. Search planners can push top-k
     /// into HNSW/FTS for append-only PK data, but must switch to
@@ -433,6 +449,7 @@ impl Default for IndexStore {
             fts_indexes: HashMap::new(),
             pk_index: None,
             indexed_count: AtomicUsize::new(0),
+            visible_count: AtomicUsize::new(0),
             pk_has_overrides: AtomicBool::new(false),
         }
     }
@@ -852,9 +869,17 @@ impl IndexStore {
         let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?;
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update the indexed prefix after every index has been updated.
+        // Update the indexed prefix after every index has been updated, and
+        // publish it.
+        //
+        // Publishing here — rather than leaving it to the writer, as the
+        // production path does — is deliberate: this entry point exists only for
+        // tests and benches, which have no writer to durably flush and then
+        // publish on their behalf. It stands in for the whole pipeline, so a
+        // fixture that indexes a batch means "this batch is indexed and durable".
         if let Some(bp) = batch_position {
             self.advance_indexed_count(bp + 1);
+            self.publish_visible(bp + 1);
         }
 
         Ok(())
@@ -1110,6 +1135,19 @@ impl IndexStore {
     pub fn indexed_count(&self) -> usize {
         self.indexed_count.load(Ordering::Acquire)
     }
+
+    /// The prefix of this memtable that readers may see: indexed **and** durable.
+    /// Snapshot this, never `indexed_count`.
+    pub fn visible_count(&self) -> usize {
+        self.visible_count.load(Ordering::Acquire)
+    }
+
+    /// Publish a prefix to readers. Called by the writer once the batches are
+    /// both indexed and WAL-durable. Monotonic — a late or out-of-order flush
+    /// completion can never retract what readers have already been shown.
+    pub(crate) fn publish_visible(&self, count: usize) {
+        self.visible_count.fetch_max(count, Ordering::AcqRel);
+    }
 }
 
 #[cfg(test)]
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
index 7ab0d00a273..e0a06ff85b9 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
@@ -439,7 +439,7 @@ impl MemTableScanner {
         // Snapshot the visibility cursor at construction time. The cursor is
         // advanced by `flush_from_batch_store` after the WAL append succeeds,
         // so this snapshot reflects WAL-durable data.
-        let visible_count = indexes.indexed_count();
+        let visible_count = indexes.visible_count();
 
         Self {
             batch_store,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
index eefec568983..e74ebda0f63 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
@@ -328,7 +328,7 @@ fn in_memory_membership(
     batch_store: &Arc<BatchStore>,
     index_store: &Arc<IndexStore>,
 ) -> GenMembership {
-    let max_visible_row = batch_store.max_visible_row(index_store.indexed_count());
+    let max_visible_row = batch_store.max_visible_row(index_store.visible_count());
     GenMembership::InMemory {
         index_store: index_store.clone(),
         max_visible_row,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
index e3345f60557..ae00cb667b5 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
@@ -322,7 +322,7 @@ mod tests {
             let (bp, off, _) = store.append(b.clone()).unwrap();
             index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
         }
-        let max_visible_row = store.max_visible_row(index.indexed_count());
+        let max_visible_row = store.max_visible_row(index.visible_count());
         GenMembership::InMemory {
             index_store: Arc::new(index),
             max_visible_row,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
index 60dc128f4d2..346cee851f4 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
@@ -98,7 +98,7 @@ fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool {
                 .get_fts_by_column(column)
                 .is_some_and(|index| !index.is_empty())
                 && batch_store
-                    .max_visible_row(index_store.indexed_count())
+                    .max_visible_row(index_store.visible_count())
                     .is_some()
         }
         _ => false,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index 245bb5f04ce..72808851d77 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -893,7 +893,7 @@ fn probe_position(
     }
     // The cursor is an exclusive count, so the last visible batch sits at
     // `count - 1`. A count of 0 means nothing is visible yet — not "batch 0".
-    let visible_count = index_store.indexed_count().min(len);
+    let visible_count = index_store.visible_count().min(len);
     let Some(last_visible_idx) = visible_count.checked_sub(1) else {
         return Ok(ProbePos::Miss);
     };
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index d7f01ea0403..a648ffa08f4 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -593,6 +593,8 @@ impl WalFlusher {
             stored_batches.iter().map(|s| s.data.clone()).collect();
 
         let appender = self.wal_appender.clone();
+        // Kept for the publish below: the index arm consumes its own clone.
+        let publish_target = indexes.clone();
         let (append_result, index_result) = if let Some(idx_registry) = indexes {
             let wal_future = async move {
                 let start = Instant::now();
@@ -621,8 +623,52 @@ impl WalFlusher {
             )
         };
 
+        // `tokio::join!` runs both arms to completion; it does not cancel the
+        // index arm when the append fails. So on a failed append the index arm has
+        // still advanced `indexed_count` — which is exactly why readers key off
+        // `visible_count` instead, and why nothing is published until both `?`
+        // below have passed.
         let (append_result, wal_io_duration) = append_result?;
-        let (index_update_duration, index_update_duration_breakdown) = index_result?;
+
+        // An index-apply failure is terminal for the writer.
+        //
+        // `insert_batches` joins every index thread unconditionally, so a failure
+        // leaves the *other* indexes fully applied — FTS and BTree landed, HNSW
+        // did not — and none of them can be rolled back: HNSW has no delete (its
+        // nodes are already linked into their neighbours' adjacency lists), FTS
+        // has already incremented `doc_count`/`total_tokens`/`df`, and BTree has
+        // linked into an append-only arena skiplist. From a partial apply there is
+        // no coherent way to continue: retry and the next attempt re-covers the
+        // range, re-inserting into the indexes that *did* succeed (duplicate HNSW
+        // nodes, doubled BM25 statistics, a permanently spurious
+        // `pk_has_overrides`); skip it and the failed index is permanently short
+        // rows the others have, on a shard that reports healthy.
+        //
+        // So discard instead: poison the writer and let reopen rebuild the
+        // indexes from the WAL. The indexes are pure derived state and replay
+        // already re-derives them, so the rollback primitive exists — it is
+        // `open()`. This is safe only because a *deterministic* index failure is
+        // impossible on data that reached the batch store: `validate_index_configs`
+        // rejects a config that disagrees with the schema at open, and the
+        // memtable's schema-equality gate rejects a batch that disagrees with it.
+        // What is left is transient (allocation, a panicking thread), and that
+        // recovers on replay.
+        let (index_update_duration, index_update_duration_breakdown) =
+            index_result.map_err(|e| {
+                Error::writer_poisoned(format!(
+                    "in-memory index update failed and a partially-applied index cannot be \
+                     rolled back; reopen the shard to rebuild the indexes from the WAL: {e}"
+                ))
+            })?;
+
+        // Both arms succeeded: the range is indexed *and* durable. Publish it.
+        //
+        // This is the only place visibility advances, and it is downstream of both
+        // `?`s. A failed append therefore leaves the rows unreadable, so a reader
+        // can never observe a row that replay will not reproduce.
+        if let Some(indexes) = publish_target {
+            indexes.publish_visible(end_batch_position);
+        }
 
         // Advance the writer-global durability cursor and wake waiters. The
         // range we just appended was `[start, end)` *local to this store*, so it
@@ -2077,6 +2123,142 @@ mod tests {
         );
     }
 
+    /// A failed WAL append must not publish rows to readers, even though the
+    /// index arm has already applied them.
+    ///
+    /// `tokio::join!` runs both arms to completion — it does not cancel the index
+    /// arm when the append fails — so `indexed_count` *does* advance on a failed
+    /// flush. That is fine, and unavoidable: the indexes are derived state, and
+    /// replay rebuilds them. What must not happen is for that to make the rows
+    /// readable, because they are not in the WAL and replay will not reproduce
+    /// them. A reader that saw them would be reading a row that no longer exists
+    /// after a reopen.
+    ///
+    /// Readers therefore key off `visible_count`, which the writer advances only
+    /// downstream of *both* arms succeeding.
+    #[tokio::test]
+    async fn test_failed_append_indexes_but_does_not_publish() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.fail_wal_puts(usize::MAX);
+        let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2));
+        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
+        let appender = Arc::new(WalAppender::with_claimed_epoch(
+            store,
+            base,
+            shard_id,
+            manifest_store,
+            epoch,
+            0,
+            WalRetryConfig {
+                max_retries: 1,
+                base_delay: Duration::from_millis(1),
+            },
+        ));
+        let flusher = WalFlusher::new(appender);
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+
+        let mut idx = IndexStore::new();
+        idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        let indexes = Arc::new(idx);
+
+        let source = batch_store_source_with_indexes(&batch_store, &indexes);
+        let err = flusher
+            .flush(&source, batch_store.len())
+            .await
+            .expect_err("the WAL PUT is failing, so the flush must fail");
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // The index arm ran to completion regardless — that is the join's nature.
+        assert_eq!(
+            indexes.indexed_count(),
+            1,
+            "the index arm applies even when the append fails"
+        );
+
+        // But nothing was published, so no reader can see the row.
+        assert_eq!(
+            indexes.visible_count(),
+            0,
+            "a row whose WAL append failed must never become readable"
+        );
+        assert_eq!(flusher.durable(), 0);
+    }
+
+    /// An index-apply failure poisons the writer.
+    ///
+    /// A partial apply cannot be rolled back (see the note at the `map_err` in
+    /// `flush_from_batch_store`), and continuing would re-cover the range on the
+    /// next flush and corrupt the indexes that *did* succeed. So the failure is
+    /// terminal: the writer poisons, reads and writes fail fast, and recovery is
+    /// reopen -> replay, which rebuilds the indexes from the WAL.
+    ///
+    /// The batches were WAL-durable before the index failed, so this is not data
+    /// loss — it is a rebuild.
+    #[tokio::test]
+    async fn test_index_failure_poisons_the_writer() {
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
+            store.clone(),
+            &base_path,
+            shard_id,
+            2,
+        ));
+        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
+        let appender = Arc::new(WalAppender::with_claimed_epoch(
+            store,
+            base_path,
+            shard_id,
+            manifest_store,
+            epoch,
+            0,
+            WalRetryConfig::default(),
+        ));
+        let flusher = WalFlusher::new(appender);
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+
+        // An HNSW index on `id`, which is an Int32 and not a vector. Every insert
+        // of this batch fails deterministically. (`validate_index_configs` rejects
+        // this at shard open — that is what makes poison-and-replay terminating —
+        // so we have to build the store by hand to reach the failure at all.)
+        let mut idx = IndexStore::new();
+        idx.add_hnsw(
+            "bad_hnsw".to_string(),
+            0,
+            "id".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            128,
+            8,
+        );
+        let indexes = Arc::new(idx);
+
+        let source = batch_store_source_with_indexes(&batch_store, &indexes);
+        let err = flusher
+            .flush(&source, batch_store.len())
+            .await
+            .expect_err("indexing an Int32 column as a vector must fail");
+
+        assert_eq!(
+            err.fence_reason(),
+            Some(FenceReason::PersistenceFailure),
+            "an index failure must be typed as terminal, not left as an ordinary error"
+        );
+        assert!(
+            flusher.check_poisoned().is_err(),
+            "the writer must be poisoned so it cannot limp on with a corrupt index"
+        );
+
+        // Nothing was published, even though the WAL append itself succeeded.
+        assert_eq!(indexes.visible_count(), 0);
+    }
+
     // A persistence failure during flush latches the poison: the flush result,
     // `check_poisoned`, and the durability watcher all report the typed error
     // (rather than the watcher hanging on a watermark that never advances).
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 790a89cb057..0d79aa7274e 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -1554,6 +1554,15 @@ impl ShardWriter {
         // 0 and the durable count is just its batch count.
         wal_flusher.advance_durable(memtable.batch_count());
 
+        // ...and publish them to readers. Replayed batches are durable by
+        // construction (they came out of the WAL) and were just re-indexed above,
+        // so they satisfy both halves of `visible`. Publishing is normally the
+        // flush's job; replay bypasses the flush, so it must do it here or the
+        // recovered rows stay invisible.
+        if let Some(indexes) = memtable.indexes_arc() {
+            indexes.publish_visible(memtable.batch_count());
+        }
+
         wal_flusher
             .wal_appender()
             .seed_next_position(next_wal_position)

From 55ab6c9b89066e1bf329762b7ad2ce3e3aec6daa Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 15:20:33 -0500
Subject: [PATCH 09/19] feat(mem_wal): run the index apply on its own task, not
 as an arm of the WAL flush
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The WAL append and the index apply were welded together under one `tokio::join!`
on one schedule. They have nothing in common: one is an in-memory write measured
in microseconds, the other a ~100ms S3 PUT billed per call. Batching exists to
bound API cost, which an in-memory index apply does not incur — so the index was
being dragged onto the WAL's schedule for no reason, and that is what made
read-your-writes impossible without durability.

Split them into two tasks with two channels, each a single sequential consumer:

- The index-apply task is triggered per put, in both modes. Being a single
  consumer is the safety property: `HnswGraph::insert_batch` hard-rejects any
  range whose start is not its `indexed_len`, and a single consumer guarantees
  contiguous in-order ranges however many putters race behind it. Ordering comes
  from the task, not from a flush interval — so triggering per-put is exactly as
  safe as triggering on a timer, and a put becomes visible in milliseconds rather
  than waiting on an S3 round-trip.
- The WAL-append task keeps the expensive work and is now append-only. The
  `join!` is gone, and with it the dirty read it caused: a failed append can no
  longer publish rows through an index arm that ran anyway.

They need separate channels, not two message types on one: `TaskDispatcher::run`
awaits `handle()` inline, so sharing would queue every index apply behind an S3
PUT.

Visibility becomes derived rather than published. `WriterCursors` holds the
writer-global `durable` count; each memtable's `IndexStore` holds its own
`indexed` count; and `visible` is computed on demand as `min(indexed, durable)`
under `durable_write`, or just `indexed` without it. Nothing caches it, which is
deliberate: a cached `min` recomputed by two independent tasks is the classic
store-buffer race — under Release/Acquire both tasks can read the other's
pre-store value, both compute a minimum below the true one, and a max-clamped
publish leaves it permanently short, hanging any put blocked on it. With nothing
cached there is nothing to leave stale. The notify channel is a bare wake-up and
every waiter recomputes.

What this buys: `durable_write: false` now costs the caller durability *only*,
not visibility. A non-durable put is read-your-writes through every arm, indexed
ones included — previously the row sat unindexed in the batch store until some
later flush happened along, so a full scan could find it while every
index-accelerated query could not.

Also: `close()` drains both tasks, `freeze` triggers the index apply for the
outgoing store, and every memtable's `IndexStore` is bound to the cursors —
including the empty one built when no indexes are configured, which would
otherwise fall back to `visible == indexed` and publish before durability.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/index.rs |  56 +-
 rust/lance/src/dataset/mem_wal/wal.rs   | 716 ++++++++++++++----------
 rust/lance/src/dataset/mem_wal/write.rs | 342 ++++++++---
 3 files changed, 691 insertions(+), 423 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index bda064e12df..9814cfe74ab 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -27,6 +27,7 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use datafusion::common::ScalarValue;
 
 use super::memtable::batch_store::StoredBatch;
+use super::wal::WriterCursors;
 use arrow_array::RecordBatch;
 use arrow_schema::{DataType, Schema as ArrowSchema};
 use lance_core::datatypes::Schema as LanceSchema;
@@ -419,21 +420,12 @@ pub struct IndexStore {
     /// writer's to make — see `visible_count`.
     indexed_count: AtomicUsize,
 
-    /// How many batches of this memtable are **published to readers**. An
-    /// exclusive count; 0 means none.
+    /// The writer's cursors, and this memtable's coordinate within them. `None`
+    /// for a bare `IndexStore` (tests, benches), where visibility is just the
+    /// indexed prefix.
     ///
-    /// Distinct from `indexed_count`, and the distinction is the whole point.
-    /// `indexed_count` is the *apply* cursor: it tracks what the index layer has
-    /// ingested, and the HNSW graph requires it to advance contiguously. It is
-    /// advanced by the index insert itself — including on a flush whose WAL
-    /// append *failed*, since the two run concurrently and the append error only
-    /// surfaces after both have completed.
-    ///
-    /// Readers must never key off that. `visible_count` is advanced by the
-    /// writer only once a batch is both indexed **and** WAL-durable, so
-    /// `visible => durable` holds and a failed append can never publish a row
-    /// that replay will not reproduce.
-    visible_count: AtomicUsize,
+    /// Visibility is **derived, never stored**: see `visible_count`.
+    durability: Option<(Arc<WriterCursors>, usize)>,
     /// Conservative flag set once this memtable has observed any primary-key
     /// rewrite while maintaining a search index. Search planners can push top-k
     /// into HNSW/FTS for append-only PK data, but must switch to
@@ -449,7 +441,7 @@ impl Default for IndexStore {
             fts_indexes: HashMap::new(),
             pk_index: None,
             indexed_count: AtomicUsize::new(0),
-            visible_count: AtomicUsize::new(0),
+            durability: None,
             pk_has_overrides: AtomicBool::new(false),
         }
     }
@@ -869,17 +861,9 @@ impl IndexStore {
         let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?;
         self.mark_pk_overrides_if_needed(had_existing);
 
-        // Update the indexed prefix after every index has been updated, and
-        // publish it.
-        //
-        // Publishing here — rather than leaving it to the writer, as the
-        // production path does — is deliberate: this entry point exists only for
-        // tests and benches, which have no writer to durably flush and then
-        // publish on their behalf. It stands in for the whole pipeline, so a
-        // fixture that indexes a batch means "this batch is indexed and durable".
+        // Update the indexed prefix after every index has been updated.
         if let Some(bp) = batch_position {
             self.advance_indexed_count(bp + 1);
-            self.publish_visible(bp + 1);
         }
 
         Ok(())
@@ -1136,17 +1120,25 @@ impl IndexStore {
         self.indexed_count.load(Ordering::Acquire)
     }
 
-    /// The prefix of this memtable that readers may see: indexed **and** durable.
-    /// Snapshot this, never `indexed_count`.
+    /// The prefix of this memtable that readers may see. Snapshot this, never
+    /// `indexed_count`.
+    ///
+    /// Derived on every call from the two cursors rather than cached, so there is
+    /// no published value that can be left stale by a race between the two tasks
+    /// that advance them. A bare `IndexStore` has no writer, so its visible
+    /// prefix is simply what has been indexed.
     pub fn visible_count(&self) -> usize {
-        self.visible_count.load(Ordering::Acquire)
+        let indexed = self.indexed_count();
+        match &self.durability {
+            Some((cursors, global_offset)) => cursors.visible_count(indexed, *global_offset),
+            None => indexed,
+        }
     }
 
-    /// Publish a prefix to readers. Called by the writer once the batches are
-    /// both indexed and WAL-durable. Monotonic — a late or out-of-order flush
-    /// completion can never retract what readers have already been shown.
-    pub(crate) fn publish_visible(&self, count: usize) {
-        self.visible_count.fetch_max(count, Ordering::AcqRel);
+    /// Bind this memtable's indexes to the writer's cursors. Called once at
+    /// construction, before the memtable is published.
+    pub(crate) fn set_durability(&mut self, cursors: Arc<WriterCursors>, global_offset: usize) {
+        self.durability = Some((cursors, global_offset));
     }
 }
 
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index a648ffa08f4..64d7f87cc9d 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -9,7 +9,7 @@
 use std::io::Cursor;
 use std::sync::Arc;
 use std::sync::Mutex as StdMutex;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::time::Instant;
 
 use std::time::Duration;
@@ -87,68 +87,186 @@ impl WalFlushFailure {
     }
 }
 
-/// Watcher for batch durability using watermark-based tracking.
+/// The writer's cursors, shared by the WAL-append task, the index-apply task,
+/// every memtable's `IndexStore`, and every `put` waiting to become visible.
 ///
-/// Uses a shared watch channel that broadcasts the durable watermark.
-/// The watcher waits until the watermark reaches or exceeds its target batch ID.
-#[derive(Clone)]
-pub struct BatchDurableWatcher {
-    /// Watch receiver for the durable watermark.
-    rx: watch::Receiver<usize>,
-    /// Target batch ID to wait for.
-    target_batch_position: usize,
-    /// Terminal flush failure shared with the flusher. When set, the watermark
-    /// can never reach the target, so `wait` returns this typed error instead of
-    /// blocking forever.
+/// Two cursors are stored, one view is derived:
+///
+/// - `durable` — writer-global count of WAL-durable batches, advanced by the
+///   WAL-append task. Exclusive; 0 means none.
+/// - `indexed` — per-memtable count of indexed batches, advanced by the
+///   index-apply task. It lives on the memtable's own `IndexStore`, not here,
+///   because each memtable has its own indexes.
+/// - `visible` — **derived, never stored**: a batch is visible once it is
+///   indexed and, under `durable_write`, also durable.
+///
+/// Deriving `visible` rather than caching it is deliberate. A cached
+/// `min(indexed, durable)` recomputed by two independent tasks is the classic
+/// store-buffer race: with `Release`/`Acquire` each task can read the other's
+/// *pre-store* value, so both compute a minimum below the true one, and a
+/// max-clamped publish then leaves the cached value permanently short. A `put`
+/// blocked on it would hang until some unrelated write happened to move a cursor
+/// again. With nothing cached there is nothing to leave stale — `notify` is a
+/// bare wake-up and every waiter recomputes from the cursors themselves.
+pub struct WriterCursors {
+    durable: AtomicUsize,
+    /// Bumped whenever a cursor advances or the writer poisons. Carries no
+    /// value; it exists only to wake waiters, which then recompute.
+    notify_tx: watch::Sender<u64>,
+    notify_rx: watch::Receiver<u64>,
+    /// First terminal failure. Shared so a poisoned writer wakes every waiter
+    /// with the typed error, instead of leaving it blocked on a cursor that can
+    /// never advance again.
     terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
+    /// Whether durability is part of visibility. Per-writer, not per-write:
+    /// `visible` is a writer-wide definition, so a mix would need two visibility
+    /// views over one memtable.
+    durable_write: bool,
+}
+
+impl WriterCursors {
+    pub fn new(durable_write: bool) -> Self {
+        let (notify_tx, notify_rx) = watch::channel(0);
+        Self {
+            durable: AtomicUsize::new(0),
+            notify_tx,
+            notify_rx,
+            terminal_error: Arc::new(StdMutex::new(None)),
+            durable_write,
+        }
+    }
+
+    /// Writer-global count of WAL-durable batches.
+    pub fn durable(&self) -> usize {
+        self.durable.load(Ordering::Acquire)
+    }
+
+    pub fn durable_write(&self) -> bool {
+        self.durable_write
+    }
+
+    /// Advance the durability cursor. Monotonic, so an out-of-order completion
+    /// can never walk it backwards.
+    pub(crate) fn advance_durable(&self, global_count: usize) {
+        self.durable.fetch_max(global_count, Ordering::AcqRel);
+        self.wake();
+    }
+
+    /// Wake every waiter so it recomputes. Called after any cursor advances, and
+    /// after the writer poisons.
+    pub(crate) fn wake(&self) {
+        self.notify_tx.send_modify(|version| *version += 1);
+    }
+
+    /// The visible prefix of one memtable, given its indexed prefix and its
+    /// writer-global coordinate.
+    pub fn visible_count(&self, indexed_count: usize, global_offset: usize) -> usize {
+        if !self.durable_write {
+            return indexed_count;
+        }
+        indexed_count.min(self.durable().saturating_sub(global_offset))
+    }
+
+    pub(crate) fn check_poisoned(&self) -> Result<()> {
+        if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
+            return Err(failure.into_error());
+        }
+        Ok(())
+    }
+
+    pub(crate) fn mark_terminal_failure(&self, error: &Error) {
+        {
+            let mut slot = self.terminal_error.lock().unwrap();
+            if slot.is_none() {
+                *slot = Some(WalFlushFailure::from_error(error));
+            }
+        }
+        // Wake waiters without advancing anything: each re-checks `terminal_error`
+        // and returns the error rather than blocking on a cursor that can no
+        // longer move.
+        self.wake();
+    }
+}
+
+/// Blocks a `put` until its batches are visible: indexed, and — in durable mode
+/// — WAL-durable too.
+///
+/// Recomputes the condition on every wake instead of comparing against a cached
+/// watermark. `notify` only says "something moved"; this decides what that means.
+pub struct BatchDurableWatcher {
+    cursors: Arc<WriterCursors>,
+    rx: watch::Receiver<u64>,
+    /// The memtable the batches landed in; its `indexed_count` is the apply
+    /// cursor being waited on. `None` in WAL-only mode, which has no indexes.
+    indexes: Option<Arc<IndexStore>>,
+    /// Local exclusive count this write needs indexed.
+    target_indexed: usize,
+    /// Writer-global exclusive count this write needs durable. Batch positions
+    /// restart at 0 in every memtable while the durability cursor spans the
+    /// writer's whole life, so the caller must globalize this.
+    target_durable: usize,
 }
 
 impl BatchDurableWatcher {
-    /// Create a new watcher for a specific batch ID.
     pub fn new(
-        rx: watch::Receiver<usize>,
-        target_batch_position: usize,
-        terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
+        cursors: Arc<WriterCursors>,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
     ) -> Self {
+        let rx = cursors.notify_rx.clone();
         Self {
+            cursors,
             rx,
-            target_batch_position,
-            terminal_error,
+            indexes,
+            target_indexed,
+            target_durable,
         }
     }
 
-    /// Wait until the batch is durable.
-    ///
-    /// Returns Ok(()) when `durable_watermark >= target_batch_position`, or
-    /// Err if a terminal flush failure (e.g. a fence) means the watermark can
-    /// never reach the target.
+    /// Whether the write is readable yet.
+    fn is_visible(&self) -> bool {
+        // WAL-only mode has no indexes, so there is nothing to index-wait on.
+        let indexed = match &self.indexes {
+            Some(indexes) => indexes.indexed_count(),
+            None => self.target_indexed,
+        };
+        if indexed < self.target_indexed {
+            return false;
+        }
+        !self.cursors.durable_write() || self.cursors.durable() >= self.target_durable
+    }
+
+    /// Wait until the write is visible, or until the writer poisons — in which
+    /// case no cursor will ever reach the target, so surface the typed error
+    /// rather than blocking forever.
     pub async fn wait(&mut self) -> Result<()> {
         loop {
-            if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
-                return Err(failure.into_error());
-            }
-            let current = *self.rx.borrow();
-            if current >= self.target_batch_position {
+            // Mark the current version seen *before* testing, so a wake-up landing
+            // between the test and `changed()` below is not lost.
+            self.rx.borrow_and_update();
+            self.cursors.check_poisoned()?;
+            if self.is_visible() {
                 return Ok(());
             }
             self.rx
                 .changed()
                 .await
-                .map_err(|_| Error::io("Durable watermark channel closed"))?;
+                .map_err(|_| Error::io("Writer cursor channel closed"))?;
         }
     }
 
-    /// Check if the batch is already durable (non-blocking).
+    /// Non-blocking check.
     pub fn is_durable(&self) -> bool {
-        *self.rx.borrow() >= self.target_batch_position
+        self.is_visible()
     }
 }
 
 impl std::fmt::Debug for BatchDurableWatcher {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("BatchDurableWatcher")
-            .field("target_batch_position", &self.target_batch_position)
-            .field("current_watermark", &*self.rx.borrow())
+            .field("target_indexed", &self.target_indexed)
+            .field("target_durable", &self.target_durable)
             .finish()
     }
 }
@@ -185,13 +303,10 @@ pub struct WalFlushResult {
 /// Source for a WAL flush — either a `BatchStore` range (MemTable mode) or
 /// a drainable in-memory pending queue (WAL-only mode).
 pub enum WalFlushSource {
-    /// MemTable mode: read a `[max_flushed+1, end_batch_position)` range
-    /// from a `BatchStore`. Indexes are updated in parallel with the WAL
-    /// append.
-    BatchStore {
-        batch_store: Arc<BatchStore>,
-        indexes: Option<Arc<IndexStore>>,
-    },
+    /// MemTable mode: append the `[durable, end_batch_position)` range of a
+    /// `BatchStore` to the WAL. Append-only — the index apply runs on its own
+    /// task, so a failed append can no longer publish rows through it.
+    BatchStore { batch_store: Arc<BatchStore> },
     /// WAL-only mode: drain all pending batches from the shared
     /// `WalOnlyState`. There are no in-memory indexes to update.
     WalOnly { state: Arc<WalOnlyState> },
@@ -206,6 +321,78 @@ impl WalFlushSource {
     }
 }
 
+/// Message to trigger an index apply.
+///
+/// Carries the store the batches actually landed in, captured on the put path
+/// *before* any freeze can rotate the memtable — pairing a new store with an old
+/// store's end position would index the wrong range.
+#[derive(Clone)]
+pub struct TriggerIndexApply {
+    pub batch_store: Arc<BatchStore>,
+    pub indexes: Arc<IndexStore>,
+    /// Local exclusive end of the range to cover.
+    pub end_batch_position: usize,
+}
+
+impl std::fmt::Debug for TriggerIndexApply {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TriggerIndexApply")
+            .field("end_batch_position", &self.end_batch_position)
+            .finish()
+    }
+}
+
+/// Apply a contiguous range of batches to a memtable's in-memory indexes.
+///
+/// Runs on its own task, as the single sequential consumer of its own channel.
+/// Being a single consumer is what makes it safe: it guarantees in-order,
+/// contiguous ranges, which is exactly what `HnswGraph::insert_batch` requires —
+/// it hard-rejects any range whose start is not `indexed_len`. Ordering comes
+/// from the task, not from the flush interval, so triggering per-put is exactly
+/// as safe as triggering on a timer.
+///
+/// It has its own channel rather than sharing the WAL flusher's, because
+/// `TaskDispatcher::run` awaits `handle()` inline: a shared channel would put a
+/// ~100ms S3 PUT in front of every latency-sensitive index apply.
+pub async fn apply_index_range(
+    cursors: &Arc<WriterCursors>,
+    message: TriggerIndexApply,
+) -> Result<()> {
+    let TriggerIndexApply {
+        batch_store,
+        indexes,
+        end_batch_position,
+    } = message;
+
+    // Self-batching: a message handled while more puts queue behind it covers
+    // everything committed so far, so the ones behind it find their range already
+    // applied. Redundant messages are therefore *routine*, not exceptional, and
+    // must be a clean no-op — never a call into `insert_batches`, whose HNSW arm
+    // hard-rejects a non-contiguous start, and which is now terminal for the
+    // writer.
+    let start = indexes.indexed_count();
+    if end_batch_position <= start {
+        return Ok(());
+    }
+
+    let stored: Vec<StoredBatch> = (start..end_batch_position)
+        .filter_map(|position| batch_store.get(position).cloned())
+        .collect();
+    if stored.is_empty() {
+        return Ok(());
+    }
+
+    // `insert_batches` advances `indexed_count` itself, once every index has
+    // taken the batch.
+    tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
+        .await
+        .map_err(|e| Error::internal(format!("Index apply task panicked: {e}")))??;
+
+    // Wake anything waiting to become visible.
+    cursors.wake();
+    Ok(())
+}
+
 /// Message to trigger a WAL flush.
 ///
 /// Carries a `source` describing where to read batches from (BatchStore range
@@ -356,11 +543,9 @@ impl WalOnlyState {
 /// shared `WalAppender`. The flusher delegates the actual WAL write to the
 /// appender, optionally running a parallel index update in MemTable mode.
 pub struct WalFlusher {
-    /// Watch channel sender for durable watermark.
-    /// Broadcasts the highest batch_position that is now durable.
-    durable_watermark_tx: watch::Sender<usize>,
-    /// Watch channel receiver for creating new watchers.
-    durable_watermark_rx: watch::Receiver<usize>,
+    /// The writer's cursors. Shared with the index-apply task and every
+    /// memtable's `IndexStore`, so all three agree on what is visible.
+    cursors: Arc<WriterCursors>,
     /// Underlying WAL append primitive — owns object store, epoch, and
     /// position discovery.
     wal_appender: Arc<WalAppender>,
@@ -372,35 +557,37 @@ pub struct WalFlusher {
     /// Created at construction and recreated after each flush.
     /// Used by backpressure to wait for WAL flushes.
     wal_flush_cell: std::sync::Mutex<Option<WatchableOnceCell<super::write::DurabilityResult>>>,
-    /// First terminal flush failure, shared with every `BatchDurableWatcher`. It
-    /// wakes durability waiters (the watermark never advances) and is read by
-    /// `check_poisoned` so the write path fails fast.
-    terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
 }
 
 impl WalFlusher {
     /// Create a new WAL flusher backed by an existing `WalAppender`.
     ///
     /// The appender owns object store, epoch, and position state. The
-    /// flusher adds the durability watermark, trigger channel, and
-    /// completion cell on top.
+    /// flusher adds the trigger channel and completion cell on top, and shares
+    /// the writer's cursors.
     pub fn new(wal_appender: Arc<WalAppender>) -> Self {
+        // Defaults to durable visibility; the writer replaces this with cursors
+        // built from its own config.
+        Self::with_cursors(wal_appender, Arc::new(WriterCursors::new(true)))
+    }
+
+    pub fn with_cursors(wal_appender: Arc<WalAppender>, cursors: Arc<WriterCursors>) -> Self {
         let shard_id = wal_appender.shard_id();
-        // Initialize durable watermark at 0 (no batches durable yet)
-        let (durable_watermark_tx, durable_watermark_rx) = watch::channel(0);
-        // Create initial WAL flush cell for backpressure
         let wal_flush_cell = WatchableOnceCell::new();
         Self {
-            durable_watermark_tx,
-            durable_watermark_rx,
+            cursors,
             wal_appender,
             shard_id,
             flush_tx: None,
             wal_flush_cell: std::sync::Mutex::new(Some(wal_flush_cell)),
-            terminal_error: Arc::new(StdMutex::new(None)),
         }
     }
 
+    /// The writer's cursors, for the index-apply task and the memtables.
+    pub fn cursors(&self) -> &Arc<WriterCursors> {
+        &self.cursors
+    }
+
     /// Set the flush channel for background flush handler.
     pub fn set_flush_channel(&mut self, tx: mpsc::UnboundedSender<TriggerWalFlush>) {
         self.flush_tx = Some(tx);
@@ -415,62 +602,65 @@ impl WalFlusher {
     ///
     /// Returns a `BatchDurableWatcher` that can be awaited for durability.
     /// The actual batch data is stored in the BatchStore.
-    pub fn track_batch(&self, target_durable_count: usize) -> BatchDurableWatcher {
-        // `target_durable_count` is writer-global and exclusive: "N batches into
-        // this writer's batch sequence are durable". It must NOT be a memtable-
-        // local batch position — positions restart at 0 in every memtable, while
-        // this channel spans the writer's whole life, so a local target would be
-        // satisfied by a *previous* memtable's appends and ack a write that was
-        // never persisted. Callers globalize via `BatchStore::global_offset`.
+    /// Watch a write until it becomes visible.
+    ///
+    /// `target_indexed` is a **memtable-local** exclusive count; `target_durable`
+    /// is a **writer-global** one. They are different coordinate spaces on
+    /// purpose: batch positions restart at 0 in every memtable, while the
+    /// durability cursor spans the writer's whole life. A local durable target
+    /// would already be satisfied by a *previous* memtable's appends, and would
+    /// ack a write that never reached the WAL. Callers globalize via
+    /// `BatchStore::global_offset`.
+    pub fn track_batch(
+        &self,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
+    ) -> BatchDurableWatcher {
         BatchDurableWatcher::new(
-            self.durable_watermark_rx.clone(),
-            target_durable_count,
-            Arc::clone(&self.terminal_error),
+            Arc::clone(&self.cursors),
+            indexes,
+            target_indexed,
+            target_durable,
         )
     }
 
     /// The writer-global WAL durability cursor: how many batches of this
     /// writer's batch sequence are durable. Exclusive count; 0 means none.
     pub fn durable(&self) -> usize {
-        *self.durable_watermark_rx.borrow()
+        self.cursors.durable()
     }
 
-    /// Advance the durability cursor and wake waiters. Monotonic: a stale or
-    /// out-of-order completion can never walk the cursor backwards.
+    /// Advance the durability cursor and wake waiters.
     pub(crate) fn advance_durable(&self, global_count: usize) {
-        self.durable_watermark_tx
-            .send_modify(|current| *current = (*current).max(global_count));
+        self.cursors.advance_durable(global_count);
     }
 
-    /// Latch a terminal flush failure and wake every durability waiter (the
-    /// watermark never advances, so they must observe the error, not block).
+    /// Latch a terminal flush failure and wake every waiter (no cursor will
+    /// advance again, so they must observe the error rather than block).
     /// Idempotent: only the first failure is retained.
     fn mark_terminal_failure(&self, error: &Error) {
-        {
-            let mut slot = self.terminal_error.lock().unwrap();
-            if slot.is_none() {
-                *slot = Some(WalFlushFailure::from_error(error));
-            }
-        }
-        // Wake `wait`ers without advancing the watermark; each re-checks
-        // `terminal_error` and returns the error.
-        self.durable_watermark_tx.send_modify(|_| {});
+        self.cursors.mark_terminal_failure(error);
+    }
+
+    /// Latch a terminal failure from outside the flush path (the index-apply
+    /// task). Same effect: reads and writes fail fast, waiters wake with the
+    /// typed error, and recovery is reopen -> replay.
+    pub(crate) fn poison(&self, error: &Error) {
+        self.cursors.mark_terminal_failure(error);
     }
 
     /// Fail fast with the typed error if this writer has been fenced (by a peer
-    /// or its own persistence failure). The write path calls this before touching
-    /// the memtable so a poisoned writer can't diverge further. Recovery is to
-    /// reopen the shard (replay the WAL).
+    /// or its own persistence failure). Both the read and write paths call this
+    /// so a poisoned writer can neither diverge further nor serve a snapshot
+    /// that replay will not reproduce. Recovery is to reopen and replay.
     pub fn check_poisoned(&self) -> Result<()> {
-        if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
-            return Err(failure.into_error());
-        }
-        Ok(())
+        self.cursors.check_poisoned()
     }
 
     /// Get the current durable watermark.
     pub fn durable_watermark(&self) -> usize {
-        *self.durable_watermark_rx.borrow()
+        self.cursors.durable()
     }
 
     /// Get a watcher for WAL flush completion.
@@ -539,11 +729,8 @@ impl WalFlusher {
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
         let result = match source {
-            WalFlushSource::BatchStore {
-                batch_store,
-                indexes,
-            } => {
-                self.flush_from_batch_store(batch_store, indexes.clone(), end_batch_position)
+            WalFlushSource::BatchStore { batch_store } => {
+                self.flush_from_batch_store(batch_store, end_batch_position)
                     .await
             }
             WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await,
@@ -558,10 +745,18 @@ impl WalFlusher {
         result
     }
 
+    /// Append this store's un-appended suffix to the WAL. **Append-only.**
+    ///
+    /// The index apply used to run here, concurrently, under a `tokio::join!`.
+    /// That was the source of the dirty read: `join!` runs both arms to
+    /// completion and does not cancel the index arm when the append fails, so a
+    /// failed append still advanced the cursor readers keyed off. The two
+    /// operations have nothing in common — one is an in-memory microsecond write,
+    /// the other a ~100ms S3 PUT billed per call — so they now run on separate
+    /// tasks with separate cursors, and this one only ever touches the WAL.
     async fn flush_from_batch_store(
         &self,
         batch_store: &BatchStore,
-        indexes: Option<Arc<IndexStore>>,
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
         // Where this store's un-appended suffix begins, derived from the
@@ -569,114 +764,38 @@ impl WalFlusher {
         // predates this memtable to 0 and one past its end to `committed_len`.
         let start_batch_position = batch_store.local_end(self.durable());
 
-        // If we've already flushed past this end, nothing to do
+        // Already appended past this end: nothing to do. Redundant triggers are
+        // routine (a put and the freeze can both target the same range), so this
+        // is the common case, not an error.
         if start_batch_position >= end_batch_position {
             return Ok(empty_flush_result());
         }
 
-        // Collect batches in range [start_batch_position, end_batch_position)
         let mut stored_batches: Vec<StoredBatch> =
             Vec::with_capacity(end_batch_position - start_batch_position);
-
         for batch_position in start_batch_position..end_batch_position {
             if let Some(stored) = batch_store.get(batch_position) {
                 stored_batches.push(stored.clone());
             }
         }
-
         if stored_batches.is_empty() {
             return Ok(empty_flush_result());
         }
 
-        let rows_to_index: usize = stored_batches.iter().map(|b| b.num_rows).sum();
+        let num_rows: usize = stored_batches.iter().map(|b| b.num_rows).sum();
         let record_batches: Vec<RecordBatch> =
             stored_batches.iter().map(|s| s.data.clone()).collect();
 
-        let appender = self.wal_appender.clone();
-        // Kept for the publish below: the index arm consumes its own clone.
-        let publish_target = indexes.clone();
-        let (append_result, index_result) = if let Some(idx_registry) = indexes {
-            let wal_future = async move {
-                let start = Instant::now();
-                let r = appender.append(record_batches).await?;
-                Ok::<_, Error>((r, start.elapsed()))
-            };
-            let index_future = async {
-                let start = Instant::now();
-                let per_index = tokio::task::spawn_blocking(move || {
-                    idx_registry.insert_batches(&stored_batches)
-                })
-                .await
-                .map_err(|e| Error::internal(format!("Index update task panicked: {}", e)))??;
-                Ok::<_, Error>((start.elapsed(), per_index))
-            };
-            tokio::join!(wal_future, index_future)
-        } else {
-            let wal_future = async move {
-                let start = Instant::now();
-                let r = appender.append(record_batches).await?;
-                Ok::<_, Error>((r, start.elapsed()))
-            };
-            (
-                wal_future.await,
-                Ok((std::time::Duration::ZERO, std::collections::HashMap::new())),
-            )
-        };
-
-        // `tokio::join!` runs both arms to completion; it does not cancel the
-        // index arm when the append fails. So on a failed append the index arm has
-        // still advanced `indexed_count` — which is exactly why readers key off
-        // `visible_count` instead, and why nothing is published until both `?`
-        // below have passed.
-        let (append_result, wal_io_duration) = append_result?;
-
-        // An index-apply failure is terminal for the writer.
-        //
-        // `insert_batches` joins every index thread unconditionally, so a failure
-        // leaves the *other* indexes fully applied — FTS and BTree landed, HNSW
-        // did not — and none of them can be rolled back: HNSW has no delete (its
-        // nodes are already linked into their neighbours' adjacency lists), FTS
-        // has already incremented `doc_count`/`total_tokens`/`df`, and BTree has
-        // linked into an append-only arena skiplist. From a partial apply there is
-        // no coherent way to continue: retry and the next attempt re-covers the
-        // range, re-inserting into the indexes that *did* succeed (duplicate HNSW
-        // nodes, doubled BM25 statistics, a permanently spurious
-        // `pk_has_overrides`); skip it and the failed index is permanently short
-        // rows the others have, on a shard that reports healthy.
-        //
-        // So discard instead: poison the writer and let reopen rebuild the
-        // indexes from the WAL. The indexes are pure derived state and replay
-        // already re-derives them, so the rollback primitive exists — it is
-        // `open()`. This is safe only because a *deterministic* index failure is
-        // impossible on data that reached the batch store: `validate_index_configs`
-        // rejects a config that disagrees with the schema at open, and the
-        // memtable's schema-equality gate rejects a batch that disagrees with it.
-        // What is left is transient (allocation, a panicking thread), and that
-        // recovers on replay.
-        let (index_update_duration, index_update_duration_breakdown) =
-            index_result.map_err(|e| {
-                Error::writer_poisoned(format!(
-                    "in-memory index update failed and a partially-applied index cannot be \
-                     rolled back; reopen the shard to rebuild the indexes from the WAL: {e}"
-                ))
-            })?;
-
-        // Both arms succeeded: the range is indexed *and* durable. Publish it.
-        //
-        // This is the only place visibility advances, and it is downstream of both
-        // `?`s. A failed append therefore leaves the rows unreadable, so a reader
-        // can never observe a row that replay will not reproduce.
-        if let Some(indexes) = publish_target {
-            indexes.publish_visible(end_batch_position);
-        }
+        let start = Instant::now();
+        let append_result = self.wal_appender.append(record_batches).await?;
+        let wal_io_duration = start.elapsed();
 
-        // Advance the writer-global durability cursor and wake waiters. The
-        // range we just appended was `[start, end)` *local to this store*, so it
-        // must be lifted into the writer's coordinate space before it is
-        // published — otherwise a fresh memtable's small local end would be
-        // compared against a channel carrying a previous memtable's larger one.
+        // Advance the writer-global durability cursor and wake waiters. The range
+        // just appended is `[start, end)` *local to this store*, so it must be
+        // lifted into the writer's coordinate space before it is published —
+        // otherwise a fresh memtable's small local end would be compared against a
+        // cursor carrying a previous memtable's larger one.
         self.advance_durable(batch_store.global_offset() + end_batch_position);
-        // Signal WAL flush completion for backpressure waiters
         self.signal_wal_flush_complete();
 
         Ok(WalFlushResult {
@@ -686,9 +805,9 @@ impl WalFlusher {
                 num_batches: append_result.num_batches,
             }),
             wal_io_duration,
-            index_update_duration,
-            index_update_duration_breakdown,
-            rows_indexed: rows_to_index,
+            index_update_duration: std::time::Duration::ZERO,
+            index_update_duration_breakdown: std::collections::HashMap::new(),
+            rows_indexed: num_rows,
             wal_bytes: append_result.wal_bytes,
         })
     }
@@ -1612,18 +1731,21 @@ mod tests {
     fn batch_store_source(batch_store: &Arc<BatchStore>) -> WalFlushSource {
         WalFlushSource::BatchStore {
             batch_store: batch_store.clone(),
-            indexes: None,
         }
     }
 
-    fn batch_store_source_with_indexes(
-        batch_store: &Arc<BatchStore>,
-        indexes: &Arc<IndexStore>,
-    ) -> WalFlushSource {
-        WalFlushSource::BatchStore {
-            batch_store: batch_store.clone(),
-            indexes: Some(indexes.clone()),
-        }
+    /// Run the index-apply task's body, as the writer's index task would.
+    async fn apply_all(batch_store: &Arc<BatchStore>, indexes: &Arc<IndexStore>) -> Result<()> {
+        let cursors = Arc::new(WriterCursors::new(true));
+        apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len(),
+            },
+        )
+        .await
     }
 
     #[tokio::test]
@@ -1633,7 +1755,7 @@ mod tests {
         let buffer = build_test_flusher(store, &base_path, shard_id, 1);
 
         // Track a batch
-        let watcher = buffer.track_batch(1);
+        let watcher = buffer.track_batch(None, 0, 1);
 
         // Watcher should not be durable yet
         assert!(!watcher.is_durable());
@@ -1652,7 +1774,7 @@ mod tests {
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 10)).unwrap();
 
-        let mut watcher = flusher.track_batch(1);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         // wait() must NOT resolve before the flush happens
         let result =
@@ -1685,8 +1807,8 @@ mod tests {
         batch_store.append(batch2).unwrap();
 
         // Track batch IDs in WAL flusher
-        let mut watcher1 = buffer.track_batch(1);
-        let mut watcher2 = buffer.track_batch(2);
+        let mut watcher1 = buffer.track_batch(None, 0, 1);
+        let mut watcher2 = buffer.track_batch(None, 0, 2);
 
         // Verify initial state
         assert!(!watcher1.is_durable());
@@ -1715,39 +1837,19 @@ mod tests {
 
     // Regression test for the visibility-cursor bug: with an empty IndexStore
     // (the common case for WAL-managed tables that mirror an index-less base
-    // dataset), a WAL flush must still advance `visible_count` so
-    // scanners can see every batch up to the durable position — not just
-    // batch 0. Before the fix, the cursor stayed at 0 for the lifetime of the
-    // memtable and scanners returned only the first row.
-    #[tokio::test]
-    async fn test_wal_flush_advances_visibility_with_empty_indexes() {
-        let (store, base_path, _temp_dir) = create_local_store().await;
-        let shard_id = Uuid::new_v4();
-        let flusher = build_test_flusher(store, &base_path, shard_id, 1);
-
-        let schema = create_test_schema();
-        let batch_store = Arc::new(BatchStore::with_capacity(10));
-        for _ in 0..3 {
-            batch_store.append(create_test_batch(&schema, 5)).unwrap();
-        }
-
-        // Empty registry, mimicking a memtable with `index_configs = []`.
-        let indexes = Arc::new(IndexStore::new());
-        assert_eq!(indexes.indexed_count(), 0);
-
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
-        flusher.flush(&source, batch_store.len()).await.unwrap();
-
-        // All three batches indexed => the indexed prefix is 3 (exclusive count).
-        assert_eq!(indexes.indexed_count(), 3);
-        assert_eq!(flusher.durable(), 3);
-    }
-
-    // Regression guard for the indexed path: with at least one BTree index
-    // configured, the cursor advance still fires (this was already working
-    // before the fix — keeping the test to lock in the behavior).
+    /// The index apply and the WAL append are separate tasks with separate
+    /// cursors. Appending makes a range durable; it does not index it. Indexing
+    /// makes it indexed; it does not make it durable. Only both together make it
+    /// visible.
+    ///
+    /// This also covers the empty-registry case (a memtable with no configured
+    /// indexes), which used to be skipped entirely by the flush's index arm and
+    /// so left the cursor stuck at 0 for the memtable's whole life.
+    #[rstest::rstest]
+    #[case::no_indexes(false)]
+    #[case::btree_index(true)]
     #[tokio::test]
-    async fn test_wal_flush_advances_visibility_with_btree_index() {
+    async fn test_append_and_index_advance_separate_cursors(#[case] with_btree: bool) {
         let (store, base_path, _temp_dir) = create_local_store().await;
         let shard_id = Uuid::new_v4();
         let flusher = build_test_flusher(store, &base_path, shard_id, 1);
@@ -1759,14 +1861,22 @@ mod tests {
         }
 
         let mut idx = IndexStore::new();
-        idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        if with_btree {
+            idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        }
         let indexes = Arc::new(idx);
 
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
-        flusher.flush(&source, batch_store.len()).await.unwrap();
+        // The append alone makes the range durable and indexes nothing.
+        flusher
+            .flush(&batch_store_source(&batch_store), batch_store.len())
+            .await
+            .unwrap();
+        assert_eq!(flusher.durable(), 3);
+        assert_eq!(indexes.indexed_count(), 0);
 
+        // The index apply alone advances the index cursor.
+        apply_all(&batch_store, &indexes).await.unwrap();
         assert_eq!(indexes.indexed_count(), 3);
-        assert_eq!(flusher.durable(), 3);
     }
 
     #[tokio::test]
@@ -1782,8 +1892,8 @@ mod tests {
         batch_store.append(create_test_batch(&schema, 5)).unwrap();
 
         // Track batch IDs and flush all pending batches
-        let _watcher1 = buffer.track_batch(1);
-        let _watcher2 = buffer.track_batch(2);
+        let _watcher1 = buffer.track_batch(None, 0, 1);
+        let _watcher2 = buffer.track_batch(None, 0, 2);
         let source = batch_store_source(&batch_store);
         let result = buffer.flush(&source, batch_store.len()).await.unwrap();
         let entry = result.entry.unwrap();
@@ -1961,7 +2071,7 @@ mod tests {
         // A durable put on the predecessor: stage a batch and track it.
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(1);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         // Flushing collides with the sentinel and fences. Both the flush result
         // and the watcher must report the fence — and the watcher must resolve
@@ -2123,21 +2233,20 @@ mod tests {
         );
     }
 
-    /// A failed WAL append must not publish rows to readers, even though the
-    /// index arm has already applied them.
+    /// A failed WAL append must not make rows visible, even when the index apply
+    /// has already taken them.
     ///
-    /// `tokio::join!` runs both arms to completion — it does not cancel the index
-    /// arm when the append fails — so `indexed_count` *does* advance on a failed
-    /// flush. That is fine, and unavoidable: the indexes are derived state, and
-    /// replay rebuilds them. What must not happen is for that to make the rows
-    /// readable, because they are not in the WAL and replay will not reproduce
-    /// them. A reader that saw them would be reading a row that no longer exists
-    /// after a reopen.
+    /// The two now run on separate tasks, so an index apply that lands while the
+    /// append is failing advances `indexed_count` — which is fine, and
+    /// unavoidable: indexes are derived state and replay rebuilds them. What must
+    /// not happen is for that to make the rows *readable*, because they are not in
+    /// the WAL and replay will not reproduce them.
     ///
-    /// Readers therefore key off `visible_count`, which the writer advances only
-    /// downstream of *both* arms succeeding.
+    /// Visibility is derived, not published, so this holds by construction: with
+    /// `durable_write`, `visible = min(indexed, durable)`, and a failed append
+    /// leaves `durable` at 0.
     #[tokio::test]
-    async fn test_failed_append_indexes_but_does_not_publish() {
+    async fn test_failed_append_indexes_but_stays_invisible() {
         let (store, base, controls) = failing_memory_store().await;
         let shard_id = Uuid::new_v4();
         controls.fail_wal_puts(usize::MAX);
@@ -2155,7 +2264,8 @@ mod tests {
                 base_delay: Duration::from_millis(1),
             },
         ));
-        let flusher = WalFlusher::new(appender);
+        let cursors = Arc::new(WriterCursors::new(true));
+        let flusher = WalFlusher::with_cursors(appender, Arc::clone(&cursors));
 
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
@@ -2163,71 +2273,61 @@ mod tests {
 
         let mut idx = IndexStore::new();
         idx.add_btree("id_idx".to_string(), 0, "id".to_string());
+        idx.set_durability(Arc::clone(&cursors), 0);
         let indexes = Arc::new(idx);
 
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
+        // The index apply succeeds.
+        apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: 1,
+            },
+        )
+        .await
+        .unwrap();
+        assert_eq!(indexes.indexed_count(), 1);
+
+        // The append does not.
         let err = flusher
-            .flush(&source, batch_store.len())
+            .flush(&batch_store_source(&batch_store), batch_store.len())
             .await
-            .expect_err("the WAL PUT is failing, so the flush must fail");
+            .expect_err("the WAL PUT is failing, so the append must fail");
         assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
 
-        // The index arm ran to completion regardless — that is the join's nature.
-        assert_eq!(
-            indexes.indexed_count(),
-            1,
-            "the index arm applies even when the append fails"
-        );
-
-        // But nothing was published, so no reader can see the row.
+        // Indexed, but not durable — so not visible.
+        assert_eq!(flusher.durable(), 0);
         assert_eq!(
             indexes.visible_count(),
             0,
             "a row whose WAL append failed must never become readable"
         );
-        assert_eq!(flusher.durable(), 0);
     }
 
     /// An index-apply failure poisons the writer.
     ///
-    /// A partial apply cannot be rolled back (see the note at the `map_err` in
-    /// `flush_from_batch_store`), and continuing would re-cover the range on the
-    /// next flush and corrupt the indexes that *did* succeed. So the failure is
-    /// terminal: the writer poisons, reads and writes fail fast, and recovery is
+    /// A partial apply cannot be rolled back — `insert_batches` joins every index
+    /// thread unconditionally, so a failure leaves the others fully applied, and
+    /// none of HNSW, FTS or BTree has a delete. Continuing would re-cover the
+    /// range on the next attempt and corrupt the indexes that *did* succeed. So
+    /// the failure is terminal: reads and writes fail fast, and recovery is
     /// reopen -> replay, which rebuilds the indexes from the WAL.
-    ///
-    /// The batches were WAL-durable before the index failed, so this is not data
-    /// loss — it is a rebuild.
     #[tokio::test]
     async fn test_index_failure_poisons_the_writer() {
         let (store, base_path, _temp_dir) = create_local_store().await;
         let shard_id = Uuid::new_v4();
-        let manifest_store = Arc::new(ShardManifestStore::new(
-            store.clone(),
-            &base_path,
-            shard_id,
-            2,
-        ));
-        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
-        let appender = Arc::new(WalAppender::with_claimed_epoch(
-            store,
-            base_path,
-            shard_id,
-            manifest_store,
-            epoch,
-            0,
-            WalRetryConfig::default(),
-        ));
-        let flusher = WalFlusher::new(appender);
+        let flusher = build_test_flusher(store, &base_path, shard_id, 1);
+        let cursors = Arc::clone(flusher.cursors());
 
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
 
-        // An HNSW index on `id`, which is an Int32 and not a vector. Every insert
-        // of this batch fails deterministically. (`validate_index_configs` rejects
-        // this at shard open — that is what makes poison-and-replay terminating —
-        // so we have to build the store by hand to reach the failure at all.)
+        // An HNSW index on `id`, which is an Int32 and not a vector, so every
+        // insert of this batch fails deterministically. `validate_index_configs`
+        // rejects this at shard open — that is what makes poison-and-replay
+        // terminating — so the store has to be built by hand to reach it at all.
         let mut idx = IndexStore::new();
         idx.add_hnsw(
             "bad_hnsw".to_string(),
@@ -2239,23 +2339,23 @@ mod tests {
         );
         let indexes = Arc::new(idx);
 
-        let source = batch_store_source_with_indexes(&batch_store, &indexes);
-        let err = flusher
-            .flush(&source, batch_store.len())
-            .await
-            .expect_err("indexing an Int32 column as a vector must fail");
+        let err = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: 1,
+            },
+        )
+        .await
+        .expect_err("indexing an Int32 column as a vector must fail");
 
-        assert_eq!(
-            err.fence_reason(),
-            Some(FenceReason::PersistenceFailure),
-            "an index failure must be typed as terminal, not left as an ordinary error"
-        );
+        // The index task latches it, exactly as `IndexApplyHandler` does.
+        flusher.poison(&err);
         assert!(
             flusher.check_poisoned().is_err(),
-            "the writer must be poisoned so it cannot limp on with a corrupt index"
+            "the writer must be poisoned rather than limp on with a corrupt index"
         );
-
-        // Nothing was published, even though the WAL append itself succeeded.
         assert_eq!(indexes.visible_count(), 0);
     }
 
@@ -2286,7 +2386,7 @@ mod tests {
         let schema = create_test_schema();
         let batch_store = Arc::new(BatchStore::with_capacity(10));
         batch_store.append(create_test_batch(&schema, 1)).unwrap();
-        let mut watcher = flusher.track_batch(1);
+        let mut watcher = flusher.track_batch(None, 0, 1);
 
         let source = batch_store_source(&batch_store);
         let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err();
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 0d79aa7274e..0404cc6a768 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -50,8 +50,8 @@ pub use super::wal::{WalEntry, WalEntryData, WalFlushFailure, WalFlushResult, Wa
 use super::memtable::flush::TriggerMemTableFlush;
 use super::scanner::GenerationWarmer;
 use super::wal::{
-    BatchDurableWatcher, TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState,
-    WalRetryConfig, WalTailer, empty_flush_result,
+    BatchDurableWatcher, TriggerIndexApply, TriggerWalFlush, WalAppender, WalFlushSource,
+    WalOnlyState, WalRetryConfig, WalTailer, WriterCursors, apply_index_range, empty_flush_result,
 };
 use super::{TOMBSTONE, schema_with_tombstone};
 use crate::session::Session;
@@ -1016,6 +1016,10 @@ struct SharedWriterState {
     state: Arc<RwLock<WriterState>>,
     wal_flusher: Arc<WalFlusher>,
     wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
+    /// The index-apply task's channel. Separate from the WAL flusher's on
+    /// purpose: `TaskDispatcher::run` awaits `handle()` inline, so sharing one
+    /// would put a ~100ms S3 PUT in front of every latency-sensitive index apply.
+    index_apply_tx: mpsc::UnboundedSender<TriggerIndexApply>,
     memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
     config: ShardWriterConfig,
     schema: Arc<ArrowSchema>,
@@ -1034,6 +1038,7 @@ impl SharedWriterState {
         state: Arc<RwLock<WriterState>>,
         wal_flusher: Arc<WalFlusher>,
         wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
+        index_apply_tx: mpsc::UnboundedSender<TriggerIndexApply>,
         memtable_flush_tx: mpsc::UnboundedSender<TriggerMemTableFlush>,
         config: ShardWriterConfig,
         schema: Arc<ArrowSchema>,
@@ -1047,6 +1052,7 @@ impl SharedWriterState {
             state,
             wal_flusher,
             wal_flush_tx,
+            index_apply_tx,
             memtable_flush_tx,
             config,
             schema,
@@ -1058,6 +1064,25 @@ impl SharedWriterState {
         }
     }
 
+    /// Ask the index-apply task to cover `[indexed, end_batch_position)` of this
+    /// store. Cheap and idempotent: a range already covered is a no-op, which is
+    /// the common case under load, since one apply coalesces the puts queued
+    /// behind it.
+    fn trigger_index_apply(
+        &self,
+        batch_store: Arc<BatchStore>,
+        indexes: Arc<IndexStore>,
+        end_batch_position: usize,
+    ) -> Result<()> {
+        self.index_apply_tx
+            .send(TriggerIndexApply {
+                batch_store,
+                indexes,
+                end_batch_position,
+            })
+            .map_err(|_| Error::io("index apply channel closed"))
+    }
+
     /// Freeze the current memtable and send it to the flush handler.
     ///
     /// Takes `&mut WriterState` directly since caller already holds the lock.
@@ -1070,7 +1095,6 @@ impl SharedWriterState {
         let last_wal_entry_position = state.last_flushed_wal_entry_position;
 
         let old_batch_store = state.memtable.batch_store();
-        let old_indexes = state.memtable.indexes_arc();
 
         let next_generation = state.memtable.generation() + 1;
         // The incoming memtable's batch 0 continues the writer's batch sequence
@@ -1087,21 +1111,36 @@ impl SharedWriterState {
             next_global_offset,
         )?;
 
-        // Build an IndexStore when there are user indexes *or* a primary key:
-        // the PK dedup index (and its flushed on-disk sidecar) is required for
-        // cross-generation dedup even when no secondary index is configured.
-        if !self.index_configs.is_empty() || !self.pk_columns.is_empty() {
-            let mut indexes = IndexStore::from_configs(
-                &self.index_configs,
-                self.max_memtable_rows,
-                self.max_memtable_batches,
-            )?;
+        // Always build and bind an IndexStore, even with no user indexes and no
+        // primary key. It is what carries the memtable's `indexed_count`, and
+        // binding it to the writer's cursors is what lets a reader derive the
+        // visible prefix — so an index-less memtable that skipped this would fall
+        // back to `visible == indexed` and publish rows before they were durable.
+        // (A PK memtable also needs the PK dedup index and its flushed sidecar.)
+        let mut indexes = IndexStore::from_configs(
+            &self.index_configs,
+            self.max_memtable_rows,
+            self.max_memtable_batches,
+        )?;
+        if !self.pk_columns.is_empty() {
             indexes.enable_pk_index(&pk_index_columns(&self.pk_columns, &self.pk_field_ids));
-            new_memtable.set_indexes_arc(Arc::new(indexes));
         }
+        indexes.set_durability(Arc::clone(self.wal_flusher.cursors()), next_global_offset);
+        new_memtable.set_indexes_arc(Arc::new(indexes));
 
         let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable);
         old_memtable.freeze(last_wal_entry_position);
+
+        // The outgoing memtable may still owe an index apply — the puts that
+        // filled it triggered one, but the task need not have drained yet, and
+        // this is the last chance to name that store. Its L0 flush is gated on
+        // the WAL append (below), not on indexing, so without this its tail could
+        // stay unindexed and invisible for the rest of its life.
+        if let Some(old_indexes) = old_memtable.indexes_arc()
+            && old_indexes.indexed_count() < old_batch_store.len()
+        {
+            self.trigger_index_apply(old_batch_store.clone(), old_indexes, old_batch_store.len())?;
+        }
         let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion();
 
         if pending_wal_range.is_some() {
@@ -1115,7 +1154,6 @@ impl SharedWriterState {
             self.wal_flusher.trigger_flush(
                 WalFlushSource::BatchStore {
                     batch_store: old_batch_store,
-                    indexes: old_indexes,
                 },
                 end_batch_position,
                 Some(completion_cell),
@@ -1156,13 +1194,19 @@ impl SharedWriterState {
         Ok(next_generation)
     }
 
-    /// Watch for a write to become WAL-durable.
+    /// Watch for a write to become visible: indexed, and — in durable mode —
+    /// WAL-durable too.
     ///
-    /// `target_durable_count` is a writer-global, exclusive batch count — not a
-    /// memtable-local position. See the caller for why that distinction is
-    /// load-bearing.
-    fn track_batch_for_wal(&self, target_durable_count: usize) -> super::wal::BatchDurableWatcher {
-        self.wal_flusher.track_batch(target_durable_count)
+    /// `target_indexed` is memtable-local; `target_durable` is writer-global.
+    /// Different coordinate spaces on purpose — see `WalFlusher::track_batch`.
+    fn track_batch_for_wal(
+        &self,
+        indexes: Option<Arc<IndexStore>>,
+        target_indexed: usize,
+        target_durable: usize,
+    ) -> super::wal::BatchDurableWatcher {
+        self.wal_flusher
+            .track_batch(indexes, target_indexed, target_durable)
     }
 
     /// Check if memtable flush is needed and trigger if so.
@@ -1193,7 +1237,6 @@ impl SharedWriterState {
         let batch_count = state.memtable.batch_count();
         let total_bytes = state.memtable.estimated_size();
         let batch_store = state.memtable.batch_store();
-        let indexes = state.memtable.indexes_arc();
 
         // Check if there are any unflushed batches
         let has_pending = batch_store.pending_wal_flush_count(self.wal_flusher.durable()) > 0;
@@ -1225,10 +1268,7 @@ impl SharedWriterState {
         // If time trigger fired, send a flush message
         if time_trigger.is_some() {
             let _ = self.wal_flush_tx.send(TriggerWalFlush {
-                source: WalFlushSource::BatchStore {
-                    batch_store,
-                    indexes,
-                },
+                source: WalFlushSource::BatchStore { batch_store },
                 end_batch_position: batch_count,
                 done: None,
             });
@@ -1253,7 +1293,6 @@ impl SharedWriterState {
             let _ = self.wal_flush_tx.send(TriggerWalFlush {
                 source: WalFlushSource::BatchStore {
                     batch_store: batch_store.clone(),
-                    indexes: indexes.clone(),
                 },
                 end_batch_position: batch_count,
                 done: None,
@@ -1418,7 +1457,12 @@ impl ShardWriter {
         }
 
         // Create WAL flusher backed by the shared appender.
-        let mut wal_flusher = WalFlusher::new(wal_appender);
+        // Build the cursors from *this writer's* config. `durable_write` is what
+        // decides whether durability is part of visibility, so a flusher that
+        // defaulted it would leave a non-durable put waiting on a durability
+        // cursor nothing ever advances.
+        let cursors = Arc::new(WriterCursors::new(config.durable_write));
+        let mut wal_flusher = WalFlusher::with_cursors(wal_appender, cursors);
 
         let (wal_flush_tx, wal_flush_rx) = mpsc::unbounded_channel();
         wal_flusher.set_flush_channel(wal_flush_tx.clone());
@@ -1506,19 +1550,20 @@ impl ShardWriter {
             config.max_memtable_batches,
         )?;
 
-        // Create indexes if configured and set them on the MemTable. The
-        // PK-position index is enabled before any WAL replay below so replayed
-        // rows are recorded in it. A primary key alone (no secondary index)
-        // still needs the PK index so flush writes its on-disk dedup sidecar.
-        if !index_configs.is_empty() || !pk_columns.is_empty() {
-            let mut indexes = IndexStore::from_configs(
-                index_configs,
-                config.max_memtable_rows,
-                config.max_memtable_batches,
-            )?;
+        // Always build and bind an IndexStore — see the matching note in
+        // `freeze_memtable`. The PK-position index is enabled before any WAL
+        // replay below so replayed rows are recorded in it.
+        let mut indexes = IndexStore::from_configs(
+            index_configs,
+            config.max_memtable_rows,
+            config.max_memtable_batches,
+        )?;
+        if !pk_columns.is_empty() {
             indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
-            memtable.set_indexes_arc(Arc::new(indexes));
         }
+        // The writer's first memtable starts at coordinate 0.
+        indexes.set_durability(Arc::clone(wal_flusher.cursors()), 0);
+        memtable.set_indexes_arc(Arc::new(indexes));
 
         // Replay any WAL entries written after the last successfully-flushed
         // generation. Each entry's writer_epoch is checked against ours; an
@@ -1554,15 +1599,6 @@ impl ShardWriter {
         // 0 and the durable count is just its batch count.
         wal_flusher.advance_durable(memtable.batch_count());
 
-        // ...and publish them to readers. Replayed batches are durable by
-        // construction (they came out of the WAL) and were just re-indexed above,
-        // so they satisfy both halves of `visible`. Publishing is normally the
-        // flush's job; replay bypasses the flush, so it must do it here or the
-        // recovered rows stay invisible.
-        if let Some(indexes) = memtable.indexes_arc() {
-            indexes.publish_visible(memtable.batch_count());
-        }
-
         wal_flusher
             .wal_appender()
             .seed_next_position(next_wal_position)
@@ -1627,11 +1663,26 @@ impl ShardWriter {
             memtable_flush_rx,
         )?;
 
+        // The index-apply task. Its own channel and its own dispatcher: the
+        // dispatcher awaits `handle()` inline, so sharing the WAL flusher's
+        // channel would queue every index apply behind a ~100ms S3 PUT.
+        let (index_apply_tx, index_apply_rx) = mpsc::unbounded_channel();
+        let index_handler = IndexApplyHandler {
+            cursors: Arc::clone(wal_flusher.cursors()),
+            wal_flusher: wal_flusher.clone(),
+        };
+        task_executor.add_handler(
+            "index_applier".to_string(),
+            Box::new(index_handler),
+            index_apply_rx,
+        )?;
+
         // Shared state used by `put()` to dispatch trigger checks.
         let writer_state = Arc::new(SharedWriterState::new(
             state.clone(),
             wal_flusher,
             wal_flush_tx,
+            index_apply_tx,
             memtable_flush_tx,
             config.clone(),
             schema.clone(),
@@ -1926,15 +1977,22 @@ impl ShardWriter {
             let end_pos = results.last().map(|(pos, _, _)| pos + 1).unwrap_or(0);
             let batch_positions = start_pos..end_pos;
 
-            // 3. Track this write for WAL durability. The target is writer-global
-            //    and exclusive. It must not be a memtable-local position: batch
-            //    positions restart at 0 in every memtable, while the durability
-            //    channel spans the writer's whole life. A local target is
-            //    therefore already satisfied by a *previous* memtable's appends,
-            //    so the first N puts into every post-rotation memtable would ack
-            //    as durable without a WAL append ever happening.
-            let durable_watcher =
-                writer_state.track_batch_for_wal(batch_store.global_offset() + end_pos);
+            // 3. Watch for this write to become *visible*: indexed, and — under
+            //    `durable_write` — WAL-durable too.
+            //
+            //    The two targets live in different coordinate spaces. `end_pos`
+            //    is memtable-local, which is what the index apply works in. The
+            //    durability cursor is writer-global, because batch positions
+            //    restart at 0 in every memtable while that cursor spans the
+            //    writer's whole life — a local durable target would already be
+            //    satisfied by a *previous* memtable's appends, so the first N
+            //    puts into every post-rotation memtable would ack as durable with
+            //    no WAL append ever happening.
+            let durable_watcher = writer_state.track_batch_for_wal(
+                indexes.clone(),
+                end_pos,
+                batch_store.global_offset() + end_pos,
+            );
 
             // 4. Check if WAL flush should be triggered
             writer_state.maybe_trigger_wal_flush(&mut state);
@@ -1949,21 +2007,33 @@ impl ShardWriter {
 
         self.stats.record_put(start.elapsed());
 
-        // Trigger the flush here (outside the lock) so the watcher can resolve;
-        // only the `wait()` is the caller's to schedule.
-        let watcher = if self.config.durable_write {
+        // Trigger the index apply, in **both** modes. This is what makes reads
+        // read-your-writes regardless of `durable_write`: skipping durability now
+        // costs the caller durability only, not visibility. It is cheap
+        // (in-memory, ~ms), so there is no reason to batch it onto the WAL's
+        // schedule — that schedule exists to bound S3 API cost, which an
+        // in-memory index apply does not incur.
+        if let Some(indexes) = &indexes {
+            writer_state.trigger_index_apply(
+                batch_store.clone(),
+                indexes.clone(),
+                batch_positions.end,
+            )?;
+        }
+
+        // Trigger the WAL flush here (outside the lock) so the watcher can
+        // resolve; only the `wait()` is the caller's to schedule.
+        if self.config.durable_write {
             self.wal_flusher.trigger_flush(
-                WalFlushSource::BatchStore {
-                    batch_store,
-                    indexes,
-                },
+                WalFlushSource::BatchStore { batch_store },
                 batch_positions.end,
                 None,
             )?;
-            Some(durable_watcher)
-        } else {
-            None
-        };
+        }
+
+        // The watcher is returned in both modes now. A non-durable put still
+        // waits — for its index apply (~ms), not for an S3 PUT (~100ms).
+        let watcher = Some(durable_watcher);
 
         Ok((WriteResult { batch_positions }, watcher))
     }
@@ -2364,23 +2434,41 @@ impl ShardWriter {
                 writer_state,
                 ..
             } => {
-                // Send final WAL flush message and wait for completion
+                // Drain *both* tasks against the active memtable. The index apply
+                // and the WAL append are independent now, so closing has to
+                // settle both: the L0 flush below turns this memtable into a
+                // Lance generation, and a generation whose indexes never saw the
+                // tail is a generation with a hole in it.
                 let st = state.read().await;
                 let batch_store = st.memtable.batch_store();
                 let indexes = st.memtable.indexes_arc();
                 let batch_count = st.memtable.batch_count();
                 drop(st);
 
+                if batch_count > 0
+                    && let Some(indexes) = indexes
+                    && indexes.indexed_count() < batch_count
+                {
+                    let mut watcher = self.wal_flusher.track_batch(
+                        Some(Arc::clone(&indexes)),
+                        batch_count,
+                        0, // durability is settled by the WAL flush below
+                    );
+                    writer_state.trigger_index_apply(
+                        Arc::clone(&batch_store),
+                        indexes,
+                        batch_count,
+                    )?;
+                    watcher.wait().await?;
+                }
+
                 if batch_count > 0 {
                     let done = WatchableOnceCell::new();
                     let reader = done.reader();
                     if writer_state
                         .wal_flush_tx
                         .send(TriggerWalFlush {
-                            source: WalFlushSource::BatchStore {
-                                batch_store,
-                                indexes,
-                            },
+                            source: WalFlushSource::BatchStore { batch_store },
                             end_batch_position: batch_count,
                             done: Some(done),
                         })
@@ -2491,6 +2579,34 @@ pub struct WalStats {
 ///
 /// This handler does parallel WAL I/O + index updates during flush.
 /// Indexes are passed through the TriggerWalFlush message.
+/// The index-apply task: one sequential consumer of the index-apply channel.
+///
+/// Sequential consumption is the safety property. `HnswGraph::insert_batch` hard-
+/// rejects any range whose start is not its `indexed_len`, so the apply must see
+/// contiguous, in-order ranges — and a single consumer guarantees that
+/// regardless of how many putters race behind it. Ordering comes from the task,
+/// not from a flush interval, which is why triggering per-put is exactly as safe
+/// as triggering on a timer, and lets a put become visible in milliseconds
+/// instead of waiting on an S3 round-trip.
+struct IndexApplyHandler {
+    cursors: Arc<WriterCursors>,
+    wal_flusher: Arc<WalFlusher>,
+}
+
+#[async_trait]
+impl MessageHandler<TriggerIndexApply> for IndexApplyHandler {
+    async fn handle(&mut self, message: TriggerIndexApply) -> Result<()> {
+        if let Err(e) = apply_index_range(&self.cursors, message).await {
+            // An index apply cannot be partially rolled back, so a failure is
+            // terminal: poison, and let reopen rebuild the indexes from the WAL.
+            // See the note in `WalFlusher::flush_from_batch_store`.
+            self.wal_flusher.poison(&e);
+            return Err(e);
+        }
+        Ok(())
+    }
+}
+
 struct WalFlushHandler {
     wal_flusher: Arc<WalFlusher>,
     /// MemTable-mode writer state, used to detect "frozen vs active" flushes
@@ -2576,13 +2692,9 @@ impl WalFlushHandler {
         // a BatchStore source carrying a non-empty `IndexStore` does. Used
         // to gate the `record_index_update` stat so WAL-only flushes don't
         // pollute the index-update counters.
-        let has_indexes = matches!(
-            &source,
-            WalFlushSource::BatchStore {
-                indexes: Some(_),
-                ..
-            }
-        );
+        // The WAL flush no longer touches indexes at all — the index apply runs
+        // on its own task — so it never records an index-update stat.
+        let has_indexes = false;
 
         // Early-out for BatchStore sources where the watermark already
         // covers the requested end position. Detection of "frozen flush"
@@ -3411,7 +3523,7 @@ mod tests {
     /// await), but the tombstone still lands in the in-memory tier. The delete
     /// analog of `test_put_no_wait_non_durable_returns_no_watcher`.
     #[tokio::test]
-    async fn test_shard_writer_delete_no_wait_non_durable_returns_no_watcher() {
+    async fn test_non_durable_delete_is_read_your_writes() {
         let (store, base_path, base_uri, _temp) = create_local_store().await;
         let schema = create_pk_test_schema();
         let config = ShardWriterConfig {
@@ -3432,7 +3544,10 @@ mod tests {
             .delete_no_wait(vec![id_only_keys(&[2])])
             .await
             .unwrap();
-        assert!(watcher.is_none(), "non-durable delete has nothing to await");
+        // As with a put: a non-durable delete awaits its index apply, not an S3
+        // round-trip. It is read-your-writes, just not durable.
+        let mut watcher = watcher.expect("a non-durable delete awaits its index apply");
+        watcher.wait().await.unwrap();
 
         // Tombstone landed in the in-memory tier (5 rows + 1 tombstone).
         assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6);
@@ -3892,7 +4007,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_put_no_wait_non_durable_returns_no_watcher() {
+    async fn test_non_durable_put_is_read_your_writes() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
 
@@ -3915,7 +4030,21 @@ mod tests {
         let batch = create_test_batch(&schema, 0, 10);
         let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap();
         assert_eq!(result.batch_positions, 0..1);
-        assert!(watcher.is_none(), "non-durable put has nothing to await");
+
+        // A non-durable put still has something to await: its *index apply*.
+        // Skipping durability now costs the caller durability only — not
+        // visibility. Before the index apply was split off the WAL flush, a
+        // non-durable write was not read-your-writes at all: the row stayed
+        // invisible until some later flush happened to index it.
+        let mut watcher = watcher.expect("a non-durable put awaits its index apply");
+        watcher.wait().await.unwrap();
+
+        let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap();
+        assert_eq!(
+            scanned.num_rows(),
+            10,
+            "a non-durable put must be readable as soon as it returns"
+        );
 
         let stats = writer.memtable_stats().await.unwrap();
         assert_eq!(stats.row_count, 10);
@@ -5107,6 +5236,53 @@ mod tests {
         writer_b.close().await.unwrap();
     }
 
+    /// A non-durable put is readable through the **index-backed** arms the moment
+    /// it returns, not just through a full scan.
+    ///
+    /// This is what splitting the index apply off the WAL flush buys. Before, the
+    /// index apply only ran as one arm of the flush, so with `durable_write:
+    /// false` nothing triggered it on the put path at all: the row sat in the
+    /// batch store, unindexed, until some later flush happened along. A full scan
+    /// (which reads the batch store directly) could still find it, while every
+    /// index-accelerated query could not — the tiers disagreed. Now the apply is
+    /// triggered per-put in both modes, so `put` returning means "indexed".
+    #[tokio::test]
+    async fn test_non_durable_put_is_visible_through_the_index() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        let config = ShardWriterConfig {
+            durable_write: false,
+            ..memtable_config_with_pk(shard_id)
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+
+        // The indexed PK lookup must find it — this is the arm that saw nothing
+        // before, because the index apply had never run.
+        let mut scanner = writer.scan().await.unwrap();
+        scanner.filter("id = 3").unwrap();
+        let hit = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            hit.num_rows(),
+            1,
+            "an index-backed lookup must see a non-durable put as soon as it returns"
+        );
+
+        // ...and so must the unindexed full scan, i.e. the tiers agree.
+        let all = writer.scan().await.unwrap().try_into_batch().await.unwrap();
+        assert_eq!(all.num_rows(), 5);
+
+        writer.close().await.unwrap();
+    }
+
     /// The durability cursor is writer-global, so a put into a *post-rotation*
     /// memtable must still wait for its own WAL append.
     ///

From 0301767052d111cf7ec8990a57f1ae9cd39bc1a1 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 15:52:56 -0500
Subject: [PATCH 10/19] feat(mem_wal): append the WAL on a background ticker,
 resolved by cursor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`flush_interval_ms` was inert. It routed to a timer that was only ever evaluated
*on the write path*, so it could add a redundant trigger but never delay or batch
one — and since every durable put triggered its own append, tuning the knob did
nothing at all.

Give the WAL flusher a real ticker (`MessageHandler::tickers`, which the memtable
flusher already used) and take the append off the put path. The append is the
only thing on that schedule: it is an S3 PUT, billed per call, and bounding that
cost is the entire reason a flush interval exists. The index apply is not on it —
it is in-memory and free to batch, so it stays per-put on its own task.

The tick names no store. `MessageFactory` is synchronous and cannot take the
async state lock, and capturing an `Arc<BatchStore>` at handler construction
would pin the first memtable forever. So `WalFlushSource::NextPending` is
resolved when the message is *handled* — by walking the live stores oldest-first
and taking the first that still owes an append.

Oldest-first, not "the active memtable", and this is load-bearing. WAL entry
positions are assigned in append-call order; replay walks them ascending; row
positions follow; and primary-key recency is "newest visible row position wins".
So append order *is* dedup order. A tick enqueued before a freeze is handled
after it, and resolving to the active memtable would append the incoming
memtable's batches ahead of the outgoing one's tail — silently handing the key to
the stale row on the next replay. It survives the crash that caused it, and a
full scan cannot see it. Selecting by cursor makes the target a function of what
is durable rather than of when the timer fired.

Two starvation fixes in the dispatcher, both of which this makes reachable:

- The interval used the default `MissedTickBehavior::Burst`, which replays every
  tick missed while `handle()` ran. A WAL append easily outlasts its own
  interval, so missed ticks accumulate, the ticker arm is always ready, and —
  being `biased` — it starves the channel. Now `Delay`.
- The `biased` select polled the ticker *before* `rx.recv()`. A tick only ever
  adds an append a real trigger would have made anyway, whereas a message may be
  a freeze's completion cell or `close()`'s final append, which nothing else
  delivers. Messages now win.

`durable_write: true` with no ticker is rejected at open: the put path no longer
triggers its own append, so such a writer would block forever on an append that
never comes. Better a clear error than a deadlock.

Accepted cost: single-client sequential *durable* throughput drops from ~10
writes/sec (one PUT round-trip) to roughly one per tick. That is the policy
choice — the interval should mean what it says, and API cost should be bounded.
Latency-sensitive callers want `durable_write: false`, which now costs them
durability only, not visibility.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/wal.rs   |  22 ++
 rust/lance/src/dataset/mem_wal/write.rs | 373 +++++++++++++++++++++---
 2 files changed, 350 insertions(+), 45 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 64d7f87cc9d..2d9c3323584 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -307,6 +307,22 @@ pub enum WalFlushSource {
     /// `BatchStore` to the WAL. Append-only — the index apply runs on its own
     /// task, so a failed append can no longer publish rows through it.
     BatchStore { batch_store: Arc<BatchStore> },
+    /// Timer-driven: append whichever store still owes the WAL an append,
+    /// resolved when the message is *handled*, not when it is enqueued.
+    ///
+    /// Carries no store on purpose. `MessageFactory` is synchronous and cannot
+    /// take the async state lock, and capturing an `Arc<BatchStore>` once at
+    /// handler construction would pin the first memtable forever.
+    ///
+    /// Resolution walks the live stores **oldest first** and takes the first with
+    /// `global_end() > durable`. It must not simply take "the active memtable":
+    /// a tick enqueued before a freeze is handled *after* it, would resolve to
+    /// the new memtable, and would append its batches ahead of the outgoing
+    /// memtable's tail. WAL entry positions are assigned in append-call order,
+    /// replay walks them ascending, row positions follow, and primary-key recency
+    /// is "newest visible row position wins" — so an out-of-order append silently
+    /// inverts dedup after a crash: the stale row wins. A full scan looks fine.
+    NextPending,
     /// WAL-only mode: drain all pending batches from the shared
     /// `WalOnlyState`. There are no in-memory indexes to update.
     WalOnly { state: Arc<WalOnlyState> },
@@ -317,6 +333,7 @@ impl WalFlushSource {
         match self {
             Self::BatchStore { .. } => "BatchStore",
             Self::WalOnly { .. } => "WalOnly",
+            Self::NextPending => "NextPending",
         }
     }
 }
@@ -734,6 +751,11 @@ impl WalFlusher {
                     .await
             }
             WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await,
+            // The handler resolves a tick to a concrete store before it gets
+            // here, because only it can take the async state lock.
+            WalFlushSource::NextPending => Err(Error::internal(
+                "WalFlushSource::NextPending must be resolved by the flush handler",
+            )),
         };
         // A terminal failure means the watermark can never advance; latch the
         // poison so waiters wake with the typed error and later writes fail fast.
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 0404cc6a768..422c3e97bbf 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -468,7 +468,13 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
         let mut ticker_intervals: Vec<(Interval, MessageFactory<T>)> = tickers
             .into_iter()
             .map(|(duration, factory)| {
-                let interval = interval_at(tokio::time::Instant::now() + duration, duration);
+                let mut interval = interval_at(tokio::time::Instant::now() + duration, duration);
+                // `Burst` (the default) replays every tick missed while `handle()`
+                // was running. A WAL append can easily outlast its own interval, so
+                // the missed ticks pile up, the ticker arm below is always ready,
+                // and — being `biased` — it starves `rx` indefinitely: freeze
+                // completion cells and `close()`'s final append never get handled.
+                interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
                 (interval, factory)
             })
             .collect();
@@ -511,12 +517,12 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
                         debug!("Task '{}' received cancellation", self.name);
                         break Ok(());
                     }
-                    _ = first_interval.tick() => {
-                        let message = (ticker_intervals[0].1)();
-                        if let Err(e) = self.handler.handle(message).await {
-                            error!("Task '{}' error handling ticker message: {}", self.name, e);
-                        }
-                    }
+                    // Explicit messages outrank the ticker. A tick is a backstop
+                    // — it only ever *adds* an append that a real trigger would
+                    // have made anyway — whereas a message may be a freeze's
+                    // completion cell or `close()`'s final append, which nothing
+                    // else will deliver. Polling the ticker first (as this did)
+                    // lets a ready tick starve them.
                     msg = self.rx.recv() => {
                         match msg {
                             Some(message) => {
@@ -530,6 +536,12 @@ impl<T: Send + Debug + 'static> TaskDispatcher<T> {
                             }
                         }
                     }
+                    _ = first_interval.tick() => {
+                        let message = (ticker_intervals[0].1)();
+                        if let Err(e) = self.handler.handle(message).await {
+                            error!("Task '{}' error handling ticker message: {}", self.name, e);
+                        }
+                    }
                 }
             }
         };
@@ -1536,6 +1548,23 @@ impl ShardWriter {
         let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
         let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
 
+        // A durable writer with no flush ticker cannot make progress. The WAL
+        // append is timer-, size-, and freeze-driven now — the put path no longer
+        // triggers its own — so with no ticker a durable put below the size
+        // threshold waits on an append that nothing will ever make. Reject the
+        // combination rather than deadlock on it.
+        if config.durable_write
+            && config
+                .max_wal_flush_interval
+                .is_none_or(|interval| interval.is_zero())
+        {
+            return Err(Error::invalid_input(
+                "durable_write requires a non-zero max_wal_flush_interval: the WAL append is \
+                 driven by that ticker, so without it a durable put would wait forever for an \
+                 append that never happens",
+            ));
+        }
+
         // Reject an index config that disagrees with the schema *before* a
         // single row is accepted. Such a config fails deterministically on every
         // insert, including inserts replayed from the WAL — so once a row is
@@ -1638,8 +1667,12 @@ impl ShardWriter {
         let backpressure = BackpressureController::new(config.clone());
 
         // Background WAL flush handler — parallel WAL I/O + index updates.
-        let wal_handler =
-            WalFlushHandler::new(wal_flusher.clone(), Some(state.clone()), stats.clone());
+        let wal_handler = WalFlushHandler::new(
+            wal_flusher.clone(),
+            Some(state.clone()),
+            config.max_wal_flush_interval,
+            stats.clone(),
+        );
         task_executor.add_handler(
             "wal_flusher".to_string(),
             Box::new(wal_handler),
@@ -1710,7 +1743,7 @@ impl ShardWriter {
     ) -> Result<WriterMode> {
         // Background WAL flush handler — no MemTable state to consult, so
         // pass `None` for the frozen-vs-active detection.
-        let wal_handler = WalFlushHandler::new(wal_flusher, None, stats);
+        let wal_handler = WalFlushHandler::new(wal_flusher, None, None, stats);
         task_executor.add_handler(
             "wal_flusher".to_string(),
             Box::new(wal_handler),
@@ -2013,23 +2046,22 @@ impl ShardWriter {
         // (in-memory, ~ms), so there is no reason to batch it onto the WAL's
         // schedule — that schedule exists to bound S3 API cost, which an
         // in-memory index apply does not incur.
-        if let Some(indexes) = &indexes {
-            writer_state.trigger_index_apply(
-                batch_store.clone(),
-                indexes.clone(),
-                batch_positions.end,
-            )?;
+        if let Some(indexes) = indexes {
+            writer_state.trigger_index_apply(batch_store, indexes, batch_positions.end)?;
         }
 
-        // Trigger the WAL flush here (outside the lock) so the watcher can
-        // resolve; only the `wait()` is the caller's to schedule.
-        if self.config.durable_write {
-            self.wal_flusher.trigger_flush(
-                WalFlushSource::BatchStore { batch_store },
-                batch_positions.end,
-                None,
-            )?;
-        }
+        // The WAL append is *not* triggered here. It happens on the background
+        // ticker (and on the size trigger, and at freeze/close), which is the only
+        // way the flush interval can mean anything: while every durable put
+        // triggered its own append, the interval could add a redundant trigger but
+        // never delay or batch one.
+        //
+        // The cost is real and accepted: a single client's sequential *durable*
+        // throughput drops from ~10 writes/sec (one PUT round-trip) to roughly one
+        // per tick. That is a policy choice — the interval should mean what it
+        // says, and S3 API cost should be bounded. Latency-sensitive callers want
+        // `durable_write: false`, which now costs them durability only, not
+        // visibility.
 
         // The watcher is returned in both modes now. A non-durable put still
         // waits — for its index apply (~ms), not for an S3 PUT (~100ms).
@@ -2575,10 +2607,28 @@ pub struct WalStats {
     pub next_wal_entry_position: u64,
 }
 
-/// Background handler for WAL flush operations.
+/// The oldest store that still owes the WAL an append, or `None` when everything
+/// is durable.
 ///
-/// This handler does parallel WAL I/O + index updates during flush.
-/// Indexes are passed through the TriggerWalFlush message.
+/// Ordering is the whole point. WAL entry positions are assigned in append-call
+/// order; replay walks them ascending; row positions follow; and primary-key
+/// recency is "newest visible row position wins". So appending a newer memtable
+/// ahead of an older one's tail silently inverts dedup after a crash.
+///
+/// Taking "the active memtable" would do exactly that, because a timer tick
+/// enqueued before a freeze is handled after it and would resolve to the incoming
+/// memtable. Selecting by cursor instead makes the target a function of what is
+/// actually durable, not of when the timer happened to fire.
+fn next_pending_store(
+    frozen: impl Iterator<Item = Arc<BatchStore>>,
+    active: Arc<BatchStore>,
+    durable: usize,
+) -> Option<Arc<BatchStore>> {
+    frozen
+        .chain(std::iter::once(active))
+        .find(|store| store.global_end() > durable)
+}
+
 /// The index-apply task: one sequential consumer of the index-apply channel.
 ///
 /// Sequential consumption is the safety property. `HnswGraph::insert_batch` hard-
@@ -2613,6 +2663,9 @@ struct WalFlushHandler {
     /// via Arc::ptr_eq on the active batch_store. `None` when running in
     /// WAL-only mode (no MemTable, no frozen-vs-active distinction).
     memtable_state: Option<Arc<RwLock<WriterState>>>,
+    /// How often to append in the background. `None` disables the ticker, leaving
+    /// the append size-triggered (and freeze/close-triggered) only.
+    flush_interval: Option<Duration>,
     stats: SharedWriteStats,
 }
 
@@ -2620,11 +2673,13 @@ impl WalFlushHandler {
     fn new(
         wal_flusher: Arc<WalFlusher>,
         memtable_state: Option<Arc<RwLock<WriterState>>>,
+        flush_interval: Option<Duration>,
         stats: SharedWriteStats,
     ) -> Self {
         Self {
             wal_flusher,
             memtable_state,
+            flush_interval,
             stats,
         }
     }
@@ -2632,6 +2687,33 @@ impl WalFlushHandler {
 
 #[async_trait]
 impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
+    /// Append periodically in the background.
+    ///
+    /// This is what the flush interval was always supposed to mean. It routed to
+    /// a timer that was only ever *evaluated on the write path*, so it could add
+    /// a redundant trigger but never delay or batch one — with every durable put
+    /// triggering its own append, tuning the knob did nothing at all.
+    ///
+    /// The ticker exists to bound S3 API cost: an append is a PUT, billed per
+    /// call, and it is the only thing on this schedule. The index apply is not —
+    /// it is in-memory and free to batch, so it runs per-put on its own task.
+    fn tickers(&mut self) -> Vec<(Duration, MessageFactory<TriggerWalFlush>)> {
+        // No interval => no ticker. A zero interval would panic in tokio.
+        let Some(interval) = self.flush_interval.filter(|d| !d.is_zero()) else {
+            return vec![];
+        };
+        // The tick names no store: `MessageFactory` is synchronous and cannot take
+        // the async state lock. `handle()` resolves it against the cursor.
+        vec![(
+            interval,
+            Box::new(|| TriggerWalFlush {
+                source: WalFlushSource::NextPending,
+                end_batch_position: 0,
+                done: None,
+            }),
+        )]
+    }
+
     async fn handle(&mut self, message: TriggerWalFlush) -> Result<()> {
         let TriggerWalFlush {
             source,
@@ -2639,6 +2721,16 @@ impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
             done,
         } = message;
 
+        // A timer tick names no store — resolve it now, at handle time.
+        let (source, end_batch_position) = match source {
+            WalFlushSource::NextPending => match self.resolve_next_pending().await {
+                Some(resolved) => resolved,
+                // Everything is already durable; the tick has nothing to do.
+                None => return Ok(()),
+            },
+            other => (other, end_batch_position),
+        };
+
         let result = self.do_flush(source, end_batch_position).await;
 
         // Propagate the just-appended WAL entry position back into the
@@ -2669,6 +2761,44 @@ impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
 }
 
 impl WalFlushHandler {
+    /// Pick the store the WAL still owes an append, oldest first.
+    ///
+    /// **WAL entries must be appended in global batch-position order for the
+    /// writer's lifetime.** `WalAppender::append` assigns each entry's position
+    /// from its own counter, in call order; replay walks those positions
+    /// ascending and assigns row positions in that order; and primary-key recency
+    /// is "newest visible row position wins". So append order fixes dedup order.
+    /// Append two memtables out of order and replay silently hands the dedup to
+    /// the *stale* row — corruption that survives the crash that caused it, and
+    /// that a full scan cannot see.
+    ///
+    /// Resolving to "the active memtable" would break exactly that: a tick
+    /// enqueued before a freeze is handled after it, resolves to the incoming
+    /// memtable, and appends its batches ahead of the outgoing memtable's tail.
+    /// So the target is a function of `durable`, not of when the timer fired.
+    ///
+    /// Safe to walk the frozen list because a store that still owes an append
+    /// cannot be swept: its L0 flush is blocked on the completion cell that only
+    /// that append fires.
+    async fn resolve_next_pending(&self) -> Option<(WalFlushSource, usize)> {
+        let state_lock = self.memtable_state.as_ref()?;
+        let state = state_lock.read().await;
+        let durable = self.wal_flusher.durable();
+
+        next_pending_store(
+            state
+                .frozen_memtables
+                .iter()
+                .map(|frozen| frozen.memtable.batch_store()),
+            state.memtable.batch_store(),
+            durable,
+        )
+        .map(|store| {
+            let end = store.len();
+            (WalFlushSource::BatchStore { batch_store: store }, end)
+        })
+    }
+
     /// Unified flush method for both active and frozen memtables and for
     /// WAL-only mode.
     ///
@@ -3937,7 +4067,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -3980,7 +4110,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4017,7 +4147,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4063,7 +4193,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4098,7 +4228,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4150,7 +4280,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4239,7 +4369,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 1024, // Very small - will trigger flush quickly
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4346,7 +4476,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -4391,7 +4521,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold — every batch crosses it.
             max_memtable_size: 64,
             manifest_scan_batch_size: 2,
@@ -4458,7 +4588,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: usize::MAX,
             max_unflushed_memtable_bytes: usize::MAX,
             manifest_scan_batch_size: 2,
@@ -4539,7 +4669,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold so a few batches cross it.
             max_memtable_size: 1024,
             manifest_scan_batch_size: 2,
@@ -4680,7 +4810,7 @@ mod tests {
             durable_write: true,
             enable_memtable: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             manifest_scan_batch_size: 2,
             ..Default::default()
         }
@@ -4973,7 +5103,7 @@ mod tests {
             durable_write: true,
             sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -5366,6 +5496,159 @@ mod tests {
         );
     }
 
+    /// A background tick must append the **oldest** store that still owes the WAL,
+    /// never "whatever memtable is active".
+    ///
+    /// A tick carries no store: it is enqueued by a timer and resolved when it is
+    /// handled. So a tick enqueued before a freeze is handled *after* it, and
+    /// resolving to the active memtable would append the incoming memtable's
+    /// batches ahead of the outgoing memtable's tail. WAL entry positions are
+    /// assigned in append-call order, replay walks them ascending, row positions
+    /// follow, and primary-key recency is "newest visible row position wins" — so
+    /// that inverts dedup after a crash, handing the key to the stale row. It
+    /// survives the crash that caused it, and a full scan cannot see it.
+    #[test]
+    fn test_next_pending_store_picks_the_oldest_owing_an_append() {
+        let schema = create_test_schema();
+
+        // A frozen store of 2 batches at coordinate 0, and the active store that
+        // rotated in behind it at coordinate 2.
+        let frozen = Arc::new(BatchStore::with_capacity(4));
+        frozen.append(create_test_batch(&schema, 0, 1)).unwrap();
+        frozen.append(create_test_batch(&schema, 1, 1)).unwrap();
+        let active = Arc::new(BatchStore::with_capacity_at(4, 2));
+        active.append(create_test_batch(&schema, 2, 1)).unwrap();
+
+        let frozen_list = || std::iter::once(Arc::clone(&frozen));
+
+        // Nothing durable: the frozen store owes the oldest append, so it wins —
+        // even though the active memtable also has un-appended batches.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 0).unwrap();
+        assert!(
+            Arc::ptr_eq(&picked, &frozen),
+            "the outgoing memtable's tail must be appended before the incoming one's head"
+        );
+
+        // Still true partway through the frozen store.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 1).unwrap();
+        assert!(Arc::ptr_eq(&picked, &frozen));
+
+        // Once the frozen store is fully durable, the active one is next.
+        let picked = next_pending_store(frozen_list(), Arc::clone(&active), 2).unwrap();
+        assert!(Arc::ptr_eq(&picked, &active));
+
+        // Everything durable: nothing to do.
+        assert!(next_pending_store(frozen_list(), Arc::clone(&active), 3).is_none());
+    }
+
+    /// A durable writer with no flush ticker cannot make progress, so `open()`
+    /// rejects it rather than letting a put block forever.
+    #[tokio::test]
+    async fn test_open_rejects_durable_write_without_a_ticker() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        let config = ShardWriterConfig {
+            durable_write: true,
+            max_wal_flush_interval: None,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let Err(err) = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]).await
+        else {
+            panic!("durable_write with no ticker must be rejected");
+        };
+        assert!(
+            err.to_string().contains("max_wal_flush_interval"),
+            "the error must name the knob, got: {err}"
+        );
+    }
+
+    /// WAL entries must be appended in global batch-position order across a
+    /// memtable rotation, because append order *is* primary-key recency order.
+    ///
+    /// `WalAppender::append` assigns each entry's position from its own counter,
+    /// in call order. Replay walks those positions ascending and assigns row
+    /// positions in that order. Primary-key recency is "newest visible row
+    /// position wins". So an out-of-order append silently inverts dedup after a
+    /// crash — the stale row wins — and a full scan cannot see it.
+    ///
+    /// The hazard is the background ticker. If it resolved to "whatever memtable
+    /// is active" rather than to the oldest store still owing an append, a tick
+    /// enqueued before a freeze but handled after it would append the *incoming*
+    /// memtable's batches ahead of the outgoing memtable's tail. So the target is
+    /// resolved from the durability cursor, not from wall-clock timing.
+    #[tokio::test]
+    async fn test_wal_append_order_preserves_pk_recency_across_rotation() {
+        use crate::dataset::mem_wal::wal::WalTailer;
+
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Two batches per memtable, so the second put fills it and rotates.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // Memtable 1: ids 7 then 20 (the second fills it and triggers the freeze).
+        writer
+            .put(vec![create_test_batch(&schema, 7, 1)])
+            .await
+            .unwrap();
+        writer
+            .put(vec![create_test_batch(&schema, 20, 1)])
+            .await
+            .unwrap();
+        // Memtable 2 overwrites id=7 with a newer row. Its append must land in the
+        // WAL *after* memtable 1's, or replay would resolve id=7 to the stale copy.
+        writer
+            .put(vec![create_test_batch(&schema, 30, 1)])
+            .await
+            .unwrap();
+        writer.close().await.unwrap();
+
+        // Walk the WAL in entry order and collect the ids as replay would see them.
+        let tailer = WalTailer::new(store, base_path, shard_id);
+        let first = tailer.first_position().await.unwrap();
+        let next = tailer.next_position().await.unwrap();
+        let mut ids: Vec<i32> = Vec::new();
+        for position in first..next {
+            let Some(entry) = tailer.read_entry(position).await.unwrap() else {
+                continue;
+            };
+            for batch in &entry.batches {
+                let column = batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap();
+                ids.extend((0..column.len()).map(|i| column.value(i)));
+            }
+        }
+
+        assert_eq!(
+            ids,
+            vec![7, 20, 30],
+            "WAL entries must follow global batch-position order; memtable 1's rows must \
+             precede memtable 2's, or replay inverts primary-key recency"
+        );
+    }
+
     /// An index config that disagrees with the schema fails `open()` outright.
     /// It must not be allowed to accept writes: the insert would fail
     /// deterministically on every batch, including batches replayed from the
@@ -5866,7 +6149,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -5921,7 +6204,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             ..Default::default()
@@ -5976,7 +6259,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             // Short grace so the sweep is observable without a slow test.
@@ -6036,7 +6319,7 @@ mod tests {
             durable_write: false,
             sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
-            max_wal_flush_interval: None,
+            max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
             frozen_memtable_grace: Duration::ZERO,

From 2464a4625acc73b56b8ac46c9d13a401bfaa0d78 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 20:32:47 -0500
Subject: [PATCH 11/19] fix(mem_wal): rotate memtables during replay instead of
 overflowing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A single memtable holds at most `max_memtable_batches` batches, but a WAL is
unbounded. Replay stuffed every WAL entry into one memtable, so a shard whose WAL
had grown past one memtable's capacity failed `open()` outright with "MemTable
batch store is full" — permanently unopenable.

Replay now rotates exactly as the live write path does, on the same trigger. When
the active memtable reaches the flush threshold it is sealed and — because the
data is already durable in the WAL — flushed straight to a Lance generation with
the same `MemTableFlusher::flush` the live path uses, rather than held in memory
until open finishes. That bounds resident memory to ~two memtables and truncates
the WAL as it goes, so a later reopen replays only the unflushed tail. Rotation is
at WAL-entry boundaries, so each sealed generation covers a clean range of
complete entries and stamps the last as its `replay_after_wal_entry_position`.

The flush trigger is now one predicate, `memtable_reached_flush_threshold`, shared
by the live path (post-insert, "room for one more batch?") and replay (pre-insert,
"room for the next entry?"). It carries both criteria — `max_memtable_size` bytes
and batch-store capacity. The byte trigger matters beyond avoiding overflow: it is
what keeps a memtable under `max_memtable_rows`, and therefore keeps the in-memory
HNSW index (sized to `max_memtable_rows`) from exhausting its capacity when the
final active memtable is indexed. A single predicate is what stops the two paths
from drifting — the exact class of bug this change set is about.

`MemTable::is_batch_store_full` is deleted: its one production caller now goes
through the shared predicate, and the two remaining test callers use the public
`batch_store().is_full()` directly.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/memtable.rs |   9 +-
 rust/lance/src/dataset/mem_wal/write.rs    | 394 +++++++++++++++++----
 2 files changed, 321 insertions(+), 82 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index f0c5299bbdf..dec498f0ef6 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -749,11 +749,6 @@ impl MemTable {
         self.batch_store.remaining_capacity()
     }
 
-    /// Check if batch store is full.
-    pub fn is_batch_store_full(&self) -> bool {
-        self.batch_store.is_full()
-    }
-
     /// Create a scanner for querying this MemTable.
     ///
     /// # Arguments
@@ -1056,7 +1051,7 @@ mod tests {
 
         assert_eq!(memtable.batch_capacity(), 3);
         assert_eq!(memtable.remaining_batch_capacity(), 3);
-        assert!(!memtable.is_batch_store_full());
+        assert!(!memtable.batch_store().is_full());
 
         // Fill up the store
         memtable
@@ -1072,7 +1067,7 @@ mod tests {
             .await
             .unwrap();
 
-        assert!(memtable.is_batch_store_full());
+        assert!(memtable.batch_store().is_full());
         assert_eq!(memtable.remaining_batch_capacity(), 0);
 
         // Next insert should fail
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 422c3e97bbf..16dc856a871 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -863,14 +863,46 @@ fn now_millis() -> u64 {
 /// Aborts with an error if any replayed entry's `writer_epoch` is strictly
 /// greater than `our_epoch` — that indicates a successor writer claimed the
 /// shard between our `claim_epoch` and this replay, fencing us.
+/// Outcome of replaying a shard's WAL into memory.
+struct ReplayResult {
+    /// The active memtable — the final, partial one replay left unsealed. A fresh
+    /// shard yields an empty one; every sealed memtable was flushed to a Lance
+    /// generation during replay and is not returned.
+    active: MemTable,
+    /// One past the highest WAL entry position observed — the next write position.
+    next_wal_position: u64,
+}
+
+/// Replay a shard's WAL into memory, flushing sealed memtables as the batch store
+/// fills.
+///
+/// A single memtable holds at most `max_memtable_batches` batches, but a WAL is
+/// unbounded — so replay must rotate exactly as the live write path does. It
+/// seals a full memtable and, because the data is already durable, flushes it to
+/// a Lance generation right here (the same `MemTableFlusher::flush` the live path
+/// uses), rather than holding every sealed memtable in memory until open
+/// finishes. That bounds resident memory to ~two memtables and truncates the WAL
+/// as it goes, so a later reopen replays only the unflushed tail. Only the final
+/// partial memtable is returned, as the active one.
+///
+/// `make_memtable(generation, global_offset)` builds a fresh, cursor-bound
+/// memtable. Rotation happens at WAL-entry boundaries, never mid-entry, so each
+/// sealed memtable covers a clean range of complete entries and stamps the last
+/// one as its flushed generation's `replay_after_wal_entry_position`.
+#[allow(clippy::too_many_arguments)]
 async fn replay_memtable_from_wal(
     object_store: Arc<ObjectStore>,
     base_path: Path,
     shard_id: Uuid,
     our_epoch: u64,
     manifest: &ShardManifest,
-    memtable: &mut MemTable,
-) -> Result<u64> {
+    base_generation: u64,
+    mut make_memtable: impl FnMut(u64, usize) -> Result<MemTable>,
+    flusher: &MemTableFlusher,
+    wal_flusher: &WalFlusher,
+    index_configs: &[MemIndexConfig],
+    max_memtable_size: usize,
+) -> Result<ReplayResult> {
     // WAL positions are 1-based (see `FIRST_WAL_ENTRY_POSITION`), so a
     // cursor of 0 means "no flush has ever stamped this shard" and replay
     // starts at position 1. After flushing position N the cursor holds N
@@ -881,14 +913,11 @@ async fn replay_memtable_from_wal(
     // contents into the base table.
     let start_position = manifest.replay_after_wal_entry_position.saturating_add(1);
 
-    // The MemTable is always freshly built before this function runs, so
-    // any existing BatchStore entries can only have come from this replay
-    // pass. We index everything in `[0, batch_count)` at the end.
-    debug_assert_eq!(memtable.batch_count(), 0);
-
     let tailer = WalTailer::new(object_store, base_path, shard_id);
     let mut position = start_position;
 
+    let mut active = make_memtable(base_generation, 0)?;
+
     loop {
         match tailer.read_entry(position).await? {
             // The first NotFound proves the WAL tip is at `position`, which
@@ -907,13 +936,60 @@ async fn replay_memtable_from_wal(
                     // Entries written before deletes existed lack `_tombstone`;
                     // inject `false` so they match the extended memtable schema.
                     // Normal entries already carry it and pass through unchanged.
-                    let target_schema = memtable.schema().clone();
+                    let target_schema = active.schema().clone();
                     let batches = entry
                         .batches
                         .into_iter()
                         .map(|b| ensure_tombstone_column(b, &target_schema))
                         .collect::<Result<Vec<_>>>()?;
-                    memtable.insert_batches_only(batches).await?;
+
+                    // Seal + flush at the entry boundary on the *same* criteria the
+                    // live path uses (`maybe_trigger_memtable_flush`): the memtable
+                    // is at or over `max_memtable_size` bytes, or this whole entry
+                    // won't fit the batch store. The byte trigger is the one that
+                    // matters beyond avoiding overflow — it is what keeps a memtable
+                    // under `max_memtable_rows`, and therefore keeps the in-memory
+                    // HNSW index (sized to `max_memtable_rows`) from exhausting its
+                    // capacity when the final active memtable is indexed.
+                    //
+                    // Rotate at the entry boundary so no entry is split across two
+                    // memtables and each sealed one covers a clean range of complete
+                    // entries. Never rotate an empty memtable — if a single entry
+                    // has more batches than a memtable can hold, a fresh one would
+                    // overflow too, the same hard limit the live put path has, left
+                    // to the insert below to surface.
+                    if !active.batch_store().is_empty()
+                        && memtable_reached_flush_threshold(
+                            &active,
+                            max_memtable_size,
+                            batches.len(),
+                        )
+                    {
+                        let store = active.batch_store();
+                        // The last entry this memtable fully absorbed is the one
+                        // before the entry about to be inserted.
+                        let covered = position.saturating_sub(1);
+                        let generation = active.generation() + 1;
+                        let global_end = store.global_end();
+
+                        // The sealed data is already durable in the WAL — mark it
+                        // so the flush's `all_flushed_to_wal` precondition holds and
+                        // no WAL re-append is attempted.
+                        wal_flusher.advance_durable(global_end);
+                        flush_replayed_memtable(
+                            flusher,
+                            &active,
+                            our_epoch,
+                            covered,
+                            global_end,
+                            index_configs,
+                        )
+                        .await?;
+
+                        active = make_memtable(generation, global_end)?;
+                    }
+
+                    active.insert_batches_only(batches).await?;
                 }
                 position = position.checked_add(1).ok_or_else(|| {
                     Error::io(format!(
@@ -925,20 +1001,17 @@ async fn replay_memtable_from_wal(
         }
     }
 
-    // Update in-memory indexes with the replayed batches so readers see them
-    // through the index path (matching what would have happened on the
-    // pre-crash writer's WAL flush). Indexes from the previous writer don't
-    // persist; this rebuilds them from the WAL.
-    if let Some(indexes) = memtable.indexes_arc() {
-        let batches_after = memtable.batch_count();
-        if batches_after > 0 {
-            let store = memtable.batch_store();
-            let mut stored: Vec<StoredBatch> = Vec::with_capacity(batches_after);
-            for pos in 0..batches_after {
-                if let Some(s) = store.get(pos) {
-                    stored.push(s.clone());
-                }
-            }
+    // Rebuild the active memtable's in-memory indexes from the batches just
+    // replayed, so readers see them through the index path — matching what the
+    // pre-crash writer's flush would have done. Sealed memtables needed no
+    // in-memory index build: they were flushed straight to disk and are gone.
+    if let Some(indexes) = active.indexes_arc() {
+        let batch_count = active.batch_count();
+        if batch_count > 0 {
+            let store = active.batch_store();
+            let stored: Vec<StoredBatch> = (0..batch_count)
+                .filter_map(|pos| store.get(pos).cloned())
+                .collect();
             tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
                 .await
                 .map_err(|e| {
@@ -947,7 +1020,50 @@ async fn replay_memtable_from_wal(
         }
     }
 
-    Ok(position)
+    Ok(ReplayResult {
+        active,
+        next_wal_position: position,
+    })
+}
+
+/// Whether a memtable has reached the threshold at which it should be sealed and
+/// flushed: at or over `max_memtable_size` bytes, or without room in its batch
+/// store for `incoming_batches` more.
+///
+/// The single source of truth for the flush trigger, shared by the live put path
+/// (`maybe_trigger_memtable_flush`, checking post-insert with `incoming_batches =
+/// 1` — "is there room for the next batch") and by replay (checking pre-insert
+/// with the next WAL entry's batch count). Keeping one predicate is what stops the
+/// two from drifting — e.g. someone adding a third criterion to one and not the
+/// other, which is the exact class of bug this whole change set is about.
+fn memtable_reached_flush_threshold(
+    memtable: &MemTable,
+    max_memtable_size: usize,
+    incoming_batches: usize,
+) -> bool {
+    memtable.estimated_size() >= max_memtable_size
+        || memtable.batch_store().remaining_capacity() < incoming_batches
+}
+
+/// Flush a sealed replay memtable to a Lance generation, choosing the indexed
+/// path when secondary indexes are configured (mirroring the live memtable-flush
+/// handler). Commits the manifest, stamping `covered` as the generation's
+/// `replay_after_wal_entry_position` so a later reopen skips these entries.
+async fn flush_replayed_memtable(
+    flusher: &MemTableFlusher,
+    memtable: &MemTable,
+    epoch: u64,
+    covered: u64,
+    durable: usize,
+    index_configs: &[MemIndexConfig],
+) -> Result<()> {
+    if index_configs.is_empty() {
+        flusher.flush(memtable, epoch, covered, durable).await?;
+    } else {
+        Box::pin(flusher.flush_with_indexes(memtable, epoch, index_configs, covered, durable))
+            .await?;
+    }
+    Ok(())
 }
 
 /// Pair each primary-key column name with its field id (both derived from the
@@ -1229,8 +1345,10 @@ impl SharedWriterState {
             return Ok(());
         }
 
-        let should_flush = state.memtable.estimated_size() >= self.config.max_memtable_size
-            || state.memtable.is_batch_store_full();
+        // Checked post-insert: flush if there is no longer room for even one more
+        // batch (or the byte threshold is crossed). Same predicate replay uses.
+        let should_flush =
+            memtable_reached_flush_threshold(&state.memtable, self.config.max_memtable_size, 1);
 
         if should_flush {
             state.flush_requested = true;
@@ -1571,62 +1689,88 @@ impl ShardWriter {
         // durable the shard can never reopen. Fail the open instead.
         validate_index_configs(index_configs, schema.as_ref(), &pk_columns)?;
 
-        let mut memtable = MemTable::with_capacity(
-            schema.clone(),
-            manifest.current_generation,
-            pk_field_ids.clone(),
-            CacheConfig::default(),
-            config.max_memtable_batches,
-        )?;
+        // Build a fresh, cursor-bound memtable at a given generation and
+        // writer-global coordinate. Replay calls this for the first memtable and
+        // after every rotation. Always builds and binds an `IndexStore`, even
+        // with no user indexes and no primary key — see the note in
+        // `freeze_memtable` for why an index-less memtable still needs one.
+        let make_bound_memtable = |generation: u64, global_offset: usize| -> Result<MemTable> {
+            let mut memtable = MemTable::with_capacity_at(
+                schema.clone(),
+                generation,
+                pk_field_ids.clone(),
+                CacheConfig::default(),
+                config.max_memtable_batches,
+                global_offset,
+            )?;
+            let mut indexes = IndexStore::from_configs(
+                index_configs,
+                config.max_memtable_rows,
+                config.max_memtable_batches,
+            )?;
+            if !pk_columns.is_empty() {
+                indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
+            }
+            indexes.set_durability(Arc::clone(wal_flusher.cursors()), global_offset);
+            memtable.set_indexes_arc(Arc::new(indexes));
+            Ok(memtable)
+        };
 
-        // Always build and bind an IndexStore — see the matching note in
-        // `freeze_memtable`. The PK-position index is enabled before any WAL
-        // replay below so replayed rows are recorded in it.
-        let mut indexes = IndexStore::from_configs(
-            index_configs,
-            config.max_memtable_rows,
-            config.max_memtable_batches,
-        )?;
-        if !pk_columns.is_empty() {
-            indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
-        }
-        // The writer's first memtable starts at coordinate 0.
-        indexes.set_durability(Arc::clone(wal_flusher.cursors()), 0);
-        memtable.set_indexes_arc(Arc::new(indexes));
+        // The flusher writes sealed memtables to Lance generations — both the
+        // ones replay seals below and the ones the live path freezes later.
+        let flusher = Arc::new(
+            MemTableFlusher::new(
+                object_store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                shard_id,
+                manifest_store.clone(),
+            )
+            .with_warmer(config.warmer.clone())
+            .with_storage_context(config.store_params.clone(), config.session.clone()),
+        );
 
         // Replay any WAL entries written after the last successfully-flushed
-        // generation. Each entry's writer_epoch is checked against ours; an
-        // entry with a strictly greater epoch indicates a successor writer
-        // claimed the shard between our `claim_epoch` and replay, so we
-        // abort the open with a fence error. The replay walked the tailer
-        // up to the WAL tip, so we hand the discovered next-write position
-        // straight to the appender — its first append skips the
-        // discover_next_position probe entirely.
-        let next_wal_position = replay_memtable_from_wal(
+        // generation, flushing sealed memtables to Lance generations as the batch
+        // store fills. Each entry's writer_epoch is checked against ours; an entry
+        // with a strictly greater epoch means a successor claimed the shard
+        // between our `claim_epoch` and replay, so we abort with a fence error.
+        // Replay walks the tailer to the WAL tip and returns the discovered
+        // next-write position, so the appender's first append skips the
+        // discover_next_position probe.
+        let ReplayResult {
+            active: memtable,
+            next_wal_position,
+        } = replay_memtable_from_wal(
             object_store.clone(),
             base_path.clone(),
             shard_id,
             epoch,
             manifest,
-            &mut memtable,
+            manifest.current_generation,
+            make_bound_memtable,
+            &flusher,
+            &wal_flusher,
+            index_configs,
+            config.max_memtable_size,
         )
         .await?;
 
-        // Mark the replayed batches durable. They came *from* the WAL, and the
-        // replay above has already re-derived the indexes over them.
+        // Mark the active memtable's replayed batches durable. They came *from*
+        // the WAL, and replay has already re-derived its indexes over them.
         //
-        // Without this the durability cursor stays at 0, so the next WAL flush
-        // re-covers [0, end): it re-appends the already-durable rows *and*
-        // re-inserts every replayed row into the indexes. None of the three
-        // in-memory indexes is idempotent (HNSW mints fresh node ids for the
-        // same row, FTS increments doc_count/df rather than recomputing them,
-        // BTree is a multiset), so a full scan keeps looking healthy while every
-        // index-accelerated query silently returns duplicates — and it compounds,
-        // because the WAL now holds those rows twice.
+        // Without this the durability cursor stays at the last sealed generation,
+        // so the next WAL flush re-covers the active tail: it re-appends the
+        // already-durable rows *and* re-inserts every replayed row into the
+        // indexes. None of the three in-memory indexes is idempotent (HNSW mints
+        // fresh node ids for the same row, FTS increments doc_count/df rather than
+        // recomputing them, BTree is a multiset), so a full scan keeps looking
+        // healthy while every index-accelerated query silently returns duplicates
+        // — and it compounds, because the WAL now holds those rows twice.
         //
-        // The replayed memtable is the writer's first, so its `global_offset` is
-        // 0 and the durable count is just its batch count.
-        wal_flusher.advance_durable(memtable.batch_count());
+        // `global_end()` is the writer-global batch count through this memtable,
+        // since its coordinate continues where the last sealed generation ended.
+        wal_flusher.advance_durable(memtable.batch_store().global_end());
 
         wal_flusher
             .wal_appender()
@@ -1658,12 +1802,6 @@ impl ShardWriter {
 
         let (memtable_flush_tx, memtable_flush_rx) = mpsc::unbounded_channel();
 
-        let flusher = Arc::new(
-            MemTableFlusher::new(object_store, base_path, base_uri, shard_id, manifest_store)
-                .with_warmer(config.warmer.clone())
-                .with_storage_context(config.store_params.clone(), config.session.clone()),
-        );
-
         let backpressure = BackpressureController::new(config.clone());
 
         // Background WAL flush handler — parallel WAL I/O + index updates.
@@ -5206,6 +5344,112 @@ mod tests {
             .unwrap();
     }
 
+    /// A WAL holding more batches than one memtable's capacity must reopen.
+    ///
+    /// One memtable holds at most `max_memtable_batches` batches, but a WAL is
+    /// unbounded, so replay has to rotate — seal the full memtable, start a fresh
+    /// one — exactly as the live write path does. Before, replay stuffed
+    /// everything into a single memtable and `open()` failed outright with
+    /// "MemTable batch store is full", leaving the shard permanently unopenable.
+    #[tokio::test]
+    async fn test_replay_rotates_when_wal_exceeds_one_memtable() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        const N: i32 = 8;
+
+        // Writer A has a *large* capacity, so its eight one-batch puts all land in
+        // a single memtable and it never freezes or flushes a generation of its
+        // own. Dropping it without close leaves an eight-entry WAL and no
+        // generations — a WAL that no single small memtable could hold.
+        let writer_a_config = ShardWriterConfig {
+            max_memtable_batches: 1000,
+            ..memtable_config_with_pk(shard_id)
+        };
+        // Writer B has a *two-batch* capacity, so replaying that eight-entry WAL is
+        // exactly what must rotate. Keeping the configs distinct isolates replay
+        // rotation from the live rotation writer A would otherwise do concurrently.
+        let config = ShardWriterConfig {
+            max_memtable_batches: 2,
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        {
+            let writer_a = ShardWriter::open(
+                store.clone(),
+                base_path.clone(),
+                base_uri.clone(),
+                writer_a_config,
+                schema.clone(),
+                vec![],
+            )
+            .await
+            .unwrap();
+            for id in 0..N {
+                writer_a
+                    .put(vec![create_test_batch(&schema, id, 1)])
+                    .await
+                    .unwrap();
+            }
+            // Drop without close: only the WAL survives, and it holds more batches
+            // than writer B's memtable can.
+        }
+
+        // Total rows across the active memtable plus every flushed generation.
+        // Distinct ids, so no cross-generation dedup — a plain sum is exact.
+        async fn total_rows(writer: &ShardWriter, base_uri: &str, shard_id: Uuid) -> usize {
+            let mut rows = writer.memtable_stats().await.unwrap().row_count;
+            let manifest = writer.manifest().await.unwrap().unwrap();
+            for fg in &manifest.flushed_generations {
+                let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, fg.path);
+                let dataset = crate::Dataset::open(&gen_uri).await.unwrap();
+                rows += dataset.count_rows(None).await.unwrap();
+            }
+            rows
+        }
+
+        // Reopen. This used to fail with a full-batch-store error.
+        let writer_b = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            config.clone(),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .expect("a WAL larger than one memtable must still reopen");
+
+        // Rotation produced sealed memtables, and replay flushed each to a Lance
+        // generation rather than holding it in memory or leaving it in the WAL.
+        let manifest = writer_b.manifest().await.unwrap().unwrap();
+        assert!(
+            !manifest.flushed_generations.is_empty(),
+            "replay must have sealed and flushed at least one full memtable"
+        );
+
+        // Every row survived, split between the flushed generations and the
+        // active (partial) memtable.
+        assert_eq!(
+            total_rows(&writer_b, &base_uri, shard_id).await as i32,
+            N,
+            "every replayed row must be durable, across generations and the active memtable"
+        );
+        writer_b.close().await.unwrap();
+
+        // Because the sealed memtables were flushed, the manifest's replay cursor
+        // advanced past their WAL entries — so a second reopen replays only the
+        // tail and still accounts for every row. The WAL truncates across reopens
+        // rather than growing without bound.
+        let writer_c =
+            ShardWriter::open(store, base_path, base_uri.clone(), config, schema, vec![])
+                .await
+                .unwrap();
+        assert_eq!(total_rows(&writer_c, &base_uri, shard_id).await as i32, N);
+        writer_c.close().await.unwrap();
+    }
+
     /// Replay-on-open recovers durable WAL entries that were never flushed
     /// to a Lance generation. Setup: writer A durably writes batches, drops
     /// without close (so MemTable freeze never runs); writer B reopens and

From c815974e01c5ab110107cdf47785b44de6042e0c Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 21:33:21 -0500
Subject: [PATCH 12/19] fix(mem_wal): update Python/Java stats bindings for the
 writer-global cursor

Making the durability cursor writer-global renamed `MemTableStats::
max_flushed_batch_position` (per-memtable, inclusive, Option) to
`durable_batch_count` + `global_offset` (writer-global, exclusive), but the
Python and Java bindings still referenced the old field, so neither excluded
crate compiled.

Expose the new fields: Python's stats dict gains `durable_batch_count` and
`global_offset` in place of `max_flushed_batch_position`, and Java's
`MemTableStats` replaces `maxFlushedBatchPosition()` (Optional<Long>) with
`durableBatchCount()` and `globalOffset()` (primitive longs), with the JNI
constructor updated to match.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 java/lance-jni/src/mem_wal.rs                 |  6 ++---
 .../java/org/lance/memwal/MemTableStats.java  | 23 +++++++++++++------
 python/src/mem_wal.rs                         |  9 ++++----
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs
index c4f56d7b97d..8c97d4aee2e 100644
--- a/java/lance-jni/src/mem_wal.rs
+++ b/java/lance-jni/src/mem_wal.rs
@@ -1368,19 +1368,19 @@ fn write_stats_to_java<'a>(
 
 fn memtable_stats_to_java<'a>(env: &mut JNIEnv<'a>, stats: &MemTableStats) -> Result<JObject<'a>> {
     let max_buffered = box_u64_opt(env, stats.max_buffered_batch_position)?;
-    let max_flushed = box_u64_opt(env, stats.max_flushed_batch_position)?;
     let pending_start = box_u64_opt(env, stats.pending_wal_start_batch_position)?;
     let pending_end = box_u64_opt(env, stats.pending_wal_end_batch_position)?;
     Ok(env.new_object(
         "org/lance/memwal/MemTableStats",
-        "(JJJJLjava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;JJJ)V",
+        "(JJJJLjava/lang/Long;JJLjava/lang/Long;Ljava/lang/Long;JJJ)V",
         &[
             JValueGen::Long(stats.row_count as i64),
             JValueGen::Long(stats.batch_count as i64),
             JValueGen::Long(stats.estimated_size as i64),
             JValueGen::Long(stats.generation as i64),
             JValueGen::Object(&max_buffered),
-            JValueGen::Object(&max_flushed),
+            JValueGen::Long(stats.durable_batch_count as i64),
+            JValueGen::Long(stats.global_offset as i64),
             JValueGen::Object(&pending_start),
             JValueGen::Object(&pending_end),
             JValueGen::Long(stats.pending_wal_batch_count as i64),
diff --git a/java/src/main/java/org/lance/memwal/MemTableStats.java b/java/src/main/java/org/lance/memwal/MemTableStats.java
index a993d771b42..58533e48194 100644
--- a/java/src/main/java/org/lance/memwal/MemTableStats.java
+++ b/java/src/main/java/org/lance/memwal/MemTableStats.java
@@ -24,7 +24,8 @@ public class MemTableStats {
   private final long estimatedSizeBytes;
   private final long generation;
   private final Optional<Long> maxBufferedBatchPosition;
-  private final Optional<Long> maxFlushedBatchPosition;
+  private final long durableBatchCount;
+  private final long globalOffset;
   private final Optional<Long> pendingWalStartBatchPosition;
   private final Optional<Long> pendingWalEndBatchPosition;
   private final long pendingWalBatchCount;
@@ -37,7 +38,8 @@ public MemTableStats(
       long estimatedSizeBytes,
       long generation,
       Long maxBufferedBatchPosition,
-      Long maxFlushedBatchPosition,
+      long durableBatchCount,
+      long globalOffset,
       Long pendingWalStartBatchPosition,
       Long pendingWalEndBatchPosition,
       long pendingWalBatchCount,
@@ -48,7 +50,8 @@ public MemTableStats(
     this.estimatedSizeBytes = estimatedSizeBytes;
     this.generation = generation;
     this.maxBufferedBatchPosition = Optional.ofNullable(maxBufferedBatchPosition);
-    this.maxFlushedBatchPosition = Optional.ofNullable(maxFlushedBatchPosition);
+    this.durableBatchCount = durableBatchCount;
+    this.globalOffset = globalOffset;
     this.pendingWalStartBatchPosition = Optional.ofNullable(pendingWalStartBatchPosition);
     this.pendingWalEndBatchPosition = Optional.ofNullable(pendingWalEndBatchPosition);
     this.pendingWalBatchCount = pendingWalBatchCount;
@@ -81,9 +84,14 @@ public Optional<Long> maxBufferedBatchPosition() {
     return maxBufferedBatchPosition;
   }
 
-  /** Highest WAL batch position flushed from the MemTable, if any. */
-  public Optional<Long> maxFlushedBatchPosition() {
-    return maxFlushedBatchPosition;
+  /** Writer-global count of WAL-durable batches (exclusive). */
+  public long durableBatchCount() {
+    return durableBatchCount;
+  }
+
+  /** Writer-global coordinate of this MemTable's batch 0. */
+  public long globalOffset() {
+    return globalOffset;
   }
 
   /** First WAL batch position pending flush, if any. */
@@ -119,7 +127,8 @@ public String toString() {
         .add("estimatedSizeBytes", estimatedSizeBytes)
         .add("generation", generation)
         .add("maxBufferedBatchPosition", maxBufferedBatchPosition.orElse(null))
-        .add("maxFlushedBatchPosition", maxFlushedBatchPosition.orElse(null))
+        .add("durableBatchCount", durableBatchCount)
+        .add("globalOffset", globalOffset)
         .add("pendingWalStartBatchPosition", pendingWalStartBatchPosition.orElse(null))
         .add("pendingWalEndBatchPosition", pendingWalEndBatchPosition.orElse(null))
         .add("pendingWalBatchCount", pendingWalBatchCount)
diff --git a/python/src/mem_wal.rs b/python/src/mem_wal.rs
index e415acfc38d..bbac185e625 100644
--- a/python/src/mem_wal.rs
+++ b/python/src/mem_wal.rs
@@ -949,10 +949,8 @@ fn memtable_stats_to_pydict(py: Python<'_>, stats: &MemTableStats) -> PyResult<P
         "max_buffered_batch_position",
         stats.max_buffered_batch_position,
     )?;
-    dict.set_item(
-        "max_flushed_batch_position",
-        stats.max_flushed_batch_position,
-    )?;
+    dict.set_item("durable_batch_count", stats.durable_batch_count)?;
+    dict.set_item("global_offset", stats.global_offset)?;
     dict.set_item(
         "pending_wal_start_batch_position",
         stats.pending_wal_start_batch_position,
@@ -1045,7 +1043,8 @@ fn closed_memtable_stats(stats_before_close: MemTableStats) -> MemTableStats {
         estimated_size: 0,
         generation: stats_before_close.generation.saturating_add(1),
         max_buffered_batch_position: None,
-        max_flushed_batch_position: None,
+        durable_batch_count: 0,
+        global_offset: 0,
         pending_wal_start_batch_position: None,
         pending_wal_end_batch_position: None,
         pending_wal_batch_count: 0,

From 6a9ab9a3f72095358d7c88db53de0810ded59476 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 21:33:58 -0500
Subject: [PATCH 13/19] refactor(mem_wal): remove the inert sync_indexed_write
 config family
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`sync_indexed_write`, `async_index_buffer_rows`, and `async_index_interval` had
no behavioral consumers — they were only serialized into the config-defaults
metadata map. They existed to configure async index-update buffering, which the
index-apply-task split replaced with unconditional per-put indexing: a write is
now read-your-writes through the index in every mode, which is exactly what
`sync_indexed_write` promised. The knobs no longer control anything.

Remove all three across every surface: the Rust core field, builders, defaults,
and metadata serialization; the write-throughput bench's now-meaningless
sync/async index axis; and the Python and Java binding parameters.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 java/lance-jni/src/mem_wal.rs                 |   9 --
 .../org/lance/memwal/ShardWriterConfig.java   |  41 -----
 .../java/org/lance/memwal/MemWalTest.java     |   1 -
 python/python/lance/dataset.py                |  18 ---
 python/python/tests/test_mem_wal.py           |   7 +-
 python/src/dataset.rs                         |  33 ----
 .../mem_wal/fts/mem_wal_fineweb_fts.rs        |  12 +-
 .../mem_wal/fts/mem_wal_fts_read_bench.rs     |   1 -
 .../mem_wal/kv/mem_wal_kv_point_lookup.rs     |   1 -
 .../mem_wal_point_lookup_bench.rs             |   1 -
 .../vector/hnsw/mem_wal_recall_hnsw.rs        |   1 -
 .../mem_wal/vector/mem_wal_index_micro.rs     |   1 -
 .../mem_wal/vector/mem_wal_vector_bench.rs    |   1 -
 .../benches/mem_wal/write/mem_wal_replay.rs   |   1 -
 .../mem_wal_shard_writer_backpressure.rs      |  10 --
 .../benches/mem_wal/write/mem_wal_write.rs    | 150 +++++++-----------
 rust/lance/src/dataset/mem_wal/api.rs         |  12 --
 rust/lance/src/dataset/mem_wal/write.rs       |  92 +----------
 18 files changed, 69 insertions(+), 323 deletions(-)

diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs
index 8c97d4aee2e..9047c6288b0 100644
--- a/java/lance-jni/src/mem_wal.rs
+++ b/java/lance-jni/src/mem_wal.rs
@@ -1265,9 +1265,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result<ShardWriter
     if let Some(v) = read_optional_bool(env, config, "durableWrite")? {
         writer_config = writer_config.with_durable_write(v);
     }
-    if let Some(v) = read_optional_bool(env, config, "syncIndexedWrite")? {
-        writer_config = writer_config.with_sync_indexed_write(v);
-    }
     if let Some(v) = read_optional_u64(env, config, "maxWalBufferSize")? {
         writer_config = writer_config.with_max_wal_buffer_size(v as usize);
     }
@@ -1289,12 +1286,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result<ShardWriter
     if let Some(v) = read_optional_u64(env, config, "manifestScanBatchSize")? {
         writer_config = writer_config.with_manifest_scan_batch_size(v as usize);
     }
-    if let Some(v) = read_optional_u64(env, config, "asyncIndexBufferRows")? {
-        writer_config = writer_config.with_async_index_buffer_rows(v as usize);
-    }
-    if let Some(v) = read_optional_u64(env, config, "asyncIndexIntervalMs")? {
-        writer_config = writer_config.with_async_index_interval(Duration::from_millis(v));
-    }
     if let Some(v) = read_optional_u64(env, config, "backpressureLogIntervalMs")? {
         writer_config = writer_config.with_backpressure_log_interval(Duration::from_millis(v));
     }
diff --git a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
index 40310907d3e..d6bf2aeb383 100644
--- a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
+++ b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java
@@ -28,7 +28,6 @@
  */
 public class ShardWriterConfig {
   private Optional<Boolean> durableWrite = Optional.empty();
-  private Optional<Boolean> syncIndexedWrite = Optional.empty();
   private Optional<Long> maxWalBufferSize = Optional.empty();
   private Optional<Long> maxWalFlushIntervalMs = Optional.empty();
   private Optional<Long> maxMemtableSize = Optional.empty();
@@ -36,8 +35,6 @@ public class ShardWriterConfig {
   private Optional<Long> maxMemtableBatches = Optional.empty();
   private Optional<Long> maxUnflushedMemtableBytes = Optional.empty();
   private Optional<Long> manifestScanBatchSize = Optional.empty();
-  private Optional<Long> asyncIndexBufferRows = Optional.empty();
-  private Optional<Long> asyncIndexIntervalMs = Optional.empty();
   private Optional<Long> backpressureLogIntervalMs = Optional.empty();
   private Optional<Long> statsLogIntervalMs = Optional.empty();
   private List<MemWalHnswParams> hnswParams = Collections.emptyList();
@@ -48,12 +45,6 @@ public ShardWriterConfig withDurableWrite(boolean durableWrite) {
     return this;
   }
 
-  /** Whether indexed writes are applied synchronously. */
-  public ShardWriterConfig withSyncIndexedWrite(boolean syncIndexedWrite) {
-    this.syncIndexedWrite = Optional.of(syncIndexedWrite);
-    return this;
-  }
-
   /** Maximum size of the in-memory WAL buffer, in bytes. */
   public ShardWriterConfig withMaxWalBufferSize(long maxWalBufferSize) {
     Preconditions.checkArgument(
@@ -118,26 +109,6 @@ public ShardWriterConfig withManifestScanBatchSize(long manifestScanBatchSize) {
     return this;
   }
 
-  /** Number of rows buffered before an asynchronous index update is triggered. */
-  public ShardWriterConfig withAsyncIndexBufferRows(long asyncIndexBufferRows) {
-    Preconditions.checkArgument(
-        asyncIndexBufferRows >= 0,
-        "asyncIndexBufferRows must not be negative, got %s",
-        asyncIndexBufferRows);
-    this.asyncIndexBufferRows = Optional.of(asyncIndexBufferRows);
-    return this;
-  }
-
-  /** Interval between asynchronous index updates, in milliseconds. */
-  public ShardWriterConfig withAsyncIndexIntervalMs(long asyncIndexIntervalMs) {
-    Preconditions.checkArgument(
-        asyncIndexIntervalMs >= 0,
-        "asyncIndexIntervalMs must not be negative, got %s",
-        asyncIndexIntervalMs);
-    this.asyncIndexIntervalMs = Optional.of(asyncIndexIntervalMs);
-    return this;
-  }
-
   /** Interval between backpressure log messages, in milliseconds. */
   public ShardWriterConfig withBackpressureLogIntervalMs(long backpressureLogIntervalMs) {
     Preconditions.checkArgument(
@@ -172,10 +143,6 @@ public Optional<Boolean> durableWrite() {
     return durableWrite;
   }
 
-  public Optional<Boolean> syncIndexedWrite() {
-    return syncIndexedWrite;
-  }
-
   public Optional<Long> maxWalBufferSize() {
     return maxWalBufferSize;
   }
@@ -204,14 +171,6 @@ public Optional<Long> manifestScanBatchSize() {
     return manifestScanBatchSize;
   }
 
-  public Optional<Long> asyncIndexBufferRows() {
-    return asyncIndexBufferRows;
-  }
-
-  public Optional<Long> asyncIndexIntervalMs() {
-    return asyncIndexIntervalMs;
-  }
-
   public Optional<Long> backpressureLogIntervalMs() {
     return backpressureLogIntervalMs;
   }
diff --git a/java/src/test/java/org/lance/memwal/MemWalTest.java b/java/src/test/java/org/lance/memwal/MemWalTest.java
index 57d0b5b81ad..3ee998733ea 100644
--- a/java/src/test/java/org/lance/memwal/MemWalTest.java
+++ b/java/src/test/java/org/lance/memwal/MemWalTest.java
@@ -409,7 +409,6 @@ void testShardWriterDeleteMasksBaseRow(@TempDir Path tempDir) throws Exception {
       ShardWriterConfig config =
           new ShardWriterConfig()
               .withDurableWrite(true)
-              .withSyncIndexedWrite(true)
               .withMaxWalBufferSize(1)
               .withMaxWalFlushIntervalMs(10);
 
diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py
index a1ce77b82b8..d57b483134c 100644
--- a/python/python/lance/dataset.py
+++ b/python/python/lance/dataset.py
@@ -5086,7 +5086,6 @@ def initialize_mem_wal(
         identity_column: Optional[str] = None,
         unsharded: bool = False,
         durable_write: Optional[bool] = None,
-        sync_indexed_write: Optional[bool] = None,
         max_wal_buffer_size: Optional[int] = None,
         max_wal_flush_interval_ms: Optional[int] = None,
         max_memtable_size: Optional[int] = None,
@@ -5094,8 +5093,6 @@ def initialize_mem_wal(
         max_memtable_batches: Optional[int] = None,
         max_unflushed_memtable_bytes: Optional[int] = None,
         manifest_scan_batch_size: Optional[int] = None,
-        async_index_buffer_rows: Optional[int] = None,
-        async_index_interval_ms: Optional[int] = None,
         backpressure_log_interval_ms: Optional[int] = None,
         stats_log_interval_ms: Optional[int] = None,
         hnsw_params: Optional[Dict[str, Dict[str, int]]] = None,
@@ -5156,7 +5153,6 @@ def initialize_mem_wal(
             identity_column=identity_column,
             unsharded=unsharded,
             durable_write=durable_write,
-            sync_indexed_write=sync_indexed_write,
             max_wal_buffer_size=max_wal_buffer_size,
             max_wal_flush_interval_ms=max_wal_flush_interval_ms,
             max_memtable_size=max_memtable_size,
@@ -5164,8 +5160,6 @@ def initialize_mem_wal(
             max_memtable_batches=max_memtable_batches,
             max_unflushed_memtable_bytes=max_unflushed_memtable_bytes,
             manifest_scan_batch_size=manifest_scan_batch_size,
-            async_index_buffer_rows=async_index_buffer_rows,
-            async_index_interval_ms=async_index_interval_ms,
             backpressure_log_interval_ms=backpressure_log_interval_ms,
             stats_log_interval_ms=stats_log_interval_ms,
             hnsw_params=hnsw_params,
@@ -5188,7 +5182,6 @@ def mem_wal_writer(
         shard_id: str,
         *,
         durable_write: Optional[bool] = None,
-        sync_indexed_write: Optional[bool] = None,
         max_wal_buffer_size: Optional[int] = None,
         max_wal_flush_interval_ms: Optional[int] = None,
         max_memtable_size: Optional[int] = None,
@@ -5196,8 +5189,6 @@ def mem_wal_writer(
         max_memtable_batches: Optional[int] = None,
         max_unflushed_memtable_bytes: Optional[int] = None,
         manifest_scan_batch_size: Optional[int] = None,
-        async_index_buffer_rows: Optional[int] = None,
-        async_index_interval_ms: Optional[int] = None,
         backpressure_log_interval_ms: Optional[int] = None,
         stats_log_interval_ms: Optional[int] = None,
         hnsw_params: Optional[Dict[str, Dict[str, int]]] = None,
@@ -5215,8 +5206,6 @@ def mem_wal_writer(
             ``str(uuid.uuid4())``).
         durable_write : bool, optional
             Whether to fsync WAL writes (default: ``True``).
-        sync_indexed_write : bool, optional
-            Whether index updates are synchronous (default: ``True``).
         max_wal_buffer_size : int, optional
             Maximum WAL buffer size in bytes (default: 10 MB).
         max_wal_flush_interval_ms : int, optional
@@ -5231,10 +5220,6 @@ def mem_wal_writer(
             Maximum unflushed bytes before backpressure (default: 1 GB).
         manifest_scan_batch_size : int, optional
             Batch size for manifest scans (default: 2).
-        async_index_buffer_rows : int, optional
-            Buffer rows for async index updates (default: 10 000).
-        async_index_interval_ms : int, optional
-            Interval for async index updates in milliseconds (default: 1000).
         backpressure_log_interval_ms : int, optional
             Interval for backpressure log messages in milliseconds
             (default: 30 000).
@@ -5282,7 +5267,6 @@ def mem_wal_writer(
             name: val
             for name, val in [
                 ("durable_write", durable_write),
-                ("sync_indexed_write", sync_indexed_write),
                 ("max_wal_buffer_size", max_wal_buffer_size),
                 ("max_wal_flush_interval_ms", max_wal_flush_interval_ms),
                 ("max_memtable_size", max_memtable_size),
@@ -5290,8 +5274,6 @@ def mem_wal_writer(
                 ("max_memtable_batches", max_memtable_batches),
                 ("max_unflushed_memtable_bytes", max_unflushed_memtable_bytes),
                 ("manifest_scan_batch_size", manifest_scan_batch_size),
-                ("async_index_buffer_rows", async_index_buffer_rows),
-                ("async_index_interval_ms", async_index_interval_ms),
                 ("backpressure_log_interval_ms", backpressure_log_interval_ms),
                 ("stats_log_interval_ms", stats_log_interval_ms),
                 ("hnsw_params", hnsw_params),
diff --git a/python/python/tests/test_mem_wal.py b/python/python/tests/test_mem_wal.py
index 2871baad986..32904ae6038 100644
--- a/python/python/tests/test_mem_wal.py
+++ b/python/python/tests/test_mem_wal.py
@@ -209,7 +209,6 @@ def test_shard_writer_delete_binding_masks_base_row(tmp_path):
     with ds.mem_wal_writer(
         shard_id,
         durable_write=True,
-        sync_indexed_write=True,
         max_wal_buffer_size=1,
         max_wal_flush_interval_ms=10,
     ) as writer:
@@ -355,7 +354,6 @@ def test_shard_writer_e2e_correctness(tmp_path):
     writer = ds.mem_wal_writer(
         shard_id,
         durable_write=True,
-        sync_indexed_write=True,
         max_wal_buffer_size=10 * 1024,  # 10 KB
         max_wal_flush_interval_ms=50,
         max_memtable_size=80,  # flush after ~80 rows
@@ -405,9 +403,7 @@ def test_shard_writer_e2e_correctness(tmp_path):
     # === New writer: write and read back via active MemTable scanner ===
     ds2 = lance.dataset(ds_path)
     shard_id2 = str(uuid.uuid4())
-    with ds2.mem_wal_writer(
-        shard_id2, durable_write=False, sync_indexed_write=True
-    ) as writer2:
+    with ds2.mem_wal_writer(shard_id2, durable_write=False) as writer2:
         verify_batch = _e2e_batch(schema, start_id=10000, num_rows=10)
         writer2.put(pa.Table.from_batches([verify_batch]))
         result = writer2.lsm_scanner().to_table()
@@ -522,7 +518,6 @@ def test_initialize_mem_wal_writer_config_defaults(tmp_path):
     # Duration knobs are recorded in milliseconds with a `_ms` suffix.
     assert defaults["max_wal_flush_interval_ms"] == "250"
     # Every ShardWriterConfig tunable is recorded once any default is set.
-    assert "sync_indexed_write" in defaults
     assert "enable_memtable" in defaults
 
 
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index a361e0b63a8..54e8b9340bc 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -164,7 +164,6 @@ fn stats_log_interval_from_millis(ms: u64) -> Option<std::time::Duration> {
 #[allow(clippy::too_many_arguments)]
 fn writer_config_from_kwargs(
     durable_write: Option<bool>,
-    sync_indexed_write: Option<bool>,
     max_wal_buffer_size: Option<usize>,
     max_wal_flush_interval_ms: Option<u64>,
     max_memtable_size: Option<usize>,
@@ -172,8 +171,6 @@ fn writer_config_from_kwargs(
     max_memtable_batches: Option<usize>,
     max_unflushed_memtable_bytes: Option<usize>,
     manifest_scan_batch_size: Option<usize>,
-    async_index_buffer_rows: Option<usize>,
-    async_index_interval_ms: Option<u64>,
     backpressure_log_interval_ms: Option<u64>,
     stats_log_interval_ms: Option<u64>,
     hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -186,10 +183,6 @@ fn writer_config_from_kwargs(
         config = config.with_durable_write(v);
         any = true;
     }
-    if let Some(v) = sync_indexed_write {
-        config = config.with_sync_indexed_write(v);
-        any = true;
-    }
     if let Some(v) = max_wal_buffer_size {
         config = config.with_max_wal_buffer_size(v);
         any = true;
@@ -218,14 +211,6 @@ fn writer_config_from_kwargs(
         config = config.with_manifest_scan_batch_size(v);
         any = true;
     }
-    if let Some(v) = async_index_buffer_rows {
-        config = config.with_async_index_buffer_rows(v);
-        any = true;
-    }
-    if let Some(v) = async_index_interval_ms {
-        config = config.with_async_index_interval(Duration::from_millis(v));
-        any = true;
-    }
     if let Some(v) = backpressure_log_interval_ms {
         config = config.with_backpressure_log_interval(Duration::from_millis(v));
         any = true;
@@ -3414,7 +3399,6 @@ impl Dataset {
         identity_column=None,
         unsharded=false,
         durable_write=None,
-        sync_indexed_write=None,
         max_wal_buffer_size=None,
         max_wal_flush_interval_ms=None,
         max_memtable_size=None,
@@ -3422,8 +3406,6 @@ impl Dataset {
         max_memtable_batches=None,
         max_unflushed_memtable_bytes=None,
         manifest_scan_batch_size=None,
-        async_index_buffer_rows=None,
-        async_index_interval_ms=None,
         backpressure_log_interval_ms=None,
         stats_log_interval_ms=None,
         hnsw_params=None,
@@ -3437,7 +3419,6 @@ impl Dataset {
         identity_column: Option<String>,
         unsharded: bool,
         durable_write: Option<bool>,
-        sync_indexed_write: Option<bool>,
         max_wal_buffer_size: Option<usize>,
         max_wal_flush_interval_ms: Option<u64>,
         max_memtable_size: Option<usize>,
@@ -3445,8 +3426,6 @@ impl Dataset {
         max_memtable_batches: Option<usize>,
         max_unflushed_memtable_bytes: Option<usize>,
         manifest_scan_batch_size: Option<usize>,
-        async_index_buffer_rows: Option<usize>,
-        async_index_interval_ms: Option<u64>,
         backpressure_log_interval_ms: Option<u64>,
         stats_log_interval_ms: Option<u64>,
         hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -3474,7 +3453,6 @@ impl Dataset {
 
         let writer_config = writer_config_from_kwargs(
             durable_write,
-            sync_indexed_write,
             max_wal_buffer_size,
             max_wal_flush_interval_ms,
             max_memtable_size,
@@ -3482,8 +3460,6 @@ impl Dataset {
             max_memtable_batches,
             max_unflushed_memtable_bytes,
             manifest_scan_batch_size,
-            async_index_buffer_rows,
-            async_index_interval_ms,
             backpressure_log_interval_ms,
             stats_log_interval_ms,
             hnsw_params,
@@ -3565,7 +3541,6 @@ impl Dataset {
         shard_id,
         *,
         durable_write=None,
-        sync_indexed_write=None,
         max_wal_buffer_size=None,
         max_wal_flush_interval_ms=None,
         max_memtable_size=None,
@@ -3573,8 +3548,6 @@ impl Dataset {
         max_memtable_batches=None,
         max_unflushed_memtable_bytes=None,
         manifest_scan_batch_size=None,
-        async_index_buffer_rows=None,
-        async_index_interval_ms=None,
         backpressure_log_interval_ms=None,
         stats_log_interval_ms=None,
         hnsw_params=None,
@@ -3584,7 +3557,6 @@ impl Dataset {
         py: Python<'_>,
         shard_id: String,
         durable_write: Option<bool>,
-        sync_indexed_write: Option<bool>,
         max_wal_buffer_size: Option<usize>,
         max_wal_flush_interval_ms: Option<u64>,
         max_memtable_size: Option<usize>,
@@ -3592,8 +3564,6 @@ impl Dataset {
         max_memtable_batches: Option<usize>,
         max_unflushed_memtable_bytes: Option<usize>,
         manifest_scan_batch_size: Option<usize>,
-        async_index_buffer_rows: Option<usize>,
-        async_index_interval_ms: Option<u64>,
         backpressure_log_interval_ms: Option<u64>,
         stats_log_interval_ms: Option<u64>,
         hnsw_params: Option<HashMap<String, HashMap<String, u32>>>,
@@ -3605,7 +3575,6 @@ impl Dataset {
 
         let config = writer_config_from_kwargs(
             durable_write,
-            sync_indexed_write,
             max_wal_buffer_size,
             max_wal_flush_interval_ms,
             max_memtable_size,
@@ -3613,8 +3582,6 @@ impl Dataset {
             max_memtable_batches,
             max_unflushed_memtable_bytes,
             manifest_scan_batch_size,
-            async_index_buffer_rows,
-            async_index_interval_ms,
             backpressure_log_interval_ms,
             stats_log_interval_ms,
             hnsw_params,
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
index 63f9368c3c5..188e2c73ce2 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
@@ -111,11 +111,6 @@ impl Mode {
     fn durable_write(self) -> bool {
         matches!(self, Self::SyncNoIndex | Self::SyncIndexed)
     }
-
-    /// Index update happens inline in `put` (only meaningful when indexed).
-    fn sync_indexed_write(self) -> bool {
-        matches!(self, Self::SyncIndexed)
-    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -428,7 +423,6 @@ fn shard_writer_config(args: &Args, shard_id: Uuid, disable_auto_flush: bool) ->
     };
     let mut config = ShardWriterConfig::new(shard_id)
         .with_durable_write(args.mode.durable_write())
-        .with_sync_indexed_write(args.mode.sync_indexed_write())
         .with_max_memtable_size(args.max_memtable_size)
         .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes)
         .with_max_memtable_rows(max_rows)
@@ -638,7 +632,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result<serde_jso
     let dataset = Dataset::open(uri).await?;
     let shard_id = Uuid::new_v4();
     // Auto-flush disabled so the MemTable holds the full read_rows.
-    // sync_indexed_write + durable_write are both forced on: the read
+    // durable_write is forced on: the read
     // panel measures FTS read latency and consistency, which are
     // properties of the *fully visible* MemTable. The MemTableScanner
     // respects the visibility watermark, and that watermark only advances
@@ -646,9 +640,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result<serde_jso
     // would query a partially-visible MemTable and report garbage. Both
     // are properties of ingestion, not of the read path, so forcing them
     // does not bias the latency/consistency measurement.
-    let config = shard_writer_config(args, shard_id, true)
-        .with_sync_indexed_write(true)
-        .with_durable_write(true);
+    let config = shard_writer_config(args, shard_id, true).with_durable_write(true);
     let writer = dataset.mem_wal_writer(shard_id, config).await?;
 
     let ingest_pool = &corpus[read_seed..];
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
index 08fdb76b95f..2c7469b3def 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
@@ -688,7 +688,6 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: args.max_memtable_rows * row_bytes * 100,
         max_memtable_rows: args.max_memtable_rows,
         max_memtable_batches: batches_per_gen,
diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
index 757c2539642..e5c6cf234e1 100644
--- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
+++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
@@ -792,7 +792,6 @@ async fn run_lance(
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: true,
-        sync_indexed_write: true,
         max_memtable_size: big,
         max_memtable_rows: args.rows * 4 + 1_000_000,
         max_memtable_batches: args.rows / args.batch_rows + 1_000_000,
diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
index cb9e6413ac1..d723cfde9c8 100644
--- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
+++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
@@ -338,7 +338,6 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: max_memtable_rows * 200,
         max_memtable_rows,
         max_wal_flush_interval: Some(Duration::from_secs(60)),
diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
index 85dd5e7ac14..a4968161647 100644
--- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
+++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
@@ -389,7 +389,6 @@ async fn run_checkpoint(
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: true,
         max_memtable_size: cp.saturating_mul(row_size_estimate).saturating_mul(4),
         max_memtable_rows: cp.saturating_mul(2),
         max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000),
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index 812535ce053..c6fb42834a3 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -214,7 +214,6 @@ async fn main() -> lance_core::Result<()> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write,
-        sync_indexed_write: true,
         max_memtable_size: max_rows.saturating_mul(row_size_estimate).saturating_mul(4),
         max_memtable_rows: max_rows.saturating_mul(2),
         max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000),
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
index 632bd37626c..a8fe8e81f29 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
@@ -520,7 +520,6 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
         wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
-        sync_indexed_write: false,
         max_memtable_size: args.max_memtable_rows * row_bytes * 2,
         max_memtable_rows: args.max_memtable_rows,
         max_unflushed_memtable_bytes: args.max_memtable_rows * row_bytes * 6,
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
index 14ec406e5e5..6fda0748006 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs
@@ -179,7 +179,6 @@ async fn populate_shard_wal(
     let dataset = Dataset::open(dataset_uri).await.unwrap();
     let mut config = ShardWriterConfig::new(shard_id);
     config.durable_write = true;
-    config.sync_indexed_write = false;
     let writer = dataset.mem_wal_writer(shard_id, config).await.unwrap();
 
     for i in 0..num_entries {
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
index 56915fb205d..79298daa872 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs
@@ -99,10 +99,6 @@ impl Mode {
     fn durable_write(self) -> bool {
         matches!(self, Self::SyncNoIndex | Self::SyncIndexed)
     }
-
-    fn sync_indexed_write(self) -> bool {
-        matches!(self, Self::SyncIndexed)
-    }
 }
 
 /// Which index the MemTable maintains in the indexed (`*_idx`) modes.
@@ -192,7 +188,6 @@ struct Args {
     max_memtable_batches: Option<usize>,
     max_wal_buffer_size: usize,
     max_wal_flush_interval_ms: u64,
-    async_index_buffer_rows: usize,
     sample_interval_ms: u64,
     target_rows_per_sec: Option<f64>,
     num_partitions: usize,
@@ -223,7 +218,6 @@ impl Default for Args {
             max_memtable_batches: None,
             max_wal_buffer_size: 10 * 1024 * 1024,
             max_wal_flush_interval_ms: 100,
-            async_index_buffer_rows: 10_000,
             sample_interval_ms: 500,
             target_rows_per_sec: None,
             num_partitions: 1,
@@ -343,11 +337,9 @@ async fn run(args: Args) -> Result<()> {
     let shard_id = Uuid::new_v4();
     let mut config = ShardWriterConfig::new(shard_id)
         .with_durable_write(args.mode.durable_write())
-        .with_sync_indexed_write(args.mode.sync_indexed_write())
         .with_max_memtable_size(args.max_memtable_size)
         .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes)
         .with_max_wal_buffer_size(args.max_wal_buffer_size)
-        .with_async_index_buffer_rows(args.async_index_buffer_rows)
         .with_max_memtable_rows(memtable_limits.rows)
         .with_max_memtable_batches(memtable_limits.batches);
     if args.max_wal_flush_interval_ms == 0 {
@@ -544,7 +536,6 @@ async fn run(args: Args) -> Result<()> {
         "max_memtable_batches": memtable_limits.batches,
         "max_wal_buffer_size": args.max_wal_buffer_size,
         "max_wal_flush_interval_ms": args.max_wal_flush_interval_ms,
-        "async_index_buffer_rows": args.async_index_buffer_rows,
         "sample_interval_ms": args.sample_interval_ms,
         "skip_close": args.skip_close,
         "setup_seconds": setup_s,
@@ -980,7 +971,6 @@ fn parse_args() -> Result<Args> {
             "--max-wal-flush-interval-ms" => {
                 args.max_wal_flush_interval_ms = parse(&flag, &value)?;
             }
-            "--async-index-buffer-rows" => args.async_index_buffer_rows = parse(&flag, &value)?,
             "--sample-interval-ms" => args.sample_interval_ms = parse(&flag, &value)?,
             "--target-rows-per-sec" => args.target_rows_per_sec = Some(parse(&flag, &value)?),
             "--num-partitions" => args.num_partitions = parse(&flag, &value)?,
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_write.rs b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
index db28c5b33ab..b5018fe09f1 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_write.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
@@ -30,7 +30,6 @@
 //! - `BATCH_SIZE`: Number of rows per write batch (default: 20)
 //! - `NUM_BATCHES`: Total number of batches to write (default: 1000)
 //! - `DURABLE_WRITE`: yes/no/both (default: no) - whether writes wait for WAL flush
-//! - `INDEXED_WRITE`: yes/no/both (default: no) - whether writes update indexes synchronously
 //! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from ShardWriterConfig)
 //! - `MAX_FLUSH_INTERVAL_MS`: WAL flush interval in milliseconds, 0 to disable (default: 1000ms)
 //! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from ShardWriterConfig)
@@ -44,8 +43,7 @@
 //!   `no` uses WAL-only mode (no MemTable, no indexes, no Lance flushes; pure WAL throughput).
 //!   `both` runs each combination twice, once per mode, side-by-side.
 //!   When `no` or `both`, the WAL-only branch always runs with
-//!   `MEMWAL_MAINTAINED_INDEXES=none` and skips `INDEXED_WRITE=yes`
-//!   (sync-indexed writes require a MemTable).
+//!   `MEMWAL_MAINTAINED_INDEXES=none`.
 //! - `SAMPLE_SIZE`: Number of benchmark iterations (default: 10, minimum: 10)
 
 #![allow(clippy::print_stdout, clippy::print_stderr)]
@@ -119,11 +117,6 @@ fn get_durable_write_options() -> Vec<bool> {
     parse_yes_no_both("DURABLE_WRITE", "no")
 }
 
-/// Get indexed write settings from environment.
-fn get_indexed_write_options() -> Vec<bool> {
-    parse_yes_no_both("INDEXED_WRITE", "no")
-}
-
 /// Get enable_memtable settings from environment. Default `yes` keeps
 /// existing benchmark behavior; `no` runs WAL-only mode; `both` runs both
 /// modes side-by-side for comparison.
@@ -442,31 +435,26 @@ fn build_label(
     num_batches: usize,
     batch_size: usize,
     durable: bool,
-    indexed: bool,
     enable_memtable: bool,
     storage: &str,
 ) -> String {
     let durable_str = if durable { "durable" } else { "nondurable" };
-    // sync_indexed_write controls sync vs async index updates
-    let indexed_str = if indexed { "sync_idx" } else { "async_idx" };
     let mode_str = if enable_memtable {
         "memtable"
     } else {
         "wal_only"
     };
     format!(
-        "{}x{} {} {} {} ({})",
-        num_batches, batch_size, mode_str, durable_str, indexed_str, storage
+        "{}x{} {} {} ({})",
+        num_batches, batch_size, mode_str, durable_str, storage
     )
 }
 
 /// Build dataset name prefix from config options.
-fn build_name_prefix(durable: bool, indexed: bool, enable_memtable: bool) -> String {
+fn build_name_prefix(durable: bool, enable_memtable: bool) -> String {
     let d = if durable { "d" } else { "nd" };
-    // sync_indexed_write: sync (si) vs async (ai)
-    let i = if indexed { "si" } else { "ai" };
     let m = if enable_memtable { "mt" } else { "wo" };
-    format!("{}_{}_{}", m, d, i)
+    format!("{}_{}", m, d)
 }
 
 /// Benchmark Lance MemWAL write throughput.
@@ -490,7 +478,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
     let maintained_indexes = get_maintained_indexes();
 
     let durable_options = get_durable_write_options();
-    let indexed_options = get_indexed_write_options();
     let enable_memtable_options = get_enable_memtable_options();
     let max_wal_buffer_size = get_max_wal_buffer_size();
     let max_flush_interval = get_max_flush_interval();
@@ -550,70 +537,56 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
     // Generate benchmarks for all combinations
     for &enable_memtable in &enable_memtable_options {
         for &durable in &durable_options {
-            for &indexed in &indexed_options {
-                if !enable_memtable && indexed {
-                    eprintln!(
-                        "Skipping wal_only + sync_idx (sync_indexed_write requires a MemTable)"
-                    );
-                    continue;
-                }
-
-                let label = build_label(
-                    num_batches,
-                    batch_size,
-                    durable,
-                    indexed,
-                    enable_memtable,
-                    storage_label,
-                );
-                let name_prefix = build_name_prefix(durable, indexed, enable_memtable);
-
-                // WAL-only mode never uses indexes; force the dataset
-                // setup to skip the MemWAL index list.
-                let effective_indexes: Vec<String> = if enable_memtable {
-                    maintained_indexes.clone()
-                } else {
-                    Vec::new()
-                };
-
-                // Create dataset ONCE before benchmark iterations
-                // Each iteration will use a different shard on the same dataset
-                let dataset = rt.block_on(create_dataset(
-                    &schema,
-                    &name_prefix,
-                    vector_dim,
-                    &effective_indexes,
-                    &dataset_prefix,
-                ));
-                let dataset_uri = dataset.uri().to_string();
-
-                // Pre-generate all batches before timing (outside iter_custom)
-                let batches: Arc<Vec<RecordBatch>> = Arc::new(
-                    (0..num_batches)
-                        .map(|i| {
-                            create_test_batch(
-                                &schema,
-                                (i * batch_size) as i64,
-                                batch_size,
-                                vector_dim,
-                            )
-                        })
-                        .collect(),
-                );
-
-                println!("Running: {}", label);
-
-                // Track if we've printed stats (only print once across all samples)
-                let stats_printed = Arc::new(AtomicBool::new(false));
-
-                group.bench_with_input(
-                    BenchmarkId::new("Lance MemWAL", &label),
-                    &(batch_size, num_batches, durable, indexed, row_size_bytes),
-                    |b, &(_batch_size, _num_batches, durable, indexed, row_size_bytes)| {
-                        let dataset_uri = dataset_uri.clone();
-                        let batches = batches.clone();
-                        let stats_printed = stats_printed.clone();
-                        b.to_async(&rt).iter_custom(|iters| {
+            let label = build_label(
+                num_batches,
+                batch_size,
+                durable,
+                enable_memtable,
+                storage_label,
+            );
+            let name_prefix = build_name_prefix(durable, enable_memtable);
+
+            // WAL-only mode never uses indexes; force the dataset
+            // setup to skip the MemWAL index list.
+            let effective_indexes: Vec<String> = if enable_memtable {
+                maintained_indexes.clone()
+            } else {
+                Vec::new()
+            };
+
+            // Create dataset ONCE before benchmark iterations
+            // Each iteration will use a different shard on the same dataset
+            let dataset = rt.block_on(create_dataset(
+                &schema,
+                &name_prefix,
+                vector_dim,
+                &effective_indexes,
+                &dataset_prefix,
+            ));
+            let dataset_uri = dataset.uri().to_string();
+
+            // Pre-generate all batches before timing (outside iter_custom)
+            let batches: Arc<Vec<RecordBatch>> = Arc::new(
+                (0..num_batches)
+                    .map(|i| {
+                        create_test_batch(&schema, (i * batch_size) as i64, batch_size, vector_dim)
+                    })
+                    .collect(),
+            );
+
+            println!("Running: {}", label);
+
+            // Track if we've printed stats (only print once across all samples)
+            let stats_printed = Arc::new(AtomicBool::new(false));
+
+            group.bench_with_input(
+                BenchmarkId::new("Lance MemWAL", &label),
+                &(batch_size, num_batches, durable, row_size_bytes),
+                |b, &(_batch_size, _num_batches, durable, row_size_bytes)| {
+                    let dataset_uri = dataset_uri.clone();
+                    let batches = batches.clone();
+                    let stats_printed = stats_printed.clone();
+                    b.to_async(&rt).iter_custom(|iters| {
                         let dataset_uri = dataset_uri.clone();
                         let batches = batches.clone();
                         let stats_printed = stats_printed.clone();
@@ -631,10 +604,10 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                     shard_id,
                                     shard_spec_id: 0,
                                     max_wal_persist_retries: 3,
-                                    wal_persist_retry_base_delay:
-                                        std::time::Duration::from_millis(50),
+                                    wal_persist_retry_base_delay: std::time::Duration::from_millis(
+                                        50,
+                                    ),
                                     durable_write: durable,
-                                    sync_indexed_write: indexed,
                                     max_wal_buffer_size: max_wal_buffer_size
                                         .unwrap_or(default_config.max_wal_buffer_size),
                                     max_wal_flush_interval: max_flush_interval
@@ -643,8 +616,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                         .unwrap_or(default_config.max_memtable_size),
                                     max_memtable_rows: default_config.max_memtable_rows,
                                     max_memtable_batches: default_config.max_memtable_batches,
-                                    async_index_buffer_rows: default_config.async_index_buffer_rows,
-                                    async_index_interval: default_config.async_index_interval,
                                     manifest_scan_batch_size: default_config
                                         .manifest_scan_batch_size,
                                     max_unflushed_memtable_bytes: default_config
@@ -706,9 +677,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                             total_duration
                         }
                     })
-                    },
-                );
-            }
+                },
+            );
         }
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs
index 1bd95092e7c..4351ecc95b7 100644
--- a/rust/lance/src/dataset/mem_wal/api.rs
+++ b/rust/lance/src/dataset/mem_wal/api.rs
@@ -392,10 +392,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap<String, Stri
             "durable_write".to_string(),
             config.durable_write.to_string(),
         ),
-        (
-            "sync_indexed_write".to_string(),
-            config.sync_indexed_write.to_string(),
-        ),
         (
             "max_wal_buffer_size".to_string(),
             config.max_wal_buffer_size.to_string(),
@@ -432,14 +428,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap<String, Stri
             "backpressure_log_interval_ms".to_string(),
             config.backpressure_log_interval.as_millis().to_string(),
         ),
-        (
-            "async_index_buffer_rows".to_string(),
-            config.async_index_buffer_rows.to_string(),
-        ),
-        (
-            "async_index_interval_ms".to_string(),
-            config.async_index_interval.as_millis().to_string(),
-        ),
         (
             "enable_memtable".to_string(),
             config.enable_memtable.to_string(),
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 16dc856a871..a2a40f86fc3 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -85,17 +85,6 @@ pub struct ShardWriterConfig {
     /// - Lower latency, batched S3 operations
     pub durable_write: bool,
 
-    /// Whether to update indexes synchronously on each write.
-    ///
-    /// When true:
-    /// - Newly written data is immediately searchable via indexes
-    /// - Higher latency due to index update overhead
-    ///
-    /// When false:
-    /// - Index updates are deferred
-    /// - New data may not appear in index-accelerated queries immediately
-    pub sync_indexed_write: bool,
-
     /// Maximum WAL buffer size in bytes before triggering a flush.
     ///
     /// This is a soft threshold - write batches are atomic and won't be split.
@@ -167,23 +156,6 @@ pub struct ShardWriterConfig {
     /// Default: 30 seconds
     pub backpressure_log_interval: Duration,
 
-    /// Maximum rows to buffer before flushing to async indexes.
-    ///
-    /// Only applies when `sync_indexed_write` is false. Larger values enable
-    /// better vectorization but increase memory usage and latency before data
-    /// becomes searchable.
-    ///
-    /// Default: 10,000 rows
-    pub async_index_buffer_rows: usize,
-
-    /// Maximum time to buffer before flushing to async indexes.
-    ///
-    /// Only applies when `sync_indexed_write` is false. Ensures bounded latency
-    /// for data to become searchable even during low write throughput.
-    ///
-    /// Default: 1 second
-    pub async_index_interval: Duration,
-
     /// Interval for periodic stats logging.
     ///
     /// Stats (write throughput, backpressure events, memtable size) are logged
@@ -225,8 +197,7 @@ pub struct ShardWriterConfig {
     ///   `durable_write` settings as MemTable mode.
     ///
     /// MemTable-tied tunables (`max_memtable_size`, `max_memtable_rows`,
-    /// `max_memtable_batches`, `sync_indexed_write`, `async_index_buffer_rows`,
-    /// `async_index_interval`) are ignored when `enable_memtable == false`.
+    /// `max_memtable_batches`) are ignored when `enable_memtable == false`.
     ///
     /// For raw single-entry synchronous atomic appends with no buffering and
     /// no background tasks, use `WalAppender` directly — it is a strictly
@@ -270,7 +241,6 @@ impl Default for ShardWriterConfig {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: true,
             max_wal_buffer_size: 10 * 1024 * 1024, // 10MB
             max_wal_flush_interval: Some(Duration::from_millis(100)), // 100ms
             max_wal_persist_retries: 3,
@@ -281,8 +251,6 @@ impl Default for ShardWriterConfig {
             manifest_scan_batch_size: 2,
             max_unflushed_memtable_bytes: 1024 * 1024 * 1024, // 1GB
             backpressure_log_interval: Duration::from_secs(30),
-            async_index_buffer_rows: 10_000,
-            async_index_interval: Duration::from_secs(1),
             stats_log_interval: Some(Duration::from_secs(60)), // 1 minute
             frozen_memtable_grace: Duration::ZERO,
             enable_memtable: true,
@@ -315,12 +283,6 @@ impl ShardWriterConfig {
         self
     }
 
-    /// Set indexed writes mode.
-    pub fn with_sync_indexed_write(mut self, indexed: bool) -> Self {
-        self.sync_indexed_write = indexed;
-        self
-    }
-
     /// Set maximum WAL buffer size.
     pub fn with_max_wal_buffer_size(mut self, size: usize) -> Self {
         self.max_wal_buffer_size = size;
@@ -383,18 +345,6 @@ impl ShardWriterConfig {
         self
     }
 
-    /// Set async index buffer rows.
-    pub fn with_async_index_buffer_rows(mut self, rows: usize) -> Self {
-        self.async_index_buffer_rows = rows;
-        self
-    }
-
-    /// Set async index interval.
-    pub fn with_async_index_interval(mut self, interval: Duration) -> Self {
-        self.async_index_interval = interval;
-        self
-    }
-
     /// Set stats logging interval. Use None to disable periodic stats logging.
     pub fn with_stats_log_interval(mut self, interval: Option<Duration>) -> Self {
         self.stats_log_interval = interval;
@@ -3878,7 +3828,6 @@ mod tests {
         ShardWriterConfig {
             shard_id,
             durable_write: false,
-            sync_indexed_write: true,
             manifest_scan_batch_size: 2,
             ..Default::default()
         }
@@ -4203,7 +4152,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4246,7 +4194,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4283,7 +4230,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4329,7 +4275,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4364,7 +4309,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4416,7 +4360,6 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: true,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -4505,7 +4448,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 1024, // Very small - will trigger flush quickly
@@ -4612,7 +4554,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64,
@@ -4657,7 +4598,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold — every batch crosses it.
@@ -4724,7 +4664,6 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: usize::MAX,
@@ -4805,7 +4744,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             // Tiny size threshold so a few batches cross it.
@@ -5239,7 +5177,6 @@ mod tests {
             shard_id,
             shard_spec_id: 0,
             durable_write: true,
-            sync_indexed_write: false,
             max_wal_buffer_size: 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -6391,7 +6328,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -6446,7 +6382,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -6501,7 +6436,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -6561,7 +6495,6 @@ mod tests {
             shard_id: Uuid::new_v4(),
             shard_spec_id: 0,
             durable_write: false,
-            sync_indexed_write: false,
             max_wal_buffer_size: 64 * 1024 * 1024,
             max_wal_flush_interval: Some(Duration::from_millis(10)),
             max_memtable_size: 64 * 1024 * 1024,
@@ -6824,9 +6757,7 @@ mod shard_writer_tests {
             Some("100")
         );
         // Every tunable field is present.
-        assert!(defaults.contains_key("sync_indexed_write"));
         assert!(defaults.contains_key("enable_memtable"));
-        assert!(defaults.contains_key("async_index_interval_ms"));
         // add_writer_config_default records arbitrary keys.
         assert_eq!(
             defaults.get("custom_knob").map(String::as_str),
@@ -7033,9 +6964,7 @@ mod shard_writer_tests {
             .expect("Failed to initialize MemWAL");
 
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(true)
-            .with_sync_indexed_write(true);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(true);
         let writer = dataset
             .mem_wal_writer(shard_id, config)
             .await
@@ -7368,9 +7297,7 @@ mod shard_writer_tests {
 
         // Create shard writer
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(false)
-            .with_sync_indexed_write(false);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(false);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -7423,9 +7350,7 @@ mod shard_writer_tests {
             .expect("Failed to initialize MemWAL");
 
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(true)
-            .with_sync_indexed_write(true);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(true);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -7564,9 +7489,7 @@ mod shard_writer_tests {
 
         // Create shard writer with default config
         let shard_id = Uuid::new_v4();
-        let config = ShardWriterConfig::new(shard_id)
-            .with_durable_write(false)
-            .with_sync_indexed_write(false);
+        let config = ShardWriterConfig::new(shard_id).with_durable_write(false);
 
         let writer = dataset
             .mem_wal_writer(shard_id, config)
@@ -7643,7 +7566,6 @@ mod shard_writer_tests {
         let shard_id = Uuid::new_v4();
         let config = ShardWriterConfig::new(shard_id)
             .with_durable_write(true) // Ensure WAL files are written
-            .with_sync_indexed_write(true)
             .with_max_memtable_size(50 * 1024) // 50KB - triggers flush after ~8 batches
             .with_max_wal_buffer_size(10 * 1024) // 10KB WAL buffer
             .with_max_wal_flush_interval(Duration::from_millis(50)); // Fast flush
@@ -7773,9 +7695,7 @@ mod shard_writer_tests {
         // rows stay invisible until the next flush. This test used to pass with
         // `durable_write(false)` only because an un-advanced cursor of 0 was
         // misread as "batch 0 is visible" — it was asserting the dirty read.
-        let new_config = ShardWriterConfig::new(new_shard_id)
-            .with_durable_write(true)
-            .with_sync_indexed_write(true);
+        let new_config = ShardWriterConfig::new(new_shard_id).with_durable_write(true);
 
         let new_writer = dataset
             .mem_wal_writer(new_shard_id, new_config)

From cd6741afc3b21ef5548d3f5e0ee1a1504c000ade Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Tue, 14 Jul 2026 22:19:05 -0500
Subject: [PATCH 14/19] docs(mem_wal): drop private intra-doc link in
 insert_batches

The public `insert_batches` doc linked to the private `PARALLEL_INDEX_MIN_ROWS`
const, which `-D rustdoc::private-intra-doc-links` rejects. Demote the link to a
plain code span; the surrounding prose already explains the threshold.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/index.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 9814cfe74ab..3f4dd47c89f 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -891,7 +891,7 @@ impl IndexStore {
 
     /// Insert multiple batches into every index.
     ///
-    /// Above [`PARALLEL_INDEX_MIN_ROWS`] rows each index runs on its own thread, which
+    /// Above `PARALLEL_INDEX_MIN_ROWS` rows each index runs on its own thread, which
     /// maximizes parallelism when several indexes are maintained. At or below it they run
     /// inline on the calling thread: the spawn is one OS thread *per index*, and for a
     /// handful of rows that costs more than the indexing itself.

From 1da8f9c5d94f0534ca453a5888ac6665a8182395 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Wed, 15 Jul 2026 11:24:32 -0500
Subject: [PATCH 15/19] fix(mem_wal): validate single-column PKs and restore
 index-apply stats

Address CodeRabbit review feedback on the WAL visibility branch:

- validate_index_configs now rejects a single-column primary key on a
  column absent from the schema, closing the gap where such a config
  passed validation and then failed deterministically on every index
  build and WAL replay. Existence is checked for all PK columns; the
  order-preserving encodable-type check stays gated on composite keys.

- Restore the index-update statistics. Index application moved onto its
  own task, leaving do_flush's record_index_update branch dead (gated on
  a hardcoded false) so index_update_count/time/rows and
  log_wal_breakdown() reported a permanent zero. apply_index_range now
  returns IndexApplyStats { rows_indexed, duration } and
  IndexApplyHandler records real applies, skipping coalesced no-ops. Drop
  the dead do_flush gate and the vestigial index fields on WalFlushResult.

- Fix stale docs that still called indexed_count a durable visibility
  watermark, and correct the point-lookup BTree test comment.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/index.rs       | 56 ++++++------
 .../dataset/mem_wal/scanner/point_lookup.rs   |  5 +-
 rust/lance/src/dataset/mem_wal/wal.rs         | 50 ++++++-----
 rust/lance/src/dataset/mem_wal/write.rs       | 85 ++++++++++++++-----
 4 files changed, 127 insertions(+), 69 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 3f4dd47c89f..a2690cd5082 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -187,22 +187,21 @@ pub fn validate_index_configs(
         }
     }
 
-    // A single-column PK aliases a BTree entry (any type). Only a *composite* PK
-    // builds an order-preserving encoded key, and only some types encode.
-    if pk_columns.len() > 1 {
-        for column in pk_columns {
-            let field = schema.field_with_name(column).map_err(|_| {
-                Error::invalid_input(format!(
-                    "primary-key column '{column}' is not in the shard schema"
-                ))
-            })?;
-            if !is_encodable_pk_type(field.data_type()) {
-                return Err(Error::invalid_input(format!(
-                    "composite primary-key column '{column}' has type {:?}, which has no \
-                     order-preserving key encoding",
-                    field.data_type()
-                )));
-            }
+    // Every PK column must exist in the schema. A single-column PK aliases a
+    // BTree entry (any type); only a *composite* PK builds an order-preserving
+    // encoded key, and only some types encode.
+    for column in pk_columns {
+        let field = schema.field_with_name(column).map_err(|_| {
+            Error::invalid_input(format!(
+                "primary-key column '{column}' is not in the shard schema"
+            ))
+        })?;
+        if pk_columns.len() > 1 && !is_encodable_pk_type(field.data_type()) {
+            return Err(Error::invalid_input(format!(
+                "composite primary-key column '{column}' has type {:?}, which has no \
+                 order-preserving key encoding",
+                field.data_type()
+            )));
         }
     }
 
@@ -406,9 +405,6 @@ pub struct IndexStore {
     /// primary key. Queried via [`Self::pk_newest_visible`] (see
     /// [`Self::enable_pk_index`]).
     pk_index: Option<PkIndex>,
-    /// Maximum batch position that is durable in the WAL and therefore
-    /// visible to scanners. Advanced unconditionally after a WAL append
-    /// succeeds; not gated on whether any indexes are configured.
     /// How many batches of this memtable have been fully indexed. An exclusive
     /// count: 0 means none.
     ///
@@ -1109,13 +1105,13 @@ impl IndexStore {
         self.btree_indexes.len() + self.hnsw_indexes.len() + self.fts_indexes.len()
     }
 
-    /// Get the visibility watermark (max batch position safe to read).
-    ///
-    /// Returns the highest batch position whose data is durable in the WAL
-    /// and therefore visible to scanners. Scanners snapshot this at plan
-    /// construction time so every plan runs against a stable cursor.
+    /// How many batches of this memtable have been fully indexed (exclusive
+    /// count; 0 before any batch is indexed).
     ///
-    /// Returns 0 before any WAL flush has advanced the watermark.
+    /// This is the *indexed* cursor, not the visibility watermark: it advances
+    /// once every index insert for a batch completes, regardless of WAL
+    /// durability. Readers must snapshot [`Self::visible_count`], which derives
+    /// what is safe to read from this cursor and the writer's durability cursor.
     pub fn indexed_count(&self) -> usize {
         self.indexed_count.load(Ordering::Acquire)
     }
@@ -1750,6 +1746,16 @@ mod tests {
         // A single-column PK of the same type is fine: it aliases a BTree.
         validate_index_configs(&[], &schema, &["coords".into()])
             .expect("single-column PK aliases a BTree and accepts any type");
+
+        // But every PK column must exist. A single-column PK naming an absent
+        // column is rejected here, not left to fail deterministically on every
+        // later index build and WAL replay.
+        let err = validate_index_configs(&[], &schema, &["missing".into()])
+            .expect_err("a single-column PK on an absent column must be rejected");
+        assert!(
+            err.to_string().contains("not in the shard schema"),
+            "error must name the missing column, got {err}"
+        );
     }
 
     #[test]
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index 72808851d77..6270b84c4a0 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -1393,8 +1393,9 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
-        // BTree on the PK so that `visible_count` advances as
-        // we insert, otherwise the scanner sees no batches at all.
+        // BTree on the PK: the point lookup resolves keys through the indexed PK
+        // path, which this exercises. (`indexed_count`/`visible_count` advance
+        // from the batch position regardless of whether any index is configured.)
         index_store.add_btree("id_idx".to_string(), 0, "id".to_string());
 
         // Two writes to pk=1, then an unrelated pk=2. The "new" row goes
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 2d9c3323584..40afb2db3a5 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -282,20 +282,15 @@ pub struct WalEntry {
     pub num_batches: usize,
 }
 
-/// Result of a parallel WAL flush with index update.
+/// Result of a WAL flush. Append-only: index application runs on its own task
+/// (see [`apply_index_range`]), which records its own stats, so this no longer
+/// carries index-update timing or row counts.
 #[derive(Debug, Clone)]
 pub struct WalFlushResult {
     /// WAL entry that was written (if any).
     pub entry: Option<WalEntry>,
     /// Duration of WAL I/O operation.
     pub wal_io_duration: std::time::Duration,
-    /// Overall wall-clock duration of the index update operation.
-    /// This includes any overhead from thread scheduling and context switching.
-    pub index_update_duration: std::time::Duration,
-    /// Per-index update durations. Key is index name, value is duration.
-    pub index_update_duration_breakdown: std::collections::HashMap<String, std::time::Duration>,
-    /// Number of rows indexed.
-    pub rows_indexed: usize,
     /// Size of WAL data written in bytes.
     pub wal_bytes: usize,
 }
@@ -359,6 +354,17 @@ impl std::fmt::Debug for TriggerIndexApply {
     }
 }
 
+/// What an [`apply_index_range`] call actually indexed. `rows_indexed == 0`
+/// marks a routine coalesced no-op (the range was already covered) that must
+/// not be recorded as an index update.
+#[derive(Debug, Default, Clone, Copy)]
+pub struct IndexApplyStats {
+    pub rows_indexed: usize,
+    /// Wall-clock time spent applying the range, including the thread-scheduling
+    /// overhead of the blocking hand-off.
+    pub duration: std::time::Duration,
+}
+
 /// Apply a contiguous range of batches to a memtable's in-memory indexes.
 ///
 /// Runs on its own task, as the single sequential consumer of its own channel.
@@ -374,7 +380,7 @@ impl std::fmt::Debug for TriggerIndexApply {
 pub async fn apply_index_range(
     cursors: &Arc<WriterCursors>,
     message: TriggerIndexApply,
-) -> Result<()> {
+) -> Result<IndexApplyStats> {
     let TriggerIndexApply {
         batch_store,
         indexes,
@@ -389,25 +395,32 @@ pub async fn apply_index_range(
     // writer.
     let start = indexes.indexed_count();
     if end_batch_position <= start {
-        return Ok(());
+        return Ok(IndexApplyStats::default());
     }
 
     let stored: Vec<StoredBatch> = (start..end_batch_position)
         .filter_map(|position| batch_store.get(position).cloned())
         .collect();
     if stored.is_empty() {
-        return Ok(());
+        return Ok(IndexApplyStats::default());
     }
 
     // `insert_batches` advances `indexed_count` itself, once every index has
-    // taken the batch.
+    // taken the batch. Time the whole hand-off so the recorded latency reflects
+    // the blocking-pool scheduling too, matching the old inline-flush measurement.
+    let rows_indexed: usize = stored.iter().map(|b| b.num_rows).sum();
+    let apply_start = Instant::now();
     tokio::task::spawn_blocking(move || indexes.insert_batches(&stored))
         .await
         .map_err(|e| Error::internal(format!("Index apply task panicked: {e}")))??;
+    let duration = apply_start.elapsed();
 
     // Wake anything waiting to become visible.
     cursors.wake();
-    Ok(())
+    Ok(IndexApplyStats {
+        rows_indexed,
+        duration,
+    })
 }
 
 /// Message to trigger a WAL flush.
@@ -804,7 +817,6 @@ impl WalFlusher {
             return Ok(empty_flush_result());
         }
 
-        let num_rows: usize = stored_batches.iter().map(|b| b.num_rows).sum();
         let record_batches: Vec<RecordBatch> =
             stored_batches.iter().map(|s| s.data.clone()).collect();
 
@@ -827,9 +839,6 @@ impl WalFlusher {
                 num_batches: append_result.num_batches,
             }),
             wal_io_duration,
-            index_update_duration: std::time::Duration::ZERO,
-            index_update_duration_breakdown: std::collections::HashMap::new(),
-            rows_indexed: num_rows,
             wal_bytes: append_result.wal_bytes,
         })
     }
@@ -862,9 +871,6 @@ impl WalFlusher {
                 num_batches: append_result.num_batches,
             }),
             wal_io_duration,
-            index_update_duration: std::time::Duration::ZERO,
-            index_update_duration_breakdown: std::collections::HashMap::new(),
-            rows_indexed: 0,
             wal_bytes: append_result.wal_bytes,
         })
     }
@@ -897,9 +903,6 @@ pub fn empty_flush_result() -> WalFlushResult {
     WalFlushResult {
         entry: None,
         wal_io_duration: std::time::Duration::ZERO,
-        index_update_duration: std::time::Duration::ZERO,
-        index_update_duration_breakdown: std::collections::HashMap::new(),
-        rows_indexed: 0,
         wal_bytes: 0,
     }
 }
@@ -1768,6 +1771,7 @@ mod tests {
             },
         )
         .await
+        .map(|_| ())
     }
 
     #[tokio::test]
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index a2a40f86fc3..0947da8159b 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -1775,7 +1775,7 @@ impl ShardWriter {
             wal_flusher.clone(),
             epoch,
             index_configs.to_vec(),
-            stats,
+            stats.clone(),
             config.frozen_memtable_grace,
         );
         task_executor.add_handler(
@@ -1791,6 +1791,7 @@ impl ShardWriter {
         let index_handler = IndexApplyHandler {
             cursors: Arc::clone(wal_flusher.cursors()),
             wal_flusher: wal_flusher.clone(),
+            stats,
         };
         task_executor.add_handler(
             "index_applier".to_string(),
@@ -2729,19 +2730,30 @@ fn next_pending_store(
 struct IndexApplyHandler {
     cursors: Arc<WriterCursors>,
     wal_flusher: Arc<WalFlusher>,
+    stats: SharedWriteStats,
 }
 
 #[async_trait]
 impl MessageHandler<TriggerIndexApply> for IndexApplyHandler {
     async fn handle(&mut self, message: TriggerIndexApply) -> Result<()> {
-        if let Err(e) = apply_index_range(&self.cursors, message).await {
+        match apply_index_range(&self.cursors, message).await {
+            Ok(applied) => {
+                // A coalesced no-op indexes nothing (`rows_indexed == 0`);
+                // recording it would inflate the count and skew avg latency.
+                if applied.rows_indexed > 0 {
+                    self.stats
+                        .record_index_update(applied.duration, applied.rows_indexed);
+                }
+                Ok(())
+            }
             // An index apply cannot be partially rolled back, so a failure is
             // terminal: poison, and let reopen rebuild the indexes from the WAL.
             // See the note in `WalFlusher::flush_from_batch_store`.
-            self.wal_flusher.poison(&e);
-            return Err(e);
+            Err(e) => {
+                self.wal_flusher.poison(&e);
+                Err(e)
+            }
         }
-        Ok(())
     }
 }
 
@@ -2906,14 +2918,6 @@ impl WalFlushHandler {
     ) -> Result<WalFlushResult> {
         let start = Instant::now();
 
-        // Whether this flush actually updates any in-memory indexes — only
-        // a BatchStore source carrying a non-empty `IndexStore` does. Used
-        // to gate the `record_index_update` stat so WAL-only flushes don't
-        // pollute the index-update counters.
-        // The WAL flush no longer touches indexes at all — the index apply runs
-        // on its own task — so it never records an index-update stat.
-        let has_indexes = false;
-
         // Early-out for BatchStore sources where the watermark already
         // covers the requested end position. Detection of "frozen flush"
         // requires the active memtable's batch_store; WAL-only handlers
@@ -2946,12 +2950,6 @@ impl WalFlushHandler {
             self.stats
                 .record_wal_flush(start.elapsed(), flush_result.wal_bytes);
             self.stats.record_wal_io(flush_result.wal_io_duration);
-            if has_indexes {
-                self.stats.record_index_update(
-                    flush_result.index_update_duration,
-                    flush_result.rows_indexed,
-                );
-            }
         }
 
         Ok(flush_result)
@@ -6317,6 +6315,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_memtable_stats_record_index_update() {
+        // MemTable mode with a BTree index: index application runs on its own
+        // task and must record an index-update stat. Regression for the stat
+        // silently reading zero after index apply moved off the WAL-flush path.
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let index_configs = vec![MemIndexConfig::BTree(BTreeIndexConfig {
+            name: "id_idx".to_string(),
+            field_id: 0,
+            column: "id".to_string(),
+        })];
+
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            flush_test_config(Uuid::new_v4()),
+            schema.clone(),
+            index_configs,
+        )
+        .await
+        .unwrap();
+        writer
+            .put(vec![create_test_batch(&schema, 0, 3)])
+            .await
+            .unwrap();
+
+        let stats_handle = writer.stats_handle();
+        // `close()` drains the index-apply task, so the apply is settled here.
+        writer.close().await.unwrap();
+
+        let snapshot = stats_handle.snapshot();
+        assert!(
+            snapshot.index_update_count >= 1,
+            "the index apply must record an index-update stat, got {}",
+            snapshot.index_update_count
+        );
+        assert_eq!(
+            snapshot.index_update_rows, 3,
+            "every indexed row must be counted exactly once, got {}",
+            snapshot.index_update_rows
+        );
+        assert!(
+            snapshot.avg_index_update_latency().is_some(),
+            "a recorded index update must expose an average latency"
+        );
+    }
+
     #[tokio::test]
     async fn test_force_seal_active_and_wait_for_flush_drain() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;

From d8bd2bfbdafb19b72f569921f52314f2d23d007e Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Wed, 15 Jul 2026 11:34:32 -0500
Subject: [PATCH 16/19] docs(mem_wal): drop private intra-doc link in
 WalFlushResult

The public `WalFlushResult` doc linked to the private `apply_index_range`
fn, which `-D rustdoc::private-intra-doc-links` rejects. Demote the link to
a plain code span; the surrounding prose already explains the reference.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/wal.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 40afb2db3a5..e371c1dbeed 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -283,7 +283,7 @@ pub struct WalEntry {
 }
 
 /// Result of a WAL flush. Append-only: index application runs on its own task
-/// (see [`apply_index_range`]), which records its own stats, so this no longer
+/// (see `apply_index_range`), which records its own stats, so this no longer
 /// carries index-update timing or row counts.
 #[derive(Debug, Clone)]
 pub struct WalFlushResult {

From b239143700477a04dfe616471dfae7d9d1f96c38 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Thu, 16 Jul 2026 11:49:11 -0500
Subject: [PATCH 17/19] fix(mem_wal): keep the latched failure across a
 poisoned terminal_error lock
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`check_poisoned` and `mark_terminal_failure` both `.unwrap()`ed the
`terminal_error` lock, so a panic anywhere under it turned every later
poison check into a panic of its own.

Ignore the poisoning instead of reporting it. The guarded data cannot be
torn: the sole writer builds the `WalFlushFailure` and assigns it whole
under an `is_none()` check, so a panic mid-section leaves the slot exactly
as it was. There is no invariant here for poisoning to protect — the flag
is pure alarm.

Mapping the poisoning to an error would be worse than the panic. This
mutex exists to carry the reason a writer is fenced, and recovery is
reopen -> replay driven by that `FenceReason`; answering "mutex was
poisoned" buries it precisely when a caller needs it. It also has nowhere
to go in `mark_terminal_failure`, which returns `()`.

Test: latch a `PersistenceFailure`, poison the mutex from a panicking
thread, then assert the typed reason and message still surface and the
latch still keeps the first failure. It panics against the old code.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/wal.rs | 48 +++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index e371c1dbeed..8d72fa5eca5 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -9,6 +9,7 @@
 use std::io::Cursor;
 use std::sync::Arc;
 use std::sync::Mutex as StdMutex;
+use std::sync::MutexGuard as StdMutexGuard;
 use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
 use std::time::Instant;
 
@@ -167,8 +168,22 @@ impl WriterCursors {
         indexed_count.min(self.durable().saturating_sub(global_offset))
     }
 
+    /// Lock `terminal_error`, ignoring mutex poisoning.
+    ///
+    /// A panic under this lock cannot tear the `Option` it guards: the sole
+    /// writer builds the value first and assigns it whole, so a panic mid-section
+    /// leaves the slot exactly as it was. Surfacing poison as an error instead
+    /// would mask the latched `FenceReason` behind an unrelated "mutex poisoned"
+    /// precisely when a caller needs the real reason — and would strand
+    /// `mark_terminal_failure`, which has no error to return.
+    fn lock_terminal_error(&self) -> StdMutexGuard<'_, Option<WalFlushFailure>> {
+        self.terminal_error
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+    }
+
     pub(crate) fn check_poisoned(&self) -> Result<()> {
-        if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
+        if let Some(failure) = self.lock_terminal_error().clone() {
             return Err(failure.into_error());
         }
         Ok(())
@@ -176,7 +191,7 @@ impl WriterCursors {
 
     pub(crate) fn mark_terminal_failure(&self, error: &Error) {
         {
-            let mut slot = self.terminal_error.lock().unwrap();
+            let mut slot = self.lock_terminal_error();
             if slot.is_none() {
                 *slot = Some(WalFlushFailure::from_error(error));
             }
@@ -2432,4 +2447,33 @@ mod tests {
             .expect_err("watcher must surface the poison");
         assert_eq!(waited.fence_reason(), Some(FenceReason::PersistenceFailure));
     }
+
+    // A panic under the `terminal_error` lock must not cost the writer its
+    // latched failure. Recovery is reopen -> replay, which is driven by the real
+    // `FenceReason`; reporting the mutex poisoning instead would bury it.
+    #[tokio::test]
+    async fn test_poisoned_terminal_error_mutex_still_reports_typed_failure() {
+        let cursors = Arc::new(WriterCursors::new(true));
+        cursors.mark_terminal_failure(&Error::writer_poisoned("injected persistence failure"));
+
+        let terminal_error = Arc::clone(&cursors.terminal_error);
+        let panicked = std::thread::spawn(move || {
+            let _guard = terminal_error.lock().unwrap();
+            panic!("poison the terminal error mutex");
+        })
+        .join();
+        assert!(panicked.is_err());
+        assert!(cursors.terminal_error.is_poisoned());
+
+        let error = cursors.check_poisoned().unwrap_err();
+        assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(error.to_string().contains("injected persistence failure"));
+
+        // The latch still takes writes, and still keeps the first failure.
+        cursors.mark_terminal_failure(&Error::fenced_by_peer("later peer fence"));
+        assert_eq!(
+            cursors.check_poisoned().unwrap_err().fence_reason(),
+            Some(FenceReason::PersistenceFailure)
+        );
+    }
 }

From ea916e81c2c803b50b2e945e602ba1539e9ad37f Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Fri, 17 Jul 2026 11:21:52 -0500
Subject: [PATCH 18/19] fix(mem_wal): harden shard-open validation and freeze
 failure handling

Three fixes from PR review, each confirmed by reproducing the failure in a
test that fails on the pre-fix tree:

- open(): derive PK metadata and run the durable-write/flush-interval and
  index-config validation *before* claim_epoch. These checks ran after the
  epoch was claimed (and, for a successor, after the predecessor was fenced),
  so an open doomed by purely local input knocked the healthy incumbent off
  the shard with a PeerClaimedEpoch fence.

- validate_index_configs(): reject a config whose field_id names a different
  column than its `column`. Index selection keys off field_id alone (a
  single-column PK reuses the BTree whose field_id matches), so a mismatch
  bound the wrong index under a valid-looking name -- stale reads plus the
  wrong column flushed into the durable PK sidecar. Coupled with open()'s
  validation move because the signature and its only caller change together.

- freeze_memtable(): retain the outgoing memtable in the read view before the
  fallible index-apply/WAL-flush dispatches, and poison on a failed send. The
  active memtable was already replaced, so a failed dispatch dropped the table
  and its accepted rows silently vanished -- a zero-row scan with no error, on
  a branch whose whole point is to poison instead.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/index.rs |  46 ++++-
 rust/lance/src/dataset/mem_wal/write.rs | 256 +++++++++++++++++++-----
 2 files changed, 246 insertions(+), 56 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index a2690cd5082..fc2ae180c81 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -110,9 +110,17 @@ enum PkIndex {
 /// through `put`: `MemTable::insert_batches_only` does a full `Arc<Schema>`
 /// equality check, so a batch that would trip one is rejected before it reaches
 /// the batch store, let alone the WAL.
+///
+/// It also rejects a config whose `field_id` names a different column than its
+/// `column`. Index *selection* keys off `field_id` — a single-column PK reuses
+/// the BTree whose `field_id` matches its key — so a config resolved only by name
+/// could be bound under the wrong identity, serving stale reads and flushing the
+/// wrong column into the durable PK sidecar. `lance_schema` supplies the
+/// authoritative name→id mapping.
 pub fn validate_index_configs(
     configs: &[MemIndexConfig],
     schema: &ArrowSchema,
+    lance_schema: &LanceSchema,
     pk_columns: &[String],
 ) -> Result<()> {
     for config in configs {
@@ -185,6 +193,25 @@ pub fn validate_index_configs(
                 }
             },
         }
+
+        // The column resolves, but index selection keys off `field_id`, not name.
+        // A config whose `field_id` identifies a *different* column would be bound
+        // under the wrong identity (e.g. reused as the single-column PK index), so
+        // reject any `field_id` that does not name the resolved column.
+        let resolved_field_id = lance_schema
+            .field(column)
+            .expect("column resolved in the Arrow schema is present in the Lance schema")
+            .id;
+        if resolved_field_id != config.field_id() {
+            return Err(Error::invalid_input(format!(
+                "index '{}' is configured with field_id {} but its column '{}' has field_id {} \
+                 in the shard schema",
+                config.name(),
+                config.field_id(),
+                column,
+                resolved_field_id,
+            )));
+        }
     }
 
     // Every PK column must exist in the schema. A single-column PK aliases a
@@ -1677,6 +1704,10 @@ mod tests {
     #[case::btree_missing_column(MemIndexConfig::BTree(BTreeIndexConfig {
         name: "idx".into(), field_id: 9, column: "nope".into(),
     }), Some("not in the shard schema"))]
+    // Column exists, but its field_id names a *different* column ("id" is 0, not 1).
+    #[case::btree_field_id_column_mismatch(MemIndexConfig::BTree(BTreeIndexConfig {
+        name: "idx".into(), field_id: 1, column: "id".into(),
+    }), Some("has field_id 0"))]
     #[case::fts_ok(MemIndexConfig::Fts(FtsIndexConfig::new(
         "idx".into(), 1, "description".into(),
     )), None)]
@@ -1703,7 +1734,8 @@ mod tests {
         #[case] expected_error: Option<&str>,
     ) {
         let schema = vector_schema();
-        let result = validate_index_configs(&[config], &schema, &[]);
+        let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap();
+        let result = validate_index_configs(&[config], &schema, &lance_schema, &[]);
         match expected_error {
             None => result.expect("valid config must pass validation"),
             Some(fragment) => {
@@ -1732,25 +1764,27 @@ mod tests {
                 true,
             ),
         ]));
+        let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap();
 
-        validate_index_configs(&[], &schema, &["id".into(), "name".into()])
+        validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "name".into()])
             .expect("Int32 + Utf8 composite PK must be encodable");
 
-        let err = validate_index_configs(&[], &schema, &["id".into(), "coords".into()])
-            .expect_err("a FixedSizeList PK column has no order-preserving encoding");
+        let err =
+            validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "coords".into()])
+                .expect_err("a FixedSizeList PK column has no order-preserving encoding");
         assert!(
             err.to_string().contains("order-preserving key encoding"),
             "error must name the reason, got {err}"
         );
 
         // A single-column PK of the same type is fine: it aliases a BTree.
-        validate_index_configs(&[], &schema, &["coords".into()])
+        validate_index_configs(&[], &schema, &lance_schema, &["coords".into()])
             .expect("single-column PK aliases a BTree and accepts any type");
 
         // But every PK column must exist. A single-column PK naming an absent
         // column is rejected here, not left to fail deterministically on every
         // later index build and WAL replay.
-        let err = validate_index_configs(&[], &schema, &["missing".into()])
+        let err = validate_index_configs(&[], &schema, &lance_schema, &["missing".into()])
             .expect_err("a single-column PK on an absent column must be rejected");
         assert!(
             err.to_string().contains("not in the shard schema"),
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 0947da8159b..a26a09eb2c8 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -1209,34 +1209,33 @@ impl SharedWriterState {
         let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable);
         old_memtable.freeze(last_wal_entry_position);
 
+        // Set up completion tracking on the outgoing table before it is retained
+        // and before any fallible dispatch, so the retained table already carries
+        // its cells and a failed send below can poison-and-return without leaving
+        // partial state to unwind.
+        let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion();
+
         // The outgoing memtable may still owe an index apply — the puts that
         // filled it triggered one, but the task need not have drained yet, and
         // this is the last chance to name that store. Its L0 flush is gated on
         // the WAL append (below), not on indexing, so without this its tail could
         // stay unindexed and invisible for the rest of its life.
-        if let Some(old_indexes) = old_memtable.indexes_arc()
-            && old_indexes.indexed_count() < old_batch_store.len()
-        {
-            self.trigger_index_apply(old_batch_store.clone(), old_indexes, old_batch_store.len())?;
-        }
-        let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion();
+        let pending_index_apply = match old_memtable.indexes_arc() {
+            Some(old_indexes) if old_indexes.indexed_count() < old_batch_store.len() => {
+                Some((old_batch_store.clone(), old_indexes, old_batch_store.len()))
+            }
+            _ => None,
+        };
 
-        if pending_wal_range.is_some() {
+        let pending_wal_flush = if pending_wal_range.is_some() {
             let completion_cell: WatchableOnceCell<
                 std::result::Result<WalFlushResult, WalFlushFailure>,
             > = WatchableOnceCell::new();
-            let completion_reader = completion_cell.reader();
-            old_memtable.set_wal_flush_completion(completion_reader);
-
-            let end_batch_position = old_batch_store.len();
-            self.wal_flusher.trigger_flush(
-                WalFlushSource::BatchStore {
-                    batch_store: old_batch_store,
-                },
-                end_batch_position,
-                Some(completion_cell),
-            )?;
-        }
+            old_memtable.set_wal_flush_completion(completion_cell.reader());
+            Some((old_batch_store.len(), completion_cell))
+        } else {
+            None
+        };
 
         let frozen_size = old_memtable.estimated_size();
         state.frozen_memtable_bytes += frozen_size;
@@ -1250,14 +1249,38 @@ impl SharedWriterState {
 
         let frozen_memtable = Arc::new(old_memtable);
 
-        // Keep this generation queryable past its manifest commit (swept after
-        // the grace by `SweepExpired`). Arc refcount, not a copy — the flush
-        // task holds it alive for the whole drain anyway.
+        // Retain the outgoing table in the read view *before* the fallible
+        // dispatches below. `state.memtable` was already replaced, so a failed
+        // send that returned here without this push would drop the table and its
+        // accepted rows would silently vanish from every scan. Keep it queryable
+        // past its manifest commit too (swept after the grace by `SweepExpired`);
+        // Arc refcount, not a copy — the flush task holds it alive anyway.
         state.frozen_memtables.push_back(FrozenMemTable {
             memtable: frozen_memtable.clone(),
             flushed_at_ms: None,
         });
 
+        // Dispatch can only fail if a background task's channel is already closed,
+        // i.e. the writer is being torn down. Poison so the read path fails fast
+        // with the typed error instead of serving the retained-but-never-durable
+        // tail, then return — the table stays in the read view.
+        if let Some((batch_store, indexes, end_batch_position)) = pending_index_apply {
+            self.trigger_index_apply(batch_store, indexes, end_batch_position)
+                .inspect_err(|e| self.wal_flusher.poison(e))?;
+        }
+
+        if let Some((end_batch_position, completion_cell)) = pending_wal_flush {
+            self.wal_flusher
+                .trigger_flush(
+                    WalFlushSource::BatchStore {
+                        batch_store: old_batch_store,
+                    },
+                    end_batch_position,
+                    Some(completion_cell),
+                )
+                .inspect_err(|e| self.wal_flusher.poison(e))?;
+        }
+
         debug!(
             "Frozen memtable generation {}, pending_count = {}",
             next_generation - 1,
@@ -1496,6 +1519,47 @@ impl ShardWriter {
             config.manifest_scan_batch_size,
         ));
 
+        // Derive PK metadata and run every side-effect-free validation *before*
+        // claiming the epoch. `claim_epoch` durably bumps the stored epoch and,
+        // for a successor, `write_fence_sentinel` fences the predecessor — so an
+        // open doomed by purely local input (a bad flush interval, an index config
+        // that disagrees with the schema) must fail here, before it can knock the
+        // healthy incumbent off the shard. Memtable-only: WAL-only mode has no
+        // indexes and no memtable ticker to validate.
+        let memtable_validation = if config.enable_memtable {
+            let lance_schema = Schema::try_from(schema.as_ref())?;
+            let pk_fields = lance_schema.unenforced_primary_key();
+            let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
+            let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
+
+            // A durable writer with no flush ticker cannot make progress. The WAL
+            // append is timer-, size-, and freeze-driven now — the put path no
+            // longer triggers its own — so with no ticker a durable put below the
+            // size threshold waits on an append that nothing will ever make. Reject
+            // the combination rather than deadlock on it.
+            if config.durable_write
+                && config
+                    .max_wal_flush_interval
+                    .is_none_or(|interval| interval.is_zero())
+            {
+                return Err(Error::invalid_input(
+                    "durable_write requires a non-zero max_wal_flush_interval: the WAL append is \
+                     driven by that ticker, so without it a durable put would wait forever for an \
+                     append that never happens",
+                ));
+            }
+
+            // Reject an index config that disagrees with the schema *before* a
+            // single row is accepted. Such a config fails deterministically on
+            // every insert, including inserts replayed from the WAL — so once a row
+            // is durable the shard can never reopen. Fail the open instead.
+            validate_index_configs(&index_configs, schema.as_ref(), &lance_schema, &pk_columns)?;
+
+            Some((pk_field_ids, pk_columns))
+        } else {
+            None
+        };
+
         // Claim the shard (epoch-based fencing) — done once, then shared
         // with the WalAppender via `with_claimed_epoch`.
         let (epoch, manifest) = manifest_store.claim_epoch(config.shard_spec_id).await?;
@@ -1552,11 +1616,15 @@ impl ShardWriter {
         let task_executor = Arc::new(TaskExecutor::new());
 
         let mode = if config.enable_memtable {
+            let (pk_field_ids, pk_columns) = memtable_validation
+                .expect("memtable_validation is Some when enable_memtable is true");
             Self::open_memtable_mode(
                 &config,
                 &schema,
                 &manifest,
                 &index_configs,
+                pk_field_ids,
+                pk_columns,
                 wal_flusher.clone(),
                 wal_flush_tx,
                 wal_flush_rx,
@@ -1598,6 +1666,8 @@ impl ShardWriter {
         schema: &Arc<ArrowSchema>,
         manifest: &ShardManifest,
         index_configs: &[MemIndexConfig],
+        pk_field_ids: Vec<i32>,
+        pk_columns: Vec<String>,
         wal_flusher: Arc<WalFlusher>,
         wal_flush_tx: mpsc::UnboundedSender<TriggerWalFlush>,
         wal_flush_rx: mpsc::UnboundedReceiver<TriggerWalFlush>,
@@ -1610,34 +1680,9 @@ impl ShardWriter {
         stats: SharedWriteStats,
         task_executor: &Arc<TaskExecutor>,
     ) -> Result<WriterMode> {
-        // Create MemTable with primary key field IDs from schema
-        let lance_schema = Schema::try_from(schema.as_ref())?;
-        let pk_fields = lance_schema.unenforced_primary_key();
-        let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
-        let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
-
-        // A durable writer with no flush ticker cannot make progress. The WAL
-        // append is timer-, size-, and freeze-driven now — the put path no longer
-        // triggers its own — so with no ticker a durable put below the size
-        // threshold waits on an append that nothing will ever make. Reject the
-        // combination rather than deadlock on it.
-        if config.durable_write
-            && config
-                .max_wal_flush_interval
-                .is_none_or(|interval| interval.is_zero())
-        {
-            return Err(Error::invalid_input(
-                "durable_write requires a non-zero max_wal_flush_interval: the WAL append is \
-                 driven by that ticker, so without it a durable put would wait forever for an \
-                 append that never happens",
-            ));
-        }
-
-        // Reject an index config that disagrees with the schema *before* a
-        // single row is accepted. Such a config fails deterministically on every
-        // insert, including inserts replayed from the WAL — so once a row is
-        // durable the shard can never reopen. Fail the open instead.
-        validate_index_configs(index_configs, schema.as_ref(), &pk_columns)?;
+        // PK metadata and index/interval validation were resolved in `open`
+        // before the epoch was claimed (a doomed open must not fence the
+        // incumbent first).
 
         // Build a fresh, cursor-bound memtable at a given generation and
         // writer-global coordinate. Replay calls this for the first memtable and
@@ -5279,6 +5324,117 @@ mod tests {
             .unwrap();
     }
 
+    /// A doomed open must fail on local validation *before* it claims the epoch,
+    /// so it cannot fence the healthy writer already serving the shard. The
+    /// durable-write/flush-interval and index-config checks used to run *after*
+    /// `claim_epoch` (and, for a successor, after `write_fence_sentinel`), so a
+    /// rejected open still bumped the stored epoch and fenced the incumbent.
+    #[tokio::test]
+    async fn test_doomed_open_does_not_fence_incumbent() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        let writer_a = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 0, 1)])
+            .await
+            .unwrap();
+
+        // A durable writer with no flush interval is rejected — the put path no
+        // longer triggers its own append, so it would deadlock. On the old path
+        // this rejection landed only after the epoch had already been claimed.
+        let doomed_config = ShardWriterConfig {
+            max_wal_flush_interval: None,
+            ..memtable_config_with_pk(shard_id)
+        };
+        let err = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            doomed_config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .map(|_| ())
+        .expect_err("durable_write with no flush interval must be rejected");
+        assert!(
+            err.to_string().contains("max_wal_flush_interval"),
+            "unexpected error: {err}"
+        );
+
+        // The incumbent is untouched: not fenced, still accepting writes.
+        writer_a.check_fenced().await.unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 1, 1)])
+            .await
+            .unwrap();
+        writer_a.close().await.unwrap();
+    }
+
+    /// A failed dispatch during `freeze_memtable` must not drop the outgoing
+    /// table's rows from the read view. The active memtable is replaced before
+    /// the WAL-flush and index-apply sends; a send that failed (background tasks
+    /// gone) used to return before the outgoing table was retained in
+    /// `frozen_memtables`, so its accepted rows silently vanished — a scan
+    /// returned 0 rows with no error. The writer must instead retain the table
+    /// and poison, so reads fail fast rather than serve a divergent snapshot.
+    #[tokio::test]
+    async fn test_freeze_dispatch_failure_retains_rows_and_poisons() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+
+        // Non-durable + no ticker: the put is read-your-writes (waits for its
+        // index apply) but nothing is WAL-flushed, so the freeze below still owes
+        // a WAL append.
+        let config = ShardWriterConfig {
+            durable_write: false,
+            max_wal_flush_interval: None,
+            ..memtable_config_with_pk(shard_id)
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        assert_eq!(writer.memtable_stats().await.unwrap().row_count, 10);
+
+        // Tear the background tasks down out from under the writer, so the
+        // freeze's dispatch sends hit closed channels.
+        writer.abort().await.unwrap();
+
+        let err = writer
+            .force_seal_active()
+            .await
+            .expect_err("force_seal_active must surface the failed dispatch");
+        assert!(
+            err.to_string().contains("channel closed"),
+            "unexpected error: {err}"
+        );
+
+        // The failure poisoned the writer: reads fail fast instead of returning a
+        // silent zero-row snapshot of a shard whose rows were dropped.
+        assert!(
+            writer.scan().await.is_err(),
+            "a poisoned writer must reject reads, not serve a divergent snapshot"
+        );
+        assert!(writer.in_memory_memtable_refs().await.is_err());
+    }
+
     /// A WAL holding more batches than one memtable's capacity must reopen.
     ///
     /// One memtable holds at most `max_memtable_batches` batches, but a WAL is

From a7caba3383285bce2eb5b3934c0a910af710af89 Mon Sep 17 00:00:00 2001
From: Daniel Rammer <hamersaw@protonmail.com>
Date: Fri, 17 Jul 2026 12:10:55 -0500
Subject: [PATCH 19/19] fix(mem_wal): reject a batch position past
 committed_len instead of skipping it

`apply_index_range` and `flush_from_batch_store` walked `[start, end)` and
silently dropped any position `BatchStore::get` returned `None` for. The store
is append-only (`get` returns `None` only past `committed_len`), so a hole means
the caller asked to cover a batch that was never committed. Skipping it while
still advancing the cursor was silent corruption: the WAL path advanced
durability to `end_batch_position` even though a batch was never appended (lost
on replay), and the index path advanced `indexed_count` past a never-indexed
batch (rows counted visible but absent from every index).

Both now error on a missing position, naming the range and committed length. The
flush path returns a terminal (writer_poisoned) error so `flush` poisons the
writer before durability moves; the index path's error poisons via its handler.
Reopen replays the WAL. Holes are impossible today, but this fails loudly the
day eviction lands rather than diverging silently.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 rust/lance/src/dataset/mem_wal/wal.rs | 114 ++++++++++++++++++++++----
 1 file changed, 99 insertions(+), 15 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 8d72fa5eca5..4082cd7ece9 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -413,12 +413,23 @@ pub async fn apply_index_range(
         return Ok(IndexApplyStats::default());
     }
 
+    // Every position in the range must exist. `get` returns `None` only for a
+    // position past `committed_len`, so a hole means the caller asked to index a
+    // batch the store never committed. Silently skipping it would let
+    // `insert_batches` advance `indexed_count` past a never-indexed batch —
+    // rows counted visible but absent from every index. Fail loudly; the handler
+    // poisons and reopen rebuilds the indexes from the WAL.
     let stored: Vec<StoredBatch> = (start..end_batch_position)
-        .filter_map(|position| batch_store.get(position).cloned())
-        .collect();
-    if stored.is_empty() {
-        return Ok(IndexApplyStats::default());
-    }
+        .map(|position| {
+            batch_store.get(position).cloned().ok_or_else(|| {
+                Error::internal(format!(
+                    "index apply range [{start}, {end_batch_position}) is missing batch \
+                     position {position}; batch_store committed_len is {}",
+                    batch_store.len()
+                ))
+            })
+        })
+        .collect::<Result<_>>()?;
 
     // `insert_batches` advances `indexed_count` itself, once every index has
     // taken the batch. Time the whole hand-off so the recorded latency reflects
@@ -821,16 +832,24 @@ impl WalFlusher {
             return Ok(empty_flush_result());
         }
 
-        let mut stored_batches: Vec<StoredBatch> =
-            Vec::with_capacity(end_batch_position - start_batch_position);
-        for batch_position in start_batch_position..end_batch_position {
-            if let Some(stored) = batch_store.get(batch_position) {
-                stored_batches.push(stored.clone());
-            }
-        }
-        if stored_batches.is_empty() {
-            return Ok(empty_flush_result());
-        }
+        // Every position in the range must exist. `get` returns `None` only for a
+        // position past `committed_len`, so a hole means we were asked to append a
+        // batch the store never committed. Silently skipping it while still
+        // advancing durability to `end_batch_position` (below) would mark an
+        // un-appended batch durable and lose it on replay — a divergence that
+        // survives the crash and hides from a full scan. Return a terminal error
+        // so `flush` poisons the writer; reopen replays the WAL.
+        let stored_batches: Vec<StoredBatch> = (start_batch_position..end_batch_position)
+            .map(|batch_position| {
+                batch_store.get(batch_position).cloned().ok_or_else(|| {
+                    Error::writer_poisoned(format!(
+                        "WAL flush range [{start_batch_position}, {end_batch_position}) is \
+                         missing batch position {batch_position}; batch_store committed_len is {}",
+                        batch_store.len()
+                    ))
+                })
+            })
+            .collect::<Result<_>>()?;
 
         let record_batches: Vec<RecordBatch> =
             stored_batches.iter().map(|s| s.data.clone()).collect();
@@ -1920,6 +1939,71 @@ mod tests {
         assert_eq!(indexes.indexed_count(), 3);
     }
 
+    /// A WAL flush asked to cover a range past the store's committed length must
+    /// fail terminally, not silently short-append. The store is append-only, so
+    /// `get` returns `None` only past `committed_len`; skipping that position
+    /// while still advancing durability to `end_batch_position` would mark an
+    /// un-appended batch durable and lose it on replay. The flush must poison
+    /// instead, and durability must not move.
+    #[tokio::test]
+    async fn test_flush_rejects_range_past_committed_len() {
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let flusher = build_test_flusher(store, &base_path, shard_id, 1);
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+
+        // Two batches committed (positions 0, 1); ask to flush through position 2.
+        let err = flusher
+            .flush(&batch_store_source(&batch_store), batch_store.len() + 1)
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(
+            err.to_string().contains("missing batch position 2"),
+            "unexpected error: {err}"
+        );
+
+        // Terminal: the flusher is poisoned and durability never advanced past
+        // what was actually appended.
+        assert!(flusher.check_poisoned().is_err());
+        assert_eq!(flusher.durable(), 0);
+    }
+
+    /// The index-apply path has the same invariant: a range past the committed
+    /// length must error rather than silently under-index and advance the cursor
+    /// past a batch that was never inserted.
+    #[tokio::test]
+    async fn test_index_apply_rejects_range_past_committed_len() {
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 5)).unwrap();
+
+        let indexes = Arc::new(IndexStore::new());
+        let cursors = Arc::new(WriterCursors::new(true));
+
+        // One batch committed (position 0); ask to index through position 1.
+        let err = apply_index_range(
+            &cursors,
+            TriggerIndexApply {
+                batch_store: batch_store.clone(),
+                indexes: indexes.clone(),
+                end_batch_position: batch_store.len() + 1,
+            },
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            err.to_string().contains("missing batch position 1"),
+            "unexpected error: {err}"
+        );
+        // The cursor did not advance past the hole.
+        assert_eq!(indexes.indexed_count(), 0);
+    }
+
     #[tokio::test]
     async fn test_wal_entry_read() {
         let (store, base_path, _temp_dir) = create_local_store().await;