From 1c65d26dafb61f1c8c1d96e097dec2a29902be46 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 8 Jul 2026 17:49:22 +0800
Subject: [PATCH 01/19] feat: add code analyzer for FTS

---
 protos/index_old.proto                        |  30 ++
 python/python/tests/test_scalar_index.py      | 246 +++++++++
 python/src/dataset.rs                         | 150 ++++--
 rust/lance-index/src/scalar/inverted/index.rs | 307 +++++++++--
 .../src/scalar/inverted/tokenizer.rs          | 482 ++++++++++++++++--
 rust/lance-tokenizer/src/code_tokenizer.rs    | 140 +++++
 rust/lance-tokenizer/src/lib.rs               |   4 +
 .../src/word_delimiter_filter.rs              | 279 ++++++++++
 rust/lance/src/dataset/mem_wal/index.rs       |  13 +-
 rust/lance/src/dataset/mem_wal/index/fts.rs   | 148 +++++-
 rust/lance/src/index/scalar/inverted.rs       |  44 +-
 rust/lance/src/io/exec/fts.rs                 |   5 +-
 12 files changed, 1684 insertions(+), 164 deletions(-)
 create mode 100644 rust/lance-tokenizer/src/code_tokenizer.rs
 create mode 100644 rust/lance-tokenizer/src/word_delimiter_filter.rs

diff --git a/protos/index_old.proto b/protos/index_old.proto
index 601aa2681da..d9024183b55 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -26,6 +26,19 @@ message LabelListIndexDetails {}
 message NGramIndexDetails {}
 message ZoneMapIndexDetails {}
 message InvertedIndexDetails {
+  // High-level analyzer profile. It selects the semantic defaults for the FTS
+  // pipeline, e.g. "text" for natural-language text or "code" for code search.
+  // Absent is a legacy "text" profile unless base_tokenizer is the legacy
+  // "code" alias. Readers must canonicalize this before comparing details.
+  optional string analyzer = 12;
+  // Document-level tokenizer. It extracts searchable text from the stored value,
+  // e.g. "text" for a plain string or "json" for JSON content. The extracted
+  // text is then passed to base_tokenizer. Absent means text / auto-inferred
+  // legacy behavior.
+  optional string lance_tokenizer = 13;
+  // Lexical tokenizer used after the analyzer/document layer has selected the
+  // text to tokenize. This is an implementation component such as "simple",
+  // "icu", "ngram", or "code"; analyzer="code" owns the code-search defaults.
   // Marking this field as optional as old versions of the index store blank details and we
   // need to make sure we have a proper optional field to detect this.
   optional string base_tokenizer = 1;
@@ -39,4 +52,21 @@ message InvertedIndexDetails {
   uint32 min_ngram_length = 9;
   uint32 max_ngram_length = 10;
   bool prefix_only = 11;
+  // Code analyzer flag. Split one lexical identifier into subwords, e.g.
+  // getUserName -> get/user/name. This changes the indexed term space and must
+  // be persisted for query-time tokenizer reconstruction and segment checks.
+  bool split_identifiers = 14;
+  // Code analyzer flag. Split identifier subwords across letter/number
+  // boundaries, e.g. HTML2JSON -> html/2/json. This changes the indexed term
+  // space and must be persisted with the index.
+  bool split_on_numerics = 15;
+  // Code analyzer flag. Keep the complete lexical identifier in addition to
+  // subwords, e.g. user_name plus user/name. This affects BM25 term statistics
+  // and must be persisted with the index.
+  bool preserve_original = 16;
+  // Code analyzer flag. Index operator tokens such as "::", "->", and "!=".
+  // Disabled by default because operators are often high-frequency noise, but
+  // when enabled it changes query semantics and must be persisted.
+  bool index_operators = 17;
+  repeated string custom_stop_words = 19;
 }
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index a2e35e6749f..52af81bc7f7 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -21,6 +21,7 @@
 from lance.query import (
     BooleanQuery,
     BoostQuery,
+    FullTextOperator,
     MatchQuery,
     MultiMatchQuery,
     Occur,
@@ -806,6 +807,239 @@ def test_full_text_search(dataset, with_position, base_tokenizer):
         )
 
 
+def test_code_analyzer_full_text_search(tmp_path):
+    table = pa.table(
+        {
+            "code": [
+                "getUserName",
+                "set_user_name",
+                "user-name",
+                "username",
+                "other",
+            ]
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+
+    results = ds.to_table(
+        columns=["code"],
+        full_text_query=MatchQuery("user", "code"),
+    )
+    assert set(results["code"].to_pylist()) == {
+        "getUserName",
+        "set_user_name",
+        "user-name",
+    }
+
+    stats = ds.stats.index_stats("code_idx")["indices"][0]
+    params = stats["params"]
+    assert params["analyzer"] == "code"
+    assert params["base_tokenizer"] == "code"
+    assert params["split_identifiers"] is True
+    assert params["split_on_numerics"] is True
+    assert params["preserve_original"] is True
+    assert params["stem"] is False
+    assert params["remove_stop_words"] is False
+
+
+def test_code_analyzer_complex_code_constructs(tmp_path):
+    table = pa.table(
+        {
+            "path": [
+                "edge/trait.rs",
+                "edge/impl.rs",
+                "edge/fn_pointer.rs",
+                "edge/unit_result.rs",
+                "edge/hrtb.rs",
+                "edge/associated.rs",
+                "edge/operators.rs",
+            ],
+            "code": [
+                """
+pub trait EdgeAsyncRepository<'a, T: Send + Sync>
+where
+    T: TryFrom<&'a str, Error = EdgeParseError>,
+{
+    type Output<'b>: Iterator<Item = Result<T, EdgeRepoError>>
+    where
+        Self: 'b;
+
+    async fn fetch_by_key<const N: usize>(
+        &'a self,
+        key: [u8; N],
+    ) -> Result<Self::Output<'a>, EdgeRepoError>;
+}
+""",
+                """
+impl<'a, T, S> EdgeAsyncRepository<'a, T> for EdgeStore<S>
+where
+    T: TryFrom<&'a str, Error = EdgeParseError> + Clone + Send + Sync,
+    S: EdgeBackend<T> + ?Sized,
+{
+    type Output<'b> = std::vec::IntoIter<Result<T, EdgeRepoError>> where Self: 'b;
+
+    async fn fetch_by_key<const N: usize>(
+        &'a self,
+        key: [u8; N],
+    ) -> Result<Self::Output<'a>, EdgeRepoError> {
+        self.backend.fetch::<T, N>(key).await
+    }
+}
+""",
+                """
+pub fn build_edge_handler<T, F>(
+    factory: F,
+) -> impl Fn() -> Result<EdgeHandler<T>, EdgeError>
+where
+    F: FnOnce() -> Result<T, EdgeError> + Send + 'static,
+    T: Default + Send + Sync + 'static,
+{
+    move || factory().map(EdgeHandler::new)
+}
+""",
+                """
+pub fn edge_unit_result_callback() -> Result<()> {
+    Ok(())
+}
+""",
+                """
+pub fn edge_higher_ranked<'a, T>(
+    visitor: impl for<'b> Fn(&'b T) -> Result<&'b str, EdgeVisitError>,
+    value: &'a T,
+) -> Result<&'a str, EdgeVisitError> {
+    visitor(value)
+}
+""",
+                """
+pub fn edge_collect_stream<I, E>(items: I) -> Result<Vec<E::Item>, E::Error>
+where
+    I: IntoIterator<Item = E>,
+    E: EdgeExtract,
+{
+    items.into_iter().map(E::extract).collect()
+}
+""",
+                """
+pub fn edge_operator_arrow() -> Result<EdgeArrow, EdgeError> {
+    let variant = EdgeModule::EdgeVariant;
+    if variant != EdgeModule::Default && EdgeMask::enabled() {
+        return Ok(EdgeArrow::new(variant));
+    }
+    Err(EdgeError::empty())
+}
+""",
+            ],
+        }
+    )
+    table = table.append_column("code_ops", table["code"])
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+    ds.create_scalar_index(
+        "code_ops",
+        index_type="INVERTED",
+        analyzer="code",
+        index_operators=True,
+    )
+
+    ds.insert(
+        pa.table(
+            {
+                "path": ["edge/flat_unindexed.rs"],
+                "code": [
+                    """
+pub async fn edge_flat_generic_return<T, E>() -> Result<T, EdgeFlatError>
+where
+    T: TryFrom<String, Error = E> + Send,
+    E: Into<EdgeFlatError>,
+{
+    T::try_from(String::new()).map_err(Into::into)
+}
+"""
+                ],
+                "code_ops": [
+                    """
+pub async fn edge_flat_operator() -> Result<EdgeFlat, EdgeFlatError> {
+    EdgeFlat::try_new() -> Result<EdgeFlat, EdgeFlatError>
+}
+"""
+                ],
+            }
+        )
+    )
+    ds = lance.dataset(tmp_path)
+
+    def assert_search(column, query, expected_path, operator=FullTextOperator.AND):
+        result = ds.scanner(
+            columns=["path", "_score"],
+            full_text_query=MatchQuery(query, column, operator=operator),
+            limit=50,
+        ).to_table()
+        assert expected_path in result["path"].to_pylist()
+
+    assert_search(
+        "code",
+        "EdgeAsyncRepository fetch_by_key TryFrom EdgeRepoError",
+        "edge/trait.rs",
+    )
+    assert_search(
+        "code",
+        "EdgeStore fetch_by_key const usize where Result",
+        "edge/impl.rs",
+    )
+    assert_search(
+        "code",
+        "build_edge_handler FnOnce Result EdgeHandler",
+        "edge/fn_pointer.rs",
+    )
+    assert_search(
+        "code",
+        "edge_unit_result_callback fn () -> Result",
+        "edge/unit_result.rs",
+    )
+    assert_search(
+        "code",
+        "edge_higher_ranked for Fn EdgeVisitError Result",
+        "edge/hrtb.rs",
+    )
+    assert_search(
+        "code",
+        "edge_collect_stream IntoIterator Item Error Result",
+        "edge/associated.rs",
+    )
+    assert_search(
+        "code",
+        "edge_flat_generic_return TryFrom EdgeFlatError Result",
+        "edge/flat_unindexed.rs",
+    )
+    assert_search(
+        "code_ops",
+        "edge_operator_arrow -> Result",
+        "edge/operators.rs",
+    )
+    assert_search(
+        "code_ops",
+        "EdgeModule :: EdgeVariant !=",
+        "edge/operators.rs",
+    )
+    assert_search(
+        "code_ops",
+        "edge_flat_operator -> Result EdgeFlatError",
+        "edge/flat_unindexed.rs",
+    )
+
+    default_operator_results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery("->", "code", operator=FullTextOperator.OR),
+    ).to_table()
+    operator_results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery("->", "code_ops", operator=FullTextOperator.OR),
+    ).to_table()
+    assert default_operator_results.num_rows == 0
+    assert operator_results.num_rows > 0
+
+
 def test_unindexed_full_text_search_on_empty_index(tmp_path):
     # Create fts index on empty table.
     schema = pa.schema({"text": pa.string()})
@@ -871,6 +1105,18 @@ def test_fts_custom_stop_words(tmp_path):
     assert len(results["_rowid"].to_pylist()) == 1
 
 
+def test_fts_custom_stop_words_none(tmp_path):
+    data = pa.table({"text": ["alpha beta", "gamma"]})
+    ds = lance.write_dataset(data, tmp_path)
+    ds.create_scalar_index("text", "INVERTED", custom_stop_words=None)
+
+    results = ds.to_table(full_text_query="alpha")
+    assert results.num_rows == 1
+
+    stats = ds.stats.index_stats("text_idx")["indices"][0]
+    assert stats["params"]["custom_stop_words"] is None
+
+
 def test_fts_stop_words_respect_language_for_simple_tokenizer(tmp_path):
     data = pa.table({"text": ["the lance data", "的 lance data"]})
     ds = lance.write_dataset(data, tmp_path, mode="overwrite")
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index a7e8b52fb1f..3442071e965 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -78,7 +78,6 @@ use lance_index::{
     FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
     progress::{IndexBuildProgress, NoopIndexBuildProgress},
-    scalar::inverted::InvertedListFormatVersion,
     scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams},
     vector::{
         ApproxMode, DEFAULT_QUERY_PARALLELISM, Query as VectorQuery,
@@ -2402,75 +2401,130 @@ impl Dataset {
                 })
             }
             "INVERTED" | "FTS" => {
-                let mut params = InvertedIndexParams::default();
+                let mut params_json = serde_json::Map::new();
                 if let Some(kwargs) = kwargs {
-                    if let Some(with_position) = kwargs.get_item("with_position")? {
-                        params = params.with_position(with_position.extract()?);
+                    let allowed_kwargs = [
+                        "analyzer",
+                        "with_position",
+                        "base_tokenizer",
+                        "language",
+                        "max_token_length",
+                        "lower_case",
+                        "stem",
+                        "remove_stop_words",
+                        "custom_stop_words",
+                        "ascii_folding",
+                        "min_ngram_length",
+                        "max_ngram_length",
+                        "prefix_only",
+                        "split_identifiers",
+                        "split_on_numerics",
+                        "preserve_original",
+                        "index_operators",
+                        "memory_limit",
+                        "num_workers",
+                        "format_version",
+                        "fragment_ids",
+                        "index_uuid",
+                        "progress_callback",
+                    ];
+                    for (key, _) in kwargs.iter() {
+                        let key: String = key.extract()?;
+                        if !allowed_kwargs.contains(&key.as_str()) {
+                            return Err(PyValueError::new_err(format!(
+                                "unknown FTS index parameter '{}'",
+                                key
+                            )));
+                        }
+                    }
+                    macro_rules! insert_param {
+                        ($name:literal, $ty:ty) => {
+                            if let Some(value) = kwargs.get_item($name)? {
+                                let value: $ty = value.extract()?;
+                                params_json.insert(
+                                    $name.to_string(),
+                                    serde_json::to_value(value).map_err(|err| {
+                                        PyValueError::new_err(format!(
+                                            "failed to serialize FTS parameter {}: {}",
+                                            $name, err
+                                        ))
+                                    })?,
+                                );
+                            }
+                        };
                     }
-                    if let Some(base_tokenizer) = kwargs.get_item("base_tokenizer")? {
-                        params = params.base_tokenizer(base_tokenizer.extract()?);
+                    insert_param!("analyzer", String);
+                    if let Some(with_position) = kwargs.get_item("with_position")? {
+                        params_json.insert(
+                            "with_position".to_string(),
+                            serde_json::Value::Bool(with_position.extract()?),
+                        );
                     }
+                    insert_param!("base_tokenizer", String);
                     if let Some(language) = kwargs.get_item("language")? {
                         let language: PyBackedStr =
                             language.cast::<PyString>()?.clone().try_into()?;
-                        params = params.language(&language).map_err(|e| {
-                            PyValueError::new_err(format!(
-                                "can't set tokenizer language to {}: {:?}",
-                                language, e
-                            ))
-                        })?;
+                        params_json.insert(
+                            "language".to_string(),
+                            serde_json::Value::String(language.to_string()),
+                        );
                     }
                     if let Some(max_token_length) = kwargs.get_item("max_token_length")? {
-                        params = params.max_token_length(max_token_length.extract()?);
+                        let max_token_length: Option<usize> = max_token_length.extract()?;
+                        params_json.insert(
+                            "max_token_length".to_string(),
+                            serde_json::to_value(max_token_length).map_err(|err| {
+                                PyValueError::new_err(format!(
+                                    "failed to serialize FTS parameter max_token_length: {}",
+                                    err
+                                ))
+                            })?,
+                        );
                     }
-                    if let Some(lower_case) = kwargs.get_item("lower_case")? {
-                        params = params.lower_case(lower_case.extract()?);
-                    }
-                    if let Some(stem) = kwargs.get_item("stem")? {
-                        params = params.stem(stem.extract()?);
-                    }
-                    if let Some(remove_stop_words) = kwargs.get_item("remove_stop_words")? {
-                        params = params.remove_stop_words(remove_stop_words.extract()?);
-                    }
-                    if let Some(stop_words_file) = kwargs.get_item("custom_stop_words")? {
-                        params = params.custom_stop_words(stop_words_file.extract()?);
-                    }
-                    if let Some(ascii_folding) = kwargs.get_item("ascii_folding")? {
-                        params = params.ascii_folding(ascii_folding.extract()?);
-                    }
-                    if let Some(min_ngram_length) = kwargs.get_item("min_ngram_length")? {
-                        params = params.ngram_min_length(min_ngram_length.extract()?);
-                    }
-                    if let Some(max_ngram_length) = kwargs.get_item("max_ngram_length")? {
-                        params = params.ngram_max_length(max_ngram_length.extract()?);
-                    }
-                    if let Some(prefix_only) = kwargs.get_item("prefix_only")? {
-                        params = params.ngram_prefix_only(prefix_only.extract()?);
-                    }
-                    if let Some(memory_limit) = kwargs.get_item("memory_limit")? {
-                        params = params.memory_limit_mb(memory_limit.extract()?);
-                    }
-                    if let Some(num_workers) = kwargs.get_item("num_workers")? {
-                        params = params.num_workers(num_workers.extract()?);
+                    insert_param!("lower_case", bool);
+                    insert_param!("stem", bool);
+                    insert_param!("remove_stop_words", bool);
+                    if let Some(custom_stop_words) = kwargs.get_item("custom_stop_words")? {
+                        let custom_stop_words: Option<Vec<String>> =
+                            custom_stop_words.extract()?;
+                        params_json.insert(
+                            "custom_stop_words".to_string(),
+                            serde_json::to_value(custom_stop_words).map_err(|err| {
+                                PyValueError::new_err(format!(
+                                    "failed to serialize FTS parameter custom_stop_words: {}",
+                                    err
+                                ))
+                            })?,
+                        );
                     }
+                    insert_param!("ascii_folding", bool);
+                    insert_param!("min_ngram_length", u32);
+                    insert_param!("max_ngram_length", u32);
+                    insert_param!("prefix_only", bool);
+                    insert_param!("split_identifiers", bool);
+                    insert_param!("split_on_numerics", bool);
+                    insert_param!("preserve_original", bool);
+                    insert_param!("index_operators", bool);
+                    insert_param!("memory_limit", u64);
+                    insert_param!("num_workers", usize);
                     if let Some(format_version) = kwargs.get_item("format_version")?
                         && !format_version.is_none()
                     {
                         let value = if let Ok(value) = format_version.cast::<PyString>() {
-                            value.to_string_lossy().to_string()
+                            serde_json::Value::String(value.to_string_lossy().to_string())
                         } else if let Ok(value) = format_version.extract::<u32>() {
-                            value.to_string()
+                            serde_json::Value::from(value)
                         } else {
                             return Err(PyValueError::new_err(
                                 "format_version must be 1, 2, 'v1', or 'v2'",
                             ));
                         };
-                        let format_version = value
-                            .parse::<InvertedListFormatVersion>()
-                            .map_err(|err| PyValueError::new_err(err.to_string()))?;
-                        params = params.format_version(format_version);
+                        params_json.insert("format_version".to_string(), value);
                     }
                 }
+                let params: InvertedIndexParams =
+                    serde_json::from_value(serde_json::Value::Object(params_json))
+                        .map_err(|err| PyValueError::new_err(err.to_string()))?;
                 Box::new(params)
             }
             _ => {
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index e1aab1c2843..0fff5b15c38 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -1758,7 +1758,6 @@ impl InvertedPartition {
         operator: Operator,
         metrics: &dyn MetricsCollector,
     ) -> Result<LoadedPostings> {
-        let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0);
         let is_phrase_query = params.phrase_slop.is_some();
         let is_and_query = operator == Operator::And;
         let required_positions = (is_and_query || is_phrase_query).then(|| {
@@ -1784,9 +1783,6 @@ impl InvertedPartition {
                     matched_positions.insert(position);
                 }
                 token_ids.push((token_id, token, position));
-            } else if is_phrase_query || is_and_query {
-                // if the token is not found, we can't do phrase or AND query
-                return Ok(LoadedPostings::empty());
             }
         }
         if token_ids.is_empty() {
@@ -1799,16 +1795,8 @@ impl InvertedPartition {
             return Ok(LoadedPostings::empty());
         }
 
-        let is_fuzzy_and_query = is_fuzzy && is_and_query && !is_phrase_query;
-        if !is_phrase_query {
-            if is_fuzzy_and_query {
-                token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id));
-                token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2);
-            } else {
-                token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id);
-                token_ids.dedup_by_key(|(token_id, _, _)| *token_id);
-            }
-        }
+        token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id));
+        token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2);
 
         let num_docs = self.docs.len();
         let loaded_postings = stream::iter(token_ids)
@@ -1824,35 +1812,6 @@ impl InvertedPartition {
             .try_collect::<Vec<_>>()
             .await?;
 
-        if (is_and_query || is_phrase_query)
-            && !is_fuzzy_and_query
-            && loaded_postings
-                .iter()
-                .any(|(_, _, _, posting)| posting.is_empty())
-        {
-            return Ok(LoadedPostings::empty());
-        }
-
-        if !is_fuzzy_and_query {
-            return Ok(LoadedPostings {
-                postings: loaded_postings
-                    .into_iter()
-                    .map(|(token_id, token, position, posting)| {
-                        let query_weight = idf(posting.len(), num_docs);
-                        PostingIterator::with_query_weight(
-                            token,
-                            token_id,
-                            position,
-                            query_weight,
-                            posting,
-                            num_docs,
-                        )
-                    })
-                    .collect(),
-                grouped_expansions: Vec::new(),
-            });
-        }
-
         let needs_union = loaded_postings
             .windows(2)
             .any(|window| window[0].2 == window[1].2);
@@ -1880,6 +1839,10 @@ impl InvertedPartition {
             } else {
                 let token_id = group[0].0;
                 let token = group[0].1.clone();
+                let query_weight = group
+                    .iter()
+                    .map(|(_, _, posting)| idf(posting.len(), num_docs))
+                    .sum::<f32>();
                 grouped_expansions.push(GroupedExpansionTerms {
                     position,
                     terms: group
@@ -1895,12 +1858,26 @@ impl InvertedPartition {
                     postings,
                     docs_for_union
                         .as_deref()
-                        .expect("union docs must be loaded for grouped fuzzy AND"),
+                        .expect("union docs must be loaded for grouped query terms"),
                 )?;
-                (token_id, token, posting)
+                if posting.is_empty() && (is_and_query || is_phrase_query) {
+                    return Ok(LoadedPostings::empty());
+                }
+                grouped_postings.push(PostingIterator::with_query_weight(
+                    token,
+                    token_id,
+                    position,
+                    query_weight,
+                    posting,
+                    num_docs,
+                ));
+                continue;
             };
             if posting.is_empty() {
-                return Ok(LoadedPostings::empty());
+                if is_and_query || is_phrase_query {
+                    return Ok(LoadedPostings::empty());
+                }
+                continue;
             }
 
             let query_weight = idf(posting.len(), num_docs);
@@ -5575,6 +5552,7 @@ async fn tokenize_and_count(
         ),
     ]));
     let output_schema_clone = output_schema.clone();
+    let query_token_indices = Arc::new(query_token_indices(query_tokens.as_ref()));
     let bytes_accumulated = Arc::new(AtomicU64::new(0));
     let bytes_warning_emitted = Arc::new(AtomicBool::new(false));
 
@@ -5583,6 +5561,7 @@ async fn tokenize_and_count(
             let mut tokenizer = tokenizer.box_clone();
             let output_schema = output_schema.clone();
             let query_tokens = query_tokens.clone();
+            let query_token_indices = query_token_indices.clone();
             let bytes_accumulated = bytes_accumulated.clone();
             let bytes_warning_emitted = bytes_warning_emitted.clone();
             let elapsed_compute = elapsed_compute.clone();
@@ -5606,8 +5585,10 @@ async fn tokenize_and_count(
                     let mut all_tokens = 0;
                     while let Some(token) = stream.next() {
                         all_tokens += 1;
-                        if let Some(token_index) = query_tokens.token_index(&token.text) {
-                            temp_query_token_counts[token_index] += 1;
+                        if let Some(token_indices) = query_token_indices.get(&token.text) {
+                            for token_index in token_indices {
+                                temp_query_token_counts[*token_index] += 1;
+                            }
                         }
                     }
                     all_tokens
@@ -5737,6 +5718,17 @@ fn tokenize_and_count_list<ListOffset: OffsetSizeTrait>(
     Ok(())
 }
 
+fn query_token_indices(query_tokens: &Tokens) -> HashMap<String, Vec<usize>> {
+    let mut indices = HashMap::new();
+    for idx in 0..query_tokens.len() {
+        indices
+            .entry(query_tokens.get_token(idx).to_string())
+            .or_insert_with(Vec::new)
+            .push(idx);
+    }
+    indices
+}
+
 /// Initialize the BM25 scorer
 ///
 /// In order to calculate BM25 scores we need to know token counts for the entire corpus.  We extract these from the
@@ -5800,9 +5792,11 @@ fn flat_bm25_score(
     query_tokens: &Tokens,
     counted_input: &RecordBatch,
     scorer: &MemBM25Scorer,
+    operator: Operator,
 ) -> Result<RecordBatch> {
     let mut row_ids_builder = UInt64Builder::with_capacity(counted_input.num_rows());
     let mut scores_builder = Float32Builder::with_capacity(counted_input.num_rows());
+    let query_groups = query_position_groups(query_tokens);
 
     let mut row_ids_iter = counted_input
         .column(FLAT_ROW_ID_COL_IDX)
@@ -5827,16 +5821,24 @@ fn flat_bm25_score(
     for _ in 0..counted_input.num_rows() {
         let num_tokens_in_doc = all_token_counts_iter.next().expect_ok()?;
         let row_id = row_ids_iter.next().expect_ok()?;
+        let mut query_token_counts = Vec::with_capacity(query_tokens.len());
+        for _ in query_tokens {
+            query_token_counts.push(query_token_counts_iter.next().expect_ok()?);
+        }
         if num_tokens_in_doc == 0 {
-            for _ in query_tokens {
-                query_token_counts_iter.next().expect_ok()?;
-            }
+            continue;
+        }
+        if operator == Operator::And
+            && !query_groups
+                .iter()
+                .all(|group| group.iter().any(|idx| query_token_counts[*idx] > 0))
+        {
             continue;
         }
         let doc_norm = K1 * (1.0 - B + B * num_tokens_in_doc as f32 / scorer.avg_doc_length());
         let mut score = 0.0;
-        for token in query_tokens {
-            let freq = query_token_counts_iter.next().expect_ok()? as f32;
+        for (token, freq) in query_tokens.into_iter().zip(query_token_counts.into_iter()) {
+            let freq = freq as f32;
             let idf = idf(scorer.num_docs_containing_token(token), scorer.num_docs());
             score += idf * (freq * (K1 + 1.0) / (freq + doc_norm));
         }
@@ -5855,6 +5857,23 @@ fn flat_bm25_score(
     Ok(batch)
 }
 
+fn query_position_groups(query_tokens: &Tokens) -> Vec<Vec<usize>> {
+    let mut groups = Vec::new();
+    let mut current_position = None;
+    for idx in 0..query_tokens.len() {
+        let position = query_tokens.position(idx);
+        if current_position != Some(position) {
+            current_position = Some(position);
+            groups.push(Vec::new());
+        }
+        groups
+            .last_mut()
+            .expect("a group should exist after pushing for position")
+            .push(idx);
+    }
+    groups
+}
+
 #[deprecated(
     note = "use `flat_bm25_search_stream_with_metrics` to record CPU compute \
             time on a metric handle; pass `None` for the old behavior"
@@ -5892,6 +5911,32 @@ pub async fn flat_bm25_search_stream_with_metrics(
     base_scorer: Option<MemBM25Scorer>,
     target_batch_size: usize,
     elapsed_compute: Option<Time>,
+) -> DataFusionResult<SendableRecordBatchStream> {
+    flat_bm25_search_stream_with_metrics_and_operator(
+        input,
+        doc_col,
+        query,
+        tokenizer,
+        base_scorer,
+        target_batch_size,
+        Operator::Or,
+        elapsed_compute,
+    )
+    .await
+}
+
+/// Same as [`flat_bm25_search_stream_with_metrics`] but applies the provided
+/// match operator when deciding whether a flat-scanned row is a hit.
+#[allow(clippy::too_many_arguments)]
+pub async fn flat_bm25_search_stream_with_metrics_and_operator(
+    input: SendableRecordBatchStream,
+    doc_col: String,
+    query: String,
+    tokenizer: Box<dyn LanceTokenizer>,
+    base_scorer: Option<MemBM25Scorer>,
+    target_batch_size: usize,
+    operator: Operator,
+    elapsed_compute: Option<Time>,
 ) -> DataFusionResult<SendableRecordBatchStream> {
     let mut tokenizer = tokenizer;
 
@@ -5947,7 +5992,7 @@ pub async fn flat_bm25_search_stream_with_metrics(
     // All post-await work is synchronous; time the scorer + score + slicing loop together.
     let post_await_start = std::time::Instant::now();
     let scorer = initialize_scorer(base_scorer.as_ref(), query_tokens.as_ref(), &counted_input);
-    let scores = flat_bm25_score(query_tokens.as_ref(), &counted_input, &scorer)?;
+    let scores = flat_bm25_score(query_tokens.as_ref(), &counted_input, &scorer, operator)?;
 
     // Finally we emit batches according to the target batch size
     let num_out_batches = scores.num_rows().div_ceil(target_batch_size);
@@ -8090,6 +8135,66 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_and_query_accepts_same_position_alternatives() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token in ["getusername", "get", "user", "name"] {
+            builder.tokens.add(token.to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+        }
+        // Doc 0 only has the split words. Doc 1 has both the complete
+        // identifier and split words. A grouped AND query should accept either
+        // `getusername` or `get` at position 0.
+        builder.posting_lists[1].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[3].add(0, PositionRecorder::Count(1));
+        builder.docs.append(100, 3);
+
+        builder.posting_lists[0].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[1].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[3].add(1, PositionRecorder::Count(1));
+        builder.docs.append(101, 4);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::code()).await;
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec![
+                "getusername".to_string(),
+                "get".to_string(),
+                "user".to_string(),
+                "name".to_string(),
+            ],
+            vec![0, 0, 1, 2],
+            DocType::Text,
+        ));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let (mut row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        row_ids.sort_unstable();
+        assert_eq!(row_ids, vec![100, 101]);
+    }
+
     // Enough distinct tokens that `write_posting_lists` emits several posting-list
     // batches (the default batch size is 256 rows), exercising the restructured
     // producer and async send path.
@@ -9247,6 +9352,100 @@ mod tests {
         assert_eq!(row_ids.values(), &[0]);
     }
 
+    #[tokio::test]
+    async fn flat_bm25_search_code_and_uses_position_groups() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "get user name",
+                    "getUserName",
+                    "get user",
+                    "username",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer = InvertedIndexParams::code().build().unwrap();
+
+        let result_stream = flat_bm25_search_stream_with_metrics_and_operator(
+            input,
+            "code".to_string(),
+            "getUserName".to_string(),
+            tokenizer,
+            None,
+            100,
+            Operator::And,
+            None,
+        )
+        .await
+        .unwrap();
+
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let mut row_ids = scored[ROW_ID]
+            .as_primitive::<UInt64Type>()
+            .values()
+            .to_vec();
+        row_ids.sort_unstable();
+
+        assert_eq!(row_ids, vec![0, 1]);
+    }
+
+    #[tokio::test]
+    async fn flat_bm25_search_code_and_counts_repeated_subwords() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1])),
+                Arc::new(StringArray::from(vec![
+                    "pub fn edge_flat_generic_return<T>() -> Result<T, EdgeFlatError> where T: TryFrom<String> { todo!() }",
+                    "pub fn edge_flat_generic_return<T>() -> Result<T> { todo!() }",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer = InvertedIndexParams::code().build().unwrap();
+
+        let result_stream = flat_bm25_search_stream_with_metrics_and_operator(
+            input,
+            "code".to_string(),
+            "edge_flat_generic_return TryFrom EdgeFlatError Result".to_string(),
+            tokenizer,
+            None,
+            100,
+            Operator::And,
+            None,
+        )
+        .await
+        .unwrap();
+
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let row_ids = scored[ROW_ID].as_primitive::<UInt64Type>().values();
+
+        assert_eq!(row_ids, &[0]);
+    }
+
     /// An [`IndexReader`] wrapper that hides the posting-group-offsets schema
     /// metadata key, so a [`PostingListReader`] opened on it takes the
     /// pre-grouping per-token fallback path (issue #7040).
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 2c7a17465f8..265d2d4ccf7 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -27,23 +27,42 @@ use crate::scalar::inverted::{
 };
 pub use lance_tokenizer::Language;
 use lance_tokenizer::{
-    AsciiFoldingFilter, IcuTokenizer, LowerCaser, NgramTokenizer, RawTokenizer, RemoveLongFilter,
-    SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer, TextAnalyzerBuilder,
-    WhitespaceTokenizer,
+    AsciiFoldingFilter, CodeLexTokenizer, IcuTokenizer, LowerCaser, NgramTokenizer, RawTokenizer,
+    RemoveLongFilter, SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer, TextAnalyzerBuilder,
+    WhitespaceTokenizer, WordDelimiterFilter,
 };
 
 /// Tokenizer configs
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, PartialEq)]
 pub struct InvertedIndexParams {
-    /// lance tokenizer takes care of different data types, such as text, json, etc.
-    /// - 'text': parsing input documents into tokens
-    /// - 'json': parsing input json string into tokens
-    /// - none: auto type inference
+    /// High-level analyzer profile.
+    ///
+    /// This is the user-facing semantic preset. `text` keeps the existing
+    /// natural-language defaults. `code` selects code-search defaults such as
+    /// code lexical tokenization, identifier splitting, preserving whole
+    /// identifiers, and disabling stemming / stop-word removal.
+    pub(crate) analyzer: String,
+
+    /// Document-level tokenizer.
+    ///
+    /// This decides how Lance extracts searchable text from the stored value:
+    /// - `text`: plain string documents
+    /// - `json`: JSON string documents
+    /// - `None`: infer from the Arrow field type during index build
+    ///
+    /// The extracted text is then passed to `base_tokenizer`.
     pub(crate) lance_tokenizer: Option<String>,
-    /// base tokenizer:
+
+    /// Lexical tokenizer used inside the selected analyzer profile.
+    ///
+    /// This is a lower-level implementation component, not the product-level
+    /// search mode. For code search, prefer `analyzer = "code"`;
+    /// `base_tokenizer = "code"` is accepted as a compatibility alias and
+    /// resolves to the code analyzer profile.
     /// - `simple`: splits tokens on whitespace and punctuation
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
+    /// - `code`: code-aware lexical tokenization
     /// - `icu`: ICU dictionary-based word segmentation
     /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
     /// - `lindera/*`: Lindera tokenizer
@@ -60,7 +79,6 @@ pub struct InvertedIndexParams {
     /// This can significantly increase the size of the index
     /// If false, only store the frequency of the term in the document
     /// Default is false
-    #[serde(default)]
     pub(crate) with_position: bool,
 
     /// maximum token length
@@ -69,15 +87,12 @@ pub struct InvertedIndexParams {
     pub(crate) max_token_length: Option<usize>,
 
     /// whether lower case tokens
-    #[serde(default = "bool_true")]
     pub(crate) lower_case: bool,
 
     /// whether apply stemming
-    #[serde(default = "bool_true")]
     pub(crate) stem: bool,
 
     /// whether remove stop words
-    #[serde(default = "bool_true")]
     pub(crate) remove_stop_words: bool,
 
     /// use customized stop words.
@@ -86,21 +101,29 @@ pub struct InvertedIndexParams {
     pub(crate) custom_stop_words: Option<Vec<String>>,
 
     /// ascii folding
-    #[serde(default = "bool_true")]
     pub(crate) ascii_folding: bool,
 
     /// min ngram length
-    #[serde(default = "default_min_ngram_length")]
     pub(crate) min_ngram_length: u32,
 
     /// max ngram length
-    #[serde(default = "default_max_ngram_length")]
     pub(crate) max_ngram_length: u32,
 
     /// whether prefix only
-    #[serde(default)]
     pub(crate) prefix_only: bool,
 
+    /// Split code identifiers into subwords.
+    pub(crate) split_identifiers: bool,
+
+    /// Split identifier subwords at letter/number boundaries.
+    pub(crate) split_on_numerics: bool,
+
+    /// Index a complete identifier in addition to subwords.
+    pub(crate) preserve_original: bool,
+
+    /// Index code operators such as `::`, `->`, and `!=`.
+    pub(crate) index_operators: bool,
+
     /// Total memory limit in MiB for the build stage.
     ///
     /// This is split evenly across FTS workers at build time. By default Lance
@@ -137,11 +160,168 @@ pub struct InvertedIndexParams {
     pub(crate) format_version: Option<InvertedListFormatVersion>,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawInvertedIndexParams {
+    analyzer: Option<String>,
+    lance_tokenizer: Option<String>,
+    base_tokenizer: Option<String>,
+    language: Option<Language>,
+    with_position: Option<bool>,
+    max_token_length: Option<Option<usize>>,
+    lower_case: Option<bool>,
+    stem: Option<bool>,
+    remove_stop_words: Option<bool>,
+    custom_stop_words: Option<Vec<String>>,
+    ascii_folding: Option<bool>,
+    min_ngram_length: Option<u32>,
+    max_ngram_length: Option<u32>,
+    prefix_only: Option<bool>,
+    split_identifiers: Option<bool>,
+    split_on_numerics: Option<bool>,
+    preserve_original: Option<bool>,
+    index_operators: Option<bool>,
+    #[serde(rename = "memory_limit", alias = "worker_memory_limit_mb")]
+    memory_limit_mb: Option<u64>,
+    #[serde(rename = "num_workers")]
+    num_workers: Option<usize>,
+    #[serde(
+        rename = "format_version",
+        default,
+        deserialize_with = "deserialize_format_version"
+    )]
+    format_version: Option<InvertedListFormatVersion>,
+}
+
+impl<'de> Deserialize<'de> for InvertedIndexParams {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        RawInvertedIndexParams::deserialize(deserializer)?
+            .resolve()
+            .map_err(serde::de::Error::custom)
+    }
+}
+
+impl RawInvertedIndexParams {
+    fn resolve(self) -> Result<InvertedIndexParams> {
+        let analyzer = match (
+            self.analyzer.as_deref().map(normalize_analyzer),
+            self.base_tokenizer.as_deref(),
+        ) {
+            (Some(Ok(analyzer)), _) => analyzer,
+            (Some(Err(err)), _) => return Err(err),
+            (None, Some("code")) => "code",
+            (None, _) => "text",
+        };
+
+        if analyzer == "text" && self.base_tokenizer.as_deref() == Some("code") {
+            return Err(Error::invalid_input(
+                "base_tokenizer='code' requires analyzer='code'".to_string(),
+            ));
+        }
+        if analyzer == "code"
+            && let Some(base_tokenizer) = self.base_tokenizer.as_deref()
+            && base_tokenizer != "code"
+        {
+            return Err(Error::invalid_input(format!(
+                "analyzer='code' requires base_tokenizer='code', got '{}'",
+                base_tokenizer
+            )));
+        }
+        if analyzer == "text"
+            && matches!(
+                (
+                    self.split_identifiers,
+                    self.split_on_numerics,
+                    self.preserve_original,
+                    self.index_operators,
+                ),
+                (Some(true), _, _, _)
+                    | (_, Some(true), _, _)
+                    | (_, _, Some(true), _)
+                    | (_, _, _, Some(true))
+            )
+        {
+            return Err(Error::invalid_input(
+                "code analyzer flags require analyzer='code'".to_string(),
+            ));
+        }
+
+        let mut params = match analyzer {
+            "code" => InvertedIndexParams::code(),
+            "text" => InvertedIndexParams::default(),
+            _ => unreachable!("analyzer is normalized above"),
+        };
+
+        params.analyzer = analyzer.to_string();
+        if let Some(lance_tokenizer) = self.lance_tokenizer {
+            params.lance_tokenizer = Some(lance_tokenizer);
+        }
+        if let Some(base_tokenizer) = self.base_tokenizer {
+            params.base_tokenizer = base_tokenizer;
+        }
+        if let Some(language) = self.language {
+            params.language = language;
+        }
+        if let Some(with_position) = self.with_position {
+            params.with_position = with_position;
+        }
+        if let Some(max_token_length) = self.max_token_length {
+            params.max_token_length = max_token_length;
+        }
+        if let Some(lower_case) = self.lower_case {
+            params.lower_case = lower_case;
+        }
+        if let Some(stem) = self.stem {
+            params.stem = stem;
+        }
+        if let Some(remove_stop_words) = self.remove_stop_words {
+            params.remove_stop_words = remove_stop_words;
+        }
+        if let Some(custom_stop_words) = self.custom_stop_words {
+            params.custom_stop_words = Some(custom_stop_words);
+        }
+        if let Some(ascii_folding) = self.ascii_folding {
+            params.ascii_folding = ascii_folding;
+        }
+        if let Some(min_ngram_length) = self.min_ngram_length {
+            params.min_ngram_length = min_ngram_length;
+        }
+        if let Some(max_ngram_length) = self.max_ngram_length {
+            params.max_ngram_length = max_ngram_length;
+        }
+        if let Some(prefix_only) = self.prefix_only {
+            params.prefix_only = prefix_only;
+        }
+        if let Some(split_identifiers) = self.split_identifiers {
+            params.split_identifiers = split_identifiers;
+        }
+        if let Some(split_on_numerics) = self.split_on_numerics {
+            params.split_on_numerics = split_on_numerics;
+        }
+        if let Some(preserve_original) = self.preserve_original {
+            params.preserve_original = preserve_original;
+        }
+        if let Some(index_operators) = self.index_operators {
+            params.index_operators = index_operators;
+        }
+        params.memory_limit_mb = self.memory_limit_mb;
+        params.num_workers = self.num_workers;
+        params.format_version = self.format_version;
+        params.validate()?;
+        Ok(params)
+    }
+}
+
 impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
     type Error = Error;
 
     fn try_from(params: &InvertedIndexParams) -> Result<Self> {
         Ok(Self {
+            analyzer: Some(params.analyzer.clone()),
+            lance_tokenizer: params.lance_tokenizer.clone(),
             base_tokenizer: Some(params.base_tokenizer.clone()),
             language: serde_json::to_string(&params.language)?,
             with_position: params.with_position,
@@ -153,6 +333,11 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             min_ngram_length: params.min_ngram_length,
             max_ngram_length: params.max_ngram_length,
             prefix_only: params.prefix_only,
+            split_identifiers: params.split_identifiers,
+            split_on_numerics: params.split_on_numerics,
+            preserve_original: params.preserve_original,
+            index_operators: params.index_operators,
+            custom_stop_words: params.custom_stop_words.clone().unwrap_or_default(),
         })
     }
 }
@@ -161,34 +346,68 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
     type Error = Error;
 
     fn try_from(details: &pbold::InvertedIndexDetails) -> Result<Self> {
-        let defaults = Self::default();
-        Ok(Self {
-            lance_tokenizer: defaults.lance_tokenizer,
-            base_tokenizer: details
-                .base_tokenizer
-                .as_ref()
-                .cloned()
-                .unwrap_or_else(|| "simple".to_string()),
-            language: serde_json::from_str(details.language.as_str())?,
-            with_position: details.with_position,
-            max_token_length: details.max_token_length.map(|l| l as usize),
-            lower_case: details.lower_case,
-            stem: details.stem,
-            remove_stop_words: details.remove_stop_words,
-            custom_stop_words: defaults.custom_stop_words,
-            ascii_folding: details.ascii_folding,
-            min_ngram_length: details.min_ngram_length,
-            max_ngram_length: details.max_ngram_length,
-            prefix_only: details.prefix_only,
-            memory_limit_mb: defaults.memory_limit_mb,
-            num_workers: defaults.num_workers,
-            format_version: defaults.format_version,
-        })
+        if details.base_tokenizer.is_none() && details.language.is_empty() {
+            return Ok(Self::default());
+        }
+
+        let analyzer = details
+            .analyzer
+            .as_deref()
+            .map(normalize_analyzer)
+            .transpose()?
+            .unwrap_or_else(|| {
+                if details.base_tokenizer.as_deref() == Some("code") {
+                    "code"
+                } else {
+                    "text"
+                }
+            });
+        let mut params = match analyzer {
+            "code" => Self::code(),
+            "text" => Self::default(),
+            _ => unreachable!("analyzer is normalized above"),
+        };
+        params.analyzer = analyzer.to_string();
+        params.lance_tokenizer = details.lance_tokenizer.clone();
+        params.base_tokenizer = details
+            .base_tokenizer
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| params.base_tokenizer.clone());
+        params.language = if details.language.is_empty() {
+            params.language
+        } else {
+            serde_json::from_str(details.language.as_str())?
+        };
+        params.with_position = details.with_position;
+        params.max_token_length = details.max_token_length.map(|l| l as usize);
+        params.lower_case = details.lower_case;
+        params.stem = details.stem;
+        params.remove_stop_words = details.remove_stop_words;
+        params.custom_stop_words =
+            (!details.custom_stop_words.is_empty()).then(|| details.custom_stop_words.clone());
+        params.ascii_folding = details.ascii_folding;
+        params.min_ngram_length = details.min_ngram_length;
+        params.max_ngram_length = details.max_ngram_length;
+        params.prefix_only = details.prefix_only;
+        params.split_identifiers = details.split_identifiers;
+        params.split_on_numerics = details.split_on_numerics;
+        params.preserve_original = details.preserve_original;
+        params.index_operators = details.index_operators;
+        params.validate()?;
+        Ok(params)
     }
 }
 
-fn bool_true() -> bool {
-    true
+fn normalize_analyzer(analyzer: &str) -> Result<&'static str> {
+    match analyzer {
+        "text" => Ok("text"),
+        "code" => Ok("code"),
+        other => Err(Error::invalid_input(format!(
+            "unknown analyzer '{}', expected 'text' or 'code'",
+            other
+        ))),
+    }
 }
 
 fn default_min_ngram_length() -> u32 {
@@ -243,6 +462,7 @@ impl InvertedIndexParams {
     /// - `simple`: splits tokens on whitespace and punctuation, default
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
+    /// - `code`: code identifier tokenization
     /// - `ngram`: N-Gram tokenizer
     /// - `icu`: ICU dictionary-based word segmentation
     /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
@@ -254,6 +474,7 @@ impl InvertedIndexParams {
     /// Default to `English`.
     pub fn new(base_tokenizer: String, language: Language) -> Self {
         Self {
+            analyzer: "text".to_string(),
             lance_tokenizer: None,
             base_tokenizer,
             language,
@@ -267,12 +488,40 @@ impl InvertedIndexParams {
             min_ngram_length: default_min_ngram_length(),
             max_ngram_length: default_max_ngram_length(),
             prefix_only: false,
+            split_identifiers: false,
+            split_on_numerics: false,
+            preserve_original: false,
+            index_operators: false,
             memory_limit_mb: None,
             num_workers: None,
             format_version: None,
         }
     }
 
+    /// Create parameters for the code analyzer profile.
+    pub fn code() -> Self {
+        Self::new("code".to_string(), Language::English)
+            .analyzer("code")
+            .expect("code analyzer should be valid")
+    }
+
+    /// Set analyzer profile.
+    pub fn analyzer(mut self, analyzer: &str) -> Result<Self> {
+        self.analyzer = normalize_analyzer(analyzer)?.to_string();
+        if self.analyzer == "code" {
+            self.base_tokenizer = "code".to_string();
+            self.split_identifiers = true;
+            self.split_on_numerics = true;
+            self.preserve_original = true;
+            self.lower_case = true;
+            self.ascii_folding = true;
+            self.stem = false;
+            self.remove_stop_words = false;
+            self.index_operators = false;
+        }
+        Ok(self)
+    }
+
     pub fn lance_tokenizer(mut self, lance_tokenizer: String) -> Self {
         self.lance_tokenizer = Some(lance_tokenizer);
         self
@@ -280,6 +529,11 @@ impl InvertedIndexParams {
 
     pub fn base_tokenizer(mut self, base_tokenizer: String) -> Self {
         self.base_tokenizer = base_tokenizer;
+        if self.base_tokenizer == "code" {
+            self = self
+                .analyzer("code")
+                .expect("code analyzer should be valid");
+        }
         self
     }
 
@@ -335,6 +589,26 @@ impl InvertedIndexParams {
         self
     }
 
+    pub fn split_identifiers(mut self, split_identifiers: bool) -> Self {
+        self.split_identifiers = split_identifiers;
+        self
+    }
+
+    pub fn split_on_numerics(mut self, split_on_numerics: bool) -> Self {
+        self.split_on_numerics = split_on_numerics;
+        self
+    }
+
+    pub fn preserve_original(mut self, preserve_original: bool) -> Self {
+        self.preserve_original = preserve_original;
+        self
+    }
+
+    pub fn index_operators(mut self, index_operators: bool) -> Self {
+        self.index_operators = index_operators;
+        self
+    }
+
     /// Set the minimum N-Gram length, only works when `base_tokenizer` is `ngram`.
     /// Must be greater than 0 and not greater than `max_ngram_length`.
     /// Default to 3.
@@ -415,6 +689,7 @@ impl InvertedIndexParams {
     }
 
     pub fn build(&self) -> Result<Box<dyn LanceTokenizer>> {
+        self.validate()?;
         let mut builder = self.build_base_tokenizer()?;
         if let Some(max_token_length) = self.max_token_length {
             builder = builder.filter_dynamic(RemoveLongFilter::limit(max_token_length));
@@ -459,11 +734,46 @@ impl InvertedIndexParams {
         }
     }
 
+    fn validate(&self) -> Result<()> {
+        normalize_analyzer(&self.analyzer)?;
+        match self.analyzer.as_str() {
+            "code" if self.base_tokenizer != "code" => Err(Error::invalid_input(format!(
+                "analyzer='code' requires base_tokenizer='code', got '{}'",
+                self.base_tokenizer
+            ))),
+            "text" if self.base_tokenizer == "code" => Err(Error::invalid_input(
+                "base_tokenizer='code' requires analyzer='code'".to_string(),
+            )),
+            "text"
+                if self.split_identifiers
+                    || self.split_on_numerics
+                    || self.preserve_original
+                    || self.index_operators =>
+            {
+                Err(Error::invalid_input(
+                    "code analyzer flags require analyzer='code'".to_string(),
+                ))
+            }
+            _ => Ok(()),
+        }
+    }
+
     fn build_base_tokenizer(&self) -> Result<TextAnalyzerBuilder> {
         match self.base_tokenizer.as_str() {
             "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()),
             "whitespace" => Ok(TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()),
             "raw" => Ok(TextAnalyzer::builder(RawTokenizer::default()).dynamic()),
+            "code" => {
+                let mut builder =
+                    TextAnalyzer::builder(CodeLexTokenizer::new(self.index_operators)).dynamic();
+                if self.split_identifiers {
+                    builder = builder.filter_dynamic(WordDelimiterFilter::new(
+                        self.preserve_original,
+                        self.split_on_numerics,
+                    ));
+                }
+                Ok(builder)
+            }
             "icu" => Ok(TextAnalyzer::builder(IcuTokenizer::default()).dynamic()),
             "icu/split" => {
                 Ok(TextAnalyzer::builder(IcuTokenizer::default().with_simple_split()).dynamic())
@@ -522,6 +832,7 @@ mod tests {
     use super::{InvertedIndexParams, InvertedListFormatVersion};
     use lance_tokenizer::TokenStream;
     use rstest::rstest;
+    use serde_json::json;
 
     #[test]
     fn test_build_only_fields_are_not_serialized() {
@@ -588,6 +899,93 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_code_analyzer_resolves_defaults() {
+        let params: InvertedIndexParams = serde_json::from_value(json!({
+            "analyzer": "code"
+        }))
+        .unwrap();
+        assert_eq!(params.analyzer, "code");
+        assert_eq!(params.base_tokenizer, "code");
+        assert!(params.split_identifiers);
+        assert!(params.split_on_numerics);
+        assert!(params.preserve_original);
+        assert!(params.lower_case);
+        assert!(params.ascii_folding);
+        assert!(!params.stem);
+        assert!(!params.remove_stop_words);
+        assert!(!params.index_operators);
+    }
+
+    #[test]
+    fn test_code_base_tokenizer_alias_resolves_analyzer() {
+        let params: InvertedIndexParams = serde_json::from_value(json!({
+            "base_tokenizer": "code"
+        }))
+        .unwrap();
+        assert_eq!(params.analyzer, "code");
+        assert_eq!(params.base_tokenizer, "code");
+    }
+
+    #[test]
+    fn test_code_analyzer_rejects_conflicting_base_tokenizer() {
+        let err = serde_json::from_value::<InvertedIndexParams>(json!({
+            "analyzer": "code",
+            "base_tokenizer": "simple"
+        }))
+        .unwrap_err();
+        assert!(err.to_string().contains("requires base_tokenizer='code'"));
+    }
+
+    #[test]
+    fn test_build_code_tokenizer() {
+        let mut tokenizer = InvertedIndexParams::code().build().unwrap();
+        let mut stream = tokenizer.token_stream_for_doc(
+            "getUserName XMLHttpRequest parseHTML2JSON utf8_reader SCREAMING_SNAKE_CASE",
+        );
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| {
+            tokens.push((token.text.clone(), token.position, token.position_length))
+        });
+        assert_eq!(
+            tokens,
+            vec![
+                ("getusername".to_string(), 0, 3),
+                ("get".to_string(), 0, 1),
+                ("user".to_string(), 1, 1),
+                ("name".to_string(), 2, 1),
+                ("xmlhttprequest".to_string(), 3, 3),
+                ("xml".to_string(), 3, 1),
+                ("http".to_string(), 4, 1),
+                ("request".to_string(), 5, 1),
+                ("parsehtml2json".to_string(), 6, 4),
+                ("parse".to_string(), 6, 1),
+                ("html".to_string(), 7, 1),
+                ("2".to_string(), 8, 1),
+                ("json".to_string(), 9, 1),
+                ("utf8_reader".to_string(), 10, 3),
+                ("utf".to_string(), 10, 1),
+                ("8".to_string(), 11, 1),
+                ("reader".to_string(), 12, 1),
+                ("screaming_snake_case".to_string(), 13, 3),
+                ("screaming".to_string(), 13, 1),
+                ("snake".to_string(), 14, 1),
+                ("case".to_string(), 15, 1),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_inverted_details_round_trip_code_params() {
+        let params = InvertedIndexParams::code()
+            .with_position(true)
+            .index_operators(true)
+            .preserve_original(false);
+        let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
+        assert_eq!(round_tripped, params);
+    }
+
     #[test]
     fn test_build_icu_tokenizer() {
         let mut tokenizer = InvertedIndexParams::default()
diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs
new file mode 100644
index 00000000000..a378eebb879
--- /dev/null
+++ b/rust/lance-tokenizer/src/code_tokenizer.rs
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{iter::Peekable, str::CharIndices};
+
+use crate::{Token, TokenStream, Tokenizer};
+
+/// Tokenizer for code-like text.
+///
+/// Identifiers are Unicode alphanumeric characters plus `_`. Other characters
+/// are lexical boundaries. When operator indexing is enabled, contiguous
+/// operator characters are emitted as tokens too.
+#[derive(Clone, Default)]
+pub struct CodeLexTokenizer {
+    index_operators: bool,
+    token: Token,
+}
+
+impl CodeLexTokenizer {
+    pub fn new(index_operators: bool) -> Self {
+        Self {
+            index_operators,
+            token: Token::default(),
+        }
+    }
+}
+
+pub struct CodeLexTokenStream<'a> {
+    text: &'a str,
+    chars: Peekable<CharIndices<'a>>,
+    token: &'a mut Token,
+    index_operators: bool,
+}
+
+impl Tokenizer for CodeLexTokenizer {
+    type TokenStream<'a> = CodeLexTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        self.token.reset();
+        CodeLexTokenStream {
+            text,
+            chars: text.char_indices().peekable(),
+            token: &mut self.token,
+            index_operators: self.index_operators,
+        }
+    }
+}
+
+fn is_identifier_char(ch: char) -> bool {
+    ch == '_' || ch.is_alphanumeric()
+}
+
+fn is_operator_char(ch: char) -> bool {
+    matches!(
+        ch,
+        '!' | '%' | '&' | '*' | '+' | '-' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '|' | '~'
+    )
+}
+
+impl CodeLexTokenStream<'_> {
+    fn search_token_end(&mut self, predicate: impl Fn(char) -> bool) -> usize {
+        while let Some((_, ch)) = self.chars.peek() {
+            if !predicate(*ch) {
+                break;
+            }
+            self.chars.next();
+        }
+        self.chars
+            .peek()
+            .map(|(offset, _)| *offset)
+            .unwrap_or(self.text.len())
+    }
+}
+
+impl TokenStream for CodeLexTokenStream<'_> {
+    fn advance(&mut self) -> bool {
+        self.token.text.clear();
+        while let Some((offset_from, ch)) = self.chars.next() {
+            let token_end = if is_identifier_char(ch) {
+                self.search_token_end(is_identifier_char)
+            } else if self.index_operators && is_operator_char(ch) {
+                self.search_token_end(is_operator_char)
+            } else {
+                continue;
+            };
+
+            self.token.position = self.token.position.wrapping_add(1);
+            self.token.position_length = 1;
+            self.token.offset_from = offset_from;
+            self.token.offset_to = token_end;
+            self.token.text.push_str(&self.text[offset_from..token_end]);
+            return true;
+        }
+        false
+    }
+
+    fn token(&self) -> &Token {
+        self.token
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.token
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{CodeLexTokenizer, TextAnalyzer, Token};
+
+    fn collect_tokens(text: &str, index_operators: bool) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(index_operators)).build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_code_lex_tokenizer_identifiers() {
+        let tokens = collect_tokens("std::vector user-name parse.HTML2JSON", false);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            texts,
+            vec!["std", "vector", "user", "name", "parse", "HTML2JSON"]
+        );
+    }
+
+    #[test]
+    fn test_code_lex_tokenizer_operators() {
+        let tokens = collect_tokens("a::b != c->d", true);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(texts, vec!["a", "::", "b", "!=", "c", "->", "d"]);
+    }
+}
diff --git a/rust/lance-tokenizer/src/lib.rs b/rust/lance-tokenizer/src/lib.rs
index 0708022b384..fc1d9512cfd 100644
--- a/rust/lance-tokenizer/src/lib.rs
+++ b/rust/lance-tokenizer/src/lib.rs
@@ -4,6 +4,7 @@
 mod alphanum_only;
 mod analyzer;
 mod ascii_folding_filter;
+mod code_tokenizer;
 mod icu;
 #[cfg(feature = "tokenizer-jieba")]
 mod jieba;
@@ -16,6 +17,7 @@ mod stemmer;
 mod stop_word_filter;
 mod tokenizer_api;
 mod whitespace_tokenizer;
+mod word_delimiter_filter;
 
 #[cfg(feature = "tokenizer-lindera")]
 mod lindera;
@@ -23,6 +25,7 @@ mod lindera;
 pub use alphanum_only::AlphaNumOnlyFilter;
 pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder};
 pub use ascii_folding_filter::AsciiFoldingFilter;
+pub use code_tokenizer::CodeLexTokenizer;
 pub use icu::IcuTokenizer;
 #[cfg(feature = "tokenizer-jieba")]
 pub use jieba::JiebaTokenizer;
@@ -37,3 +40,4 @@ pub use stemmer::{Language, Stemmer};
 pub use stop_word_filter::StopWordFilter;
 pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer};
 pub use whitespace_tokenizer::WhitespaceTokenizer;
+pub use word_delimiter_filter::WordDelimiterFilter;
diff --git a/rust/lance-tokenizer/src/word_delimiter_filter.rs b/rust/lance-tokenizer/src/word_delimiter_filter.rs
new file mode 100644
index 00000000000..b03834c03f1
--- /dev/null
+++ b/rust/lance-tokenizer/src/word_delimiter_filter.rs
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::VecDeque;
+
+use crate::{Token, TokenFilter, TokenStream, Tokenizer};
+
+/// Splits code identifiers into subword tokens.
+///
+/// The original identifier can be preserved at the first subword position. If
+/// a token is not a compound identifier then it is passed through unchanged.
+#[derive(Clone)]
+pub struct WordDelimiterFilter {
+    preserve_original: bool,
+    split_on_numerics: bool,
+}
+
+impl WordDelimiterFilter {
+    pub fn new(preserve_original: bool, split_on_numerics: bool) -> Self {
+        Self {
+            preserve_original,
+            split_on_numerics,
+        }
+    }
+}
+
+impl TokenFilter for WordDelimiterFilter {
+    type Tokenizer<T: Tokenizer> = WordDelimiterFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        WordDelimiterFilterWrapper {
+            tokenizer,
+            preserve_original: self.preserve_original,
+            split_on_numerics: self.split_on_numerics,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct WordDelimiterFilterWrapper<T> {
+    tokenizer: T,
+    preserve_original: bool,
+    split_on_numerics: bool,
+}
+
+impl<T: Tokenizer> Tokenizer for WordDelimiterFilterWrapper<T> {
+    type TokenStream<'a> = WordDelimiterTokenStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        WordDelimiterTokenStream {
+            tail: self.tokenizer.token_stream(text),
+            current: Token::default(),
+            pending: VecDeque::new(),
+            preserve_original: self.preserve_original,
+            split_on_numerics: self.split_on_numerics,
+            position_offset: 0,
+        }
+    }
+}
+
+pub struct WordDelimiterTokenStream<T> {
+    tail: T,
+    current: Token,
+    pending: VecDeque<Token>,
+    preserve_original: bool,
+    split_on_numerics: bool,
+    position_offset: usize,
+}
+
+#[derive(Clone, Copy)]
+struct CharInfo {
+    offset: usize,
+    ch: char,
+}
+
+fn is_identifier_text(text: &str) -> bool {
+    text.chars().any(|ch| ch == '_' || ch.is_alphanumeric())
+}
+
+fn split_identifier(text: &str, split_on_numerics: bool) -> Vec<(usize, usize)> {
+    let mut pieces = Vec::new();
+    let mut segment_start = None;
+    for (offset, ch) in text.char_indices() {
+        if ch == '_' {
+            if let Some(start) = segment_start.take() {
+                split_segment(text, start, offset, split_on_numerics, &mut pieces);
+            }
+        } else if segment_start.is_none() {
+            segment_start = Some(offset);
+        }
+    }
+    if let Some(start) = segment_start {
+        split_segment(text, start, text.len(), split_on_numerics, &mut pieces);
+    }
+    pieces
+}
+
+fn split_segment(
+    text: &str,
+    start: usize,
+    end: usize,
+    split_on_numerics: bool,
+    pieces: &mut Vec<(usize, usize)>,
+) {
+    if start == end {
+        return;
+    }
+    let chars = text[start..end]
+        .char_indices()
+        .map(|(offset, ch)| CharInfo {
+            offset: start + offset,
+            ch,
+        })
+        .collect::<Vec<_>>();
+    let mut piece_start = start;
+    for idx in 1..chars.len() {
+        if is_boundary(&chars, idx, split_on_numerics) {
+            pieces.push((piece_start, chars[idx].offset));
+            piece_start = chars[idx].offset;
+        }
+    }
+    pieces.push((piece_start, end));
+}
+
+fn is_boundary(chars: &[CharInfo], idx: usize, split_on_numerics: bool) -> bool {
+    let prev = chars[idx - 1].ch;
+    let cur = chars[idx].ch;
+    if split_on_numerics && prev.is_ascii_digit() != cur.is_ascii_digit() {
+        return true;
+    }
+    if prev.is_lowercase() && cur.is_uppercase() {
+        return true;
+    }
+    if prev.is_uppercase()
+        && cur.is_uppercase()
+        && chars
+            .get(idx + 1)
+            .is_some_and(|next| next.ch.is_lowercase())
+    {
+        return true;
+    }
+    false
+}
+
+impl<T: TokenStream> WordDelimiterTokenStream<T> {
+    fn refill(&mut self) -> bool {
+        while self.pending.is_empty() {
+            if !self.tail.advance() {
+                return false;
+            }
+            self.enqueue_current_tail_token();
+        }
+        true
+    }
+
+    fn enqueue_current_tail_token(&mut self) {
+        let token = self.tail.token();
+        let adjusted_position = token.position.saturating_add(self.position_offset);
+        if !is_identifier_text(&token.text) {
+            let mut token = token.clone();
+            token.position = adjusted_position;
+            self.pending.push_back(token);
+            return;
+        }
+
+        let parts = split_identifier(&token.text, self.split_on_numerics);
+        if parts.len() <= 1 {
+            let mut token = token.clone();
+            token.position = adjusted_position;
+            self.pending.push_back(token);
+            return;
+        }
+
+        if self.preserve_original {
+            let mut original = token.clone();
+            original.position = adjusted_position;
+            original.position_length = parts.len();
+            self.pending.push_back(original);
+        }
+
+        for (part_idx, (start, end)) in parts.iter().copied().enumerate() {
+            let mut part = token.clone();
+            part.offset_from = token.offset_from + start;
+            part.offset_to = token.offset_from + end;
+            part.position = adjusted_position + part_idx;
+            part.position_length = 1;
+            part.text.clear();
+            part.text.push_str(&token.text[start..end]);
+            self.pending.push_back(part);
+        }
+        self.position_offset += parts.len() - 1;
+    }
+}
+
+impl<T: TokenStream> TokenStream for WordDelimiterTokenStream<T> {
+    fn advance(&mut self) -> bool {
+        if !self.refill() {
+            return false;
+        }
+        self.current = self
+            .pending
+            .pop_front()
+            .expect("pending token should be available after refill");
+        true
+    }
+
+    fn token(&self) -> &Token {
+        &self.current
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        &mut self.current
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{CodeLexTokenizer, TextAnalyzer, Token, WordDelimiterFilter};
+
+    fn collect_tokens(text: &str) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false))
+            .filter(WordDelimiterFilter::new(true, true))
+            .build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_word_delimiter_code_identifiers() {
+        let tokens = collect_tokens("getUserName XMLHttpRequest parseHTML2JSON utf8_reader");
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            texts,
+            vec![
+                "getUserName",
+                "get",
+                "User",
+                "Name",
+                "XMLHttpRequest",
+                "XML",
+                "Http",
+                "Request",
+                "parseHTML2JSON",
+                "parse",
+                "HTML",
+                "2",
+                "JSON",
+                "utf8_reader",
+                "utf",
+                "8",
+                "reader",
+            ]
+        );
+    }
+
+    #[test]
+    fn test_word_delimiter_positions_span_compounds() {
+        let tokens = collect_tokens("getUserName next");
+        let positions = tokens
+            .iter()
+            .map(|token| (token.text.as_str(), token.position, token.position_length))
+            .collect::<Vec<_>>();
+        assert_eq!(
+            positions,
+            vec![
+                ("getUserName", 0, 3),
+                ("get", 0, 1),
+                ("User", 1, 1),
+                ("Name", 2, 1),
+                ("next", 3, 1),
+            ]
+        );
+    }
+}
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index d16d3105551..3c6f862d79a 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -139,11 +139,14 @@ impl MemIndexConfig {
 
         // Extract InvertedIndexParams from index_details if available
         let params = if let Some(details_any) = &index_meta.index_details {
-            if let Ok(details) = pbold::InvertedIndexDetails::decode(details_any.value.as_slice()) {
-                InvertedIndexParams::try_from(&details)?
-            } else {
-                InvertedIndexParams::default()
-            }
+            let details = pbold::InvertedIndexDetails::decode(details_any.value.as_slice())
+                .map_err(|err| {
+                    Error::io(format!(
+                        "failed to decode InvertedIndexDetails for MemWAL FTS index '{}': {}",
+                        index_meta.name, err
+                    ))
+                })?;
+            InvertedIndexParams::try_from(&details)?
         } else {
             InvertedIndexParams::default()
         };
diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 61c797db041..8a25625a460 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -59,8 +59,8 @@ use fst::{Map, Streamer};
 use lance_bitpacking::{BitPacker, BitPacker4x};
 use lance_core::{Error, Result};
 use lance_index::scalar::InvertedIndexParams;
-use lance_index::scalar::inverted::query::Operator;
-use lance_index::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
+use lance_index::scalar::inverted::query::{Operator, Tokens};
+use lance_index::scalar::inverted::tokenizer::document_tokenizer::{DocType, LanceTokenizer};
 use lance_index::scalar::inverted::{DocSet, MemBM25Scorer, Scorer, TokenSet};
 use lance_tokenizer::TokenStream;
 use rayon::prelude::*;
@@ -371,6 +371,37 @@ fn char_prefix(term: &str, prefix_length: u32) -> &str {
         .unwrap_or(term)
 }
 
+fn query_tokens_to_vec(tokens: &Tokens) -> Vec<String> {
+    (0..tokens.len())
+        .map(|idx| tokens.get_token(idx).to_string())
+        .collect()
+}
+
+fn has_grouped_positions(tokens: &Tokens) -> bool {
+    let mut seen = HashSet::new();
+    (0..tokens.len()).any(|idx| !seen.insert(tokens.position(idx)))
+}
+
+fn query_position_groups(tokens: &Tokens) -> Vec<Vec<String>> {
+    let mut groups = Vec::new();
+    let mut current_position = None;
+    for idx in 0..tokens.len() {
+        let position = tokens.position(idx);
+        if current_position != Some(position) {
+            current_position = Some(position);
+            groups.push(Vec::new());
+        }
+        let group = groups
+            .last_mut()
+            .expect("a group should exist after pushing for position");
+        let token = tokens.get_token(idx).to_string();
+        if !group.contains(&token) {
+            group.push(token);
+        }
+    }
+    groups
+}
+
 /// Builder for constructing Boolean queries.
 #[derive(Debug, Clone, Default)]
 pub struct BooleanQueryBuilder {
@@ -1267,7 +1298,7 @@ impl FtsMemIndex {
     /// use `search_with_options` for sorted/limited output.
     pub fn search(&self, term: &str) -> Vec<FtsEntry> {
         let st = self.state.load_full();
-        let tokens = self.tokenize_for_search(term);
+        let tokens = self.analyze_for_search(term);
         self.search_match(&st, &tokens, Operator::Or, None, true, true)
     }
 
@@ -1322,6 +1353,22 @@ impl FtsMemIndex {
     /// shared rising threshold (instead of every partition cold-starting).
     /// Without a limit, an exact O(matches) scan across partitions + tail.
     fn search_match(
+        &self,
+        st: &IndexState,
+        query_tokens: &Tokens,
+        operator: Operator,
+        limit: Option<usize>,
+        include_tail: bool,
+        tail_skip: bool,
+    ) -> Vec<FtsEntry> {
+        if operator == Operator::And && has_grouped_positions(query_tokens) {
+            return self.search_grouped_and(st, query_tokens, limit, include_tail, tail_skip);
+        }
+        let tokens = query_tokens_to_vec(query_tokens);
+        self.search_match_strings(st, &tokens, operator, limit, include_tail, tail_skip)
+    }
+
+    fn search_match_strings(
         &self,
         st: &IndexState,
         tokens: &[String],
@@ -1390,6 +1437,58 @@ impl FtsMemIndex {
         }
     }
 
+    fn search_grouped_and(
+        &self,
+        st: &IndexState,
+        query_tokens: &Tokens,
+        limit: Option<usize>,
+        include_tail: bool,
+        tail_skip: bool,
+    ) -> Vec<FtsEntry> {
+        let mut result_map: Option<HashMap<RowPosition, f32>> = None;
+        for group in query_position_groups(query_tokens) {
+            let group_results =
+                self.search_match_strings(st, &group, Operator::Or, None, include_tail, tail_skip);
+            let group_map = group_results
+                .into_iter()
+                .map(|entry| (entry.row_position, entry.score))
+                .collect::<HashMap<_, _>>();
+            let Some(current) = result_map.as_mut() else {
+                result_map = Some(group_map);
+                continue;
+            };
+            current.retain(|row_position, score| {
+                if let Some(group_score) = group_map.get(row_position) {
+                    *score += group_score;
+                    true
+                } else {
+                    false
+                }
+            });
+            if current.is_empty() {
+                return Vec::new();
+            }
+        }
+
+        let mut results = result_map
+            .unwrap_or_default()
+            .into_iter()
+            .map(|(row_position, score)| FtsEntry {
+                row_position,
+                score,
+            })
+            .collect::<Vec<_>>();
+        if let Some(limit) = limit {
+            results.sort_by(|a, b| {
+                b.score
+                    .partial_cmp(&a.score)
+                    .unwrap_or(std::cmp::Ordering::Equal)
+            });
+            results.truncate(limit);
+        }
+        results
+    }
+
     fn search_phrase_tokens(
         &self,
         st: &IndexState,
@@ -1402,7 +1501,7 @@ impl FtsMemIndex {
         }
         if tokens.len() == 1 {
             // A single-token phrase reduces to a regular term search.
-            return self.search_match(st, tokens, Operator::Or, None, include_tail, true);
+            return self.search_match_strings(st, tokens, Operator::Or, None, include_tail, true);
         }
         // A multi-token phrase needs token positions; without them (the index
         // was built `with_position = false`) phrase search is unsupported, as
@@ -1464,7 +1563,7 @@ impl FtsMemIndex {
         if expanded.is_empty() {
             return Vec::new();
         }
-        self.search_match(st, &expanded, Operator::Or, None, include_tail, true)
+        self.search_match_strings(st, &expanded, Operator::Or, None, include_tail, true)
     }
 
     /// Expand `term` against the term dictionaries of every partition (and the
@@ -1556,7 +1655,7 @@ impl FtsMemIndex {
                 operator,
                 boost,
             } => {
-                let tokens = self.tokenize_for_search(query);
+                let tokens = self.analyze_for_search(query);
                 let mut results =
                     self.search_match(st, &tokens, *operator, limit, include_tail, tail_skip);
                 apply_boost(&mut results, *boost);
@@ -1736,13 +1835,19 @@ impl FtsMemIndex {
     }
 
     fn tokenize_for_search(&self, text: &str) -> Vec<String> {
+        query_tokens_to_vec(&self.analyze_for_search(text))
+    }
+
+    fn analyze_for_search(&self, text: &str) -> Tokens {
         let mut tok = PooledTokenizer::new(&self.tokenizer_pool);
         let mut stream = tok.get_mut().token_stream_for_search(text);
-        let mut out = Vec::new();
+        let mut tokens = Vec::new();
+        let mut positions = Vec::new();
         while let Some(t) = stream.next() {
-            out.push(t.text.clone());
+            tokens.push(t.text.clone());
+            positions.push(t.position as u32);
         }
-        out
+        Tokens::with_positions(tokens, positions, DocType::Text)
     }
 
     // ------------------------------------------------------------------
@@ -3652,6 +3757,31 @@ mod tests {
         assert!(entries.is_empty());
     }
 
+    #[test]
+    fn test_code_analyzer_and_query_uses_position_alternatives() {
+        let schema = create_test_schema();
+        let index =
+            FtsMemIndex::with_params(1, "description".to_string(), InvertedIndexParams::code());
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1])),
+                Arc::new(StringArray::from(vec!["get user name", "getUserName"])),
+            ],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+
+        let query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And);
+        let mut rows = index
+            .search_query(&query)
+            .into_iter()
+            .map(|entry| entry.row_position)
+            .collect::<Vec<_>>();
+        rows.sort_unstable();
+        assert_eq!(rows, vec![0, 1]);
+    }
+
     fn create_phrase_test_batch(schema: &ArrowSchema) -> RecordBatch {
         RecordBatch::try_new(
             Arc::new(schema.clone()),
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index 000d2c3139c..5aebf66f627 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -11,7 +11,7 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use lance_core::ROW_ID;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::pbold::InvertedIndexDetails;
-use lance_index::scalar::inverted::InvertedIndex;
+use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexParams};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
 use lance_table::format::IndexMetadata;
@@ -182,10 +182,13 @@ pub async fn load_segments(dataset: &Dataset, column: &str) -> Result<Option<Vec
 /// Load and validate the shared [`InvertedIndexDetails`] across committed
 /// segments returned by [`load_segments`].
 ///
-/// All segments are required to agree on their decoded `InvertedIndexDetails`
+/// All segments are required to agree on their semantic `InvertedIndexDetails`
 /// payload (analyzer, tokenizer, position settings, etc.); inconsistent
-/// segments return an error. Returns the canonical details that may be used
-/// when constructing a tokenizer or running a query against the index.
+/// segments return an error. Details are canonicalized before comparison so
+/// legacy segments that omit default fields (for example, missing
+/// `analyzer = "text"`) remain compatible with newly written text FTS
+/// segments. Returns the canonical details that may be used when constructing a
+/// tokenizer or running a query against the index.
 pub async fn load_segment_details(
     dataset: &Dataset,
     column: &str,
@@ -200,6 +203,7 @@ pub async fn load_segment_details(
                     "failed to decode InvertedIndexDetails payload: {err}"
                 ))
             })?;
+        let details = canonicalize_inverted_index_details(details)?;
         match &expected_details {
             Some(expected) if expected != &details => {
                 return Err(Error::invalid_input(format!(
@@ -219,6 +223,13 @@ pub async fn load_segment_details(
     })
 }
 
+fn canonicalize_inverted_index_details(
+    details: InvertedIndexDetails,
+) -> Result<InvertedIndexDetails> {
+    let params = InvertedIndexParams::try_from(&details)?;
+    InvertedIndexDetails::try_from(&params)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -231,4 +242,29 @@ mod tests {
         let decoded = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap();
         assert_eq!(decoded, InvertedIndexDetails::default());
     }
+
+    #[test]
+    fn canonicalize_inverted_details_accepts_legacy_missing_analyzer() {
+        let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
+        let mut legacy = current.clone();
+        legacy.analyzer = None;
+
+        assert_ne!(legacy, current);
+        assert_eq!(
+            canonicalize_inverted_index_details(legacy).unwrap(),
+            canonicalize_inverted_index_details(current).unwrap()
+        );
+    }
+
+    #[test]
+    fn canonicalize_inverted_details_accepts_legacy_empty_details() {
+        let legacy = InvertedIndexDetails::default();
+        let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
+
+        assert_ne!(legacy, current);
+        assert_eq!(
+            canonicalize_inverted_index_details(legacy).unwrap(),
+            canonicalize_inverted_index_details(current).unwrap()
+        );
+    }
 }
diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs
index add864c0ea9..1c0ffdba9e4 100644
--- a/rust/lance/src/io/exec/fts.rs
+++ b/rust/lance/src/io/exec/fts.rs
@@ -51,7 +51,7 @@ use lance_index::scalar::inverted::query::{
 use lance_index::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer;
 use lance_index::scalar::inverted::{
     FTS_SCHEMA, InvertedIndex, MemBM25Scorer, SCORE_COL, build_global_bm25_scorer,
-    flat_bm25_search_stream_with_metrics,
+    flat_bm25_search_stream_with_metrics_and_operator,
 };
 use lance_index::{prefilter::PreFilter, scalar::inverted::query::BooleanQuery};
 use lance_tokenizer::{SimpleTokenizer, TextAnalyzer};
@@ -1095,13 +1095,14 @@ impl ExecutionPlan for FlatMatchQueryExec {
                 ),
             };
 
-            flat_bm25_search_stream_with_metrics(
+            flat_bm25_search_stream_with_metrics_and_operator(
                 unindexed_input,
                 column,
                 query.terms,
                 tokenizer,
                 base_scorer,
                 target_batch_size,
+                query.operator,
                 Some(elapsed_compute),
             )
             .await

From 43e45945cb4c21462799d2a1b0936471588386e5 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 8 Jul 2026 17:58:32 +0800
Subject: [PATCH 02/19] fix: avoid persisting unrelated FTS stop words

---
 protos/index_old.proto                            |  1 -
 python/python/tests/test_scalar_index.py          | 13 -------------
 rust/lance-index/src/scalar/inverted/tokenizer.rs |  3 ---
 3 files changed, 17 deletions(-)

diff --git a/protos/index_old.proto b/protos/index_old.proto
index d9024183b55..575e58d24e9 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -68,5 +68,4 @@ message InvertedIndexDetails {
   // Disabled by default because operators are often high-frequency noise, but
   // when enabled it changes query semantics and must be persisted.
   bool index_operators = 17;
-  repeated string custom_stop_words = 19;
 }
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index 52af81bc7f7..c5ffe91457c 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -1105,18 +1105,6 @@ def test_fts_custom_stop_words(tmp_path):
     assert len(results["_rowid"].to_pylist()) == 1
 
 
-def test_fts_custom_stop_words_none(tmp_path):
-    data = pa.table({"text": ["alpha beta", "gamma"]})
-    ds = lance.write_dataset(data, tmp_path)
-    ds.create_scalar_index("text", "INVERTED", custom_stop_words=None)
-
-    results = ds.to_table(full_text_query="alpha")
-    assert results.num_rows == 1
-
-    stats = ds.stats.index_stats("text_idx")["indices"][0]
-    assert stats["params"]["custom_stop_words"] is None
-
-
 def test_fts_stop_words_respect_language_for_simple_tokenizer(tmp_path):
     data = pa.table({"text": ["the lance data", "的 lance data"]})
     ds = lance.write_dataset(data, tmp_path, mode="overwrite")
@@ -5158,7 +5146,6 @@ def test_describe_indices(tmp_path, format_version, expected_format_version):
     assert details["lower_case"]
     assert details["stem"]
     assert details["remove_stop_words"]
-    assert details["custom_stop_words"] is None
     assert details["ascii_folding"]
     assert details["min_ngram_length"] == 3
     assert details["max_ngram_length"] == 3
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 265d2d4ccf7..102126c6dc7 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -337,7 +337,6 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             split_on_numerics: params.split_on_numerics,
             preserve_original: params.preserve_original,
             index_operators: params.index_operators,
-            custom_stop_words: params.custom_stop_words.clone().unwrap_or_default(),
         })
     }
 }
@@ -384,8 +383,6 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
         params.lower_case = details.lower_case;
         params.stem = details.stem;
         params.remove_stop_words = details.remove_stop_words;
-        params.custom_stop_words =
-            (!details.custom_stop_words.is_empty()).then(|| details.custom_stop_words.clone());
         params.ascii_folding = details.ascii_folding;
         params.min_ngram_length = details.min_ngram_length;
         params.max_ngram_length = details.max_ngram_length;

From 27dfd30a174e1088503098a12952a34d1581f7ee Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 8 Jul 2026 18:05:36 +0800
Subject: [PATCH 03/19] chore: simplify FTS Python parameter parsing

---
 python/src/dataset.rs | 152 +++++++++++++++++++++++++-----------------
 1 file changed, 92 insertions(+), 60 deletions(-)

diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index 3442071e965..e662050201c 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -2437,30 +2437,49 @@ impl Dataset {
                             )));
                         }
                     }
-                    macro_rules! insert_param {
-                        ($name:literal, $ty:ty) => {
-                            if let Some(value) = kwargs.get_item($name)? {
-                                let value: $ty = value.extract()?;
-                                params_json.insert(
-                                    $name.to_string(),
-                                    serde_json::to_value(value).map_err(|err| {
-                                        PyValueError::new_err(format!(
-                                            "failed to serialize FTS parameter {}: {}",
-                                            $name, err
-                                        ))
-                                    })?,
-                                );
-                            }
-                        };
+                    fn insert_fts_param<T: serde::Serialize>(
+                        params_json: &mut serde_json::Map<String, serde_json::Value>,
+                        name: &str,
+                        value: T,
+                    ) -> PyResult<()> {
+                        let value = serde_json::to_value(value).map_err(|err| {
+                            PyValueError::new_err(format!(
+                                "failed to serialize FTS parameter {}: {}",
+                                name, err
+                            ))
+                        })?;
+                        params_json.insert(name.to_string(), value);
+                        Ok(())
                     }
-                    insert_param!("analyzer", String);
-                    if let Some(with_position) = kwargs.get_item("with_position")? {
-                        params_json.insert(
-                            "with_position".to_string(),
-                            serde_json::Value::Bool(with_position.extract()?),
-                        );
+                    fn insert_optional_fts_param<'py, T>(
+                        params_json: &mut serde_json::Map<String, serde_json::Value>,
+                        kwargs: &Bound<'py, PyDict>,
+                        name: &str,
+                    ) -> PyResult<()>
+                    where
+                        for<'a> T: FromPyObject<'a, 'py> + serde::Serialize,
+                    {
+                        if let Some(value) = kwargs.get_item(name)? {
+                            let value: T = value.extract().map_err(Into::into)?;
+                            insert_fts_param(params_json, name, value)?;
+                        }
+                        Ok(())
                     }
-                    insert_param!("base_tokenizer", String);
+                    insert_optional_fts_param::<String>(
+                        &mut params_json,
+                        kwargs,
+                        "analyzer",
+                    )?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "with_position",
+                    )?;
+                    insert_optional_fts_param::<String>(
+                        &mut params_json,
+                        kwargs,
+                        "base_tokenizer",
+                    )?;
                     if let Some(language) = kwargs.get_item("language")? {
                         let language: PyBackedStr =
                             language.cast::<PyString>()?.clone().try_into()?;
@@ -2469,44 +2488,57 @@ impl Dataset {
                             serde_json::Value::String(language.to_string()),
                         );
                     }
-                    if let Some(max_token_length) = kwargs.get_item("max_token_length")? {
-                        let max_token_length: Option<usize> = max_token_length.extract()?;
-                        params_json.insert(
-                            "max_token_length".to_string(),
-                            serde_json::to_value(max_token_length).map_err(|err| {
-                                PyValueError::new_err(format!(
-                                    "failed to serialize FTS parameter max_token_length: {}",
-                                    err
-                                ))
-                            })?,
-                        );
-                    }
-                    insert_param!("lower_case", bool);
-                    insert_param!("stem", bool);
-                    insert_param!("remove_stop_words", bool);
-                    if let Some(custom_stop_words) = kwargs.get_item("custom_stop_words")? {
-                        let custom_stop_words: Option<Vec<String>> =
-                            custom_stop_words.extract()?;
-                        params_json.insert(
-                            "custom_stop_words".to_string(),
-                            serde_json::to_value(custom_stop_words).map_err(|err| {
-                                PyValueError::new_err(format!(
-                                    "failed to serialize FTS parameter custom_stop_words: {}",
-                                    err
-                                ))
-                            })?,
-                        );
-                    }
-                    insert_param!("ascii_folding", bool);
-                    insert_param!("min_ngram_length", u32);
-                    insert_param!("max_ngram_length", u32);
-                    insert_param!("prefix_only", bool);
-                    insert_param!("split_identifiers", bool);
-                    insert_param!("split_on_numerics", bool);
-                    insert_param!("preserve_original", bool);
-                    insert_param!("index_operators", bool);
-                    insert_param!("memory_limit", u64);
-                    insert_param!("num_workers", usize);
+                    insert_optional_fts_param::<Option<usize>>(
+                        &mut params_json,
+                        kwargs,
+                        "max_token_length",
+                    )?;
+                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "lower_case")?;
+                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "stem")?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "remove_stop_words",
+                    )?;
+                    insert_optional_fts_param::<Option<Vec<String>>>(
+                        &mut params_json,
+                        kwargs,
+                        "custom_stop_words",
+                    )?;
+                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "ascii_folding")?;
+                    insert_optional_fts_param::<u32>(
+                        &mut params_json,
+                        kwargs,
+                        "min_ngram_length",
+                    )?;
+                    insert_optional_fts_param::<u32>(
+                        &mut params_json,
+                        kwargs,
+                        "max_ngram_length",
+                    )?;
+                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "prefix_only")?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "split_identifiers",
+                    )?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "split_on_numerics",
+                    )?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "preserve_original",
+                    )?;
+                    insert_optional_fts_param::<bool>(
+                        &mut params_json,
+                        kwargs,
+                        "index_operators",
+                    )?;
+                    insert_optional_fts_param::<u64>(&mut params_json, kwargs, "memory_limit")?;
+                    insert_optional_fts_param::<usize>(&mut params_json, kwargs, "num_workers")?;
                     if let Some(format_version) = kwargs.get_item("format_version")?
                         && !format_version.is_none()
                     {

From 41f2b5dbe0811595e2baf33c8352850c29d1fb7b Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 8 Jul 2026 22:32:04 +0800
Subject: [PATCH 04/19] refactor: use typed FTS params in Python binding

---
 python/src/dataset.rs | 197 ++++++++++++++++++++----------------------
 1 file changed, 94 insertions(+), 103 deletions(-)

diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index e662050201c..2ce14ce4fcc 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -74,6 +74,7 @@ use lance_index::scalar::inverted::query::Occur;
 use lance_index::scalar::inverted::query::{
     BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
 };
+use lance_index::scalar::inverted::InvertedListFormatVersion;
 use lance_index::{
     FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
@@ -2401,7 +2402,7 @@ impl Dataset {
                 })
             }
             "INVERTED" | "FTS" => {
-                let mut params_json = serde_json::Map::new();
+                let mut params = InvertedIndexParams::default();
                 if let Some(kwargs) = kwargs {
                     let allowed_kwargs = [
                         "analyzer",
@@ -2437,126 +2438,116 @@ impl Dataset {
                             )));
                         }
                     }
-                    fn insert_fts_param<T: serde::Serialize>(
-                        params_json: &mut serde_json::Map<String, serde_json::Value>,
-                        name: &str,
-                        value: T,
-                    ) -> PyResult<()> {
-                        let value = serde_json::to_value(value).map_err(|err| {
-                            PyValueError::new_err(format!(
-                                "failed to serialize FTS parameter {}: {}",
-                                name, err
-                            ))
-                        })?;
-                        params_json.insert(name.to_string(), value);
-                        Ok(())
-                    }
-                    fn insert_optional_fts_param<'py, T>(
-                        params_json: &mut serde_json::Map<String, serde_json::Value>,
-                        kwargs: &Bound<'py, PyDict>,
-                        name: &str,
-                    ) -> PyResult<()>
-                    where
-                        for<'a> T: FromPyObject<'a, 'py> + serde::Serialize,
-                    {
-                        if let Some(value) = kwargs.get_item(name)? {
-                            let value: T = value.extract().map_err(Into::into)?;
-                            insert_fts_param(params_json, name, value)?;
+
+                    let analyzer: Option<String> = kwargs
+                        .get_item("analyzer")?
+                        .map(|value| value.extract())
+                        .transpose()?;
+                    let base_tokenizer: Option<String> = kwargs
+                        .get_item("base_tokenizer")?
+                        .map(|value| value.extract())
+                        .transpose()?;
+
+                    match (analyzer.as_deref(), base_tokenizer.as_deref()) {
+                        (Some("text"), Some("code")) => {
+                            return Err(PyValueError::new_err(
+                                "base_tokenizer='code' requires analyzer='code'",
+                            ));
+                        }
+                        (Some("code"), Some(base_tokenizer)) if base_tokenizer != "code" => {
+                            return Err(PyValueError::new_err(format!(
+                                "analyzer='code' requires base_tokenizer='code', got '{}'",
+                                base_tokenizer
+                            )));
                         }
-                        Ok(())
+                        _ => {}
+                    }
+
+                    if let Some(analyzer) = analyzer {
+                        params = params
+                            .analyzer(&analyzer)
+                            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+                    }
+                    if let Some(with_position) = kwargs.get_item("with_position")? {
+                        params = params.with_position(with_position.extract()?);
+                    }
+                    if let Some(base_tokenizer) = base_tokenizer {
+                        params = params.base_tokenizer(base_tokenizer);
                     }
-                    insert_optional_fts_param::<String>(
-                        &mut params_json,
-                        kwargs,
-                        "analyzer",
-                    )?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "with_position",
-                    )?;
-                    insert_optional_fts_param::<String>(
-                        &mut params_json,
-                        kwargs,
-                        "base_tokenizer",
-                    )?;
                     if let Some(language) = kwargs.get_item("language")? {
                         let language: PyBackedStr =
                             language.cast::<PyString>()?.clone().try_into()?;
-                        params_json.insert(
-                            "language".to_string(),
-                            serde_json::Value::String(language.to_string()),
-                        );
+                        params = params.language(&language).map_err(|err| {
+                            PyValueError::new_err(format!(
+                                "can't set tokenizer language to {}: {}",
+                                language, err
+                            ))
+                        })?;
+                    }
+                    if let Some(max_token_length) = kwargs.get_item("max_token_length")? {
+                        params = params.max_token_length(max_token_length.extract()?);
+                    }
+                    if let Some(lower_case) = kwargs.get_item("lower_case")? {
+                        params = params.lower_case(lower_case.extract()?);
+                    }
+                    if let Some(stem) = kwargs.get_item("stem")? {
+                        params = params.stem(stem.extract()?);
+                    }
+                    if let Some(remove_stop_words) = kwargs.get_item("remove_stop_words")? {
+                        params = params.remove_stop_words(remove_stop_words.extract()?);
+                    }
+                    if let Some(custom_stop_words) = kwargs.get_item("custom_stop_words")? {
+                        params = params.custom_stop_words(custom_stop_words.extract()?);
+                    }
+                    if let Some(ascii_folding) = kwargs.get_item("ascii_folding")? {
+                        params = params.ascii_folding(ascii_folding.extract()?);
+                    }
+                    if let Some(min_ngram_length) = kwargs.get_item("min_ngram_length")? {
+                        params = params.ngram_min_length(min_ngram_length.extract()?);
+                    }
+                    if let Some(max_ngram_length) = kwargs.get_item("max_ngram_length")? {
+                        params = params.ngram_max_length(max_ngram_length.extract()?);
+                    }
+                    if let Some(prefix_only) = kwargs.get_item("prefix_only")? {
+                        params = params.ngram_prefix_only(prefix_only.extract()?);
+                    }
+                    if let Some(split_identifiers) = kwargs.get_item("split_identifiers")? {
+                        params = params.split_identifiers(split_identifiers.extract()?);
+                    }
+                    if let Some(split_on_numerics) = kwargs.get_item("split_on_numerics")? {
+                        params = params.split_on_numerics(split_on_numerics.extract()?);
+                    }
+                    if let Some(preserve_original) = kwargs.get_item("preserve_original")? {
+                        params = params.preserve_original(preserve_original.extract()?);
+                    }
+                    if let Some(index_operators) = kwargs.get_item("index_operators")? {
+                        params = params.index_operators(index_operators.extract()?);
+                    }
+                    if let Some(memory_limit) = kwargs.get_item("memory_limit")? {
+                        params = params.memory_limit_mb(memory_limit.extract()?);
+                    }
+                    if let Some(num_workers) = kwargs.get_item("num_workers")? {
+                        params = params.num_workers(num_workers.extract()?);
                     }
-                    insert_optional_fts_param::<Option<usize>>(
-                        &mut params_json,
-                        kwargs,
-                        "max_token_length",
-                    )?;
-                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "lower_case")?;
-                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "stem")?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "remove_stop_words",
-                    )?;
-                    insert_optional_fts_param::<Option<Vec<String>>>(
-                        &mut params_json,
-                        kwargs,
-                        "custom_stop_words",
-                    )?;
-                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "ascii_folding")?;
-                    insert_optional_fts_param::<u32>(
-                        &mut params_json,
-                        kwargs,
-                        "min_ngram_length",
-                    )?;
-                    insert_optional_fts_param::<u32>(
-                        &mut params_json,
-                        kwargs,
-                        "max_ngram_length",
-                    )?;
-                    insert_optional_fts_param::<bool>(&mut params_json, kwargs, "prefix_only")?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "split_identifiers",
-                    )?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "split_on_numerics",
-                    )?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "preserve_original",
-                    )?;
-                    insert_optional_fts_param::<bool>(
-                        &mut params_json,
-                        kwargs,
-                        "index_operators",
-                    )?;
-                    insert_optional_fts_param::<u64>(&mut params_json, kwargs, "memory_limit")?;
-                    insert_optional_fts_param::<usize>(&mut params_json, kwargs, "num_workers")?;
                     if let Some(format_version) = kwargs.get_item("format_version")?
                         && !format_version.is_none()
                     {
                         let value = if let Ok(value) = format_version.cast::<PyString>() {
-                            serde_json::Value::String(value.to_string_lossy().to_string())
+                            value.to_string_lossy().to_string()
                         } else if let Ok(value) = format_version.extract::<u32>() {
-                            serde_json::Value::from(value)
+                            value.to_string()
                         } else {
                             return Err(PyValueError::new_err(
                                 "format_version must be 1, 2, 'v1', or 'v2'",
                             ));
                         };
-                        params_json.insert("format_version".to_string(), value);
+                        let format_version =
+                            value.parse::<InvertedListFormatVersion>().map_err(|err| {
+                                PyValueError::new_err(err.to_string())
+                            })?;
+                        params = params.format_version(format_version);
                     }
                 }
-                let params: InvertedIndexParams =
-                    serde_json::from_value(serde_json::Value::Object(params_json))
-                        .map_err(|err| PyValueError::new_err(err.to_string()))?;
                 Box::new(params)
             }
             _ => {

From 6d2b5880d93704ab9ca9ba7908d87aad714def7b Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 00:17:03 +0800
Subject: [PATCH 05/19] fix: stabilize FTS code analyzer details

---
 rust/lance-index/src/scalar/inverted/index.rs | 181 +++++++++++++++++-
 .../src/scalar/inverted/tokenizer.rs          |  21 ++
 rust/lance/src/index/scalar/inverted.rs       |  60 +++++-
 3 files changed, 254 insertions(+), 8 deletions(-)

diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 0fff5b15c38..dcb15a9fbc7 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -18,7 +18,7 @@ use std::{
 use crate::metrics::NoOpMetricsCollector;
 use crate::prefilter::NoFilter;
 use crate::scalar::registry::{TrainingCriteria, TrainingOrdering};
-use arrow::array::{FixedSizeListBuilder, Float32Builder};
+use arrow::array::{FixedSizeListBuilder, Float32Builder, Int32Builder};
 use arrow::datatypes::{self, Float32Type, Int32Type, UInt64Type};
 use arrow::{
     array::{
@@ -1682,6 +1682,49 @@ impl InvertedPartition {
         )))
     }
 
+    fn union_plain_posting_lists_with_positions(postings: Vec<PostingList>) -> Result<PostingList> {
+        let mut positions_by_row_id = BTreeMap::<u64, Vec<u32>>::new();
+        for posting in postings {
+            for (row_id, _, positions) in posting.iter() {
+                let positions = positions.ok_or_else(|| {
+                    Error::index("cannot union grouped phrase terms without positions".to_string())
+                })?;
+                positions_by_row_id
+                    .entry(row_id)
+                    .or_default()
+                    .extend(positions);
+            }
+        }
+        if positions_by_row_id.is_empty() {
+            return Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            )));
+        }
+
+        let mut row_ids = Vec::with_capacity(positions_by_row_id.len());
+        let mut frequencies = Vec::with_capacity(positions_by_row_id.len());
+        let mut positions_builder = ListBuilder::new(Int32Builder::new());
+        for (row_id, mut positions) in positions_by_row_id {
+            positions.sort_unstable();
+            row_ids.push(row_id);
+            frequencies.push(positions.len() as f32);
+            for position in positions {
+                positions_builder.values().append_value(position as i32);
+            }
+            positions_builder.append(true);
+        }
+
+        Ok(PostingList::Plain(PlainPostingList::new(
+            ScalarBuffer::from(row_ids),
+            ScalarBuffer::from(frequencies),
+            None,
+            Some(positions_builder.finish()),
+        )))
+    }
+
     fn union_compressed_posting_lists(
         postings: Vec<PostingList>,
         docs: &DocSet,
@@ -1725,7 +1768,59 @@ impl InvertedPartition {
         PostingList::from_batch(&batch, Some(max_score), Some(length))
     }
 
-    fn union_posting_lists(postings: Vec<PostingList>, docs: &DocSet) -> Result<PostingList> {
+    fn union_compressed_posting_lists_with_positions(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+    ) -> Result<PostingList> {
+        let mut positions_by_doc_id = BTreeMap::<u32, Vec<u32>>::new();
+        for posting in postings {
+            for (doc_id, _, positions) in posting.iter() {
+                let doc_id = u32::try_from(doc_id).map_err(|_| {
+                    Error::index(format!(
+                        "compressed posting doc id {} exceeds u32::MAX",
+                        doc_id
+                    ))
+                })?;
+                let positions = positions.ok_or_else(|| {
+                    Error::index("cannot union grouped phrase terms without positions".to_string())
+                })?;
+                positions_by_doc_id
+                    .entry(doc_id)
+                    .or_default()
+                    .extend(positions);
+            }
+        }
+        if positions_by_doc_id.is_empty() {
+            return Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            )));
+        }
+
+        let mut builder = PostingListBuilder::new(true);
+        let mut doc_ids = Vec::with_capacity(positions_by_doc_id.len());
+        let mut frequencies = Vec::with_capacity(positions_by_doc_id.len());
+        for (doc_id, mut positions) in positions_by_doc_id {
+            positions.sort_unstable();
+            let frequency = positions.len() as u32;
+            builder.add(doc_id, PositionRecorder::Position(positions.into()));
+            doc_ids.push(doc_id);
+            frequencies.push(frequency);
+        }
+        let block_max_scores = docs.calculate_block_max_scores(doc_ids.iter(), frequencies.iter());
+        let batch = builder.to_batch(block_max_scores)?;
+        let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+        let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+        PostingList::from_batch(&batch, Some(max_score), Some(length))
+    }
+
+    fn union_posting_lists(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        with_positions: bool,
+    ) -> Result<PostingList> {
         let has_plain = postings
             .iter()
             .any(|posting| matches!(posting, PostingList::Plain(_)));
@@ -1736,7 +1831,13 @@ impl InvertedPartition {
             (true, true) => Err(Error::index(
                 "cannot union mixed plain and compressed posting lists".to_owned(),
             )),
+            (true, false) if with_positions => {
+                Self::union_plain_posting_lists_with_positions(postings)
+            }
             (true, false) => Self::union_plain_posting_lists(postings),
+            (false, true) if with_positions => {
+                Self::union_compressed_posting_lists_with_positions(postings, docs)
+            }
             (false, true) => Self::union_compressed_posting_lists(postings, docs),
             (false, false) => Ok(PostingList::Plain(PlainPostingList::new(
                 ScalarBuffer::from(Vec::<u64>::new()),
@@ -1859,6 +1960,7 @@ impl InvertedPartition {
                     docs_for_union
                         .as_deref()
                         .expect("union docs must be loaded for grouped query terms"),
+                    is_phrase_query,
                 )?;
                 if posting.is_empty() && (is_and_query || is_phrase_query) {
                     return Ok(LoadedPostings::empty());
@@ -8195,6 +8297,81 @@ mod tests {
         assert_eq!(row_ids, vec![100, 101]);
     }
 
+    #[tokio::test]
+    async fn test_phrase_query_accepts_same_position_alternatives() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, true, TokenSetFormat::default());
+        for token in ["getusername", "get", "user", "name"] {
+            builder.tokens.add(token.to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(true));
+        }
+        // Doc 0 only has split words. Doc 1 has both the complete identifier
+        // and split words at the same position. Doc 2 has the terms but not as
+        // an exact phrase.
+        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(0, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[3].add(0, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(100, 3);
+
+        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(1, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[3].add(1, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(101, 3);
+
+        builder.posting_lists[0].add(2, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[2].add(2, PositionRecorder::Position(vec![2].into()));
+        builder.posting_lists[3].add(2, PositionRecorder::Position(vec![3].into()));
+        builder.docs.append(102, 3);
+
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(
+            &store,
+            vec![0],
+            InvertedIndexParams::code().with_position(true),
+        )
+        .await;
+        let index = InvertedIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec![
+                "getusername".to_string(),
+                "get".to_string(),
+                "user".to_string(),
+                "name".to_string(),
+            ],
+            vec![0, 0, 1, 2],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_phrase_slop(Some(0)),
+        );
+        let (mut row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        row_ids.sort_unstable();
+        assert_eq!(row_ids, vec![100, 101]);
+    }
+
     // Enough distinct tokens that `write_posting_lists` emits several posting-list
     // batches (the default batch size is 256 rows), exercising the restructured
     // producer and async send path.
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 102126c6dc7..3fcf4f2533d 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -168,6 +168,7 @@ struct RawInvertedIndexParams {
     base_tokenizer: Option<String>,
     language: Option<Language>,
     with_position: Option<bool>,
+    #[serde(default, deserialize_with = "deserialize_explicit_option")]
     max_token_length: Option<Option<usize>>,
     lower_case: Option<bool>,
     stem: Option<bool>,
@@ -446,6 +447,16 @@ where
     }
 }
 
+fn deserialize_explicit_option<'de, D, T>(
+    deserializer: D,
+) -> std::result::Result<Option<Option<T>>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+    T: Deserialize<'de>,
+{
+    Option::<T>::deserialize(deserializer).map(Some)
+}
+
 impl Default for InvertedIndexParams {
     fn default() -> Self {
         Self::new("simple".to_owned(), Language::English)
@@ -888,6 +899,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_training_json_preserves_disabled_max_token_length() {
+        let params = InvertedIndexParams::default().max_token_length(None);
+        let json = params.to_training_json().unwrap();
+        assert_eq!(json.get("max_token_length"), Some(&serde_json::Value::Null));
+
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.max_token_length, None);
+    }
+
     #[test]
     fn test_default_format_version_resolves_to_v2() {
         assert_eq!(
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index 5aebf66f627..bb253522c03 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -11,7 +11,9 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use lance_core::ROW_ID;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::pbold::InvertedIndexDetails;
-use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexParams};
+use lance_index::scalar::inverted::{
+    InvertedIndex, InvertedIndexParams, document_tokenizer::DocType,
+};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
 use lance_table::format::IndexMetadata;
@@ -203,7 +205,7 @@ pub async fn load_segment_details(
                     "failed to decode InvertedIndexDetails payload: {err}"
                 ))
             })?;
-        let details = canonicalize_inverted_index_details(details)?;
+        let details = canonicalize_inverted_index_details(dataset, column, details)?;
         match &expected_details {
             Some(expected) if expected != &details => {
                 return Err(Error::invalid_input(format!(
@@ -224,9 +226,35 @@ pub async fn load_segment_details(
 }
 
 fn canonicalize_inverted_index_details(
+    dataset: &Dataset,
+    column: &str,
+    details: InvertedIndexDetails,
+) -> Result<InvertedIndexDetails> {
+    let field = dataset.schema().field(column).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "FTS index details require column {} to exist in the dataset schema",
+            column
+        ))
+    })?;
+    let field = ArrowField::from(field);
+    canonicalize_inverted_index_details_for_field(&field, details)
+}
+
+fn canonicalize_inverted_index_details_for_field(
+    field: &ArrowField,
     details: InvertedIndexDetails,
 ) -> Result<InvertedIndexDetails> {
+    let inferred_lance_tokenizer = if details.lance_tokenizer.is_none() {
+        Some(DocType::try_from(field)?.as_ref().to_string())
+    } else {
+        None
+    };
     let params = InvertedIndexParams::try_from(&details)?;
+    let params = if let Some(lance_tokenizer) = inferred_lance_tokenizer {
+        params.lance_tokenizer(lance_tokenizer)
+    } else {
+        params
+    };
     InvertedIndexDetails::try_from(&params)
 }
 
@@ -234,6 +262,10 @@ fn canonicalize_inverted_index_details(
 mod tests {
     use super::*;
 
+    fn text_field() -> ArrowField {
+        ArrowField::new("text", arrow_schema::DataType::Utf8, true)
+    }
+
     #[test]
     fn decode_legacy_inverted_details_type_url() {
         let mut details_any = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap();
@@ -251,8 +283,8 @@ mod tests {
 
         assert_ne!(legacy, current);
         assert_eq!(
-            canonicalize_inverted_index_details(legacy).unwrap(),
-            canonicalize_inverted_index_details(current).unwrap()
+            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
+            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
         );
     }
 
@@ -263,8 +295,24 @@ mod tests {
 
         assert_ne!(legacy, current);
         assert_eq!(
-            canonicalize_inverted_index_details(legacy).unwrap(),
-            canonicalize_inverted_index_details(current).unwrap()
+            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
+            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
+        );
+    }
+
+    #[test]
+    fn canonicalize_inverted_details_infers_missing_lance_tokenizer() {
+        let mut legacy = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
+        legacy.lance_tokenizer = None;
+        let current = InvertedIndexDetails::try_from(
+            &InvertedIndexParams::default().lance_tokenizer("text".to_string()),
+        )
+        .unwrap();
+
+        assert_ne!(legacy, current);
+        assert_eq!(
+            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
+            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
         );
     }
 }

From f7329fb2dd599e46b5454c507b3a1404a98effea Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 00:23:57 +0800
Subject: [PATCH 06/19] style: format Python Rust bindings

---
 python/src/dataset.rs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index 2ce14ce4fcc..6dd893885ce 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -70,11 +70,11 @@ use lance_core::datatypes::BlobHandling;
 use lance_datafusion::utils::reader_to_stream;
 use lance_encoding::decoder::DecoderConfig;
 use lance_file::reader::FileReaderOptions;
+use lance_index::scalar::inverted::InvertedListFormatVersion;
 use lance_index::scalar::inverted::query::Occur;
 use lance_index::scalar::inverted::query::{
     BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
 };
-use lance_index::scalar::inverted::InvertedListFormatVersion;
 use lance_index::{
     FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
@@ -2541,10 +2541,9 @@ impl Dataset {
                                 "format_version must be 1, 2, 'v1', or 'v2'",
                             ));
                         };
-                        let format_version =
-                            value.parse::<InvertedListFormatVersion>().map_err(|err| {
-                                PyValueError::new_err(err.to_string())
-                            })?;
+                        let format_version = value
+                            .parse::<InvertedListFormatVersion>()
+                            .map_err(|err| PyValueError::new_err(err.to_string()))?;
                         params = params.format_version(format_version);
                     }
                 }

From a6dd5ef2c58cc198ee06901994dcaa0cc3c59b6e Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 11:57:43 +0800
Subject: [PATCH 07/19] fix: disable code identifier splitting by default

---
 python/python/tests/test_scalar_index.py      | 27 ++++++++++++--
 .../src/scalar/inverted/tokenizer.rs          | 36 ++++++++++++++++---
 2 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index c5ffe91457c..df5dfe8bedf 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -807,7 +807,25 @@ def test_full_text_search(dataset, with_position, base_tokenizer):
         )
 
 
-def test_code_analyzer_full_text_search(tmp_path):
+def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path):
+    table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]})
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+
+    results = ds.to_table(
+        columns=["code"],
+        full_text_query=MatchQuery("user", "code"),
+    )
+    assert results["code"].to_pylist() == ["user"]
+
+    stats = ds.stats.index_stats("code_idx")["indices"][0]
+    params = stats["params"]
+    assert params["analyzer"] == "code"
+    assert params["base_tokenizer"] == "code"
+    assert params["split_identifiers"] is False
+
+
+def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path):
     table = pa.table(
         {
             "code": [
@@ -820,7 +838,12 @@ def test_code_analyzer_full_text_search(tmp_path):
         }
     )
     ds = lance.write_dataset(table, tmp_path)
-    ds.create_scalar_index("code", index_type="INVERTED", analyzer="code")
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        split_identifiers=True,
+    )
 
     results = ds.to_table(
         columns=["code"],
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 3fcf4f2533d..51037c9b45f 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -39,8 +39,9 @@ pub struct InvertedIndexParams {
     ///
     /// This is the user-facing semantic preset. `text` keeps the existing
     /// natural-language defaults. `code` selects code-search defaults such as
-    /// code lexical tokenization, identifier splitting, preserving whole
-    /// identifiers, and disabling stemming / stop-word removal.
+    /// code lexical tokenization, optional identifier splitting, preserving
+    /// whole identifiers when splitting is enabled, and disabling stemming /
+    /// stop-word removal.
     pub(crate) analyzer: String,
 
     /// Document-level tokenizer.
@@ -518,7 +519,7 @@ impl InvertedIndexParams {
         self.analyzer = normalize_analyzer(analyzer)?.to_string();
         if self.analyzer == "code" {
             self.base_tokenizer = "code".to_string();
-            self.split_identifiers = true;
+            self.split_identifiers = false;
             self.split_on_numerics = true;
             self.preserve_original = true;
             self.lower_case = true;
@@ -925,7 +926,7 @@ mod tests {
         .unwrap();
         assert_eq!(params.analyzer, "code");
         assert_eq!(params.base_tokenizer, "code");
-        assert!(params.split_identifiers);
+        assert!(!params.split_identifiers);
         assert!(params.split_on_numerics);
         assert!(params.preserve_original);
         assert!(params.lower_case);
@@ -956,7 +957,7 @@ mod tests {
     }
 
     #[test]
-    fn test_build_code_tokenizer() {
+    fn test_build_code_tokenizer_does_not_split_identifiers_by_default() {
         let mut tokenizer = InvertedIndexParams::code().build().unwrap();
         let mut stream = tokenizer.token_stream_for_doc(
             "getUserName XMLHttpRequest parseHTML2JSON utf8_reader SCREAMING_SNAKE_CASE",
@@ -965,6 +966,31 @@ mod tests {
         stream.process(&mut |token| {
             tokens.push((token.text.clone(), token.position, token.position_length))
         });
+        assert_eq!(
+            tokens,
+            vec![
+                ("getusername".to_string(), 0, 1),
+                ("xmlhttprequest".to_string(), 1, 1),
+                ("parsehtml2json".to_string(), 2, 1),
+                ("utf8_reader".to_string(), 3, 1),
+                ("screaming_snake_case".to_string(), 4, 1),
+            ]
+        );
+    }
+
+    #[test]
+    fn test_build_code_tokenizer_with_identifier_splitting() {
+        let mut tokenizer = InvertedIndexParams::code()
+            .split_identifiers(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_doc(
+            "getUserName XMLHttpRequest parseHTML2JSON utf8_reader SCREAMING_SNAKE_CASE",
+        );
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| {
+            tokens.push((token.text.clone(), token.position, token.position_length))
+        });
         assert_eq!(
             tokens,
             vec![

From 4b4c63d06415377251189681d37286a3bee98e54 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 12:14:57 +0800
Subject: [PATCH 08/19] test: opt in to code identifier splitting

---
 rust/lance/src/dataset/mem_wal/index/fts.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 8a25625a460..d8022747b46 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -3760,8 +3760,11 @@ mod tests {
     #[test]
     fn test_code_analyzer_and_query_uses_position_alternatives() {
         let schema = create_test_schema();
-        let index =
-            FtsMemIndex::with_params(1, "description".to_string(), InvertedIndexParams::code());
+        let index = FtsMemIndex::with_params(
+            1,
+            "description".to_string(),
+            InvertedIndexParams::code().split_identifiers(true),
+        );
         let batch = RecordBatch::try_new(
             schema,
             vec![

From 45c9d666e2b6226b496b2a1991196b3e672e6e24 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 12:40:59 +0800
Subject: [PATCH 09/19] test: opt in to code tokenizer splitting

---
 rust/lance-index/src/scalar/inverted/index.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index dcb15a9fbc7..8d375517060 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -9553,7 +9553,10 @@ mod tests {
             schema.clone(),
             stream::iter(vec![Ok(batch)]),
         ));
-        let tokenizer = InvertedIndexParams::code().build().unwrap();
+        let tokenizer = InvertedIndexParams::code()
+            .split_identifiers(true)
+            .build()
+            .unwrap();
 
         let result_stream = flat_bm25_search_stream_with_metrics_and_operator(
             input,

From 2fbf49b8cd669de8385255406296383d1876906a Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 12:53:09 +0800
Subject: [PATCH 10/19] fix: address code analyzer review feedback

---
 protos/index_old.proto                        |  12 +-
 python/python/tests/test_scalar_index.py      |  12 ++
 python/src/dataset.rs                         |  22 ++++
 rust/lance-index/src/scalar/inverted/index.rs |  37 ++++--
 .../src/scalar/inverted/tokenizer.rs          | 111 +++++++++++++++---
 rust/lance-tokenizer/src/code_tokenizer.rs    |  15 +++
 .../src/word_delimiter_filter.rs              |  26 +++-
 7 files changed, 199 insertions(+), 36 deletions(-)

diff --git a/protos/index_old.proto b/protos/index_old.proto
index 575e58d24e9..4ebc2143e6a 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -57,13 +57,13 @@ message InvertedIndexDetails {
   // be persisted for query-time tokenizer reconstruction and segment checks.
   bool split_identifiers = 14;
   // Code analyzer flag. Split identifier subwords across letter/number
-  // boundaries, e.g. HTML2JSON -> html/2/json. This changes the indexed term
-  // space and must be persisted with the index.
-  bool split_on_numerics = 15;
+  // boundaries, e.g. HTML2JSON -> html/2/json. When absent, use the analyzer
+  // profile default; when present, persist the explicit index-time choice.
+  optional bool split_on_numerics = 15;
   // Code analyzer flag. Keep the complete lexical identifier in addition to
-  // subwords, e.g. user_name plus user/name. This affects BM25 term statistics
-  // and must be persisted with the index.
-  bool preserve_original = 16;
+  // subwords, e.g. user_name plus user/name. When absent, use the analyzer
+  // profile default; when present, persist the explicit index-time choice.
+  optional bool preserve_original = 16;
   // Code analyzer flag. Index operator tokens such as "::", "->", and "!=".
   // Disabled by default because operators are often high-frequency noise, but
   // when enabled it changes query semantics and must be persisted.
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index df5dfe8bedf..d947bb1ab62 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -866,6 +866,18 @@ def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path):
     assert params["remove_stop_words"] is False
 
 
+def test_code_analyzer_flags_require_code_analyzer(tmp_path):
+    table = pa.table({"text": ["getUserName"]})
+    ds = lance.write_dataset(table, tmp_path)
+
+    with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"):
+        ds.create_scalar_index(
+            "text",
+            index_type="INVERTED",
+            split_identifiers=True,
+        )
+
+
 def test_code_analyzer_complex_code_constructs(tmp_path):
     table = pa.table(
         {
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index 6dd893885ce..9cac162d463 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -2463,6 +2463,28 @@ impl Dataset {
                         _ => {}
                     }
 
+                    let uses_code_analyzer = match analyzer.as_deref() {
+                        Some("code") => true,
+                        Some("text") | None => base_tokenizer.as_deref() == Some("code"),
+                        Some(_) => true,
+                    };
+                    if !uses_code_analyzer {
+                        for flag in [
+                            "split_identifiers",
+                            "split_on_numerics",
+                            "preserve_original",
+                            "index_operators",
+                        ] {
+                            if let Some(value) = kwargs.get_item(flag)?
+                                && value.extract::<bool>()?
+                            {
+                                return Err(PyValueError::new_err(
+                                    "code analyzer flags require analyzer='code'",
+                                ));
+                            }
+                        }
+                    }
+
                     if let Some(analyzer) = analyzer {
                         params = params
                             .analyzer(&analyzer)
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 8d375517060..111bd2f9710 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -1955,13 +1955,10 @@ impl InvertedPartition {
                     .into_iter()
                     .map(|(_, _, posting)| posting)
                     .collect::<Vec<_>>();
-                let posting = Self::union_posting_lists(
-                    postings,
-                    docs_for_union
-                        .as_deref()
-                        .expect("union docs must be loaded for grouped query terms"),
-                    is_phrase_query,
-                )?;
+                let docs = docs_for_union.as_deref().ok_or_else(|| {
+                    Error::index("union docs were not loaded for grouped query terms".to_string())
+                })?;
+                let posting = Self::union_posting_lists(postings, docs, is_phrase_query)?;
                 if posting.is_empty() && (is_and_query || is_phrase_query) {
                     return Ok(LoadedPostings::empty());
                 }
@@ -6029,6 +6026,32 @@ pub async fn flat_bm25_search_stream_with_metrics(
 
 /// Same as [`flat_bm25_search_stream_with_metrics`] but applies the provided
 /// match operator when deciding whether a flat-scanned row is a hit.
+///
+/// # Examples
+///
+/// ```no_run
+/// # async fn example(
+/// #     input: datafusion::execution::SendableRecordBatchStream,
+/// # ) -> Result<(), Box<dyn std::error::Error>> {
+/// use lance_index::scalar::inverted::{
+///     flat_bm25_search_stream_with_metrics_and_operator, query::Operator, InvertedIndexParams,
+/// };
+///
+/// let tokenizer = InvertedIndexParams::code().build()?;
+/// let _stream = flat_bm25_search_stream_with_metrics_and_operator(
+///     input,
+///     "code".to_string(),
+///     "Result".to_string(),
+///     tokenizer,
+///     None,
+///     1024,
+///     Operator::And,
+///     None,
+/// )
+/// .await?;
+/// # Ok(())
+/// # }
+/// ```
 #[allow(clippy::too_many_arguments)]
 pub async fn flat_bm25_search_stream_with_metrics_and_operator(
     input: SendableRecordBatchStream,
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 51037c9b45f..ed56181bbea 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -336,8 +336,8 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             max_ngram_length: params.max_ngram_length,
             prefix_only: params.prefix_only,
             split_identifiers: params.split_identifiers,
-            split_on_numerics: params.split_on_numerics,
-            preserve_original: params.preserve_original,
+            split_on_numerics: Some(params.split_on_numerics),
+            preserve_original: Some(params.preserve_original),
             index_operators: params.index_operators,
         })
     }
@@ -390,8 +390,12 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
         params.max_ngram_length = details.max_ngram_length;
         params.prefix_only = details.prefix_only;
         params.split_identifiers = details.split_identifiers;
-        params.split_on_numerics = details.split_on_numerics;
-        params.preserve_original = details.preserve_original;
+        if let Some(split_on_numerics) = details.split_on_numerics {
+            params.split_on_numerics = split_on_numerics;
+        }
+        if let Some(preserve_original) = details.preserve_original {
+            params.preserve_original = preserve_original;
+        }
         params.index_operators = details.index_operators;
         params.validate()?;
         Ok(params)
@@ -507,26 +511,52 @@ impl InvertedIndexParams {
         }
     }
 
+    fn apply_code_defaults(&mut self) {
+        self.analyzer = "code".to_string();
+        self.base_tokenizer = "code".to_string();
+        self.split_identifiers = false;
+        self.split_on_numerics = true;
+        self.preserve_original = true;
+        self.lower_case = true;
+        self.ascii_folding = true;
+        self.stem = false;
+        self.remove_stop_words = false;
+        self.index_operators = false;
+    }
+
     /// Create parameters for the code analyzer profile.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let tokenizer = InvertedIndexParams::code().build();
+    /// assert!(tokenizer.is_ok());
+    /// ```
     pub fn code() -> Self {
-        Self::new("code".to_string(), Language::English)
-            .analyzer("code")
-            .expect("code analyzer should be valid")
+        let mut params = Self::new("code".to_string(), Language::English);
+        params.apply_code_defaults();
+        params
     }
 
     /// Set analyzer profile.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::default()
+    ///     .analyzer("code")?
+    ///     .split_identifiers(true);
+    /// assert!(params.build().is_ok());
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn analyzer(mut self, analyzer: &str) -> Result<Self> {
         self.analyzer = normalize_analyzer(analyzer)?.to_string();
         if self.analyzer == "code" {
-            self.base_tokenizer = "code".to_string();
-            self.split_identifiers = false;
-            self.split_on_numerics = true;
-            self.preserve_original = true;
-            self.lower_case = true;
-            self.ascii_folding = true;
-            self.stem = false;
-            self.remove_stop_words = false;
-            self.index_operators = false;
+            self.apply_code_defaults();
         }
         Ok(self)
     }
@@ -536,12 +566,24 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set the tokenizer implementation used by the analyzer.
+    ///
+    /// Setting this to `"code"` selects the code analyzer defaults.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::default()
+    ///     .base_tokenizer("code".to_string())
+    ///     .split_identifiers(true);
+    /// assert!(params.build().is_ok());
+    /// ```
     pub fn base_tokenizer(mut self, base_tokenizer: String) -> Self {
         self.base_tokenizer = base_tokenizer;
         if self.base_tokenizer == "code" {
-            self = self
-                .analyzer("code")
-                .expect("code analyzer should be valid");
+            self.apply_code_defaults();
         }
         self
     }
@@ -598,21 +640,38 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set whether code identifiers are split into subwords.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use lance_index::scalar::InvertedIndexParams;
+    ///
+    /// let params = InvertedIndexParams::code()
+    ///     .split_identifiers(true)
+    ///     .split_on_numerics(false)
+    ///     .preserve_original(false)
+    ///     .index_operators(true);
+    /// assert!(params.build().is_ok());
+    /// ```
     pub fn split_identifiers(mut self, split_identifiers: bool) -> Self {
         self.split_identifiers = split_identifiers;
         self
     }
 
+    /// Set whether identifier subwords are split at letter/number boundaries.
     pub fn split_on_numerics(mut self, split_on_numerics: bool) -> Self {
         self.split_on_numerics = split_on_numerics;
         self
     }
 
+    /// Set whether the complete identifier is indexed alongside subwords.
     pub fn preserve_original(mut self, preserve_original: bool) -> Self {
         self.preserve_original = preserve_original;
         self
     }
 
+    /// Set whether operator tokens such as `::`, `->`, and `!=` are indexed.
     pub fn index_operators(mut self, index_operators: bool) -> Self {
         self.index_operators = index_operators;
         self
@@ -1026,10 +1085,24 @@ mod tests {
             .index_operators(true)
             .preserve_original(false);
         let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        assert_eq!(details.split_on_numerics, Some(true));
+        assert_eq!(details.preserve_original, Some(false));
         let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
         assert_eq!(round_tripped, params);
     }
 
+    #[test]
+    fn test_inverted_details_uses_code_defaults_for_absent_flags() {
+        let mut details =
+            crate::pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::code()).unwrap();
+        details.split_on_numerics = None;
+        details.preserve_original = None;
+
+        let params = InvertedIndexParams::try_from(&details).unwrap();
+        assert!(params.split_on_numerics);
+        assert!(params.preserve_original);
+    }
+
     #[test]
     fn test_build_icu_tokenizer() {
         let mut tokenizer = InvertedIndexParams::default()
diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs
index a378eebb879..743084f9e05 100644
--- a/rust/lance-tokenizer/src/code_tokenizer.rs
+++ b/rust/lance-tokenizer/src/code_tokenizer.rs
@@ -10,6 +10,20 @@ use crate::{Token, TokenStream, Tokenizer};
 /// Identifiers are Unicode alphanumeric characters plus `_`. Other characters
 /// are lexical boundaries. When operator indexing is enabled, contiguous
 /// operator characters are emitted as tokens too.
+///
+/// # Examples
+///
+/// ```
+/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream};
+///
+/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(true)).build();
+/// let mut stream = analyzer.token_stream("a::b");
+///
+/// assert!(stream.advance());
+/// assert_eq!(stream.token().text, "a");
+/// assert!(stream.advance());
+/// assert_eq!(stream.token().text, "::");
+/// ```
 #[derive(Clone, Default)]
 pub struct CodeLexTokenizer {
     index_operators: bool,
@@ -25,6 +39,7 @@ impl CodeLexTokenizer {
     }
 }
 
+/// Token stream produced by [`CodeLexTokenizer`].
 pub struct CodeLexTokenStream<'a> {
     text: &'a str,
     chars: Peekable<CharIndices<'a>>,
diff --git a/rust/lance-tokenizer/src/word_delimiter_filter.rs b/rust/lance-tokenizer/src/word_delimiter_filter.rs
index b03834c03f1..70138529efe 100644
--- a/rust/lance-tokenizer/src/word_delimiter_filter.rs
+++ b/rust/lance-tokenizer/src/word_delimiter_filter.rs
@@ -9,6 +9,21 @@ use crate::{Token, TokenFilter, TokenStream, Tokenizer};
 ///
 /// The original identifier can be preserved at the first subword position. If
 /// a token is not a compound identifier then it is passed through unchanged.
+///
+/// # Examples
+///
+/// ```
+/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream, WordDelimiterFilter};
+///
+/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false))
+///     .filter(WordDelimiterFilter::new(true, true))
+///     .build();
+/// let mut stream = analyzer.token_stream("parseHTML2JSON");
+///
+/// let mut tokens = Vec::new();
+/// stream.process(&mut |token| tokens.push(token.text.clone()));
+/// assert_eq!(tokens, vec!["parseHTML2JSON", "parse", "HTML", "2", "JSON"]);
+/// ```
 #[derive(Clone)]
 pub struct WordDelimiterFilter {
     preserve_original: bool,
@@ -37,6 +52,7 @@ impl TokenFilter for WordDelimiterFilter {
 }
 
 #[derive(Clone)]
+/// Tokenizer wrapper produced by [`WordDelimiterFilter`].
 pub struct WordDelimiterFilterWrapper<T> {
     tokenizer: T,
     preserve_original: bool,
@@ -58,6 +74,7 @@ impl<T: Tokenizer> Tokenizer for WordDelimiterFilterWrapper<T> {
     }
 }
 
+/// Token stream produced by [`WordDelimiterFilterWrapper`].
 pub struct WordDelimiterTokenStream<T> {
     tail: T,
     current: Token,
@@ -197,10 +214,11 @@ impl<T: TokenStream> TokenStream for WordDelimiterTokenStream<T> {
         if !self.refill() {
             return false;
         }
-        self.current = self
-            .pending
-            .pop_front()
-            .expect("pending token should be available after refill");
+        let Some(token) = self.pending.pop_front() else {
+            debug_assert!(false, "pending token should be available after refill");
+            return false;
+        };
+        self.current = token;
         true
     }
 

From 6374862c7539013342f2d7d7cff4d9cd8a5a0b98 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 9 Jul 2026 23:34:29 +0800
Subject: [PATCH 11/19] fix: align mem-wal fts token positions

---
 rust/lance/src/dataset/mem_wal/index/fts.rs | 311 ++++++++++++++++++--
 1 file changed, 294 insertions(+), 17 deletions(-)

diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index d8022747b46..a78a9ab195b 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -402,6 +402,10 @@ fn query_position_groups(tokens: &Tokens) -> Vec<Vec<String>> {
     groups
 }
 
+fn position_groups_to_tokens(groups: &[Vec<String>]) -> Vec<String> {
+    groups.iter().flatten().cloned().collect()
+}
+
 /// Builder for constructing Boolean queries.
 #[derive(Debug, Clone, Default)]
 pub struct BooleanQueryBuilder {
@@ -1115,6 +1119,14 @@ impl FtsMemIndex {
         self.insert_batch(batch, row_offset)
     }
 
+    fn checked_token_position(row_position: RowPosition, token_position: usize) -> Result<u32> {
+        u32::try_from(token_position).map_err(|_| {
+            Error::invalid_input(format!(
+                "token position overflow for row_position={row_position}: token_position={token_position}"
+            ))
+        })
+    }
+
     fn insert_batch(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> {
         let st = self.state.load_full();
         let batch_position = st.tail.next_position();
@@ -1164,8 +1176,8 @@ impl FtsMemIndex {
 
             if let Some(text) = text_opt {
                 let mut stream = tokenizer.token_stream_for_doc(text);
-                let mut position: u32 = 0;
                 while let Some(tok) = stream.next() {
+                    let position = Self::checked_token_position(row_position, tok.position)?;
                     let term = tok.text.as_str();
                     // One hash lookup per token: extend the term's builder, or
                     // intern its `Arc<str>` once on first sight this batch.
@@ -1177,7 +1189,6 @@ impl FtsMemIndex {
                             BatchTermBuilder::with_first(row_position, position),
                         );
                     }
-                    position += 1;
                     doc_token_count += 1;
                 }
             }
@@ -1306,7 +1317,7 @@ impl FtsMemIndex {
     /// `slop` intervening tokens between consecutive query tokens.
     pub fn search_phrase(&self, phrase: &str, slop: u32) -> Vec<FtsEntry> {
         let st = self.state.load_full();
-        let tokens = self.tokenize_for_search(phrase);
+        let tokens = self.analyze_for_search(phrase);
         self.search_phrase_tokens(&st, &tokens, slop, true)
     }
 
@@ -1492,16 +1503,27 @@ impl FtsMemIndex {
     fn search_phrase_tokens(
         &self,
         st: &IndexState,
-        tokens: &[String],
+        query_tokens: &Tokens,
         slop: u32,
         include_tail: bool,
     ) -> Vec<FtsEntry> {
-        if tokens.is_empty() {
+        if query_tokens.is_empty() {
             return Vec::new();
         }
-        if tokens.len() == 1 {
+        let groups = query_position_groups(query_tokens);
+        if groups.is_empty() {
+            return Vec::new();
+        }
+        if groups.len() == 1 {
             // A single-token phrase reduces to a regular term search.
-            return self.search_match_strings(st, tokens, Operator::Or, None, include_tail, true);
+            return self.search_match_strings(
+                st,
+                &groups[0],
+                Operator::Or,
+                None,
+                include_tail,
+                true,
+            );
         }
         // A multi-token phrase needs token positions; without them (the index
         // was built `with_position = false`) phrase search is unsupported, as
@@ -1509,24 +1531,40 @@ impl FtsMemIndex {
         if !self.params.has_positions() {
             return Vec::new();
         }
+        let has_grouped_terms = groups.iter().any(|group| group.len() > 1);
+        let tokens = position_groups_to_tokens(&groups);
         let tail_snap = st.tail.snapshot();
         let scan_tail = include_tail && tail_snap.visible_count > 0;
-        let scorer = build_scorer(st, &tail_snap, tokens, include_tail);
+        let scorer = build_scorer(st, &tail_snap, &tokens, include_tail);
         if scorer.num_docs() == 0 {
             return Vec::new();
         }
         let mut results = Vec::new();
         for p in st.partitions.iter() {
-            results.extend(p.search_phrase(tokens, slop, &scorer));
+            if has_grouped_terms {
+                results.extend(p.search_phrase_groups(&groups, slop, &scorer));
+            } else {
+                results.extend(p.search_phrase(&tokens, slop, &scorer));
+            }
         }
         if scan_tail {
-            results.extend(phrase_search_tail(
-                &tail_snap,
-                &st.tail.terms,
-                tokens,
-                slop,
-                &scorer,
-            ));
+            if has_grouped_terms {
+                results.extend(phrase_search_tail_groups(
+                    &tail_snap,
+                    &st.tail.terms,
+                    &groups,
+                    slop,
+                    &scorer,
+                ));
+            } else {
+                results.extend(phrase_search_tail(
+                    &tail_snap,
+                    &st.tail.terms,
+                    &tokens,
+                    slop,
+                    &scorer,
+                ));
+            }
         }
         results
     }
@@ -1662,7 +1700,7 @@ impl FtsMemIndex {
                 results
             }
             FtsQueryExpr::Phrase { query, slop, boost } => {
-                let tokens = self.tokenize_for_search(query);
+                let tokens = self.analyze_for_search(query);
                 let mut results = self.search_phrase_tokens(st, &tokens, *slop, include_tail);
                 apply_boost(&mut results, *boost);
                 results
@@ -2369,6 +2407,126 @@ fn phrase_search_tail(
     results
 }
 
+#[derive(Default, Clone)]
+struct PhraseGroupDoc {
+    positions: Vec<u32>,
+    score: f32,
+}
+
+struct PhraseCandidate {
+    positions_by_group: Vec<Vec<u32>>,
+    score: f32,
+}
+
+fn merge_phrase_group<K>(
+    candidates: &mut Option<HashMap<K, PhraseCandidate>>,
+    group_idx: usize,
+    group_count: usize,
+    group_docs: HashMap<K, PhraseGroupDoc>,
+) -> bool
+where
+    K: Copy + Eq + std::hash::Hash,
+{
+    let Some(current) = candidates.as_mut() else {
+        *candidates = Some(
+            group_docs
+                .into_iter()
+                .map(|(doc, group_doc)| {
+                    let mut positions_by_group = vec![Vec::new(); group_count];
+                    positions_by_group[group_idx] = group_doc.positions;
+                    (
+                        doc,
+                        PhraseCandidate {
+                            positions_by_group,
+                            score: group_doc.score,
+                        },
+                    )
+                })
+                .collect(),
+        );
+        return true;
+    };
+
+    current.retain(|doc, candidate| {
+        if let Some(group_doc) = group_docs.get(doc) {
+            candidate.positions_by_group[group_idx] = group_doc.positions.clone();
+            candidate.score += group_doc.score;
+            true
+        } else {
+            false
+        }
+    });
+    !current.is_empty()
+}
+
+fn phrase_search_tail_groups(
+    snap: &Snapshot,
+    terms: &SkipMap<Arc<str>, Arc<ArcSwap<TermSlice>>>,
+    groups: &[Vec<String>],
+    slop: u32,
+    scorer: &MemBM25Scorer,
+) -> Vec<FtsEntry> {
+    let mut candidates: Option<HashMap<RowPosition, PhraseCandidate>> = None;
+    for (group_idx, group) in groups.iter().enumerate() {
+        let group_docs = tail_phrase_group_docs(snap, terms, group, scorer);
+        if group_docs.is_empty()
+            || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs)
+        {
+            return Vec::new();
+        }
+    }
+
+    candidates
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop))
+        .map(|(row_position, candidate)| FtsEntry {
+            row_position,
+            score: candidate.score,
+        })
+        .collect()
+}
+
+fn tail_phrase_group_docs(
+    snap: &Snapshot,
+    terms: &SkipMap<Arc<str>, Arc<ArcSwap<TermSlice>>>,
+    group: &[String],
+    scorer: &MemBM25Scorer,
+) -> HashMap<RowPosition, PhraseGroupDoc> {
+    let mut docs: HashMap<RowPosition, PhraseGroupDoc> = HashMap::new();
+    for token in group {
+        let Some(entry) = terms.get(token.as_str()) else {
+            continue;
+        };
+        let qw = scorer.query_weight(token);
+        let slice = entry.value().load_full();
+        for chunk in slice.chunks() {
+            if chunk.batch_position >= snap.visible_count {
+                continue;
+            }
+            let Some(meta) = snap.batch_for(chunk.batch_position) else {
+                continue;
+            };
+            let Some(positions) = &chunk.positions else {
+                continue;
+            };
+            for (i, &row_position) in chunk.row_positions.iter().enumerate() {
+                let entry = docs.entry(row_position).or_default();
+                entry
+                    .positions
+                    .extend_from_slice(positions.doc_positions(i));
+                let dl = meta.dl(row_position).unwrap_or(1);
+                entry.score += qw * scorer.doc_weight(chunk.frequencies[i], dl);
+            }
+        }
+    }
+    for doc in docs.values_mut() {
+        doc.positions.sort_unstable();
+        doc.positions.dedup();
+    }
+    docs
+}
+
 fn phrase_matches<T: AsRef<[u32]>>(positions: &[T], slop: u32) -> bool {
     if positions.is_empty() {
         return false;
@@ -3418,6 +3576,62 @@ impl Partition {
         }
         results
     }
+
+    fn search_phrase_groups(
+        &self,
+        groups: &[Vec<String>],
+        slop: u32,
+        scorer: &MemBM25Scorer,
+    ) -> Vec<FtsEntry> {
+        let mut candidates: Option<HashMap<u32, PhraseCandidate>> = None;
+        for (group_idx, group) in groups.iter().enumerate() {
+            let group_docs = self.phrase_group_docs(group, scorer);
+            if group_docs.is_empty()
+                || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs)
+            {
+                return Vec::new();
+            }
+        }
+
+        candidates
+            .unwrap_or_default()
+            .into_iter()
+            .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop))
+            .map(|(doc, candidate)| FtsEntry {
+                row_position: self.docs.row_id(doc),
+                score: candidate.score,
+            })
+            .collect()
+    }
+
+    fn phrase_group_docs(
+        &self,
+        group: &[String],
+        scorer: &MemBM25Scorer,
+    ) -> HashMap<u32, PhraseGroupDoc> {
+        let mut docs: HashMap<u32, PhraseGroupDoc> = HashMap::new();
+        for token in group {
+            let Some(term_id) = self.term_id(token) else {
+                continue;
+            };
+            let qw = scorer.query_weight(token);
+            let mut cursor = PostingCursor::new(self, term_id);
+            while let Some(doc) = cursor.cursor_doc() {
+                let positions = cursor.positions().to_vec();
+                let freq = cursor.freq();
+                let dl = self.docs.num_tokens(doc);
+                let entry = docs.entry(doc).or_default();
+                entry.positions.extend_from_slice(&positions);
+                entry.score += qw * scorer.doc_weight(freq, dl);
+                cursor.advance();
+            }
+        }
+        for doc in docs.values_mut() {
+            doc.positions.sort_unstable();
+            doc.positions.dedup();
+        }
+        docs
+    }
 }
 
 /// A scored MemTable row, ordered by score then row position (`total_cmp`,
@@ -3785,6 +3999,69 @@ mod tests {
         assert_eq!(rows, vec![0, 1]);
     }
 
+    fn tail_positions_for(index: &FtsMemIndex, term: &str, row_position: RowPosition) -> Vec<u32> {
+        let st = index.state.load_full();
+        let snap = st.tail.snapshot();
+        let entry = st.tail.terms.get(term).expect("term should be indexed");
+        let slice = entry.value().load();
+        for chunk in slice.chunks() {
+            if chunk.batch_position >= snap.visible_count {
+                continue;
+            }
+            if let Ok(doc_idx) = chunk.row_positions.binary_search(&row_position) {
+                return chunk
+                    .positions
+                    .as_ref()
+                    .expect("test index stores positions")
+                    .doc_positions(doc_idx)
+                    .to_vec();
+            }
+        }
+        panic!("term {term} should be present in row {row_position}");
+    }
+
+    #[test]
+    fn test_code_analyzer_phrase_uses_token_positions_and_alternatives() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::with_params(
+            1,
+            "description".to_string(),
+            InvertedIndexParams::code()
+                .with_position(true)
+                .split_identifiers(true),
+        );
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "get user name",
+                    "getUserName",
+                    "get user",
+                    "get fast user name",
+                ])),
+            ],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+
+        assert_eq!(tail_positions_for(&index, "getusername", 1), vec![0]);
+        assert_eq!(tail_positions_for(&index, "get", 1), vec![0]);
+        assert_eq!(tail_positions_for(&index, "user", 1), vec![1]);
+        assert_eq!(tail_positions_for(&index, "name", 1), vec![2]);
+
+        let query = FtsQueryExpr::phrase("getUserName");
+        assert_eq!(rows(index.search_phrase("getUserName", 0)), vec![0, 1]);
+        assert_eq!(rows(index.search_query(&query)), vec![0, 1]);
+
+        index.flush();
+        let partition_only = SearchOptions::new().with_include_tail(false);
+        assert_eq!(
+            rows(index.search_with_options(&query, partition_only)),
+            vec![0, 1]
+        );
+    }
+
     fn create_phrase_test_batch(schema: &ArrowSchema) -> RecordBatch {
         RecordBatch::try_new(
             Arc::new(schema.clone()),

From c8dc54536fbd44f8278fe8032f2a5c773cf4f3ce Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 10 Jul 2026 15:15:36 +0800
Subject: [PATCH 12/19] fix: avoid persisting fts analyzer profiles

---
 protos/index_old.proto                        |  30 ++--
 .../src/scalar/inverted/tokenizer.rs          | 146 +++++++++---------
 rust/lance/src/index/scalar/inverted.rs       |  22 +--
 3 files changed, 93 insertions(+), 105 deletions(-)

diff --git a/protos/index_old.proto b/protos/index_old.proto
index 4ebc2143e6a..97c9a434467 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -26,19 +26,17 @@ message LabelListIndexDetails {}
 message NGramIndexDetails {}
 message ZoneMapIndexDetails {}
 message InvertedIndexDetails {
-  // High-level analyzer profile. It selects the semantic defaults for the FTS
-  // pipeline, e.g. "text" for natural-language text or "code" for code search.
-  // Absent is a legacy "text" profile unless base_tokenizer is the legacy
-  // "code" alias. Readers must canonicalize this before comparing details.
-  optional string analyzer = 12;
+  // Analyzer profiles are input-only and are not part of persisted details.
+  reserved 12;
   // Document-level tokenizer. It extracts searchable text from the stored value,
   // e.g. "text" for a plain string or "json" for JSON content. The extracted
   // text is then passed to base_tokenizer. Absent means text / auto-inferred
   // legacy behavior.
   optional string lance_tokenizer = 13;
-  // Lexical tokenizer used after the analyzer/document layer has selected the
-  // text to tokenize. This is an implementation component such as "simple",
-  // "icu", "ngram", or "code"; analyzer="code" owns the code-search defaults.
+  // Lexical tokenizer used after document-level text extraction. This is an
+  // implementation component such as "simple", "icu", "ngram", or "code".
+  // Input-time analyzer profiles are expanded into this field and the concrete
+  // options below before these details are persisted.
   // Marking this field as optional as old versions of the index store blank details and we
   // need to make sure we have a proper optional field to detect this.
   optional string base_tokenizer = 1;
@@ -52,19 +50,19 @@ message InvertedIndexDetails {
   uint32 min_ngram_length = 9;
   uint32 max_ngram_length = 10;
   bool prefix_only = 11;
-  // Code analyzer flag. Split one lexical identifier into subwords, e.g.
+  // Code tokenizer option. Split one lexical identifier into subwords, e.g.
   // getUserName -> get/user/name. This changes the indexed term space and must
   // be persisted for query-time tokenizer reconstruction and segment checks.
   bool split_identifiers = 14;
-  // Code analyzer flag. Split identifier subwords across letter/number
-  // boundaries, e.g. HTML2JSON -> html/2/json. When absent, use the analyzer
-  // profile default; when present, persist the explicit index-time choice.
+  // Code tokenizer option. Split identifier subwords across letter/number
+  // boundaries, e.g. HTML2JSON -> html/2/json. When absent, use the default for
+  // base_tokenizer; when present, persist the explicit index-time choice.
   optional bool split_on_numerics = 15;
-  // Code analyzer flag. Keep the complete lexical identifier in addition to
-  // subwords, e.g. user_name plus user/name. When absent, use the analyzer
-  // profile default; when present, persist the explicit index-time choice.
+  // Code tokenizer option. Keep the complete lexical identifier in addition to
+  // subwords, e.g. user_name plus user/name. When absent, use the default for
+  // base_tokenizer; when present, persist the explicit index-time choice.
   optional bool preserve_original = 16;
-  // Code analyzer flag. Index operator tokens such as "::", "->", and "!=".
+  // Code tokenizer option. Index operator tokens such as "::", "->", and "!=".
   // Disabled by default because operators are often high-frequency noise, but
   // when enabled it changes query semantics and must be persisted.
   bool index_operators = 17;
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index ed56181bbea..c81d3f78908 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -35,15 +35,6 @@ use lance_tokenizer::{
 /// Tokenizer configs
 #[derive(Debug, Clone, Serialize, PartialEq)]
 pub struct InvertedIndexParams {
-    /// High-level analyzer profile.
-    ///
-    /// This is the user-facing semantic preset. `text` keeps the existing
-    /// natural-language defaults. `code` selects code-search defaults such as
-    /// code lexical tokenization, optional identifier splitting, preserving
-    /// whole identifiers when splitting is enabled, and disabling stemming /
-    /// stop-word removal.
-    pub(crate) analyzer: String,
-
     /// Document-level tokenizer.
     ///
     /// This decides how Lance extracts searchable text from the stored value:
@@ -54,12 +45,10 @@ pub struct InvertedIndexParams {
     /// The extracted text is then passed to `base_tokenizer`.
     pub(crate) lance_tokenizer: Option<String>,
 
-    /// Lexical tokenizer used inside the selected analyzer profile.
+    /// Lexical tokenizer used after document-level text extraction.
     ///
-    /// This is a lower-level implementation component, not the product-level
-    /// search mode. For code search, prefer `analyzer = "code"`;
-    /// `base_tokenizer = "code"` is accepted as a compatibility alias and
-    /// resolves to the code analyzer profile.
+    /// Client-facing analyzer profiles are resolved into this field and the
+    /// concrete filter options before params are persisted.
     /// - `simple`: splits tokens on whitespace and punctuation
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
@@ -164,6 +153,7 @@ pub struct InvertedIndexParams {
 #[derive(Debug, Deserialize)]
 #[serde(deny_unknown_fields)]
 struct RawInvertedIndexParams {
+    // Input-only preset expanded before constructing normalized params.
     analyzer: Option<String>,
     lance_tokenizer: Option<String>,
     base_tokenizer: Option<String>,
@@ -257,7 +247,6 @@ impl RawInvertedIndexParams {
             _ => unreachable!("analyzer is normalized above"),
         };
 
-        params.analyzer = analyzer.to_string();
         if let Some(lance_tokenizer) = self.lance_tokenizer {
             params.lance_tokenizer = Some(lance_tokenizer);
         }
@@ -322,7 +311,6 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
 
     fn try_from(params: &InvertedIndexParams) -> Result<Self> {
         Ok(Self {
-            analyzer: Some(params.analyzer.clone()),
             lance_tokenizer: params.lance_tokenizer.clone(),
             base_tokenizer: Some(params.base_tokenizer.clone()),
             language: serde_json::to_string(&params.language)?,
@@ -351,24 +339,10 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
             return Ok(Self::default());
         }
 
-        let analyzer = details
-            .analyzer
-            .as_deref()
-            .map(normalize_analyzer)
-            .transpose()?
-            .unwrap_or_else(|| {
-                if details.base_tokenizer.as_deref() == Some("code") {
-                    "code"
-                } else {
-                    "text"
-                }
-            });
-        let mut params = match analyzer {
-            "code" => Self::code(),
-            "text" => Self::default(),
-            _ => unreachable!("analyzer is normalized above"),
+        let mut params = match details.base_tokenizer.as_deref() {
+            Some("code") => Self::code(),
+            _ => Self::default(),
         };
-        params.analyzer = analyzer.to_string();
         params.lance_tokenizer = details.lance_tokenizer.clone();
         params.base_tokenizer = details
             .base_tokenizer
@@ -486,8 +460,7 @@ impl InvertedIndexParams {
     /// this is not used for `lindera/*` and `jieba/*` tokenizers.
     /// Default to `English`.
     pub fn new(base_tokenizer: String, language: Language) -> Self {
-        Self {
-            analyzer: "text".to_string(),
+        let mut params = Self {
             lance_tokenizer: None,
             base_tokenizer,
             language,
@@ -508,11 +481,26 @@ impl InvertedIndexParams {
             memory_limit_mb: None,
             num_workers: None,
             format_version: None,
+        };
+        if params.base_tokenizer == "code" {
+            params.apply_code_defaults();
         }
+        params
+    }
+
+    fn apply_text_defaults(&mut self) {
+        self.base_tokenizer = "simple".to_string();
+        self.split_identifiers = false;
+        self.split_on_numerics = false;
+        self.preserve_original = false;
+        self.lower_case = true;
+        self.ascii_folding = true;
+        self.stem = true;
+        self.remove_stop_words = true;
+        self.index_operators = false;
     }
 
     fn apply_code_defaults(&mut self) {
-        self.analyzer = "code".to_string();
         self.base_tokenizer = "code".to_string();
         self.split_identifiers = false;
         self.split_on_numerics = true;
@@ -535,12 +523,13 @@ impl InvertedIndexParams {
     /// assert!(tokenizer.is_ok());
     /// ```
     pub fn code() -> Self {
-        let mut params = Self::new("code".to_string(), Language::English);
-        params.apply_code_defaults();
-        params
+        Self::new("code".to_string(), Language::English)
     }
 
-    /// Set analyzer profile.
+    /// Apply an analyzer profile to the concrete tokenizer parameters.
+    ///
+    /// The profile name is an input-time preset and is not persisted. Explicit
+    /// options applied after this method override the profile defaults.
     ///
     /// # Examples
     ///
@@ -554,9 +543,10 @@ impl InvertedIndexParams {
     /// # Ok::<(), Box<dyn std::error::Error>>(())
     /// ```
     pub fn analyzer(mut self, analyzer: &str) -> Result<Self> {
-        self.analyzer = normalize_analyzer(analyzer)?.to_string();
-        if self.analyzer == "code" {
-            self.apply_code_defaults();
+        match normalize_analyzer(analyzer)? {
+            "text" => self.apply_text_defaults(),
+            "code" => self.apply_code_defaults(),
+            _ => unreachable!("analyzer is normalized above"),
         }
         Ok(self)
     }
@@ -566,7 +556,7 @@ impl InvertedIndexParams {
         self
     }
 
-    /// Set the tokenizer implementation used by the analyzer.
+    /// Set the lexical tokenizer implementation.
     ///
     /// Setting this to `"code"` selects the code analyzer defaults.
     ///
@@ -803,27 +793,17 @@ impl InvertedIndexParams {
     }
 
     fn validate(&self) -> Result<()> {
-        normalize_analyzer(&self.analyzer)?;
-        match self.analyzer.as_str() {
-            "code" if self.base_tokenizer != "code" => Err(Error::invalid_input(format!(
-                "analyzer='code' requires base_tokenizer='code', got '{}'",
-                self.base_tokenizer
-            ))),
-            "text" if self.base_tokenizer == "code" => Err(Error::invalid_input(
-                "base_tokenizer='code' requires analyzer='code'".to_string(),
-            )),
-            "text"
-                if self.split_identifiers
-                    || self.split_on_numerics
-                    || self.preserve_original
-                    || self.index_operators =>
-            {
-                Err(Error::invalid_input(
-                    "code analyzer flags require analyzer='code'".to_string(),
-                ))
-            }
-            _ => Ok(()),
+        if self.base_tokenizer != "code"
+            && (self.split_identifiers
+                || self.split_on_numerics
+                || self.preserve_original
+                || self.index_operators)
+        {
+            return Err(Error::invalid_input(
+                "code analyzer flags require base_tokenizer='code'".to_string(),
+            ));
         }
+        Ok(())
     }
 
     fn build_base_tokenizer(&self) -> Result<TextAnalyzerBuilder> {
@@ -897,7 +877,7 @@ pub fn language_model_home() -> Option<PathBuf> {
 
 #[cfg(test)]
 mod tests {
-    use super::{InvertedIndexParams, InvertedListFormatVersion};
+    use super::{InvertedIndexParams, InvertedListFormatVersion, Language};
     use lance_tokenizer::TokenStream;
     use rstest::rstest;
     use serde_json::json;
@@ -983,7 +963,6 @@ mod tests {
             "analyzer": "code"
         }))
         .unwrap();
-        assert_eq!(params.analyzer, "code");
         assert_eq!(params.base_tokenizer, "code");
         assert!(!params.split_identifiers);
         assert!(params.split_on_numerics);
@@ -996,13 +975,38 @@ mod tests {
     }
 
     #[test]
-    fn test_code_base_tokenizer_alias_resolves_analyzer() {
-        let params: InvertedIndexParams = serde_json::from_value(json!({
-            "base_tokenizer": "code"
+    fn test_analyzer_profile_resolves_to_persisted_params() {
+        let from_profile: InvertedIndexParams = serde_json::from_value(json!({
+            "analyzer": "code",
+            "split_identifiers": true,
+            "preserve_original": false
         }))
         .unwrap();
-        assert_eq!(params.analyzer, "code");
-        assert_eq!(params.base_tokenizer, "code");
+        let from_base_tokenizer: InvertedIndexParams = serde_json::from_value(json!({
+            "base_tokenizer": "code",
+            "split_identifiers": true,
+            "preserve_original": false
+        }))
+        .unwrap();
+        let from_constructor = InvertedIndexParams::new("code".to_string(), Language::English)
+            .split_identifiers(true)
+            .preserve_original(false);
+
+        assert_eq!(from_profile, from_base_tokenizer);
+        assert_eq!(from_profile, from_constructor);
+
+        let persisted = serde_json::to_value(&from_profile).unwrap();
+        assert!(persisted.get("analyzer").is_none());
+        assert_eq!(
+            serde_json::from_value::<InvertedIndexParams>(persisted).unwrap(),
+            from_profile
+        );
+    }
+
+    #[test]
+    fn test_text_analyzer_replaces_code_defaults() {
+        let params = InvertedIndexParams::code().analyzer("text").unwrap();
+        assert_eq!(params, InvertedIndexParams::default());
     }
 
     #[test]
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index bb253522c03..4c50b3a0a0e 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -185,12 +185,11 @@ pub async fn load_segments(dataset: &Dataset, column: &str) -> Result<Option<Vec
 /// segments returned by [`load_segments`].
 ///
 /// All segments are required to agree on their semantic `InvertedIndexDetails`
-/// payload (analyzer, tokenizer, position settings, etc.); inconsistent
+/// payload (tokenizer, position settings, etc.); inconsistent
 /// segments return an error. Details are canonicalized before comparison so
-/// legacy segments that omit default fields (for example, missing
-/// `analyzer = "text"`) remain compatible with newly written text FTS
-/// segments. Returns the canonical details that may be used when constructing a
-/// tokenizer or running a query against the index.
+/// legacy segments that omit default fields remain compatible with newly
+/// written text FTS segments. Returns the canonical details that may be used
+/// when constructing a tokenizer or running a query against the index.
 pub async fn load_segment_details(
     dataset: &Dataset,
     column: &str,
@@ -275,19 +274,6 @@ mod tests {
         assert_eq!(decoded, InvertedIndexDetails::default());
     }
 
-    #[test]
-    fn canonicalize_inverted_details_accepts_legacy_missing_analyzer() {
-        let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
-        let mut legacy = current.clone();
-        legacy.analyzer = None;
-
-        assert_ne!(legacy, current);
-        assert_eq!(
-            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
-            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
-        );
-    }
-
     #[test]
     fn canonicalize_inverted_details_accepts_legacy_empty_details() {
         let legacy = InvertedIndexDetails::default();

From e04732675011855c1a777c35889049eb4314a222 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 10 Jul 2026 16:27:08 +0800
Subject: [PATCH 13/19] fix: address code analyzer ci failures

---
 python/python/tests/test_scalar_index.py      | 4 ++--
 rust/lance-index/src/scalar/inverted/index.rs | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index d947bb1ab62..3ddbb842d11 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -820,7 +820,7 @@ def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path):
 
     stats = ds.stats.index_stats("code_idx")["indices"][0]
     params = stats["params"]
-    assert params["analyzer"] == "code"
+    assert "analyzer" not in params
     assert params["base_tokenizer"] == "code"
     assert params["split_identifiers"] is False
 
@@ -857,7 +857,7 @@ def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path):
 
     stats = ds.stats.index_stats("code_idx")["indices"][0]
     params = stats["params"]
-    assert params["analyzer"] == "code"
+    assert "analyzer" not in params
     assert params["base_tokenizer"] == "code"
     assert params["split_identifiers"] is True
     assert params["split_on_numerics"] is True
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 111bd2f9710..055e8738f3d 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -5936,7 +5936,7 @@ fn flat_bm25_score(
         }
         let doc_norm = K1 * (1.0 - B + B * num_tokens_in_doc as f32 / scorer.avg_doc_length());
         let mut score = 0.0;
-        for (token, freq) in query_tokens.into_iter().zip(query_token_counts.into_iter()) {
+        for (token, freq) in query_tokens.into_iter().zip(query_token_counts) {
             let freq = freq as f32;
             let idf = idf(scorer.num_docs_containing_token(token), scorer.num_docs());
             score += idf * (freq * (K1 + 1.0) / (freq + doc_norm));

From 659284efd71d78cdc44e08e52a2f6e2b11936ca9 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Tue, 14 Jul 2026 17:25:49 +0800
Subject: [PATCH 14/19] fix: clean up analyzer profile persistence

---
 protos/index_old.proto  |  2 --
 rust/lance/src/index.rs | 51 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/protos/index_old.proto b/protos/index_old.proto
index 97c9a434467..6654e871324 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -26,8 +26,6 @@ message LabelListIndexDetails {}
 message NGramIndexDetails {}
 message ZoneMapIndexDetails {}
 message InvertedIndexDetails {
-  // Analyzer profiles are input-only and are not part of persisted details.
-  reserved 12;
   // Document-level tokenizer. It extracts searchable text from the stored value,
   // e.g. "text" for a plain string or "json" for JSON content. The extracted
   // text is then passed to base_tokenizer. Absent means text / auto-inferred
diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs
index 9f2a07348f8..e3aa4a89da0 100644
--- a/rust/lance/src/index.rs
+++ b/rust/lance/src/index.rs
@@ -4958,6 +4958,57 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_optimize_empty_code_fts_index_preserves_params() {
+        let dir = TempStrDir::default();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("code", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values([0, 1])),
+                Arc::new(StringArray::from_iter_values([
+                    "fn GetUserEmail() {}",
+                    "fn ParseConfig() {}",
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(reader, &dir, None).await.unwrap();
+
+        let params = InvertedIndexParams::default()
+            .analyzer("code")
+            .unwrap()
+            .split_identifiers(true)
+            .preserve_original(false);
+        dataset
+            .create_index_builder(&["code"], IndexType::Inverted, &params)
+            .name("code_idx".to_string())
+            .train(false)
+            .await
+            .unwrap();
+
+        dataset.optimize_indices(&Default::default()).await.unwrap();
+
+        let stats: serde_json::Value =
+            serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap();
+        assert_eq!(stats["num_unindexed_rows"], 0);
+
+        let result = dataset
+            .scan()
+            .project(&["id"])
+            .unwrap()
+            .full_text_search(FullTextSearchQuery::new("email".to_string()))
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(result["id"].as_primitive::<Int32Type>().values(), &[0]);
+    }
+
     /// Helper function to check if an index is being used in a query plan
     fn assert_index_usage(plan: &str, column_name: &str, should_use_index: bool, context: &str) {
         let index_used = if column_name == "text" {

From 3f07a56086e37648cf7d81c89bec683fe068766f Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Tue, 14 Jul 2026 18:05:36 +0800
Subject: [PATCH 15/19] fix: resolve FTS merge integration

---
 rust/lance-index/src/scalar/inverted/index.rs | 42 -------------------
 .../src/scalar/inverted/tokenizer.rs          | 10 +++--
 2 files changed, 6 insertions(+), 46 deletions(-)

diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index c4a932f2c68..5e43b505325 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -11476,48 +11476,6 @@ mod tests {
         assert_eq!(row_ids, &[0]);
     }
 
-    /// An [`IndexReader`] wrapper that hides the posting-group-offsets schema
-    /// metadata key, so a [`PostingListReader`] opened on it takes the
-    /// pre-grouping per-token fallback path (issue #7040).
-    struct GroupKeyStrippingReader {
-        inner: Arc<dyn IndexReader>,
-        schema: lance_core::datatypes::Schema,
-    }
-
-    impl GroupKeyStrippingReader {
-        fn new(inner: Arc<dyn IndexReader>) -> Self {
-            let mut schema = inner.schema().clone();
-            schema.metadata.remove(POSTING_GROUP_OFFSETS_BUF_KEY);
-            Self { inner, schema }
-        }
-    }
-
-    #[async_trait]
-    impl IndexReader for GroupKeyStrippingReader {
-        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
-            self.inner.read_record_batch(n, batch_size).await
-        }
-        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
-            self.inner.read_global_buffer(index).await
-        }
-        async fn read_range(
-            &self,
-            range: std::ops::Range<usize>,
-            projection: Option<&[&str]>,
-        ) -> Result<RecordBatch> {
-            self.inner.read_range(range, projection).await
-        }
-        async fn num_batches(&self, batch_size: u64) -> u32 {
-            self.inner.num_batches(batch_size).await
-        }
-        fn num_rows(&self) -> usize {
-            self.inner.num_rows()
-        }
-        fn schema(&self) -> &lance_core::datatypes::Schema {
-            &self.schema
-        }
-    }
-
     fn posting_entries(posting: &PostingList) -> Vec<(u64, u32)> {
         posting.iter().map(|(doc, freq, _)| (doc, freq)).collect()
     }
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 46b19f0be96..95ed6fcd7f0 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -363,10 +363,12 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
 
     fn try_from(details: &pbold::InvertedIndexDetails) -> Result<Self> {
         if details.base_tokenizer.is_none() && details.language.is_empty() {
-            let mut params = Self::default();
-            params.block_size = match details.block_size {
-                Some(block_size) => validate_block_size(block_size as usize)?,
-                None => LEGACY_BLOCK_SIZE,
+            let params = Self {
+                block_size: match details.block_size {
+                    Some(block_size) => validate_block_size(block_size as usize)?,
+                    None => LEGACY_BLOCK_SIZE,
+                },
+                ..Self::default()
             };
             return Ok(params);
         }

From 163bec80c83de0da264390f39d1b360e7a11f737 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 17 Jul 2026 09:37:51 +0800
Subject: [PATCH 16/19] fix: group code tokenizer index details

---
 protos/index_old.proto                        | 42 +++++------
 .../src/scalar/inverted/tokenizer.rs          | 75 +++++++++++++------
 rust/lance/src/index/scalar/inverted.rs       | 56 +-------------
 3 files changed, 78 insertions(+), 95 deletions(-)

diff --git a/protos/index_old.proto b/protos/index_old.proto
index 2aac1b5176c..bc1c7e1a951 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -26,11 +26,23 @@ message LabelListIndexDetails {}
 message NGramIndexDetails {}
 message ZoneMapIndexDetails {}
 message InvertedIndexDetails {
-  // Document-level tokenizer. It extracts searchable text from the stored value,
-  // e.g. "text" for a plain string or "json" for JSON content. The extracted
-  // text is then passed to base_tokenizer. Absent means text / auto-inferred
-  // legacy behavior.
-  optional string lance_tokenizer = 13;
+  message CodeTokenizerConfig {
+    // Split one lexical identifier into subwords, e.g. getUserName ->
+    // get/user/name.
+    bool split_identifiers = 1;
+    // Split identifier subwords across letter/number boundaries, e.g.
+    // HTML2JSON -> html/2/json. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool split_on_numerics = 2;
+    // Keep the complete lexical identifier in addition to subwords, e.g.
+    // user_name plus user/name. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool preserve_original = 3;
+    // Index operator tokens such as "::", "->", and "!=". Operators are not
+    // indexed by default because they are often high-frequency noise.
+    bool index_operators = 4;
+  }
+
   // Lexical tokenizer used after document-level text extraction. This is an
   // implementation component such as "simple", "icu", "ngram", or "code".
   // Input-time analyzer profiles are expanded into this field and the concrete
@@ -53,20 +65,8 @@ message InvertedIndexDetails {
   // A present value records the block size used by the index; 256 is only
   // valid with format version 3.
   optional uint32 block_size = 12;
-  // Code tokenizer option. Split one lexical identifier into subwords, e.g.
-  // getUserName -> get/user/name. This changes the indexed term space and must
-  // be persisted for query-time tokenizer reconstruction and segment checks.
-  bool split_identifiers = 14;
-  // Code tokenizer option. Split identifier subwords across letter/number
-  // boundaries, e.g. HTML2JSON -> html/2/json. When absent, use the default for
-  // base_tokenizer; when present, persist the explicit index-time choice.
-  optional bool split_on_numerics = 15;
-  // Code tokenizer option. Keep the complete lexical identifier in addition to
-  // subwords, e.g. user_name plus user/name. When absent, use the default for
-  // base_tokenizer; when present, persist the explicit index-time choice.
-  optional bool preserve_original = 16;
-  // Code tokenizer option. Index operator tokens such as "::", "->", and "!=".
-  // Disabled by default because operators are often high-frequency noise, but
-  // when enabled it changes query semantics and must be persisted.
-  bool index_operators = 17;
+  // Options for base_tokenizer = "code". Presence records the code tokenizer
+  // configuration used to build the index; absence means there is no
+  // code-specific configuration to apply.
+  CodeTokenizerConfig code_config = 13;
 }
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 95ed6fcd7f0..55fa6b3576c 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -337,7 +337,6 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
 
     fn try_from(params: &InvertedIndexParams) -> Result<Self> {
         Ok(Self {
-            lance_tokenizer: params.lance_tokenizer.clone(),
             base_tokenizer: Some(params.base_tokenizer.clone()),
             language: serde_json::to_string(&params.language)?,
             with_position: params.with_position,
@@ -350,10 +349,14 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             max_ngram_length: params.max_ngram_length,
             prefix_only: params.prefix_only,
             block_size: Some(params.block_size as u32),
-            split_identifiers: params.split_identifiers,
-            split_on_numerics: Some(params.split_on_numerics),
-            preserve_original: Some(params.preserve_original),
-            index_operators: params.index_operators,
+            code_config: (params.base_tokenizer == "code").then_some(
+                pbold::inverted_index_details::CodeTokenizerConfig {
+                    split_identifiers: params.split_identifiers,
+                    split_on_numerics: Some(params.split_on_numerics),
+                    preserve_original: Some(params.preserve_original),
+                    index_operators: params.index_operators,
+                },
+            ),
         })
     }
 }
@@ -362,6 +365,11 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
     type Error = Error;
 
     fn try_from(details: &pbold::InvertedIndexDetails) -> Result<Self> {
+        if details.code_config.is_some() && details.base_tokenizer.as_deref() != Some("code") {
+            return Err(Error::invalid_input(
+                "code_config requires base_tokenizer='code'".to_string(),
+            ));
+        }
         if details.base_tokenizer.is_none() && details.language.is_empty() {
             let params = Self {
                 block_size: match details.block_size {
@@ -377,7 +385,6 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
             Some("code") => Self::code(),
             _ => Self::default(),
         };
-        params.lance_tokenizer = details.lance_tokenizer.clone();
         params.base_tokenizer = details
             .base_tokenizer
             .as_ref()
@@ -401,14 +408,16 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
             Some(block_size) => validate_block_size(block_size as usize)?,
             None => LEGACY_BLOCK_SIZE,
         };
-        params.split_identifiers = details.split_identifiers;
-        if let Some(split_on_numerics) = details.split_on_numerics {
-            params.split_on_numerics = split_on_numerics;
-        }
-        if let Some(preserve_original) = details.preserve_original {
-            params.preserve_original = preserve_original;
+        if let Some(code_config) = &details.code_config {
+            params.split_identifiers = code_config.split_identifiers;
+            if let Some(split_on_numerics) = code_config.split_on_numerics {
+                params.split_on_numerics = split_on_numerics;
+            }
+            if let Some(preserve_original) = code_config.preserve_original {
+                params.preserve_original = preserve_original;
+            }
+            params.index_operators = code_config.index_operators;
         }
-        params.index_operators = details.index_operators;
         params.validate()?;
         Ok(params)
     }
@@ -994,6 +1003,7 @@ mod tests {
     use crate::pbold;
 
     use super::{InvertedIndexParams, InvertedListFormatVersion, Language};
+    use lance_core::Error;
     use lance_tokenizer::TokenStream;
     use rstest::rstest;
     use serde_json::json;
@@ -1216,8 +1226,9 @@ mod tests {
             .index_operators(true)
             .preserve_original(false);
         let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
-        assert_eq!(details.split_on_numerics, Some(true));
-        assert_eq!(details.preserve_original, Some(false));
+        let code_config = details.code_config.as_ref().unwrap();
+        assert_eq!(code_config.split_on_numerics, Some(true));
+        assert_eq!(code_config.preserve_original, Some(false));
         let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
         assert_eq!(round_tripped, params);
     }
@@ -1226,14 +1237,38 @@ mod tests {
     fn test_inverted_details_uses_code_defaults_for_absent_flags() {
         let mut details =
             crate::pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::code()).unwrap();
-        details.split_on_numerics = None;
-        details.preserve_original = None;
+        let code_config = details.code_config.as_mut().unwrap();
+        code_config.split_on_numerics = None;
+        code_config.preserve_original = None;
 
         let params = InvertedIndexParams::try_from(&details).unwrap();
         assert!(params.split_on_numerics);
         assert!(params.preserve_original);
     }
 
+    #[test]
+    fn test_inverted_details_rejects_code_config_for_text_tokenizer() {
+        let mut details =
+            crate::pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::code()).unwrap();
+        details.base_tokenizer = Some("simple".to_string());
+
+        let err = InvertedIndexParams::try_from(&details).unwrap_err();
+        assert!(matches!(&err, Error::InvalidInput { .. }));
+        assert!(
+            err.to_string()
+                .contains("code_config requires base_tokenizer='code'")
+        );
+    }
+
+    #[test]
+    fn test_inverted_details_does_not_persist_document_tokenizer() {
+        let params = InvertedIndexParams::default().lance_tokenizer("json".to_string());
+        let details = crate::pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        let round_tripped = InvertedIndexParams::try_from(&details).unwrap();
+
+        assert_eq!(round_tripped.lance_tokenizer, None);
+    }
+
     #[test]
     fn test_format_version_must_match_block_size() {
         InvertedIndexParams::default()
@@ -1300,7 +1335,6 @@ mod tests {
         assert_eq!(details.block_size, Some(256));
 
         let old_details = pbold::InvertedIndexDetails {
-            lance_tokenizer: None,
             base_tokenizer: Some("simple".to_string()),
             language: serde_json::to_string(&Language::English).unwrap(),
             with_position: false,
@@ -1313,10 +1347,7 @@ mod tests {
             max_ngram_length: 3,
             prefix_only: false,
             block_size: None,
-            split_identifiers: false,
-            split_on_numerics: None,
-            preserve_original: None,
-            index_operators: false,
+            code_config: None,
         };
         let params = InvertedIndexParams::try_from(&old_details).unwrap();
         assert_eq!(params.block_size, 128);
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index 1e46f4822de..e49daa67b5e 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -12,9 +12,7 @@ use lance_core::ROW_ID;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::pbold::InvertedIndexDetails;
 use lance_index::scalar::index_files_to_table;
-use lance_index::scalar::inverted::{
-    InvertedIndex, InvertedIndexParams, document_tokenizer::DocType,
-};
+use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexParams};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
 use lance_table::format::IndexMetadata;
@@ -205,7 +203,7 @@ pub async fn load_segment_details(
                     "failed to decode InvertedIndexDetails payload: {err}"
                 ))
             })?;
-        let details = canonicalize_inverted_index_details(dataset, column, details)?;
+        let details = canonicalize_inverted_index_details(details)?;
         match &expected_details {
             Some(expected) if expected != &details => {
                 return Err(Error::invalid_input(format!(
@@ -226,35 +224,9 @@ pub async fn load_segment_details(
 }
 
 fn canonicalize_inverted_index_details(
-    dataset: &Dataset,
-    column: &str,
-    details: InvertedIndexDetails,
-) -> Result<InvertedIndexDetails> {
-    let field = dataset.schema().field(column).ok_or_else(|| {
-        Error::invalid_input(format!(
-            "FTS index details require column {} to exist in the dataset schema",
-            column
-        ))
-    })?;
-    let field = ArrowField::from(field);
-    canonicalize_inverted_index_details_for_field(&field, details)
-}
-
-fn canonicalize_inverted_index_details_for_field(
-    field: &ArrowField,
     details: InvertedIndexDetails,
 ) -> Result<InvertedIndexDetails> {
-    let inferred_lance_tokenizer = if details.lance_tokenizer.is_none() {
-        Some(DocType::try_from(field)?.as_ref().to_string())
-    } else {
-        None
-    };
     let params = InvertedIndexParams::try_from(&details)?;
-    let params = if let Some(lance_tokenizer) = inferred_lance_tokenizer {
-        params.lance_tokenizer(lance_tokenizer)
-    } else {
-        params
-    };
     InvertedIndexDetails::try_from(&params)
 }
 
@@ -271,10 +243,6 @@ pub async fn load_segment_params(
 mod tests {
     use super::*;
 
-    fn text_field() -> ArrowField {
-        ArrowField::new("text", arrow_schema::DataType::Utf8, true)
-    }
-
     #[test]
     fn decode_legacy_inverted_details_type_url() {
         let mut details_any = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap();
@@ -291,24 +259,8 @@ mod tests {
 
         assert_ne!(legacy, current);
         assert_eq!(
-            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
-            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
-        );
-    }
-
-    #[test]
-    fn canonicalize_inverted_details_infers_missing_lance_tokenizer() {
-        let mut legacy = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
-        legacy.lance_tokenizer = None;
-        let current = InvertedIndexDetails::try_from(
-            &InvertedIndexParams::default().lance_tokenizer("text".to_string()),
-        )
-        .unwrap();
-
-        assert_ne!(legacy, current);
-        assert_eq!(
-            canonicalize_inverted_index_details_for_field(&text_field(), legacy).unwrap(),
-            canonicalize_inverted_index_details_for_field(&text_field(), current).unwrap()
+            canonicalize_inverted_index_details(legacy).unwrap(),
+            canonicalize_inverted_index_details(current).unwrap()
         );
     }
 }

From d1bf2482fbb009bf918983dbedea9d72d4ad7b87 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Sat, 18 Jul 2026 00:38:51 +0800
Subject: [PATCH 17/19] fix: correct code FTS tokenization and scoring

---
 python/python/tests/test_scalar_index.py      |  47 +++
 rust/lance-index/src/scalar/inverted/index.rs | 325 ++++++++++++------
 rust/lance-index/src/scalar/inverted/wand.rs  |  99 +++++-
 rust/lance-tokenizer/src/code_tokenizer.rs    |  42 ++-
 4 files changed, 399 insertions(+), 114 deletions(-)

diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index 92e60a82281..cd49fb03474 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -866,6 +866,53 @@ def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path):
     assert params["remove_stop_words"] is False
 
 
+def test_code_analyzer_operator_search_matches_rust_turbofish(tmp_path):
+    table = pa.table(
+        {
+            "path": ["turbofish.rs", "comparison.rs"],
+            "code": ["value.parse::<usize>()", "value.parse<usize>()"],
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        index_operators=True,
+    )
+
+    results = ds.to_table(
+        columns=["path"],
+        full_text_query=MatchQuery("::", "code", operator=FullTextOperator.OR),
+    )
+    assert results["path"].to_pylist() == ["turbofish.rs"]
+
+
+def test_code_analyzer_exact_identifier_survives_grouped_top_k(tmp_path):
+    table = pa.table(
+        {
+            "path": ["split_0.rs", "split_1.rs", "split_2.rs", "exact.rs"],
+            "code": ["get user name", "get user name", "get user name", "getUserName"],
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    ds.create_scalar_index(
+        "code",
+        index_type="INVERTED",
+        analyzer="code",
+        split_identifiers=True,
+    )
+
+    results = ds.scanner(
+        columns=["path", "_score"],
+        full_text_query=MatchQuery(
+            "getUserName", "code", operator=FullTextOperator.AND
+        ),
+        limit=1,
+    ).to_table()
+    assert results["path"].to_pylist() == ["exact.rs"]
+
+
 def test_code_analyzer_flags_require_code_analyzer(tmp_path):
     table = pa.table({"text": ["getUserName"]})
     ds = lance.write_dataset(table, tmp_path)
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 44bad629372..49909cf0e74 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -291,6 +291,7 @@ struct LoadedPostings {
     postings: Vec<PostingIterator>,
     grouped_expansions: Vec<GroupedExpansionTerms>,
     impact_safe: bool,
+    exact_scoring_required: bool,
 }
 
 impl LoadedPostings {
@@ -299,6 +300,7 @@ impl LoadedPostings {
             postings: Vec::new(),
             grouped_expansions: Vec::new(),
             impact_safe: false,
+            exact_scoring_required: false,
         }
     }
 }
@@ -306,48 +308,7 @@ impl LoadedPostings {
 #[derive(Debug)]
 struct GroupedExpansionTerms {
     position: u32,
-    terms: Vec<ExpansionTermFreqs>,
-}
-
-fn grouped_rescore_wand_limit(
-    limit: Option<usize>,
-    grouped_expansions: &[GroupedExpansionTerms],
-) -> Option<usize> {
-    let limit = limit?;
-    // Grouped fuzzy AND rescoring needs a small candidate cushion because WAND
-    // ranks by the unioned group posting first and the exact expansion IDF later.
-    let expansion_terms = grouped_expansions
-        .iter()
-        .map(|group| group.terms.len())
-        .sum::<usize>()
-        .max(1);
-    Some(limit.saturating_mul(expansion_terms))
-}
-
-#[derive(Debug)]
-struct ExpansionTermFreqs {
-    token: String,
-    freqs_by_posting_doc_id: Vec<(u64, u32)>,
-}
-
-impl ExpansionTermFreqs {
-    fn new(token: String, posting: &PostingList) -> Self {
-        let freqs_by_posting_doc_id = posting
-            .iter()
-            .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq))
-            .collect();
-        Self {
-            token,
-            freqs_by_posting_doc_id,
-        }
-    }
-
-    fn frequency(&self, posting_doc_id: u64) -> Option<u32> {
-        self.freqs_by_posting_doc_id
-            .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id)
-            .ok()
-            .map(|idx| self.freqs_by_posting_doc_id[idx].1)
-    }
+    terms: Arc<[GroupedTermScorer]>,
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Default)]
@@ -983,6 +944,7 @@ impl InvertedIndex {
                         postings,
                         grouped_expansions,
                         impact_safe,
+                        exact_scoring_required,
                     } = loaded_postings;
                     if postings.is_empty() {
                         // No hits in this partition; its DocSet stays
@@ -1005,28 +967,17 @@ impl InvertedIndex {
                     let mask = mask.clone();
                     let metrics = metrics.clone();
                     let part_for_wand = part.clone();
-                    let has_grouped_expansions = !grouped_expansions.is_empty();
-                    let use_impact_path = impact_safe && !has_grouped_expansions;
-                    let wand_params = if has_grouped_expansions {
-                        let mut rescoring_params = params.as_ref().clone();
-                        rescoring_params.limit =
-                            grouped_rescore_wand_limit(params.limit, &grouped_expansions);
-                        Arc::new(rescoring_params)
-                    } else {
-                        params.clone()
-                    };
-                    let partition_threshold = if has_grouped_expansions {
-                        Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()))
-                    } else if use_impact_path {
+                    let use_global_scorer = impact_safe || exact_scoring_required;
+                    let partition_threshold = if use_global_scorer {
                         impact_shared_threshold
                     } else {
                         legacy_shared_threshold
                     };
-                    let wand_scorer = use_impact_path.then(|| impact_scorer.clone());
+                    let wand_scorer = use_global_scorer.then(|| impact_scorer.clone());
                     let candidates = spawn_cpu(move || {
                         let candidates = part_for_wand.bm25_search(
                             docs_for_wand.as_ref(),
-                            wand_params.as_ref(),
+                            params.as_ref(),
                             operator,
                             mask,
                             postings,
@@ -1110,19 +1061,11 @@ impl InvertedIndex {
                             * scorer.doc_weight(freq, doc_length);
                     }
                     for group in &grouped_expansions {
-                        for term in &group.terms {
+                        for term in group.terms.iter() {
                             let Some(freq) = term.frequency(posting_doc_id) else {
                                 continue;
                             };
-                            let idf_weight = match idf_cache.get(&term.token) {
-                                Some(weight) => *weight,
-                                None => {
-                                    let weight = scorer.query_weight(&term.token);
-                                    idf_cache.insert(term.token.clone(), weight);
-                                    weight
-                                }
-                            };
-                            score += idf_weight * scorer.doc_weight(freq, doc_length);
+                            score += term.query_weight() * scorer.doc_weight(freq, doc_length);
                         }
                     }
                     push_scored_candidate(&mut candidates, limit, addr, score)?;
@@ -1877,7 +1820,53 @@ impl InvertedPartition {
         }
     }
 
-    fn union_plain_posting_lists(postings: Vec<PostingList>) -> Result<PostingList> {
+    #[inline]
+    fn grouped_score_upper_bound(
+        query_weight: f32,
+        union_freq: u32,
+        doc_length: u32,
+        scorer: &MemBM25Scorer,
+    ) -> f32 {
+        // BM25's document weight is monotonic in frequency and every IDF is
+        // non-negative. Scoring the summed frequency with the summed IDF is
+        // therefore an upper bound on the sum of the individual term scores.
+        query_weight * scorer.doc_weight(union_freq, doc_length)
+    }
+
+    fn grouped_block_max_scores(
+        doc_ids: &[u32],
+        frequencies: &[u32],
+        block_size: usize,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Vec<f32> {
+        doc_ids
+            .chunks(block_size)
+            .zip(frequencies.chunks(block_size))
+            .map(|(doc_ids, frequencies)| {
+                doc_ids
+                    .iter()
+                    .zip(frequencies)
+                    .map(|(doc_id, freq)| {
+                        Self::grouped_score_upper_bound(
+                            query_weight,
+                            *freq,
+                            docs.scoring_num_tokens(*doc_id),
+                            scorer,
+                        )
+                    })
+                    .fold(0.0, f32::max)
+            })
+            .collect()
+    }
+
+    fn union_plain_posting_lists(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
         let mut freqs_by_row_id = BTreeMap::new();
         for posting in postings {
             for (row_id, freq, _) in posting.iter() {
@@ -1889,19 +1878,31 @@ impl InvertedPartition {
         }
         let mut row_ids = Vec::with_capacity(freqs_by_row_id.len());
         let mut frequencies = Vec::with_capacity(freqs_by_row_id.len());
+        let mut max_score = 0.0_f32;
         for (row_id, freq) in freqs_by_row_id {
+            max_score = max_score.max(Self::grouped_score_upper_bound(
+                query_weight,
+                freq,
+                docs.num_tokens_by_row_id(row_id),
+                scorer,
+            ));
             row_ids.push(row_id);
             frequencies.push(freq as f32);
         }
         Ok(PostingList::Plain(PlainPostingList::new(
             ScalarBuffer::from(row_ids),
             ScalarBuffer::from(frequencies),
-            None,
+            Some(max_score),
             None,
         )))
     }
 
-    fn union_plain_posting_lists_with_positions(postings: Vec<PostingList>) -> Result<PostingList> {
+    fn union_plain_posting_lists_with_positions(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
+    ) -> Result<PostingList> {
         let mut positions_by_row_id = BTreeMap::<u64, Vec<u32>>::new();
         for posting in postings {
             for (row_id, _, positions) in posting.iter() {
@@ -1926,10 +1927,18 @@ impl InvertedPartition {
         let mut row_ids = Vec::with_capacity(positions_by_row_id.len());
         let mut frequencies = Vec::with_capacity(positions_by_row_id.len());
         let mut positions_builder = ListBuilder::new(Int32Builder::new());
+        let mut max_score = 0.0_f32;
         for (row_id, mut positions) in positions_by_row_id {
             positions.sort_unstable();
+            let frequency = positions.len() as u32;
+            max_score = max_score.max(Self::grouped_score_upper_bound(
+                query_weight,
+                frequency,
+                docs.num_tokens_by_row_id(row_id),
+                scorer,
+            ));
             row_ids.push(row_id);
-            frequencies.push(positions.len() as f32);
+            frequencies.push(frequency as f32);
             for position in positions {
                 positions_builder.values().append_value(position as i32);
             }
@@ -1939,7 +1948,7 @@ impl InvertedPartition {
         Ok(PostingList::Plain(PlainPostingList::new(
             ScalarBuffer::from(row_ids),
             ScalarBuffer::from(frequencies),
-            None,
+            Some(max_score),
             Some(positions_builder.finish()),
         )))
     }
@@ -1947,6 +1956,8 @@ impl InvertedPartition {
     fn union_compressed_posting_lists(
         postings: Vec<PostingList>,
         docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
     ) -> Result<PostingList> {
         let block_size = postings
             .iter()
@@ -1987,10 +1998,13 @@ impl InvertedPartition {
             doc_ids.push(doc_id);
             frequencies.push(freq);
         }
-        let block_max_scores = docs.calculate_block_max_scores_with_block_size(
-            doc_ids.iter(),
-            frequencies.iter(),
+        let block_max_scores = Self::grouped_block_max_scores(
+            &doc_ids,
+            &frequencies,
             block_size,
+            docs,
+            query_weight,
+            scorer,
         );
         let batch = builder.to_batch(block_max_scores)?;
         let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
@@ -2001,6 +2015,8 @@ impl InvertedPartition {
     fn union_compressed_posting_lists_with_positions(
         postings: Vec<PostingList>,
         docs: &DocSet,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
     ) -> Result<PostingList> {
         let block_size = postings
             .iter()
@@ -2046,10 +2062,13 @@ impl InvertedPartition {
             doc_ids.push(doc_id);
             frequencies.push(frequency);
         }
-        let block_max_scores = docs.calculate_block_max_scores_with_block_size(
-            doc_ids.iter(),
-            frequencies.iter(),
+        let block_max_scores = Self::grouped_block_max_scores(
+            &doc_ids,
+            &frequencies,
             block_size,
+            docs,
+            query_weight,
+            scorer,
         );
         let batch = builder.to_batch(block_max_scores)?;
         let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
@@ -2061,6 +2080,8 @@ impl InvertedPartition {
         postings: Vec<PostingList>,
         docs: &DocSet,
         with_positions: bool,
+        query_weight: f32,
+        scorer: &MemBM25Scorer,
     ) -> Result<PostingList> {
         let has_plain = postings
             .iter()
@@ -2073,13 +2094,18 @@ impl InvertedPartition {
                 "cannot union mixed plain and compressed posting lists".to_owned(),
             )),
             (true, false) if with_positions => {
-                Self::union_plain_posting_lists_with_positions(postings)
+                Self::union_plain_posting_lists_with_positions(postings, docs, query_weight, scorer)
             }
-            (true, false) => Self::union_plain_posting_lists(postings),
-            (false, true) if with_positions => {
-                Self::union_compressed_posting_lists_with_positions(postings, docs)
+            (true, false) => Self::union_plain_posting_lists(postings, docs, query_weight, scorer),
+            (false, true) if with_positions => Self::union_compressed_posting_lists_with_positions(
+                postings,
+                docs,
+                query_weight,
+                scorer,
+            ),
+            (false, true) => {
+                Self::union_compressed_posting_lists(postings, docs, query_weight, scorer)
             }
-            (false, true) => Self::union_compressed_posting_lists(postings, docs),
             (false, false) => Ok(PostingList::Plain(PlainPostingList::new(
                 ScalarBuffer::from(Vec::<u64>::new()),
                 ScalarBuffer::from(Vec::<f32>::new()),
@@ -2116,6 +2142,10 @@ impl InvertedPartition {
         let token_positions = (0..tokens.len())
             .map(|index| tokens.position(index))
             .collect::<Vec<_>>();
+        let mut seen_positions = HashSet::with_capacity(token_positions.len());
+        let exact_scoring_required = token_positions
+            .iter()
+            .any(|position| !seen_positions.insert(*position));
         let mut token_ids = Vec::with_capacity(tokens.len());
         let mut matched_positions = required_positions.as_ref().map(|_| HashSet::new());
         for (index, token) in tokens.into_iter().enumerate() {
@@ -2175,23 +2205,31 @@ impl InvertedPartition {
                 postings: loaded_postings
                     .into_iter()
                     .map(|(token_id, token, position, posting)| {
-                        let query_weight = if impact_safe {
+                        let needs_scorer_upper_bound =
+                            exact_scoring_required && !posting.has_impacts();
+                        let query_weight = if impact_safe || exact_scoring_required {
                             impact_scorer.query_weight(&token)
                         } else {
                             idf(posting.len(), num_docs)
                         };
-                        PostingIterator::with_query_weight(
+                        let posting = PostingIterator::with_query_weight(
                             token,
                             token_id,
                             position,
                             query_weight,
                             posting,
                             num_docs,
-                        )
+                        );
+                        if needs_scorer_upper_bound {
+                            posting.with_scorer_upper_bound()
+                        } else {
+                            posting
+                        }
                     })
                     .collect(),
                 grouped_expansions: Vec::new(),
                 impact_safe,
+                exact_scoring_required,
             });
         }
 
@@ -2219,16 +2257,17 @@ impl InvertedPartition {
             } else {
                 let token_id = group[0].0;
                 let token = group[0].1.clone();
-                let query_weight = group
+                let terms = group
                     .iter()
-                    .map(|(_, _, posting)| idf(posting.len(), num_docs))
-                    .sum::<f32>();
+                    .map(|(_, token, posting)| {
+                        GroupedTermScorer::new(impact_scorer.query_weight(token), posting)
+                    })
+                    .collect::<Vec<_>>();
+                let terms = Arc::<[GroupedTermScorer]>::from(terms);
+                let query_weight = terms.iter().map(GroupedTermScorer::query_weight).sum();
                 grouped_expansions.push(GroupedExpansionTerms {
                     position,
-                    terms: group
-                        .iter()
-                        .map(|(_, token, posting)| ExpansionTermFreqs::new(token.clone(), posting))
-                        .collect(),
+                    terms: terms.clone(),
                 });
                 let postings = group
                     .into_iter()
@@ -2237,18 +2276,27 @@ impl InvertedPartition {
                 let docs = docs_for_union.as_deref().ok_or_else(|| {
                     Error::index("union docs were not loaded for grouped query terms".to_string())
                 })?;
-                let posting = Self::union_posting_lists(postings, docs, is_phrase_query)?;
+                let posting = Self::union_posting_lists(
+                    postings,
+                    docs,
+                    is_phrase_query,
+                    query_weight,
+                    impact_scorer,
+                )?;
                 if posting.is_empty() && (is_and_query || is_phrase_query) {
                     return Ok(LoadedPostings::empty());
                 }
-                grouped_postings.push(PostingIterator::with_query_weight(
-                    token,
-                    token_id,
-                    position,
-                    query_weight,
-                    posting,
-                    num_docs,
-                ));
+                grouped_postings.push(
+                    PostingIterator::with_query_weight(
+                        token,
+                        token_id,
+                        position,
+                        query_weight,
+                        posting,
+                        num_docs,
+                    )
+                    .with_grouped_terms(terms),
+                );
                 continue;
             };
             if posting.is_empty() {
@@ -2258,21 +2306,28 @@ impl InvertedPartition {
                 continue;
             }
 
-            let query_weight = idf(posting.len(), num_docs);
-            grouped_postings.push(PostingIterator::with_query_weight(
+            let query_weight = impact_scorer.query_weight(&token);
+            let needs_scorer_upper_bound = !posting.has_impacts();
+            let posting = PostingIterator::with_query_weight(
                 token,
                 token_id,
                 position,
                 query_weight,
                 posting,
                 num_docs,
-            ));
+            );
+            grouped_postings.push(if needs_scorer_upper_bound {
+                posting.with_scorer_upper_bound()
+            } else {
+                posting
+            });
         }
 
         Ok(LoadedPostings {
             postings: grouped_postings,
             grouped_expansions,
             impact_safe: false,
+            exact_scoring_required: true,
         })
     }
 
@@ -9795,6 +9850,7 @@ mod tests {
             postings,
             grouped_expansions,
             impact_safe,
+            exact_scoring_required,
         } = partition
             .load_posting_lists(
                 tokens,
@@ -9806,6 +9862,7 @@ mod tests {
             .await
             .unwrap();
         assert!(impact_safe);
+        assert!(!exact_scoring_required);
         assert!(grouped_expansions.is_empty());
 
         let mask = NoFilter.mask();
@@ -10655,6 +10712,64 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[case::and(Operator::And)]
+    #[case::or(Operator::Or)]
+    #[tokio::test]
+    async fn test_grouped_scoring_keeps_exact_winner_outside_proxy_window(
+        #[case] operator: Operator,
+    ) {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("common".to_owned());
+        builder.tokens.add("rare".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        for doc_id in 0..3 {
+            builder.posting_lists[0].add(doc_id, PositionRecorder::Count(1));
+            builder.docs.append(100 + doc_id as u64, 1);
+        }
+        builder.posting_lists[1].add(3, PositionRecorder::Count(1));
+        builder.docs.append(103, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::with_positions(
+            vec!["common".to_owned(), "rare".to_owned()],
+            vec![0, 0],
+            DocType::Text,
+        ));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let (row_ids, _scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                operator,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(
+            row_ids,
+            vec![103],
+            "the rare term's exact IDF must win even when proxy scoring ranks it outside the old candidate cushion"
+        );
+    }
+
     #[tokio::test]
     async fn test_fuzzy_and_grouped_rescore_keeps_wand_limit_bounded() {
         let tmpdir = TempObjDir::default();
diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs
index 465d0f8e2ab..e2eb352175b 100644
--- a/rust/lance-index/src/scalar/inverted/wand.rs
+++ b/rust/lance-index/src/scalar/inverted/wand.rs
@@ -339,6 +339,43 @@ fn scorer_upper_bound<S: Scorer + ?Sized>(query_weight: f32, scorer: &S) -> f32
     }
 }
 
+/// Per-term data retained after same-position postings are unioned.
+#[derive(Debug)]
+pub(super) struct GroupedTermScorer {
+    query_weight: f32,
+    freqs_by_posting_doc_id: Vec<(u64, u32)>,
+}
+
+impl GroupedTermScorer {
+    pub(super) fn new(query_weight: f32, posting: &PostingList) -> Self {
+        let freqs_by_posting_doc_id = posting
+            .iter()
+            .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq))
+            .collect();
+        Self {
+            query_weight,
+            freqs_by_posting_doc_id,
+        }
+    }
+
+    pub(super) fn query_weight(&self) -> f32 {
+        self.query_weight
+    }
+
+    pub(super) fn frequency(&self, posting_doc_id: u64) -> Option<u32> {
+        self.freqs_by_posting_doc_id
+            .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id)
+            .ok()
+            .map(|index| self.freqs_by_posting_doc_id[index].1)
+    }
+
+    fn score<S: Scorer + ?Sized>(&self, posting_doc_id: u64, doc_length: u32, scorer: &S) -> f32 {
+        self.frequency(posting_doc_id)
+            .map(|freq| self.query_weight * scorer.doc_weight(freq, doc_length))
+            .unwrap_or_default()
+    }
+}
+
 pub struct PostingIterator {
     token: String,
     token_id: u32,
@@ -351,6 +388,12 @@ pub struct PostingIterator {
     block_idx: usize,
     current_doc: Option<DocInfo>,
     approximate_upper_bound: f32,
+    // Stored block maxima may use partition-local statistics. Queries scored
+    // with corpus-wide statistics must use a scorer-provided bound instead.
+    use_scorer_upper_bound: bool,
+    // The union posting drives iteration and pruning; these terms produce the
+    // exact score for each candidate document.
+    grouped_terms: Option<Arc<[GroupedTermScorer]>>,
     // Position cursors temporarily own this buffer and return it on drop. This
     // keeps repeated cursor creation allocation-free without lending a slice
     // out of the interior-mutable compressed state.
@@ -788,6 +831,8 @@ impl PostingIterator {
             block_idx: 0,
             current_doc: None,
             approximate_upper_bound,
+            use_scorer_upper_bound: false,
+            grouped_terms: None,
             position_scratch: RefCell::new(Some(Vec::new())),
             compressed,
         };
@@ -795,6 +840,16 @@ impl PostingIterator {
         posting
     }
 
+    pub(super) fn with_scorer_upper_bound(mut self) -> Self {
+        self.use_scorer_upper_bound = true;
+        self
+    }
+
+    pub(super) fn with_grouped_terms(mut self, terms: Arc<[GroupedTermScorer]>) -> Self {
+        self.grouped_terms = Some(terms);
+        self
+    }
+
     #[inline]
     pub(crate) fn term_index(&self) -> u32 {
         self.position
@@ -830,6 +885,9 @@ impl PostingIterator {
                     &mut compressed.block_max_window.impact_score_cache,
                 );
         }
+        if self.use_scorer_upper_bound {
+            return scorer_upper_bound(self.query_weight, scorer);
+        }
         if let PostingList::Compressed(ref list) = self.list
             && list.block_size == MAX_POSTING_BLOCK_SIZE
         {
@@ -840,9 +898,20 @@ impl PostingIterator {
 
     #[inline]
     fn score<S: Scorer + ?Sized>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
+        if let (Some(grouped_terms), Some(doc)) = (&self.grouped_terms, self.current_doc) {
+            return grouped_terms
+                .iter()
+                .map(|term| term.score(doc.doc_id(), doc_length, scorer))
+                .sum();
+        }
         self.query_weight * scorer.doc_weight(freq, doc_length)
     }
 
+    #[inline]
+    fn has_grouped_terms(&self) -> bool {
+        self.grouped_terms.is_some()
+    }
+
     #[inline]
     fn cost(&self) -> usize {
         self.list.len()
@@ -1147,11 +1216,17 @@ impl PostingIterator {
                 if let Some(impacts) = list.impacts.as_ref() {
                     return self.impact_level0(impacts, scorer).1;
                 }
+                if self.use_scorer_upper_bound {
+                    return scorer_upper_bound(self.query_weight, scorer);
+                }
                 if list.block_size == MAX_POSTING_BLOCK_SIZE {
                     return scorer_upper_bound(self.query_weight, scorer);
                 }
                 list.block_max_score(self.block_idx)
             }
+            PostingList::Plain(_) if self.use_scorer_upper_bound => {
+                scorer_upper_bound(self.query_weight, scorer)
+            }
             PostingList::Plain(_) => self.approximate_upper_bound,
         }
     }
@@ -1177,6 +1252,12 @@ impl PostingIterator {
                         };
                     }
                 }
+                if self.use_scorer_upper_bound {
+                    return BlockMaxScore {
+                        score: scorer_upper_bound(self.query_weight, scorer),
+                        blocks_scanned: 0,
+                    };
+                }
                 let compressed = unsafe { &mut *self.compressed_state_ptr() };
                 compressed.block_max_window.max_score_up_to(
                     list,
@@ -1187,7 +1268,11 @@ impl PostingIterator {
                 )
             }
             PostingList::Plain(_) => BlockMaxScore {
-                score: self.approximate_upper_bound,
+                score: if self.use_scorer_upper_bound {
+                    scorer_upper_bound(self.query_weight, scorer)
+                } else {
+                    self.approximate_upper_bound
+                },
                 blocks_scanned: 0,
             },
         }
@@ -1738,10 +1823,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
             && self.operator == Operator::Or
             && params.phrase_slop.is_none()
             && !self.head.is_empty()
-            && self
-                .head
-                .iter()
-                .all(|posting| posting.posting.is_compressed())
+            && self.head.iter().all(|posting| {
+                posting.posting.is_compressed() && !posting.posting.has_grouped_terms()
+            })
         {
             return self.maxscore_search(params, mask, metrics);
         }
@@ -1752,7 +1836,10 @@ impl<'a, S: Scorer> Wand<'a, S> {
         // `next()` leapfrogging through boxed iterators.
         if self.operator == Operator::And
             && !self.lead.is_empty()
-            && self.lead.iter().all(|posting| posting.is_compressed())
+            && self
+                .lead
+                .iter()
+                .all(|posting| posting.is_compressed() && !posting.has_grouped_terms())
             && self
                 .bulk_and_mode_override
                 .unwrap_or_else(|| *BULK_AND_MODE)
diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs
index 743084f9e05..0d00aa2b94c 100644
--- a/rust/lance-tokenizer/src/code_tokenizer.rs
+++ b/rust/lance-tokenizer/src/code_tokenizer.rs
@@ -8,8 +8,9 @@ use crate::{Token, TokenStream, Tokenizer};
 /// Tokenizer for code-like text.
 ///
 /// Identifiers are Unicode alphanumeric characters plus `_`. Other characters
-/// are lexical boundaries. When operator indexing is enabled, contiguous
-/// operator characters are emitted as tokens too.
+/// are lexical boundaries. When operator indexing is enabled, recognized
+/// multi-character operators use longest-match tokenization and remaining
+/// operator characters are emitted individually.
 ///
 /// # Examples
 ///
@@ -72,6 +73,12 @@ fn is_operator_char(ch: char) -> bool {
     )
 }
 
+const MULTI_CHAR_OPERATORS: &[&str] = &[
+    ">>>=", "<<=", ">>=", "&&=", "||=", "??=", "**=", "//=", "===", "!==", ">>>", "<=>", "::",
+    "->", "=>", "==", "!=", "<=", ">=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "&=",
+    "|=", "^=", "<<", ">>", "**", "//", "??", ":=", "<-", "|>", "~=",
+];
+
 impl CodeLexTokenStream<'_> {
     fn search_token_end(&mut self, predicate: impl Fn(char) -> bool) -> usize {
         while let Some((_, ch)) = self.chars.peek() {
@@ -85,6 +92,25 @@ impl CodeLexTokenStream<'_> {
             .map(|(offset, _)| *offset)
             .unwrap_or(self.text.len())
     }
+
+    fn operator_token_end(&mut self, offset_from: usize) -> usize {
+        let remaining = &self.text[offset_from..];
+        let operator_len = MULTI_CHAR_OPERATORS
+            .iter()
+            .filter(|operator| remaining.starts_with(**operator))
+            .map(|operator| operator.len())
+            .max()
+            .unwrap_or(1);
+        let token_end = offset_from + operator_len;
+        while self
+            .chars
+            .peek()
+            .is_some_and(|(offset, _)| *offset < token_end)
+        {
+            self.chars.next();
+        }
+        token_end
+    }
 }
 
 impl TokenStream for CodeLexTokenStream<'_> {
@@ -94,7 +120,7 @@ impl TokenStream for CodeLexTokenStream<'_> {
             let token_end = if is_identifier_char(ch) {
                 self.search_token_end(is_identifier_char)
             } else if self.index_operators && is_operator_char(ch) {
-                self.search_token_end(is_operator_char)
+                self.operator_token_end(offset_from)
             } else {
                 continue;
             };
@@ -152,4 +178,14 @@ mod tests {
             .collect::<Vec<_>>();
         assert_eq!(texts, vec!["a", "::", "b", "!=", "c", "->", "d"]);
     }
+
+    #[test]
+    fn test_code_lex_tokenizer_splits_adjacent_operators() {
+        let tokens = collect_tokens("value.parse::<usize>()", true);
+        let texts = tokens
+            .iter()
+            .map(|token| token.text.as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(texts, vec!["value", "parse", "::", "<", "usize", ">"]);
+    }
 }

From ea89396b44ca799d69f69663a83943eef9db6147 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Sat, 18 Jul 2026 01:41:13 +0800
Subject: [PATCH 18/19] fix: bump FTS index version to v4

---
 rust/lance-index/src/scalar/inverted.rs       | 12 +++----
 rust/lance-index/src/scalar/inverted/index.rs | 32 +++++++++----------
 .../src/dataset/mem_wal/memtable/flush.rs     |  6 ++--
 rust/lance/src/index.rs                       |  9 ++++++
 4 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs
index 926bfad6a69..cce15582c9e 100644
--- a/rust/lance-index/src/scalar/inverted.rs
+++ b/rust/lance-index/src/scalar/inverted.rs
@@ -148,7 +148,6 @@ impl InvertedIndexPlugin {
         });
 
         params.validate_format_version()?;
-        let format_version = params.resolved_format_version();
         let details = pbold::InvertedIndexDetails::try_from(&params)?;
         let mut inverted_index =
             InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask)
@@ -156,7 +155,7 @@ impl InvertedIndexPlugin {
         let files = inverted_index.update(data, index_store, None).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version: format_version.index_version(),
+            index_version: INVERTED_INDEX_VERSION_V4,
             files,
         })
     }
@@ -268,7 +267,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
     }
 
     fn version(&self) -> u32 {
-        max_supported_fts_format_version().index_version()
+        INVERTED_INDEX_VERSION_V4
     }
 
     fn new_query_parser(
@@ -320,12 +319,9 @@ mod tests {
     use crate::scalar::{BuiltinIndexType, ScalarIndexParams};
 
     #[test]
-    fn test_plugin_version_tracks_max_supported_format() {
+    fn test_plugin_version_tracks_current_index_version() {
         let plugin = InvertedIndexPlugin;
-        assert_eq!(
-            plugin.version(),
-            max_supported_fts_format_version().index_version()
-        );
+        assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V4);
     }
 
     #[test]
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 49909cf0e74..0ec4b1da5d3 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -86,13 +86,18 @@ use crate::{FtsPrewarmOptions, Index};
 use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys};
 use std::str::FromStr;
 
+// Manifest compatibility versions. Versions 0-3 historically coincided with
+// physical layout revisions. Version 4 is the first version managed
+// independently from the physical posting layout.
 // Version 0: Arrow TokenSetFormat (legacy)
 // Version 1: Fst TokenSetFormat with per-doc compressed positions
 // Version 2: Fst TokenSetFormat with shared posting-list position streams.
 // Version 3: Version 2 layout with 256-document physical posting blocks.
+// Version 4: Code analyzer support.
 pub const INVERTED_INDEX_VERSION_V1: u32 = 1;
 pub const INVERTED_INDEX_VERSION_V2: u32 = 2;
 pub const INVERTED_INDEX_VERSION_V3: u32 = 3;
+pub const INVERTED_INDEX_VERSION_V4: u32 = 4;
 pub const TOKENS_FILE: &str = "tokens.lance";
 pub const INVERT_LIST_FILE: &str = "invert.lance";
 pub const DOCS_FILE: &str = "docs.lance";
@@ -546,7 +551,7 @@ impl InvertedIndex {
                 TokenSetFormat::Arrow,
                 InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2,
             ) => 0,
-            (_, format_version) => format_version.index_version(),
+            _ => INVERTED_INDEX_VERSION_V4,
         }
     }
 
@@ -11047,7 +11052,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_update_preserves_loaded_v2_format_version() -> Result<()> {
+    async fn test_update_preserves_v2_layout_with_v4_index_version() -> Result<()> {
         let src_dir = TempObjDir::default();
         let dest_dir = TempObjDir::default();
         let src_store = Arc::new(LanceIndexStore::new(
@@ -11102,7 +11107,8 @@ mod tests {
         writer.finish_with_metadata(metadata).await.unwrap();
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(index.index_version(), format_version.index_version());
+        assert_eq!(index.format_version(), format_version);
+        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V4);
 
         let schema = Arc::new(Schema::new(vec![
             Field::new("doc", DataType::Utf8, true),
@@ -11116,10 +11122,11 @@ mod tests {
             .update(Box::pin(stream), dest_store.as_ref(), None)
             .await?;
 
-        assert_eq!(created.index_version, format_version.index_version());
+        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V4);
 
         let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(updated.index_version(), format_version.index_version());
+        assert_eq!(updated.format_version(), format_version);
+        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V4);
         assert_eq!(updated.partitions.len(), 2);
         for partition in &updated.partitions {
             assert_eq!(
@@ -11172,25 +11179,16 @@ mod tests {
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
         assert_eq!(index.format_version(), InvertedListFormatVersion::V3);
-        assert_eq!(
-            index.index_version(),
-            InvertedListFormatVersion::V3.index_version()
-        );
+        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V4);
 
         let created = index
             .update(empty_doc_stream(), dest_store.as_ref(), None)
             .await?;
-        assert_eq!(
-            created.index_version,
-            InvertedListFormatVersion::V3.index_version()
-        );
+        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V4);
 
         let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
         assert_eq!(updated.format_version(), InvertedListFormatVersion::V3);
-        assert_eq!(
-            updated.index_version(),
-            InvertedListFormatVersion::V3.index_version()
-        );
+        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V4);
 
         Ok(())
     }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index e88be539e30..bcb4226a8f5 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -13,6 +13,7 @@ use lance_core::utils::deletion::DeletionVector;
 use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_index::mem_wal::{FlushedGeneration, ShardManifest};
+use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4;
 use lance_index::scalar::{IndexStore, ScalarIndexParams};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_table::format::IndexMetadata;
@@ -776,7 +777,6 @@ impl MemTableFlusher {
             })?;
 
             let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone();
-            let format_version = fts_cfg.params.resolved_format_version();
 
             let index_meta = IndexMetadata {
                 uuid: index_uuid,
@@ -785,7 +785,7 @@ impl MemTableFlusher {
                 dataset_version: dataset.version().version,
                 fragment_bitmap: Some(fragment_ids),
                 index_details: Some(Arc::new(index_details)),
-                index_version: format_version.index_version() as i32,
+                index_version: INVERTED_INDEX_VERSION_V4 as i32,
                 created_at: None,
                 base_id: None,
                 files: None,
@@ -1192,6 +1192,7 @@ mod tests {
     use super::*;
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+    use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4;
     use std::sync::Arc;
     use tempfile::TempDir;
 
@@ -2192,6 +2193,7 @@ mod tests {
 
         assert_eq!(indices.len(), 1);
         assert_eq!(indices[0].name, "text_fts");
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32);
 
         // Verify FTS query returns correct results
         // Searching for "hello" should find the first document
diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs
index c452e70f421..4d39e5e6483 100644
--- a/rust/lance/src/index.rs
+++ b/rust/lance/src/index.rs
@@ -2897,6 +2897,7 @@ mod tests {
     use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array};
     use lance_index::pbold::BTreeIndexDetails;
     use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME;
+    use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4;
     use lance_index::scalar::{
         BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
     };
@@ -4996,8 +4997,16 @@ mod tests {
             .await
             .unwrap();
 
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32);
+
         dataset.optimize_indices(&Default::default()).await.unwrap();
 
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V4 as i32);
+
         let stats: serde_json::Value =
             serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap();
         assert_eq!(stats["num_unindexed_rows"], 0);

From c9ffe977a359a50a53b7081d8ebc990c2229186f Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Sun, 19 Jul 2026 03:00:32 +0800
Subject: [PATCH 19/19] fix: use FTS v4 consistently for new indexes

---
 docs/src/guide/migration.md                   |   8 +-
 .../index/scalar/InvertedIndexParams.java     |  16 ++-
 .../index/scalar/InvertedIndexParamsTest.java |  11 ++
 protos/index_old.proto                        |   4 +-
 python/python/lance/dataset.py                |   4 +-
 python/python/tests/test_scalar_index.py      |  10 +-
 python/src/dataset.rs                         |   2 +-
 rust/lance-index/src/scalar/inverted.rs       |   3 +-
 .../src/scalar/inverted/builder.rs            |  31 +++--
 rust/lance-index/src/scalar/inverted/index.rs | 123 +++++++++++++-----
 .../src/scalar/inverted/tokenizer.rs          |  31 +++--
 rust/lance/src/dataset/mem_wal/index.rs       |   8 +-
 .../src/dataset/mem_wal/memtable/flush.rs     |   4 +-
 13 files changed, 166 insertions(+), 89 deletions(-)

diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md
index 5efd7b26b7f..6b999558e7d 100644
--- a/docs/src/guide/migration.md
+++ b/docs/src/guide/migration.md
@@ -8,15 +8,15 @@ migrate.
 
 ## 9.0.0
 
-* Newly created FTS / inverted indexes now default to format v2 instead of v1.
+* Newly created FTS / inverted indexes now default to format v4 instead of v1.
   The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the
-  format used for newly created indexes. Users who need a specific index layout
-  should pass the index creation parameter `format_version` explicitly.
+  format used for newly created indexes. Users who need a specific older index
+  layout should pass the index creation parameter `format_version` explicitly.
 
 * This affects users who create FTS / inverted indexes and need those indexes to
   be readable by older Lance versions, or who depend on the v1 index layout. In
   those cases, pass `format_version=1` when creating the index. Otherwise, newly
-  created indexes will use v2 by default, and older Lance readers may not be able
+  created indexes will use v4 by default, and older Lance readers may not be able
   to read them.
 
   ```python
diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
index 82513a89ee7..900d6b9b2fd 100755
--- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
+++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
@@ -264,16 +264,16 @@ public Builder skipMerge(boolean skipMerge) {
     /**
      * Configure the on-disk FTS format version to write when creating a new index.
      *
-     * <p>If unset, Lance writes v2 for {@code blockSize = 128} and v3 for {@code blockSize = 256}.
-     * {@code formatVersion = 3} is experimental and is only valid with {@code blockSize = 256}.
+     * <p>If unset, Lance writes v4 for either supported block size. {@code formatVersion = 3} is
+     * experimental and is only valid with {@code blockSize = 256}.
      *
-     * @param formatVersion FTS format version, must be 1, 2, or 3
+     * @param formatVersion FTS format version, must be 1, 2, 3, or 4
      * @return this builder
      * @throws IllegalArgumentException
      */
     public Builder formatVersion(int formatVersion) {
-      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) {
-        throw new IllegalArgumentException("formatVersion must be 1, 2, or 3");
+      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3 && formatVersion != 4) {
+        throw new IllegalArgumentException("formatVersion must be 1, 2, 3, or 4");
       }
       this.formatVersion = formatVersion;
       return this;
@@ -283,8 +283,10 @@ public Builder formatVersion(int formatVersion) {
     public ScalarIndexParams build() {
       if (formatVersion != null) {
         Preconditions.checkArgument(
-            (blockSize == 256 && formatVersion == 3) || (blockSize == 128 && formatVersion != 3),
-            "formatVersion 3 requires blockSize 256, and blockSize 256 requires formatVersion 3");
+            formatVersion == 4
+                || (blockSize == 256 && formatVersion == 3)
+                || (blockSize == 128 && formatVersion != 3),
+            "formatVersion 3 requires blockSize 256, and legacy formats require blockSize 128");
       }
       Map<String, Object> params = new HashMap<>();
       if (baseTokenizer != null) {
diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
index 0ccc429ec57..8618d3ce091 100644
--- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
+++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
@@ -75,4 +75,15 @@ void formatVersionThreeRequiresBlockSize256() {
         IllegalArgumentException.class,
         () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build());
   }
+
+  @Test
+  void formatVersionFourSupportsBothBlockSizes() {
+    for (int blockSize : new int[] {128, 256}) {
+      ScalarIndexParams params =
+          InvertedIndexParams.builder().blockSize(blockSize).formatVersion(4).build();
+      Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+      assertEquals(blockSize, ((Number) json.get("block_size")).intValue());
+      assertEquals(4, ((Number) json.get("format_version")).intValue());
+    }
+  }
 }
diff --git a/protos/index_old.proto b/protos/index_old.proto
index bc1c7e1a951..10b4dfa602d 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -62,8 +62,8 @@ message InvertedIndexDetails {
   bool prefix_only = 11;
   // Number of documents per compressed posting block. An absent value means
   // the index predates this field and must use the legacy block size of 128.
-  // A present value records the block size used by the index; 256 is only
-  // valid with format version 3.
+  // A present value records the block size used by the index; 256 is valid
+  // with format versions 3 and 4.
   optional uint32 block_size = 12;
   // Options for base_tokenizer = "code". Presence records the code tokenizer
   // configuration used to build the index; absence means there is no
diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py
index def635d58a6..f4d5ccd8380 100644
--- a/python/python/lance/dataset.py
+++ b/python/python/lance/dataset.py
@@ -3370,8 +3370,8 @@ def create_scalar_index(
         format_version: int or str, optional
             This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS
             format version to write when creating a new index. Accepts ``1``,
-            ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. If unset, Lance
-            writes v2 for ``block_size=128`` and v3 for ``block_size=256``.
+            ``2``, ``3``, ``4``, ``"v1"``, ``"v2"``, ``"v3"``, or ``"v4"``.
+            If unset, Lance writes v4 for either supported block size.
             ``format_version=3`` is experimental and is only valid with
             ``block_size=256``.
 
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index cd49fb03474..0c5b41592ae 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -1271,7 +1271,7 @@ def test_create_scalar_index_fts_block_size(dataset):
     )
     indices = dataset.describe_indices()
     doc_index = next(index for index in indices if index.name == "doc_idx")
-    assert doc_index.segments[0].index_version == 3
+    assert doc_index.segments[0].index_version == 4
 
     row = dataset.take(indices=[0], columns=["doc"])
     query = row.column(0)[0].as_py().split(" ")[0]
@@ -5390,7 +5390,7 @@ def test_json_inverted_match_query(tmp_path):
 
 @pytest.mark.parametrize(
     ("format_version", "expected_format_version"),
-    [(1, 1), (2, 2), ("v1", 1), ("v2", 2)],
+    [(1, 1), (2, 2), (4, 4), ("v1", 1), ("v2", 2), ("v4", 4)],
 )
 def test_describe_indices(tmp_path, format_version, expected_format_version):
     data = pa.table(
@@ -5520,7 +5520,7 @@ def test_describe_indices(tmp_path, format_version, expected_format_version):
         assert index.num_rows_indexed == 50
 
 
-def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypatch):
+def test_create_inverted_index_defaults_to_v4_and_ignores_env(tmp_path, monkeypatch):
     monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", "1")
     data = pa.table({"text": ["document about lance database"]})
     ds = lance.write_dataset(data, tmp_path)
@@ -5528,7 +5528,7 @@ def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypa
     ds.create_scalar_index("text", index_type="INVERTED")
 
     indices = ds.describe_indices()
-    assert indices[0].segments[0].index_version == 2
+    assert indices[0].segments[0].index_version == 4
 
 
 def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
@@ -5536,7 +5536,7 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
     ds = lance.write_dataset(data, tmp_path)
 
     with pytest.raises(ValueError, match="unsupported FTS format version"):
-        ds.create_scalar_index("text", index_type="INVERTED", format_version="v4")
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v5")
 
     with pytest.raises(ValueError, match="format_version=3"):
         ds.create_scalar_index("text", index_type="INVERTED", format_version="v3")
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index 07dbf327a05..eddf555fd35 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -2583,7 +2583,7 @@ impl Dataset {
                             value.to_string()
                         } else {
                             return Err(PyValueError::new_err(
-                                "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'",
+                                "format_version must be 1, 2, 3, 4, 'v1', 'v2', 'v3', or 'v4'",
                             ));
                         };
                         let format_version = value
diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs
index cce15582c9e..ca27367febc 100644
--- a/rust/lance-index/src/scalar/inverted.rs
+++ b/rust/lance-index/src/scalar/inverted.rs
@@ -148,6 +148,7 @@ impl InvertedIndexPlugin {
         });
 
         params.validate_format_version()?;
+        let format_version = params.resolved_format_version();
         let details = pbold::InvertedIndexDetails::try_from(&params)?;
         let mut inverted_index =
             InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask)
@@ -155,7 +156,7 @@ impl InvertedIndexPlugin {
         let files = inverted_index.update(data, index_store, None).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version: INVERTED_INDEX_VERSION_V4,
+            index_version: format_version.index_version(),
             files,
         })
     }
diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs
index 41804e94fd1..060e85a4d67 100644
--- a/rust/lance-index/src/scalar/inverted/builder.rs
+++ b/rust/lance-index/src/scalar/inverted/builder.rs
@@ -1825,16 +1825,16 @@ pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts(
         InvertedListFormatVersion::V1 => {
             inverted_list_schema_v1(with_position, block_size, with_impacts)
         }
-        InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => {
-            inverted_list_schema_with_tail_codec_and_position_codec(
-                with_position,
-                format_version,
-                PostingTailCodec::VarintDelta,
-                Some(PositionStreamCodec::PackedDelta),
-                block_size,
-                with_impacts,
-            )
-        }
+        InvertedListFormatVersion::V2
+        | InvertedListFormatVersion::V3
+        | InvertedListFormatVersion::V4 => inverted_list_schema_with_tail_codec_and_position_codec(
+            with_position,
+            format_version,
+            PostingTailCodec::VarintDelta,
+            Some(PositionStreamCodec::PackedDelta),
+            block_size,
+            with_impacts,
+        ),
     }
 }
 
@@ -2105,7 +2105,6 @@ async fn merge_metadata_files(
     let mut params = None;
     let mut token_set_format = None;
     let mut format_version = None;
-    let mut posting_tail_codec = None;
     let mut deleted_fragments = RoaringBitmap::new();
     progress
         .stage_start(
@@ -2147,9 +2146,6 @@ async fn merge_metadata_files(
         if format_version.is_none() {
             format_version = Some(parse_format_version_from_metadata(metadata)?);
         }
-        if posting_tail_codec.is_none() {
-            posting_tail_codec = Some(parse_posting_tail_codec(metadata)?);
-        }
 
         if reader.num_rows() > 0 {
             let metadata_batch = reader.read_range(0..1, None).await?;
@@ -2215,8 +2211,7 @@ async fn merge_metadata_files(
         None,
         deleted_fragments,
     )
-    .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1))
-    .with_posting_tail_codec(posting_tail_codec.unwrap_or(PostingTailCodec::Fixed32));
+    .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1));
     progress
         .stage_start("write_merged_metadata", Some(1), "files")
         .await?;
@@ -2922,6 +2917,10 @@ mod tests {
         expected_partitions.dedup();
         let remapped_partitions = (0..expected_partitions.len() as u64).collect::<Vec<_>>();
         assert_eq!(written_partitions, remapped_partitions);
+        assert_eq!(
+            parse_format_version_from_metadata(metadata)?,
+            InvertedListFormatVersion::V4
+        );
 
         for (new_id, old_id) in expected_partitions.iter().enumerate() {
             assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?;
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 74f20f59a0c..0ec3248be53 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -86,14 +86,11 @@ use crate::{FtsPrewarmOptions, Index};
 use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys};
 use std::str::FromStr;
 
-// Manifest compatibility versions. Versions 0-3 historically coincided with
-// physical layout revisions. Version 4 is the first version managed
-// independently from the physical posting layout.
 // Version 0: Arrow TokenSetFormat (legacy)
 // Version 1: Fst TokenSetFormat with per-doc compressed positions
 // Version 2: Fst TokenSetFormat with shared posting-list position streams.
 // Version 3: Version 2 layout with 256-document physical posting blocks.
-// Version 4: Code analyzer support.
+// Version 4: Code analyzer support with configurable posting block sizes.
 pub const INVERTED_INDEX_VERSION_V1: u32 = 1;
 pub const INVERTED_INDEX_VERSION_V2: u32 = 2;
 pub const INVERTED_INDEX_VERSION_V3: u32 = 3;
@@ -152,7 +149,7 @@ pub fn resolve_fts_format_version(
 }
 
 pub fn default_fts_format_version() -> InvertedListFormatVersion {
-    InvertedListFormatVersion::V2
+    InvertedListFormatVersion::V4
 }
 
 pub fn current_fts_format_version() -> InvertedListFormatVersion {
@@ -160,15 +157,16 @@ pub fn current_fts_format_version() -> InvertedListFormatVersion {
 }
 
 pub fn max_supported_fts_format_version() -> InvertedListFormatVersion {
-    InvertedListFormatVersion::V3
+    InvertedListFormatVersion::V4
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
 pub enum InvertedListFormatVersion {
     V1,
-    #[default]
     V2,
     V3,
+    #[default]
+    V4,
 }
 
 impl InvertedListFormatVersion {
@@ -204,25 +202,26 @@ impl InvertedListFormatVersion {
             Self::V1 => INVERTED_INDEX_VERSION_V1,
             Self::V2 => INVERTED_INDEX_VERSION_V2,
             Self::V3 => INVERTED_INDEX_VERSION_V3,
+            Self::V4 => INVERTED_INDEX_VERSION_V4,
         }
     }
 
     pub fn posting_tail_codec(self) -> PostingTailCodec {
         match self {
             Self::V1 => PostingTailCodec::Fixed32,
-            Self::V2 | Self::V3 => PostingTailCodec::VarintDelta,
+            Self::V2 | Self::V3 | Self::V4 => PostingTailCodec::VarintDelta,
         }
     }
 
     pub fn position_codec(self) -> Option<PositionStreamCodec> {
         match self {
             Self::V1 => None,
-            Self::V2 | Self::V3 => Some(PositionStreamCodec::PackedDelta),
+            Self::V2 | Self::V3 | Self::V4 => Some(PositionStreamCodec::PackedDelta),
         }
     }
 
     pub fn uses_shared_position_stream(self) -> bool {
-        matches!(self, Self::V2 | Self::V3)
+        matches!(self, Self::V2 | Self::V3 | Self::V4)
     }
 }
 
@@ -234,8 +233,9 @@ impl FromStr for InvertedListFormatVersion {
             "1" | "v1" | "V1" => Ok(Self::V1),
             "2" | "v2" | "V2" => Ok(Self::V2),
             "3" | "v3" | "V3" => Ok(Self::V3),
+            "4" | "v4" | "V4" => Ok(Self::V4),
             other => Err(Error::index(format!(
-                "unsupported FTS format version {}, expected 1, 2, or 3",
+                "unsupported FTS format version {}, expected 1, 2, 3, or 4",
                 other
             ))),
         }
@@ -246,11 +246,7 @@ pub fn default_fts_format_version_for_block_size(
     block_size: usize,
 ) -> Result<InvertedListFormatVersion> {
     validate_block_size(block_size)?;
-    match block_size {
-        LEGACY_BLOCK_SIZE => Ok(InvertedListFormatVersion::V2),
-        256 => Ok(InvertedListFormatVersion::V3),
-        _ => unreachable!("validate_block_size limits supported block sizes"),
-    }
+    Ok(InvertedListFormatVersion::V4)
 }
 
 pub fn validate_format_version_block_size(
@@ -260,7 +256,8 @@ pub fn validate_format_version_block_size(
     validate_block_size(block_size)?;
     match (format_version, block_size) {
         (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)
-        | (InvertedListFormatVersion::V3, 256) => Ok(()),
+        | (InvertedListFormatVersion::V3, 256)
+        | (InvertedListFormatVersion::V4, _) => Ok(()),
         (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, 256) => {
             Err(Error::invalid_input(format!(
                 "FTS format_version={} is incompatible with block_size=256; use format_version=3",
@@ -551,7 +548,7 @@ impl InvertedIndex {
                 TokenSetFormat::Arrow,
                 InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2,
             ) => 0,
-            _ => INVERTED_INDEX_VERSION_V4,
+            (_, format_version) => format_version.index_version(),
         }
     }
 
@@ -667,8 +664,7 @@ impl InvertedIndex {
         let mut builder = InvertedIndexBuilder::new(first.params.clone()).with_progress(progress);
         builder = builder
             .with_token_set_format(first.token_set_format)
-            .with_format_version(first.format_version())
-            .with_posting_tail_codec(first.posting_tail_codec());
+            .with_format_version(first.format_version());
         let files = builder
             .update_from_segments(new_data, dest_store, segments, old_data_filter)
             .await?;
@@ -7917,10 +7913,10 @@ mod tests {
     }
 
     #[test]
-    fn test_resolve_fts_format_version_defaults_to_v2() {
+    fn test_resolve_fts_format_version_defaults_to_v4() {
         assert_eq!(
             resolve_fts_format_version(None).unwrap(),
-            InvertedListFormatVersion::V2
+            InvertedListFormatVersion::V4
         );
         assert_eq!(
             resolve_fts_format_version(Some("2")).unwrap(),
@@ -7930,6 +7926,10 @@ mod tests {
             resolve_fts_format_version(Some("3")).unwrap(),
             InvertedListFormatVersion::V3
         );
+        assert_eq!(
+            resolve_fts_format_version(Some("4")).unwrap(),
+            InvertedListFormatVersion::V4
+        );
     }
 
     #[test]
@@ -8632,6 +8632,8 @@ mod tests {
     #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)]
     #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)]
     #[case::v3(InvertedListFormatVersion::V3, 256)]
+    #[case::v4_128(InvertedListFormatVersion::V4, LEGACY_BLOCK_SIZE)]
+    #[case::v4_256(InvertedListFormatVersion::V4, 256)]
     #[tokio::test]
     async fn test_prewarm_streams_in_chunks_preserves_content(
         #[case] format_version: InvertedListFormatVersion,
@@ -8732,7 +8734,7 @@ mod tests {
             "single partition must be streamed in more than one chunk, got {chunk_count}"
         );
 
-        if format_version == InvertedListFormatVersion::V3 {
+        if block_size == 256 {
             let (start, end) = inverted_list.group_range_for_token(0).unwrap();
             let group = inverted_list
                 .index_cache
@@ -8742,18 +8744,18 @@ mod tests {
                     inverted_list.has_impacts,
                 ))
                 .await
-                .expect("v3 prewarm should populate the packed group cache");
+                .expect("256-document blocks should populate the packed group cache");
             assert!(group.is_packed());
             let (max_score, length) = inverted_list.bulk_metadata_for_token(0);
             let PostingList::Compressed(posting) =
                 group.posting_list(0, max_score, length).unwrap().unwrap()
             else {
-                panic!("expected compressed v3 posting list");
+                panic!("expected compressed posting list");
             };
             assert_eq!(posting.block_size, 256);
             assert!(
                 posting.impacts.is_some(),
-                "v3 packed prewarm must preserve impact skip data"
+                "packed prewarm must preserve impact skip data"
             );
         }
 
@@ -11225,7 +11227,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_update_preserves_v2_layout_with_v4_index_version() -> Result<()> {
+    async fn test_update_preserves_v2_format_version() -> Result<()> {
         let src_dir = TempObjDir::default();
         let dest_dir = TempObjDir::default();
         let src_store = Arc::new(LanceIndexStore::new(
@@ -11281,7 +11283,7 @@ mod tests {
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
         assert_eq!(index.format_version(), format_version);
-        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V4);
+        assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V2);
 
         let schema = Arc::new(Schema::new(vec![
             Field::new("doc", DataType::Utf8, true),
@@ -11295,11 +11297,11 @@ mod tests {
             .update(Box::pin(stream), dest_store.as_ref(), None)
             .await?;
 
-        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V4);
+        assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V2);
 
         let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
         assert_eq!(updated.format_version(), format_version);
-        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V4);
+        assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V2);
         assert_eq!(updated.partitions.len(), 2);
         for partition in &updated.partitions {
             assert_eq!(
@@ -11312,7 +11314,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_block_size_256_writes_v3_metadata_and_index_version() -> Result<()> {
+    async fn test_block_size_256_writes_v4_metadata_and_index_version() -> Result<()> {
         let src_dir = TempObjDir::default();
         let dest_dir = TempObjDir::default();
         let src_store = Arc::new(LanceIndexStore::new(
@@ -11328,7 +11330,7 @@ mod tests {
 
         let params = InvertedIndexParams::default().block_size(256)?;
         let format_version = params.resolved_format_version();
-        assert_eq!(format_version, InvertedListFormatVersion::V3);
+        assert_eq!(format_version, InvertedListFormatVersion::V4);
 
         let mut partition = InnerBuilder::new_with_format_version_and_block_size(
             0,
@@ -11351,7 +11353,7 @@ mod tests {
         write_test_metadata(&src_store, vec![0], params).await;
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(index.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(index.format_version(), InvertedListFormatVersion::V4);
         assert_eq!(index.index_version(), INVERTED_INDEX_VERSION_V4);
 
         let created = index
@@ -11360,7 +11362,7 @@ mod tests {
         assert_eq!(created.index_version, INVERTED_INDEX_VERSION_V4);
 
         let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
-        assert_eq!(updated.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(updated.format_version(), InvertedListFormatVersion::V4);
         assert_eq!(updated.index_version(), INVERTED_INDEX_VERSION_V4);
 
         Ok(())
@@ -11383,12 +11385,13 @@ mod tests {
 
         let index = write_single_partition_index(
             src_store,
-            InvertedIndexParams::default(),
+            InvertedIndexParams::default().format_version(InvertedListFormatVersion::V2),
             TokenSetFormat::Arrow,
             "hello",
             100,
         )
         .await?;
+        assert_eq!(index.index_version(), 0);
         let created = InvertedIndex::merge_segments(
             &[index],
             empty_doc_stream(),
@@ -11400,6 +11403,7 @@ mod tests {
 
         assert_eq!(created.index_version, 0);
         let merged = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(merged.index_version(), 0);
         assert_eq!(merged.token_set_format, TokenSetFormat::Arrow);
 
         let tokens = Arc::new(Tokens::new(vec!["hello".to_string()], DocType::Text));
@@ -11414,6 +11418,55 @@ mod tests {
         Ok(())
     }
 
+    #[rstest::rstest]
+    #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)]
+    #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)]
+    #[case::v3(InvertedListFormatVersion::V3, 256)]
+    #[case::v4_128(InvertedListFormatVersion::V4, LEGACY_BLOCK_SIZE)]
+    #[case::v4_256(InvertedListFormatVersion::V4, 256)]
+    #[tokio::test]
+    async fn test_merge_segments_preserves_format_version(
+        #[case] format_version: InvertedListFormatVersion,
+        #[case] block_size: usize,
+    ) -> Result<()> {
+        let src_dir = TempObjDir::default();
+        let dest_dir = TempObjDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)?
+            .format_version(format_version);
+
+        let index =
+            write_single_partition_index(src_store, params, TokenSetFormat::Fst, "hello", 100)
+                .await?;
+        assert_eq!(index.format_version(), format_version);
+
+        let created = InvertedIndex::merge_segments(
+            &[index],
+            empty_doc_stream(),
+            dest_store.as_ref(),
+            None,
+            crate::progress::noop_progress(),
+        )
+        .await?;
+        assert_eq!(created.index_version, format_version.index_version());
+
+        let merged = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(merged.format_version(), format_version);
+        assert_eq!(merged.index_version(), format_version.index_version());
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_merge_segments_uses_memory_limit_for_old_partitions() -> Result<()> {
         let src_dir_1 = TempObjDir::default();
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 55fa6b3576c..df7bcc1a28e 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -158,8 +158,7 @@ pub struct InvertedIndexParams {
     /// On-disk FTS format version to write when creating a new index.
     ///
     /// This is a build-time only parameter and is not persisted with the index.
-    /// If unset, Lance writes v2 for `block_size = 128` and v3 for
-    /// `block_size = 256`.
+    /// If unset, Lance writes v4 for either supported block size.
     /// `format_version = 3` is experimental and is only valid with
     /// `block_size = 256`.
     #[serde(
@@ -484,7 +483,7 @@ where
         serde_json::Value::Number(value) => {
             let Some(format_version) = value.as_u64() else {
                 return Err(serde::de::Error::custom(format!(
-                    "FTS format_version must be 1, 2, or 3, got {value}"
+                    "FTS format_version must be 1, 2, 3, or 4, got {value}"
                 )));
             };
             resolve_fts_format_version(Some(&format_version.to_string()))
@@ -492,7 +491,7 @@ where
                 .map_err(serde::de::Error::custom)
         }
         other => Err(serde::de::Error::custom(format!(
-            "FTS format_version must be 1, 2, or 3, got {other}"
+            "FTS format_version must be 1, 2, 3, or 4, got {other}"
         ))),
     }
 }
@@ -797,9 +796,9 @@ impl InvertedIndexParams {
 
     /// Set the on-disk FTS format version to use when creating a new index.
     ///
-    /// If unset, Lance writes v2 for `block_size = 128` and v3 for
-    /// `block_size = 256`. Existing indexes keep their own on-disk format
-    /// during update and optimize operations.
+    /// If unset, Lance writes v4 for either supported block size. Existing
+    /// indexes keep their own on-disk format during update and optimize
+    /// operations.
     /// `format_version = 3` is experimental and is only valid with
     /// `block_size = 256`.
     pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self {
@@ -1076,10 +1075,10 @@ mod tests {
     }
 
     #[test]
-    fn test_default_format_version_resolves_to_v2() {
+    fn test_default_format_version_resolves_to_v4() {
         assert_eq!(
             InvertedIndexParams::default().resolved_format_version(),
-            InvertedListFormatVersion::V2
+            InvertedListFormatVersion::V4
         );
     }
 
@@ -1130,13 +1129,13 @@ mod tests {
     }
 
     #[test]
-    fn test_block_size_256_defaults_to_v3() {
+    fn test_block_size_256_defaults_to_v4() {
         assert_eq!(
             InvertedIndexParams::default()
                 .block_size(256)
                 .unwrap()
                 .resolved_format_version(),
-            InvertedListFormatVersion::V3
+            InvertedListFormatVersion::V4
         );
     }
 
@@ -1280,6 +1279,16 @@ mod tests {
             .unwrap()
             .validate_format_version()
             .unwrap();
+        InvertedIndexParams::default()
+            .format_version(InvertedListFormatVersion::V4)
+            .validate_format_version()
+            .unwrap();
+        InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .format_version(InvertedListFormatVersion::V4)
+            .validate_format_version()
+            .unwrap();
 
         let err = InvertedIndexParams::default()
             .block_size(256)
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index c86035c597d..e67e73d92a6 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -209,8 +209,9 @@ impl MemIndexConfig {
             0 | 1 => Ok(InvertedListFormatVersion::V1),
             2 => Ok(InvertedListFormatVersion::V2),
             3 => Ok(InvertedListFormatVersion::V3),
+            4 => Ok(InvertedListFormatVersion::V4),
             version => Err(Error::invalid_input(format!(
-                "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3",
+                "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, 3, or 4",
                 index_meta.name, version
             ))),
         }
@@ -1384,6 +1385,7 @@ mod tests {
             (0, InvertedListFormatVersion::V1),
             (1, InvertedListFormatVersion::V1),
             (2, InvertedListFormatVersion::V2),
+            (4, InvertedListFormatVersion::V4),
         ] {
             let config =
                 MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema)
@@ -1406,9 +1408,9 @@ mod tests {
         let arrow_schema = create_test_schema();
         let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
 
-        let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err();
+        let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(5), &schema).unwrap_err();
         assert!(
-            err.to_string().contains("unsupported index_version 4"),
+            err.to_string().contains("unsupported index_version 5"),
             "{err}"
         );
     }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index bcb4226a8f5..ac3e6c81602 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -13,7 +13,6 @@ use lance_core::utils::deletion::DeletionVector;
 use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_index::mem_wal::{FlushedGeneration, ShardManifest};
-use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V4;
 use lance_index::scalar::{IndexStore, ScalarIndexParams};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_table::format::IndexMetadata;
@@ -777,6 +776,7 @@ impl MemTableFlusher {
             })?;
 
             let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone();
+            let format_version = fts_cfg.params.resolved_format_version();
 
             let index_meta = IndexMetadata {
                 uuid: index_uuid,
@@ -785,7 +785,7 @@ impl MemTableFlusher {
                 dataset_version: dataset.version().version,
                 fragment_bitmap: Some(fragment_ids),
                 index_details: Some(Arc::new(index_details)),
-                index_version: INVERTED_INDEX_VERSION_V4 as i32,
+                index_version: format_version.index_version() as i32,
                 created_at: None,
                 base_id: None,
                 files: None,