feat: add code analyzer for FTS

docs/src/guide/migration.md

@@ -8,15 +8,15 @@ migrate.
 
 ## 9.0.0
 
-* Newly created FTS / inverted indexes now default to format v2 instead of v1.
+* Newly created FTS / inverted indexes now default to format v4 instead of v1.
   The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the
-  format used for newly created indexes. Users who need a specific index layout
-  should pass the index creation parameter `format_version` explicitly.
+  format used for newly created indexes. Users who need a specific older index
+  layout should pass the index creation parameter `format_version` explicitly.
 
 * This affects users who create FTS / inverted indexes and need those indexes to
   be readable by older Lance versions, or who depend on the v1 index layout. In
   those cases, pass `format_version=1` when creating the index. Otherwise, newly
-  created indexes will use v2 by default, and older Lance readers may not be able
+  created indexes will use v4 by default, and older Lance readers may not be able
   to read them.
 
   ```python

java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java

@@ -264,16 +264,16 @@ public Builder skipMerge(boolean skipMerge) {
     /**
      * Configure the on-disk FTS format version to write when creating a new index.
      *
-     * <p>If unset, Lance writes v2 for {@code blockSize = 128} and v3 for {@code blockSize = 256}.
-     * {@code formatVersion = 3} is experimental and is only valid with {@code blockSize = 256}.
+     * <p>If unset, Lance writes v4 for either supported block size. {@code formatVersion = 3} is
+     * experimental and is only valid with {@code blockSize = 256}.
      *
-     * @param formatVersion FTS format version, must be 1, 2, or 3
+     * @param formatVersion FTS format version, must be 1, 2, 3, or 4
      * @return this builder
      * @throws IllegalArgumentException
      */
     public Builder formatVersion(int formatVersion) {
-      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) {
-        throw new IllegalArgumentException("formatVersion must be 1, 2, or 3");
+      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3 && formatVersion != 4) {
+        throw new IllegalArgumentException("formatVersion must be 1, 2, 3, or 4");
       }
       this.formatVersion = formatVersion;
       return this;
@@ -283,8 +283,10 @@ public Builder formatVersion(int formatVersion) {
     public ScalarIndexParams build() {
       if (formatVersion != null) {
         Preconditions.checkArgument(
-            (blockSize == 256 && formatVersion == 3) || (blockSize == 128 && formatVersion != 3),
-            "formatVersion 3 requires blockSize 256, and blockSize 256 requires formatVersion 3");
+            formatVersion == 4
+                || (blockSize == 256 && formatVersion == 3)
+                || (blockSize == 128 && formatVersion != 3),
+            "formatVersion 3 requires blockSize 256, and legacy formats require blockSize 128");
       }
       Map<String, Object> params = new HashMap<>();
       if (baseTokenizer != null) {

java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java

@@ -75,4 +75,15 @@ void formatVersionThreeRequiresBlockSize256() {
         IllegalArgumentException.class,
         () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build());
   }
+
+  @Test
+  void formatVersionFourSupportsBothBlockSizes() {
+    for (int blockSize : new int[] {128, 256}) {
+      ScalarIndexParams params =
+          InvertedIndexParams.builder().blockSize(blockSize).formatVersion(4).build();
+      Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+      assertEquals(blockSize, ((Number) json.get("block_size")).intValue());
+      assertEquals(4, ((Number) json.get("format_version")).intValue());
+    }
+  }
 }

protos/index_old.proto

@@ -26,6 +26,27 @@ message LabelListIndexDetails {}
 message NGramIndexDetails {}
 message ZoneMapIndexDetails {}
 message InvertedIndexDetails {
+  message CodeTokenizerConfig {
+    // Split one lexical identifier into subwords, e.g. getUserName ->
+    // get/user/name.
+    bool split_identifiers = 1;
+    // Split identifier subwords across letter/number boundaries, e.g.
+    // HTML2JSON -> html/2/json. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool split_on_numerics = 2;
+    // Keep the complete lexical identifier in addition to subwords, e.g.
+    // user_name plus user/name. An absent value uses the code tokenizer default;
+    // a present value records the explicit index-time choice.
+    optional bool preserve_original = 3;
+    // Index operator tokens such as "::", "->", and "!=". Operators are not
+    // indexed by default because they are often high-frequency noise.
+    bool index_operators = 4;
+  }
+
+  // Lexical tokenizer used after document-level text extraction. This is an
+  // implementation component such as "simple", "icu", "ngram", or "code".
+  // Input-time analyzer profiles are expanded into this field and the concrete
+  // options below before these details are persisted.
   // Marking this field as optional as old versions of the index store blank details and we
   // need to make sure we have a proper optional field to detect this.
   optional string base_tokenizer = 1;
@@ -41,7 +62,11 @@ message InvertedIndexDetails {
   bool prefix_only = 11;
   // Number of documents per compressed posting block. An absent value means
   // the index predates this field and must use the legacy block size of 128.
-  // A present value records the block size used by the index; 256 is only
-  // valid with format version 3.
+  // A present value records the block size used by the index; 256 is valid
+  // with format versions 3 and 4.
   optional uint32 block_size = 12;
+  // Options for base_tokenizer = "code". Presence records the code tokenizer
+  // configuration used to build the index; absence means there is no
+  // code-specific configuration to apply.
+  CodeTokenizerConfig code_config = 13;
 }

python/python/lance/dataset.py

@@ -3370,8 +3370,8 @@ def create_scalar_index(
         format_version: int or str, optional
             This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS
             format version to write when creating a new index. Accepts ``1``,
-            ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. If unset, Lance
-            writes v2 for ``block_size=128`` and v3 for ``block_size=256``.
+            ``2``, ``3``, ``4``, ``"v1"``, ``"v2"``, ``"v3"``, or ``"v4"``.
+            If unset, Lance writes v4 for either supported block size.
             ``format_version=3`` is experimental and is only valid with
             ``block_size=256``.