feat(lance-linalg): runtime SIMD dispatch for pre-Haswell x86_64 from-source builds

.github/workflows/rust.yml

@@ -253,6 +253,42 @@ jobs:
       - name: Check benchmarks
         run: cargo check --profile ci --benches
 
+  qemu-pre-haswell:
+    # Verifies that lance-linalg's runtime SIMD dispatch still works
+    # correctly when the binary is built with the lower x86-64-v2 baseline
+    # (the legacy build path documented in CONTRIBUTING.md). Emulates a
+    # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA
+    # instructions that leak past the runtime dispatch.
+    #
+    # The published-wheel default baseline (`target-cpu=haswell`) is set in
+    # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to
+    # exercise the legacy path without affecting any other build.
+    name: pre-Haswell SIGILL check (qemu Nehalem)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      CC: clang
+      CXX: clang++
+      RUSTFLAGS: "-C target-cpu=x86-64-v2"
+      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem"
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Setup rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev qemu-user
+      - name: Build lance-linalg lib tests in release mode
+        run: |
+          cargo test --release -p lance-linalg --lib --no-run
+      - name: Run lance-linalg lib tests under qemu Nehalem
+        run: |
+          cargo test --release -p lance-linalg --lib
+
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}

CONTRIBUTING.md

@@ -25,6 +25,16 @@ Currently Lance is implemented in Rust and comes with a Python wrapper. So you'l
     a. Install pre-commit: https://pre-commit.com/#install
     b. Run `pre-commit install` in the root of the repo
 
+## Building for legacy x86_64 hosts (pre-Haswell)
+
+The default workspace build targets `haswell` (AVX2 + FMA + F16C), matching the published wheels. To build a binary that runs on pre-Haswell silicon (Sandy Bridge / Ivy Bridge / Westmere on Intel, Bulldozer / Piledriver / Steamroller on AMD — i.e. CPUs without AVX2), set the baseline yourself at build time:
+
+```sh
+RUSTFLAGS="-C target-cpu=x86-64-v2" cargo build --release
+```
+
+Runtime SIMD dispatch in `lance-linalg::distance` will then pick the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) based on the host. From Python, use `lance.simd_info()` to verify which tier was selected.
+
 ## Sample Workflow
 
 1. Fork the repo

python/python/lance/__init__.py

@@ -47,6 +47,7 @@
     ScanStatistics,
     bytes_read_counter,
     iops_counter,
+    simd_info,
 )
 from .mem_wal import (
     ExecutionPlan,
@@ -115,6 +116,7 @@
     "json_to_schema",
     "schema_to_json",
     "set_logger",
+    "simd_info",
     "write_dataset",
     "FFILanceTableProvider",
     "IndexProgress",

python/python/tests/test_lance.py

@@ -248,6 +248,28 @@ def test_io_counters(tmp_path):
     assert lance.bytes_read_counter() > starting_bytes
 
 
+def test_simd_info():
+    info = lance.simd_info()
+    assert info["tier"] in (
+        "none",
+        "sse",
+        "avx",
+        "avx_fma",
+        "avx2",
+        "avx512",
+        "avx512_fp16",
+        "neon",
+        "lsx",
+        "lasx",
+    )
+    assert isinstance(info["target_arch"], str) and info["target_arch"]
+    if info["target_arch"] == "x86_64":
+        # The x86_64 ABI mandates SSE2.
+        assert "sse2" in info["host_features"]
+    else:
+        assert info["host_features"] == []
+
+
 @pytest.mark.parametrize(
     "row_param, column_name",
     [("with_row_id", "_rowid"), ("with_row_address", "_rowaddr")],

python/src/lib.rs

@@ -329,6 +329,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(language_model_home))?;
     m.add_wrapped(wrap_pyfunction!(bytes_read_counter))?;
     m.add_wrapped(wrap_pyfunction!(iops_counter))?;
+    m.add_wrapped(wrap_pyfunction!(simd_info))?;
     m.add_wrapped(wrap_pyfunction!(stable_version))?;
     // Debug functions
     m.add_wrapped(wrap_pyfunction!(debug::format_schema))?;
@@ -347,6 +348,37 @@ fn iops_counter() -> PyResult<u64> {
     Ok(::lance::io::iops_counter())
 }
 
+/// Returns a dict describing which SIMD tier the lance runtime dispatches to
+/// on this host, plus the raw CPU feature flags it detected.
+///
+/// Mirrors `pyarrow.runtime_info()`: a cheap, transparent way to verify that
+/// the host is hitting the expected SIMD tier (e.g., `"avx512_fp16"`,
+/// `"avx2"`) when debugging vector-search performance.
+///
+/// Returns:
+///   {
+///     "tier": str,                      # e.g. "avx2", "avx_fma", "neon", "none"
+///     "target_arch": str,               # e.g. "x86_64", "aarch64", "loongarch64"
+///     "host_features": list[str],       # raw CPU feature flags (x86_64 only)
+///   }
+///
+/// Examples:
+///   >>> import lance
+///   >>> info = lance.simd_info()
+///   >>> sorted(info)
+///   ['host_features', 'target_arch', 'tier']
+///   >>> isinstance(info["tier"], str)
+///   True
+#[pyfunction]
+pub fn simd_info(py: Python<'_>) -> PyResult<Py<PyAny>> {
+    let info = lance_core::utils::cpu::simd_info();
+    let dict = pyo3::types::PyDict::new(py);
+    dict.set_item("tier", info.tier.to_string())?;
+    dict.set_item("target_arch", info.target_arch)?;
+    dict.set_item("host_features", info.host_features)?;
+    Ok(dict.into())
+}
+
 #[pyfunction(name = "bytes_read_counter")]
 fn bytes_read_counter() -> PyResult<u64> {
     Ok(::lance::io::bytes_read_counter())