feat(lance-linalg): runtime SIMD dispatch for pre-Haswell x86_64 from-source builds

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "9.0.0-beta.8"
+current_version = "9.0.0-beta.21"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",

.coderabbit.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: en-US
+early_access: false
+reviews:
+  profile: assertive
+  poem: false
+  review_status: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - WIP
+      - Draft
+    drafts: false
+    base_branches:
+      - main

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,79 @@
+name: Bug report
+description: Report incorrect, unexpected, or crashing behavior.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to file a bug report! Please search
+        [existing issues](https://github.com/lance-format/lance/issues) first to
+        avoid filing a duplicate.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: >-
+        A minimal, self-contained code snippet or sequence of steps that
+        reproduces the problem. The more we can copy-paste and run, the faster
+        we can fix it.
+      placeholder: |
+        1. Write a dataset with `...`
+        2. Scan with filter `...`
+        3. Observe `...`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: false
+  - type: input
+    id: lance-version
+    attributes:
+      label: Lance version
+      description: >-
+        Output of `python -c "import lance; print(lance.__version__)"`, or the
+        `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`.
+      placeholder: "e.g. 0.40.0"
+    validations:
+      required: true
+  - type: dropdown
+    id: language
+    attributes:
+      label: Language binding
+      multiple: true
+      options:
+        - Python
+        - Rust
+        - Java
+        - Other / not sure
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...).
+      placeholder: "e.g. Ubuntu 22.04, x86_64, S3"
+    validations:
+      required: false
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs / traceback
+      description: >-
+        Any relevant log output or stack trace. This will be automatically
+        formatted as code, so no need for backticks.
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question / usage help
+    url: https://discord.gg/lance
+    about: >-
+      For questions and general discussion, please ask in the Lance Discord
+      rather than opening an issue.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest a new capability or an improvement to an existing one.
+labels: feature
+---
+
+<!--
+Describe what you'd like Lance to do and why. There's no required format —
+mention the problem or use case, and a rough idea of the solution if you have
+one. Please search existing issues first to avoid duplicates.
+-->

.github/ISSUE_TEMPLATE/performance_issue.md

@@ -0,0 +1,12 @@
+---
+name: Performance issue
+about: Report slow operations, high memory use, or a performance regression.
+labels: performance
+---
+
+<!--
+Describe what's slow or memory-hungry. There's no required format, but concrete
+numbers (timings, memory, dataset shape) and a way to reproduce help a lot. If
+it's a regression, note the last version that performed well. Please search
+existing issues first to avoid duplicates.
+-->

.github/dependabot.yml

@@ -6,25 +6,40 @@ updates:
     schedule:
       interval: "weekly"
       day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
 
   - package-ecosystem: "cargo"
     directory: "/python"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
 
   - package-ecosystem: "cargo"
     directory: "/java/lance-jni"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
 
   - package-ecosystem: "uv"
     directory: "/python"
     versioning-strategy: lockfile-only
     schedule:
       interval: "weekly"
       day: "wednesday"
-
+    groups:
+      uv:
+        patterns:
+          - "*"

.github/labeler-issues.yml

@@ -0,0 +1,28 @@
+version: 1
+# Never remove labels a template or a human already applied.
+appendOnly: true
+# Content-based labels for issues, applied by srvaroa/labeler via
+# .github/workflows/issue-labeler.yml. This covers issues that bypass the
+# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`,
+# the REST API, or an agent, which never see the web-UI template chooser.
+#
+# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]",
+# "perf -"), which many people type by hand; a modest body keyword match is a
+# secondary fallback. Rules are OR'd (one label per matching entry), so an
+# unmatched issue gets no label and is left for human / LLM triage rather than
+# guessed at. appendOnly means matches stack harmlessly with template labels.
+labels:
+# --- Primary signal: a leading bug/feature/perf marker in the title ---
+- label: bug
+  title: "(?i)^\\s*[\\[(]?\\s*bug\\b"
+- label: feature
+  title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b"
+- label: performance
+  title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b"
+# --- Secondary fallback: keywords anywhere in the body ---
+- label: bug
+  body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)"
+- label: performance
+  body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)"
+- label: feature
+  body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)"

.github/workflows/issue-labeler.yml

@@ -0,0 +1,29 @@
+name: Issue Labeler
+
+# Applies bug / feature / performance labels to issues based on their title and
+# body. This complements the .github/ISSUE_TEMPLATE forms, which only apply
+# labels for issues opened through the web UI; this catches issues opened via
+# `gh issue create`, the REST API, or an agent, which bypass templates.
+#
+# Issues never originate from a fork, so unlike the PR labelers this uses a
+# plain `issues` trigger (no pull_request_target) with a scoped GITHUB_TOKEN.
+on:
+  issues:
+    types: [opened, edited]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+
+jobs:
+  label:
+    name: Apply issue labels
+    permissions:
+      issues: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          config_path: .github/labeler-issues.yml

.github/workflows/java-publish.yml

@@ -112,7 +112,7 @@ jobs:
             export CXX=clang++
             ldd --version
 
-            cargo build --release
+            cargo build --release --locked
           "
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
@@ -135,7 +135,7 @@ jobs:
         run: brew install protobuf
       - name: Build native lib
         working-directory: java/lance-jni
-        run: cargo build --release
+        run: cargo build --release --locked
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: liblance_jni_darwin_aarch64.zip

.github/workflows/java.yml

@@ -47,14 +47,14 @@ jobs:
         run: cargo fmt --check
       - name: Rust Clippy
         working-directory: java/lance-jni
-        run: cargo clippy --all-targets -- -D warnings
+        run: cargo clippy --all-targets --locked -- -D warnings
 
   build-and-test-java:
     runs-on: ubuntu-24.04-4x
     timeout-minutes: 60
     strategy:
       matrix:
-        java-version: [11, 17, 21]
+        java-version: [11, 17, 21, 25]
     name: Build and Test with Java ${{ matrix.java-version }}
     steps:
       - name: Checkout repository
@@ -79,7 +79,13 @@ jobs:
           distribution: temurin
           java-version: ${{ matrix.java-version }}
           cache: "maven"
+      # Skip the format gate on Java 25: google-java-format 1.22.0 (pinned via
+      # spotless) reaches into com.sun.tools.javac internals and throws
+      # NoSuchMethodError on JDK 25's javac. No google-java-format release runs
+      # on JDK 25 without also dropping JDK 11/17 support, so we exclude 25 from
+      # the (JDK-independent, already-enforced-on-11/17/21) style check instead.
       - name: Running code style check with Java ${{ matrix.java-version }}
+        if: matrix.java-version != '25'
         working-directory: java
         run: |
           mvn spotless:check
@@ -91,4 +97,7 @@ jobs:
         env:
           LANCE_INTEGRATION_TEST: "1"
         run: |
-          mvn install
+          # spotless is also bound into the build lifecycle, so on Java 25 the
+          # format gate must be skipped here too (-Dspotless.skip=true), not just
+          # in the standalone step above.
+          mvn install ${{ matrix.java-version == '25' && '-Dspotless.skip=true' || '' }}

.github/workflows/license-header-check.yml

@@ -22,11 +22,25 @@ jobs:
       - name: Check out code
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install license-header-checker
+        env:
+          # Pin to a specific release and download the asset directly from the
+          # release CDN. This avoids the flaky dynamic "latest" tag lookup in the
+          # upstream install.sh (it fetches the releases page and parses tag_name,
+          # which intermittently returns empty and fails the job).
+          LHC_VERSION: "1.5.0"
         run: |
           set -euo pipefail
-          curl -sfSL https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh -o /tmp/install-lhc.sh
-          bash /tmp/install-lhc.sh
-          rm -f /tmp/install-lhc.sh
+          mkdir -p ./bin
+          base="https://github.com/lluissm/license-header-checker/releases/download/v${LHC_VERSION}"
+          asset="license-header-checker_${LHC_VERSION}_linux_amd64.tar.gz"
+          curl -sfSL "${base}/${asset}" -o "/tmp/${asset}"
+          curl -sfSL "${base}/checksums.txt" -o /tmp/lhc-checksums.txt
+          # Verify the download against the release checksums before extracting so a
+          # corrupted or tampered asset cannot be executed. This restores the
+          # integrity check the upstream install.sh performed via hash_sha256_verify.
+          (cd /tmp && sha256sum -c --ignore-missing lhc-checksums.txt)
+          tar -xzf "/tmp/${asset}" -C ./bin license-header-checker
+          rm -f "/tmp/${asset}" /tmp/lhc-checksums.txt
       - name: Check license headers (rust)
         run: ./bin/license-header-checker -a -v ./rust/license_header.txt rust rs && [[ -z `git status -s` ]]
       - name: Check license headers (python)

.github/workflows/rust.yml

@@ -253,6 +253,42 @@ jobs:
       - name: Check benchmarks
         run: cargo check --profile ci --benches
 
+  qemu-pre-haswell:
+    # Verifies that lance-linalg's runtime SIMD dispatch still works
+    # correctly when the binary is built with the lower x86-64-v2 baseline
+    # (the legacy build path documented in CONTRIBUTING.md). Emulates a
+    # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA
+    # instructions that leak past the runtime dispatch.
+    #
+    # The published-wheel default baseline (`target-cpu=haswell`) is set in
+    # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to
+    # exercise the legacy path without affecting any other build.
+    name: pre-Haswell SIGILL check (qemu Nehalem)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      CC: clang
+      CXX: clang++
+      RUSTFLAGS: "-C target-cpu=x86-64-v2"
+      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem"
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Setup rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev qemu-user
+      - name: Build lance-linalg lib tests in release mode
+        run: |
+          cargo test --release -p lance-linalg --lib --no-run
+      - name: Run lance-linalg lib tests under qemu Nehalem
+        run: |
+          cargo test --release -p lance-linalg --lib
+
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}

.pre-commit-config.yaml

@@ -15,3 +15,16 @@ repos:
     rev: v1.26.0
     hooks:
     - id: typos
+
+  # Catch a Cargo.toml change that wasn't reflected in the matching Cargo.lock.
+  # Runs fully offline (`--frozen` = `--locked` + `--offline`) and does not compile,
+  # so it stays fast. Checks all three lockfiles since a workspace dep change can
+  # touch the excluded python/ and java/ locks too.
+  - repo: local
+    hooks:
+      - id: cargo-lock-sync
+        name: Cargo.lock in sync with Cargo.toml (offline)
+        entry: bash -c 'for m in Cargo.toml python/Cargo.toml java/lance-jni/Cargo.toml; do cargo metadata --frozen --format-version 1 --manifest-path "$m" >/dev/null 2>&1 || { echo "Cargo.lock is out of date for $m. Refresh it (e.g. cargo check --manifest-path $m) and commit the updated lockfile."; exit 1; }; done'
+        language: system
+        files: '(^|/)Cargo\.(toml|lock)$'
+        pass_filenames: false