feat(svd): implement svd in rust without using openblas as a dependency

benchmarks/full_report/report.ipynb

@@ -8,11 +8,12 @@
    "outputs": [],
    "source": [
     "import sys\n",
+    "\n",
     "sys.dont_write_bytecode = True\n",
     "\n",
     "import os\n",
     "\n",
-    "module_path = os.path.abspath(os.path.join('.'))\n",
+    "module_path = os.path.abspath(os.path.join(\".\"))\n",
     "if module_path not in sys.path:\n",
     "    sys.path.append(module_path)"
    ]
@@ -61,58 +62,91 @@
     "    gt_in_sample = knn(in_sample, data, metric, 10)\n",
     "\n",
     "    print(\"generated gt\")\n",
-    "    \n",
+    "\n",
     "    with tempfile.TemporaryDirectory() as d:\n",
     "        write_lance(d, data)\n",
     "        ds = lance.dataset(d)\n",
     "\n",
-    "        for q, target in zip(tqdm(in_sample, desc=\"checking brute force\"), gt_in_sample):\n",
-    "            res = ds.to_table(nearest={\n",
-    "                \"column\": \"vec\",\n",
-    "                \"q\": q,\n",
-    "                \"k\": 10,\n",
-    "                \"metric\": metric,\n",
-    "            }, columns=[\"id\"])\n",
+    "        for q, target in zip(\n",
+    "            tqdm(in_sample, desc=\"checking brute force\"), gt_in_sample\n",
+    "        ):\n",
+    "            res = ds.to_table(\n",
+    "                nearest={\n",
+    "                    \"column\": \"vec\",\n",
+    "                    \"q\": q,\n",
+    "                    \"k\": 10,\n",
+    "                    \"metric\": metric,\n",
+    "                },\n",
+    "                columns=[\"id\"],\n",
+    "            )\n",
     "            assert len(np.intersect1d(res[\"id\"].to_numpy(), target)) == 10\n",
-    "    \n",
-    "        ds = ds.create_index(\"vec\", \"IVF_PQ\", metric=metric, num_partitions=num_partitions, num_sub_vectors=num_sub_vectors)\n",
-    "    \n",
+    "\n",
+    "        ds = ds.create_index(\n",
+    "            \"vec\",\n",
+    "            \"IVF_PQ\",\n",
+    "            metric=metric,\n",
+    "            num_partitions=num_partitions,\n",
+    "            num_sub_vectors=num_sub_vectors,\n",
+    "        )\n",
+    "\n",
     "        recall_data = []\n",
     "        for nprobes in nprobes_list:\n",
     "            for refine_factor in refine_factor_list:\n",
     "                hits = 0\n",
     "                # check that brute force impl is correct\n",
-    "                for q, target in zip(tqdm(query, desc=f\"out of sample, nprobes={nprobes}, refine={refine_factor}\"), gt):\n",
-    "                    res = ds.to_table(nearest={\n",
-    "                        \"column\": \"vec\",\n",
-    "                        \"q\": q,\n",
-    "                        \"k\": 10,\n",
-    "                        \"nprobes\": nprobes,\n",
-    "                        \"refine_factor\": refine_factor,\n",
-    "                    }, columns=[\"id\"])[\"id\"].to_numpy()\n",
+    "                for q, target in zip(\n",
+    "                    tqdm(\n",
+    "                        query,\n",
+    "                        desc=f\"out of sample, nprobes={nprobes}, refine={refine_factor}\",\n",
+    "                    ),\n",
+    "                    gt,\n",
+    "                ):\n",
+    "                    res = ds.to_table(\n",
+    "                        nearest={\n",
+    "                            \"column\": \"vec\",\n",
+    "                            \"q\": q,\n",
+    "                            \"k\": 10,\n",
+    "                            \"nprobes\": nprobes,\n",
+    "                            \"refine_factor\": refine_factor,\n",
+    "                        },\n",
+    "                        columns=[\"id\"],\n",
+    "                    )[\"id\"].to_numpy()\n",
     "                    hits += len(np.intersect1d(res, target))\n",
-    "                recall_data.append([\n",
-    "                    \"out_of_sample\",\n",
-    "                    nprobes,\n",
-    "                    refine_factor,\n",
-    "                    hits / 10 / len(gt),\n",
-    "                ])\n",
+    "                recall_data.append(\n",
+    "                    [\n",
+    "                        \"out_of_sample\",\n",
+    "                        nprobes,\n",
+    "                        refine_factor,\n",
+    "                        hits / 10 / len(gt),\n",
+    "                    ]\n",
+    "                )\n",
     "                # check that brute force impl is correct\n",
-    "                for q, target in zip(tqdm(in_sample, desc=f\"in sample nprobes={nprobes}, refine={refine_factor}\"), gt_in_sample):\n",
-    "                    res = ds.to_table(nearest={\n",
-    "                        \"column\": \"vec\",\n",
-    "                        \"q\": q,\n",
-    "                        \"k\": 10,\n",
-    "                        \"nprobes\": nprobes,\n",
-    "                        \"refine_factor\": refine_factor,\n",
-    "                    }, columns=[\"id\"])[\"id\"].to_numpy()\n",
+    "                for q, target in zip(\n",
+    "                    tqdm(\n",
+    "                        in_sample,\n",
+    "                        desc=f\"in sample nprobes={nprobes}, refine={refine_factor}\",\n",
+    "                    ),\n",
+    "                    gt_in_sample,\n",
+    "                ):\n",
+    "                    res = ds.to_table(\n",
+    "                        nearest={\n",
+    "                            \"column\": \"vec\",\n",
+    "                            \"q\": q,\n",
+    "                            \"k\": 10,\n",
+    "                            \"nprobes\": nprobes,\n",
+    "                            \"refine_factor\": refine_factor,\n",
+    "                        },\n",
+    "                        columns=[\"id\"],\n",
+    "                    )[\"id\"].to_numpy()\n",
     "                    hits += len(np.intersect1d(res, target))\n",
-    "                recall_data.append([\n",
-    "                    \"in_sample\",\n",
-    "                    nprobes,\n",
-    "                    refine_factor,\n",
-    "                    hits / 10 / len(gt_in_sample),\n",
-    "                ])\n",
+    "                recall_data.append(\n",
+    "                    [\n",
+    "                        \"in_sample\",\n",
+    "                        nprobes,\n",
+    "                        refine_factor,\n",
+    "                        hits / 10 / len(gt_in_sample),\n",
+    "                    ]\n",
+    "                )\n",
     "    return recall_data"
    ]
   },
@@ -124,15 +158,19 @@
    "outputs": [],
    "source": [
     "def make_plot(recall_data):\n",
-    "    df = pd.DataFrame(recall_data, columns=[\"case\", \"nprobes\", \"refine_factor\", \"recall\"])\n",
-    "    \n",
+    "    df = pd.DataFrame(\n",
+    "        recall_data, columns=[\"case\", \"nprobes\", \"refine_factor\", \"recall\"]\n",
+    "    )\n",
+    "\n",
     "    num_cases = len(df[\"case\"].unique())\n",
     "    (fig, axs) = plt.subplots(1, 2, figsize=(16, 8))\n",
-    "    \n",
+    "\n",
     "    for case, ax in zip(df[\"case\"].unique(), axs):\n",
     "        current_case = df[df[\"case\"] == case]\n",
     "        sns.heatmap(\n",
-    "            current_case.drop(columns=[\"case\"]).set_index([\"nprobes\", \"refine_factor\"])[\"recall\"].unstack(),\n",
+    "            current_case.drop(columns=[\"case\"])\n",
+    "            .set_index([\"nprobes\", \"refine_factor\"])[\"recall\"]\n",
+    "            .unstack(),\n",
     "            annot=True,\n",
     "            ax=ax,\n",
     "        ).set(title=f\"Recall -- {case}\")"

benchmarks/sift/Results.ipynb

@@ -35,6 +35,7 @@
    "outputs": [],
    "source": [
     "import pandas as pd\n",
+    "\n",
     "df = pd.read_csv(\"query.csv\")"
    ]
   },
@@ -46,6 +47,7 @@
    "outputs": [],
    "source": [
     "import seaborn as sns\n",
+    "\n",
     "sns.set_style(\"darkgrid\")"
    ]
   },

ci/check_breaking_changes.py

@@ -4,6 +4,7 @@
 
 Can also be used as a library to detect breaking changes without version validation.
 """
+
 import argparse
 import os
 import sys
@@ -40,10 +41,17 @@ def detect_breaking_changes(repo, base, head):
     parser = argparse.ArgumentParser()
     parser.add_argument("base", help="Base commit/tag for comparison")
     parser.add_argument("head", help="Head commit/tag for comparison")
-    parser.add_argument("last_stable_version", nargs="?", help="Last stable version (for validation)")
-    parser.add_argument("current_version", nargs="?", help="Current version (for validation)")
-    parser.add_argument("--detect-only", action="store_true",
-                        help="Only detect breaking changes, don't validate version")
+    parser.add_argument(
+        "last_stable_version", nargs="?", help="Last stable version (for validation)"
+    )
+    parser.add_argument(
+        "current_version", nargs="?", help="Current version (for validation)"
+    )
+    parser.add_argument(
+        "--detect-only",
+        action="store_true",
+        help="Only detect breaking changes, don't validate version",
+    )
     args = parser.parse_args()
 
     repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"])

ci/setup_version.py

@@ -3,7 +3,7 @@
 This script is used to set the pre-release version for beta releases
 (e.g. 0.10.17-beta.1) when the tag indicates a beta release.
 
-With the new automated release process, stable versions are already 
+With the new automated release process, stable versions are already
 updated by bump-my-version during the release workflow.
 """
 
@@ -38,9 +38,9 @@ def main():
         print(f"Setting beta version: {current_version} -> {args.version[1:]}")
     else:
         # For stable releases, version should already match
-        assert (
-            parsed_version.release == current_version_parsed.release
-        ), f"Version mismatch for stable release: {parsed_version.release} != {current_version_parsed.release}"
+        assert parsed_version.release == current_version_parsed.release, (
+            f"Version mismatch for stable release: {parsed_version.release} != {current_version_parsed.release}"
+        )
 
     with open("python/Cargo.toml", "w") as f:
         f.writelines(lines)

memtest/tests/integration_test.rs

@@ -2,7 +2,7 @@ use libc::{c_void, size_t};
 use std::ptr;
 
 // Import from the library we're testing
-use memtest::{memtest_get_stats, memtest_reset_stats, MemtestStats};
+use memtest::{MemtestStats, memtest_get_stats, memtest_reset_stats};
 
 extern "C" {
     fn malloc(size: size_t) -> *mut c_void;

notebooks/quickstart.ipynb

@@ -221,7 +221,7 @@
     "shutil.rmtree(\"/tmp/test.lance\", ignore_errors=True)\n",
     "\n",
     "tbl = pa.Table.from_pandas(df)\n",
-    "pa.dataset.write_dataset(tbl, \"/tmp/test.parquet\", format='parquet')\n",
+    "pa.dataset.write_dataset(tbl, \"/tmp/test.parquet\", format=\"parquet\")\n",
     "\n",
     "parquet = pa.dataset.dataset(\"/tmp/test.parquet\")\n",
     "parquet.to_table().to_pandas()"
@@ -542,7 +542,7 @@
     }
    ],
    "source": [
-    "lance.dataset('/tmp/test.lance', version=1).to_table().to_pandas()"
+    "lance.dataset(\"/tmp/test.lance\", version=1).to_table().to_pandas()"
    ]
   },
   {
@@ -600,7 +600,7 @@
     }
    ],
    "source": [
-    "lance.dataset('/tmp/test.lance', version=2).to_table().to_pandas()"
+    "lance.dataset(\"/tmp/test.lance\", version=2).to_table().to_pandas()"
    ]
   },
   {
@@ -698,7 +698,7 @@
     }
    ],
    "source": [
-    "lance.dataset('/tmp/test.lance', version=\"stable\").to_table().to_pandas()"
+    "lance.dataset(\"/tmp/test.lance\", version=\"stable\").to_table().to_pandas()"
    ]
   },
   {
@@ -796,11 +796,13 @@
     "\n",
     "with open(\"sift/sift_base.fvecs\", mode=\"rb\") as fobj:\n",
     "    buf = fobj.read()\n",
-    "    data = np.array(struct.unpack(\"<128000000f\", buf[4 : 4 + 4 * 1000000 * 128])).reshape((1000000, 128))\n",
+    "    data = np.array(\n",
+    "        struct.unpack(\"<128000000f\", buf[4 : 4 + 4 * 1000000 * 128])\n",
+    "    ).reshape((1000000, 128))\n",
     "    dd = dict(zip(range(1000000), data))\n",
     "\n",
     "table = vec_to_table(dd)\n",
-    "lance.write_dataset(table, uri, max_rows_per_group=8192, max_rows_per_file=1024*1024)"
+    "lance.write_dataset(table, uri, max_rows_per_group=8192, max_rows_per_file=1024 * 1024)"
    ]
   },
   {
@@ -864,6 +866,7 @@
    ],
    "source": [
     "import duckdb\n",
+    "\n",
     "# if this segfaults make sure duckdb v0.7+ is installed\n",
     "samples = duckdb.query(\"SELECT vector FROM sift1m USING SAMPLE 100\").to_df().vector\n",
     "samples"
@@ -910,10 +913,12 @@
     "import time\n",
     "\n",
     "start = time.time()\n",
-    "tbl = sift1m.to_table(columns=[\"id\"], nearest={\"column\": \"vector\", \"q\": samples[0], \"k\": 10})\n",
+    "tbl = sift1m.to_table(\n",
+    "    columns=[\"id\"], nearest={\"column\": \"vector\", \"q\": samples[0], \"k\": 10}\n",
+    ")\n",
     "end = time.time()\n",
     "\n",
-    "print(f\"Time(sec): {end-start}\")\n",
+    "print(f\"Time(sec): {end - start}\")\n",
     "print(tbl.to_pandas())"
    ]
   },
@@ -993,7 +998,7 @@
     "\n",
     "sift1m.create_index(\n",
     "    \"vector\",\n",
-    "    index_type=\"IVF_PQ\", # IVF_PQ, IVF_HNSW_PQ and IVF_HNSW_SQ are supported\n",
+    "    index_type=\"IVF_PQ\",  # IVF_PQ, IVF_HNSW_PQ and IVF_HNSW_SQ are supported\n",
     "    num_partitions=256,  # IVF\n",
     "    num_sub_vectors=16,  # PQ\n",
     ")"
@@ -1068,7 +1073,7 @@
     "    start = time.time()\n",
     "    tbl = sift1m.to_table(nearest={\"column\": \"vector\", \"q\": q, \"k\": 10})\n",
     "    end = time.time()\n",
-    "    tot += (end - start)\n",
+    "    tot += end - start\n",
     "\n",
     "print(f\"Avg(sec): {tot / len(samples)}\")\n",
     "print(tbl.to_pandas())"
@@ -1425,7 +1430,7 @@
    "source": [
     "tbl = sift1m.to_table()\n",
     "tbl = tbl.append_column(\"item_id\", pa.array(range(len(tbl))))\n",
-    "tbl = tbl.append_column(\"revenue\", pa.array((np.random.randn(len(tbl))+5)*1000))\n",
+    "tbl = tbl.append_column(\"revenue\", pa.array((np.random.randn(len(tbl)) + 5) * 1000))\n",
     "tbl.to_pandas()"
    ]
   },
@@ -1564,7 +1569,9 @@
     }
    ],
    "source": [
-    "sift1m.to_table(columns=[\"revenue\"], nearest={\"column\": \"vector\", \"q\": samples[0], \"k\": 10}).to_pandas()"
+    "sift1m.to_table(\n",
+    "    columns=[\"revenue\"], nearest={\"column\": \"vector\", \"q\": samples[0], \"k\": 10}\n",
+    ").to_pandas()"
    ]
   }
  ],