diff --git a/cpp/vector_engine_ffi.h b/cpp/vector_engine_ffi.h index f1996df..5b68462 100644 --- a/cpp/vector_engine_ffi.h +++ b/cpp/vector_engine_ffi.h @@ -1,12 +1,27 @@ #ifndef VECTOR_ENGINE_FFI_H #define VECTOR_ENGINE_FFI_H +#include + struct NativeVectorEngine; +struct NativeSearchResults; extern "C" { -NativeVectorEngine* native_vector_engine_new(); -void native_vector_engine_free(NativeVectorEngine* engine); + // subject to change + NativeVectorEngine* native_vector_engine_new(); + void native_vector_engine_free(NativeVectorEngine* engine); + + // insert / delete / search + void native_vector_engine_insert( NativeVectorEngine* engine, const char* id, const float* vector, size_t len); + bool native_vector_engine_delete( NativeVectorEngine* engine, const char* id); + NativeSearchResults* native_vector_engine_search( const NativeVectorEngine* engine, const float* query,size_t len, size_t k); + + // working with pointers to send info back to rust + size_t native_search_results_len(const NativeSearchResults* results); + const char* native_search_results_id_at(const NativeSearchResults* results, size_t index); + float native_search_results_score_at(const NativeSearchResults* results, size_t index); + void native_search_results_free(NativeSearchResults* results); } diff --git a/crates/vdb-ffi/Cargo.lock b/crates/vdb-ffi/Cargo.lock new file mode 100644 index 0000000..e826eb0 --- /dev/null +++ b/crates/vdb-ffi/Cargo.lock @@ -0,0 +1,228 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vdb-ffi" +version = "0.1.0" +dependencies = [ + "bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/crates/vdb-ffi/Cargo.toml b/crates/vdb-ffi/Cargo.toml new file mode 100644 index 0000000..a8dd816 --- /dev/null +++ b/crates/vdb-ffi/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "vdb-ffi" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/lib.rs" diff --git a/crates/vdb-ffi/src/bindings.rs b/crates/vdb-ffi/src/bindings.rs new file mode 100644 index 0000000..41594b2 --- /dev/null +++ b/crates/vdb-ffi/src/bindings.rs @@ -0,0 +1,24 @@ +#[repr(C)] +pub struct NativeVectorEngine { + _private: [u8; 0], // rust short hand for declaring "opaque" struct +} + +#[repr(C)] +pub struct NativeSearchResults { + _private: [u8; 0], +} + +unsafe extern "C" { + pub fn native_vector_engine_new() -> *mut NativeVectorEngine; + pub fn native_vector_engine_free(engine: *mut NativeVectorEngine); + + pub fn native_vector_engine_insert( engine: *mut NativeVectorEngine, id: *const std::os::raw::c_char, vector: *const f32, len: usize, ); + + pub fn native_vector_engine_delete( engine: *mut NativeVectorEngine, id: *const std::os::raw::c_char, ) -> bool; + pub fn native_vector_engine_search( engine: *const NativeVectorEngine, query: *const f32, len: usize, k: usize, ) -> *mut NativeSearchResults; + pub fn native_search_results_len(results: *const NativeSearchResults) -> usize; + + pub fn native_search_results_id_at( results: *const NativeSearchResults, index: usize, ) -> *const std::os::raw::c_char; + pub fn native_search_results_score_at( results: *const NativeSearchResults, index: usize, ) -> f32; + pub fn native_search_results_free(results: *mut NativeSearchResults); +} diff --git a/crates/vdb-ffi/src/engine.rs b/crates/vdb-ffi/src/engine.rs new file mode 100644 index 0000000..df892cd --- /dev/null +++ b/crates/vdb-ffi/src/engine.rs @@ -0,0 +1,101 @@ +use crate::bindings::{ + native_search_results_free, + native_search_results_id_at, + native_search_results_len, + native_search_results_score_at, + native_vector_engine_delete, + native_vector_engine_free, + native_vector_engine_insert, + native_vector_engine_new, + native_vector_engine_search, + NativeSearchResults, + NativeVectorEngine, +}; +use std::ffi::{CStr, CString}; + +pub struct FfiVectorEngine { + handle: *mut NativeVectorEngine, +} + +pub struct FfiSearchResults { + handle: *mut NativeSearchResults, +} + +impl FfiVectorEngine { + pub fn new() -> Self { + let handle = unsafe { native_vector_engine_new() }; + + assert!( + !handle.is_null(), + "native_vector_engine_new went wrong somewhere" + ); + + Self { + handle + } + } + + pub fn insert(&mut self, id: &str, vector: &[f32]) { + let id = CString::new(id).expect("vector IDs must not contain null bytes"); + + unsafe { + native_vector_engine_insert( self.handle, id.as_ptr(), vector.as_ptr(), vector.len(),); + } + } + + pub fn delete(&mut self, id: &str) -> bool { + let id = CString::new(id).expect("vector IDs must not contain null bytes"); + unsafe { native_vector_engine_delete(self.handle, id.as_ptr()) } + } + + pub fn search(&self, query: &[f32], k: usize) -> FfiSearchResults { + let handle = unsafe { + native_vector_engine_search(self.handle, query.as_ptr(), query.len(), k) }; + + assert!( + !handle.is_null(), + "native_vector_engine_search returned a null results handle" + ); + + FfiSearchResults { handle } + } +} + +impl FfiSearchResults { + pub fn len(&self) -> usize { + unsafe { native_search_results_len(self.handle) } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn id_at(&self, index: usize) -> String { + let id_ptr = unsafe { native_search_results_id_at(self.handle, index) }; + + unsafe { CStr::from_ptr(id_ptr) } + .to_str() + .expect("native search result ID must be valid") + .to_string() + } + + pub fn score_at(&self, index: usize) -> f32 { + unsafe { native_search_results_score_at(self.handle, index) } + } +} + +impl Drop for FfiVectorEngine { + fn drop(&mut self) { + unsafe { + native_vector_engine_free(self.handle); + } + } +} + +impl Drop for FfiSearchResults { + fn drop(&mut self) { + unsafe { + native_search_results_free(self.handle); + } + } +} diff --git a/crates/vdb-ffi/src/lib.rs b/crates/vdb-ffi/src/lib.rs new file mode 100644 index 0000000..8c4f91f --- /dev/null +++ b/crates/vdb-ffi/src/lib.rs @@ -0,0 +1,4 @@ +mod bindings; +mod engine; + +pub use engine::{FfiSearchResults, FfiVectorEngine};