Skip to content

Commit 391dcee

Browse files
committed
fix(vec0): lock incremental persistence and registry
Snapshot HNSW nodes under read lock before incremental SQLite serialization, and key the vec0 registry by (db, schema, table) instead of table name alone to prevent cross-connection collisions.
1 parent 5a6101f commit 391dcee

3 files changed

Lines changed: 39 additions & 17 deletions

File tree

include/sqlite-vec-cpp/index/hnsw.hpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,6 +1176,22 @@ template <concepts::VectorElement StorageT, typename MetricT> class HNSWIndex {
11761176
}
11771177
}
11781178

1179+
/// Copy a selected set of nodes under the nodes read lock.
1180+
[[nodiscard]] std::vector<NodeType>
1181+
snapshotPersistableNodes(std::span<const size_t> node_ids) const {
1182+
std::shared_lock nodes_lock(nodes_mutex_);
1183+
std::vector<NodeType> snapshot;
1184+
snapshot.reserve(node_ids.size());
1185+
for (size_t node_id : node_ids) {
1186+
auto it = nodes_.find(node_id);
1187+
if (it == nodes_.end()) {
1188+
continue;
1189+
}
1190+
snapshot.push_back(it->second);
1191+
}
1192+
return snapshot;
1193+
}
1194+
11791195
/// Monotonically increasing counter, bumped on every insert/delete.
11801196
/// Used by HNSWQuantizedSearch to detect stale quantization snapshots.
11811197
[[nodiscard]] uint64_t mutation_generation() const {

include/sqlite-vec-cpp/index/hnsw_persistence.hpp

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -613,24 +613,21 @@ int save_hnsw_nodes_incremental(sqlite3* db, const char* schema, const char* tab
613613
return rc;
614614
}
615615

616-
for (size_t node_id : nodes_to_save) {
617-
const typename HNSWIndex<T, Metric>::NodeType* node = index.get_node(node_id);
618-
if (!node) {
619-
continue;
620-
}
616+
const auto node_snapshot = index.snapshotPersistableNodes(nodes_to_save);
621617

622-
auto node_blob = serialize_hnsw_node(*node);
618+
for (const auto& node : node_snapshot) {
619+
auto node_blob = serialize_hnsw_node(node);
623620

624621
sqlite3_reset(stmt);
625-
sqlite3_bind_int64(stmt, 1, node_id);
622+
sqlite3_bind_int64(stmt, 1, node.id);
626623
sqlite3_bind_blob(stmt, 2, node_blob.data(), node_blob.size(), SQLITE_TRANSIENT);
627624

628625
rc = sqlite3_step(stmt);
629626
if (rc != SQLITE_DONE) {
630627
sqlite3_finalize(stmt);
631628
sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr);
632629
if (pzErr)
633-
*pzErr = sqlite3_mprintf("Failed to save HNSW node %zu", node_id);
630+
*pzErr = sqlite3_mprintf("Failed to save HNSW node %zu", node.id);
634631
return rc;
635632
}
636633
}

include/sqlite-vec-cpp/sqlite/vec0_module.hpp

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -77,20 +77,29 @@ inline std::mutex& vec0_registry_mutex() {
7777
static std::mutex m;
7878
return m;
7979
}
80-
inline void vec0_registry_put(const std::string& key, Vec0Table* t) {
80+
inline std::string vec0_registry_key(sqlite3* db, std::string_view schema_name,
81+
std::string_view table_name) {
82+
std::ostringstream key;
83+
key << static_cast<const void*>(db) << '|' << schema_name << '|' << table_name;
84+
return key.str();
85+
}
86+
inline void vec0_registry_put(sqlite3* db, std::string_view schema_name,
87+
std::string_view table_name, Vec0Table* t) {
8188
std::lock_guard<std::mutex> lk(vec0_registry_mutex());
82-
vec0_table_registry()[key] = t;
89+
vec0_table_registry()[vec0_registry_key(db, schema_name, table_name)] = t;
8390
}
84-
inline void vec0_registry_remove(const std::string& key) {
91+
inline void vec0_registry_remove(sqlite3* db, std::string_view schema_name,
92+
std::string_view table_name) {
8593
std::lock_guard<std::mutex> lk(vec0_registry_mutex());
86-
vec0_table_registry().erase(key);
94+
vec0_table_registry().erase(vec0_registry_key(db, schema_name, table_name));
8795
}
8896
template <typename Fn>
89-
inline auto vec0_with_table(sqlite3* /*db*/, const std::string& table_name, Fn&& fn)
97+
inline auto vec0_with_table(sqlite3* db, std::string_view schema_name,
98+
std::string_view table_name, Fn&& fn)
9099
-> decltype(fn(static_cast<Vec0Table*>(nullptr))) {
91100
std::lock_guard<std::mutex> lk(vec0_registry_mutex());
92101
auto& reg = vec0_table_registry();
93-
auto it = reg.find(table_name);
102+
auto it = reg.find(vec0_registry_key(db, schema_name, table_name));
94103
if (it == reg.end()) {
95104
return fn(nullptr);
96105
}
@@ -618,7 +627,7 @@ inline int vec0Create(sqlite3* db, void* pAux, int argc, const char* const* argv
618627
}
619628

620629
*ppVTab = &table->base;
621-
vec0_registry_put(table->table_name, table);
630+
vec0_registry_put(table->db, table->schema_name, table->table_name, table);
622631
return SQLITE_OK;
623632
}
624633

@@ -633,15 +642,15 @@ inline int vec0Connect(sqlite3* db, void* pAux, int argc, const char* const* arg
633642
// xDisconnect: Called when disconnecting from table
634643
inline int vec0Disconnect(sqlite3_vtab* pVTab) {
635644
auto* table = reinterpret_cast<Vec0Table*>(pVTab);
636-
vec0_registry_remove(table->table_name);
645+
vec0_registry_remove(table->db, table->schema_name, table->table_name);
637646
delete table;
638647
return SQLITE_OK;
639648
}
640649

641650
// xDestroy: Called when DROP TABLE is executed
642651
inline int vec0Destroy(sqlite3_vtab* pVTab) {
643652
auto* table = reinterpret_cast<Vec0Table*>(pVTab);
644-
vec0_registry_remove(table->table_name);
653+
vec0_registry_remove(table->db, table->schema_name, table->table_name);
645654

646655
// Drop shadow tables
647656
std::ostringstream drop_meta;

0 commit comments

Comments
 (0)