diff --git a/helper-scripts/fetch-models.sh b/helper-scripts/fetch-models.sh index fad8262..841d947 100755 --- a/helper-scripts/fetch-models.sh +++ b/helper-scripts/fetch-models.sh @@ -40,12 +40,19 @@ if [[ -z "$DL" ]]; then fi download() { - local url="$1" dest="$2" + # Download to a temporary name and atomically rename into place. A running + # container bind-mounts this directory read-only and mmaps the GGUF files; + # writing straight to the final name would let it map a half-written file + # (and later fault with SIGBUS), and an interrupted download would leave a + # partial file that looks complete. set -e aborts before the rename if the + # transfer fails, so a broken download never lands at the real path. + local url="$1" dest="$2" tmp="$2.part" if [[ "$DL" == "curl" ]]; then - curl -L --progress-bar -o "$dest" "$url" + curl -L --fail --progress-bar -o "$tmp" "$url" else - wget --progress=bar:force:noscroll -O "$dest" "$url" + wget --progress=bar:force:noscroll -O "$tmp" "$url" fi + mv -f "$tmp" "$dest" } # ── LLM model ──────────────────────────────────────────────────────── diff --git a/public/app.js b/public/app.js index 3976eb5..7a39024 100644 --- a/public/app.js +++ b/public/app.js @@ -219,8 +219,10 @@ if (!s.llm_loaded) { setPill('err', 'LLM missing'); - setBanner('Language model not loaded — place the GGUF files in gguf_models/ and ' + - 'restart. The library stays browsable; answers are unavailable.'); + const where = s.gguf_dir || 'gguf_models/'; + setBanner('Language model not loaded — drop the GGUF files into ' + where + + ' and they load automatically within ~30s (no restart needed). ' + + 'The library stays browsable; answers are unavailable.'); } else if (s.documents_indexed === 0) { setPill('warn', 'Index empty'); setBanner(''); diff --git a/src/config.h b/src/config.h index e28b306..9d5fd5e 100644 --- a/src/config.h +++ b/src/config.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include // ── Version ────────────────────────────────────────────────────────── #define JIC_VERSION "0.3.0" @@ -70,16 +72,94 @@ inline std::string get_cors_origin() { } // ── Model paths (overridable via environment) ──────────────────────── +// The directory that holds the GGUF model files. Overridable so a deploy +// whose bind mount lands somewhere other than ./gguf_models can point the +// server at the real location instead of silently running degraded. +inline std::string get_gguf_dir() { + return env_or("JIC_GGUF_DIR", "./gguf_models"); +} + inline std::string get_llm_model_path() { - return std::string("./gguf_models/") + + return get_gguf_dir() + "/" + env_or("LLM_GGUF_FILE", "Llama-3.2-3B-Instruct-Q4_K_M.gguf"); } inline std::string get_embedding_model_path() { - return std::string("./gguf_models/") + + return get_gguf_dir() + "/" + env_or("EMBEDDING_GGUF_FILE", "nomic-embed-text-v1.5.Q4_K_M.gguf"); } +// Absolute, resolved form of the model directory — computed once (cwd is +// stable at /app in the container). Never throws: a missing directory is a +// normal state (the whole point of degraded mode), so this must be safe to +// call from the /status handler on every poll. +inline std::string resolved_gguf_dir() { + static const std::string cached = [] { + std::error_code ec; + auto abs = std::filesystem::weakly_canonical(get_gguf_dir(), ec); + return ec ? get_gguf_dir() : abs.string(); + }(); + return cached; +} + +// Human-readable diagnostic for logs when a model fails to load: the exact +// resolved path, whether the file is actually there (and its size), and if +// not, what the directory *does* contain — so "LLM missing" stops being a +// mystery. Error-code-safe throughout; never throws. +inline std::string describe_model_path(const std::string& path) { + namespace fs = std::filesystem; + std::error_code ec; + const fs::path p(path); + + auto abs = fs::weakly_canonical(p, ec); + std::string out = "resolved='" + (ec ? path : abs.string()) + "'"; + ec.clear(); + + if (fs::exists(p, ec) && !ec) { + auto sz = fs::file_size(p, ec); + out += ec ? " [present, size unknown]" + : " [present, " + std::to_string(sz) + " bytes]"; + return out; + } + + out += " [MISSING]"; + const fs::path dir = p.parent_path(); + ec.clear(); + auto dir_abs = fs::weakly_canonical(dir, ec); + const std::string dir_shown = ec ? dir.string() : dir_abs.string(); + ec.clear(); + + if (fs::is_directory(dir, ec) && !ec) { + out += "; dir '" + dir_shown + "' contains: "; + std::error_code it_ec; + // Iterate with the error_code increment(): directory_iterator's + // operator++ (used by range-for) THROWS on a mid-scan failure, which + // would break this helper's never-throws contract and, in the loader + // thread, terminate the process. increment(ec) reports instead. + fs::directory_iterator it(dir, it_ec), end; + if (it_ec) { + // e.g. the dir exists but is not readable by this uid — exactly + // the kind of thing that looks like "no models" but isn't. + out += "(unreadable: " + it_ec.message() + ")"; + } else { + int n = 0; + for (; !it_ec && it != end; it.increment(it_ec)) { + if (n++) out += ", "; + out += it->path().filename().string(); + if (n >= 20) { out += ", …"; break; } + } + if (n == 0) out += "(empty)"; + } + } else if (ec) { + // is_directory() itself failed (e.g. the parent is not traversable): + // report the actual reason instead of mislabelling it "does not exist". + out += "; dir '" + dir_shown + "' is unreadable: " + ec.message(); + } else { + out += "; dir '" + dir_shown + "' does not exist"; + } + return out; +} + inline std::string get_llm_model_name() { return env_or("LLM_MODEL", "llama3.2:3b"); } diff --git a/src/embeddings.h b/src/embeddings.h index 4da3ec9..0e2b212 100644 --- a/src/embeddings.h +++ b/src/embeddings.h @@ -38,6 +38,12 @@ class EmbeddingGenerator { } bool init() { + // Idempotent: the retry paths (server loader, ingestion wait-loop) may + // call this repeatedly. Free any partial state from a prior failed + // attempt so a reload never leaks a model/context. + if (ctx) { llama_free(ctx); ctx = nullptr; } + if (model) { llama_model_free(model); model = nullptr; } + llama_model_params mp = llama_model_default_params(); model = llama_model_load_from_file( get_embedding_model_path().c_str(), mp); @@ -50,14 +56,25 @@ class EmbeddingGenerator { return ctx != nullptr; } + // Returns an EMBEDDING_DIM-length vector on success, or an EMPTY vector if + // the text could not be embedded (context / tokenise / decode failure). + // Callers MUST treat empty as "no embedding" and skip it — never store a + // zero vector as if it were real, which silently poisons the search index. std::vector get_embedding(const std::string& text) { std::lock_guard lock(mutex); + // Defensive: if a prior reset failed and left ctx null, don't call into + // llama with a null context (crash). Try once to rebuild it first. + if (!ctx) { + reset_context(); + if (!ctx) return {}; + } + // Reset context if it's getting full int n_ctx = llama_n_ctx(ctx); if (n_past > n_ctx * 3 / 4) { reset_context(); - if (!ctx) return std::vector(EMBEDDING_DIM, 0.0f); + if (!ctx) return {}; // could not embed → empty; caller skips it } const llama_vocab* vocab = llama_model_get_vocab(model); @@ -66,7 +83,7 @@ class EmbeddingGenerator { int n_prompt = -llama_tokenize( vocab, text.c_str(), text.size(), NULL, 0, true, true); if (n_prompt <= 0) - return std::vector(EMBEDDING_DIM, 0.0f); + return {}; // could not embed → empty; caller skips it if (n_prompt > 2048) n_prompt = 2048; std::vector tokens(n_prompt); @@ -74,19 +91,19 @@ class EmbeddingGenerator { vocab, text.c_str(), text.size(), tokens.data(), tokens.size(), true, true); if (actual < 0) - return std::vector(EMBEDDING_DIM, 0.0f); + return {}; // could not embed → empty; caller skips it tokens.resize(actual); if (tokens.size() > 2048) tokens.resize(2048); llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size()); if (llama_decode(ctx, batch) != 0) - return std::vector(EMBEDDING_DIM, 0.0f); + return {}; // could not embed → empty; caller skips it n_past += tokens.size(); const float* emb = llama_get_embeddings(ctx); if (!emb) - return std::vector(EMBEDDING_DIM, 0.0f); + return {}; // could not embed → empty; caller skips it int n_embd = llama_model_n_embd(model); std::vector result(emb, emb + std::min(n_embd, EMBEDDING_DIM)); diff --git a/src/ingestion.cpp b/src/ingestion.cpp index 1857c80..c65bcb1 100644 --- a/src/ingestion.cpp +++ b/src/ingestion.cpp @@ -64,11 +64,34 @@ int main() { }, nullptr); // ── Embedding model ────────────────────────────────────────────── + // The model is host-provisioned and may not be present yet (fresh install, + // or files dropped in later). Instead of crash-looping under the restart + // policy, wait for it: retry until it loads or we're asked to stop. The + // index then starts building the moment the model appears — no restart. EmbeddingGenerator embeddings; - if (!embeddings.init()) { - std::cerr << "Failed to initialise embedding model" << std::endl; - return 1; + bool warned = false; + while (g_running.load()) { + // Only attempt to load once the file exists and has stopped changing + // (file_is_settled → false for a missing file, and for one still being + // copied in): don't mmap a half-written model, which could SIGBUS. + if (file_is_settled(get_embedding_model_path()) && embeddings.init()) + break; + if (!warned) { + std::cerr << "Embedding model not available yet — " + << describe_model_path(get_embedding_model_path()) << std::endl; + std::cerr << "Ingestion will wait and retry every " + << get_scan_interval_sec() << "s." << std::endl; + warned = true; + } + interruptible_sleep(get_scan_interval_sec()); } + if (!g_running.load()) { + std::cout << "Stopped before the embedding model became available." + << std::endl; + llama_backend_free(); + return 0; + } + std::cout << "Embedding model loaded." << std::endl; // ── SQLite index ───────────────────────────────────────────────── const std::string db_path = get_db_path(); @@ -162,12 +185,20 @@ int main() { const int BATCH = 50; std::vector batch_docs; std::vector> batch_embs; - int total_stored = 0; + int total_stored = 0; + int failed_chunks = 0; for (int ci = 0; ci < static_cast(chunks.size()); ci++) { if (!g_running.load()) break; auto emb = embeddings.get_embedding(chunks[ci]); - if (static_cast(emb.size()) != EMBEDDING_DIM) continue; + // Empty = the model failed to embed this chunk. Skip it + // (never store a zero vector — that poisons search) and + // remember the failure so we don't prematurely mark the + // whole file done. + if (static_cast(emb.size()) != EMBEDDING_DIM) { + failed_chunks++; + continue; + } batch_docs.push_back({rel_path, chunks[ci], -1, ci}); batch_embs.push_back(std::move(emb)); @@ -201,6 +232,29 @@ int main() { break; } + // Every chunk failed to embed → the model is unhealthy, not + // the document. Do NOT mark it processed (that would be + // permanent); leave it for the next scan once the model + // recovers. A partial success still commits what stored. + if (total_stored == 0 && failed_chunks > 0) { + std::cerr << " ⚠ " << rel_path << ": all " + << failed_chunks << " chunk(s) failed to embed" + << " — leaving unmarked to retry next scan" + << std::endl; + continue; + } + + // Partial failure: commit what stored (re-ingesting would + // duplicate rows — add_batch is INSERT, not upsert), but + // surface it so a half-indexed document is observable + // instead of silently missing content. + if (failed_chunks > 0) { + std::cerr << " ⚠ " << rel_path << ": " << failed_chunks + << " of " << chunks.size() + << " chunk(s) failed to embed and were dropped" + << std::endl; + } + index.mark_file_processed(rel_path, total_stored); std::cout << " ✓ " << rel_path << ": " << total_stored << " chunks indexed" << std::endl; diff --git a/src/llm.h b/src/llm.h index 8fbcdcc..469714b 100644 --- a/src/llm.h +++ b/src/llm.h @@ -22,6 +22,12 @@ class LLMGenerator { } bool init() { + // Idempotent: the background loader retries this until it succeeds, + // so free any partially-initialised state from a prior failed attempt + // before reloading — otherwise a reload would leak a model/context. + if (ctx) { llama_free(ctx); ctx = nullptr; } + if (model) { llama_model_free(model); model = nullptr; } + llama_model_params model_params = llama_model_default_params(); model = llama_model_load_from_file(get_llm_model_path().c_str(), model_params); if (!model) { diff --git a/src/server.cpp b/src/server.cpp index a68250c..8eef77e 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -36,9 +36,15 @@ using json = nlohmann::json; // Global state // ═════════════════════════════════════════════════════════════════════ -static SQLiteVecIndex* g_index = nullptr; -static EmbeddingGenerator* g_embeddings = nullptr; -static LLMGenerator* g_llm = nullptr; +static SQLiteVecIndex* g_index = nullptr; // set once before listen() + +// The models are loaded by a background thread (see model_loader) while request +// handlers may already be running on cpp-httplib's worker pool. These pointers +// only ever transition null → loaded exactly once and never revert, so plain +// acquire/release atomics are a correct, lock-free way to publish them: a reader +// sees either null (→ 503 / no context) or a fully-constructed object. +static std::atomic g_embeddings{nullptr}; +static std::atomic g_llm{nullptr}; static httplib::Server* g_server = nullptr; static std::atomic g_running{true}; @@ -106,11 +112,17 @@ static void send_error(httplib::Response& res, int status, const std::string& ms static void handle_query(const httplib::Request& req, httplib::Response& res) { // Degraded mode: the server runs without GGUF models so the UI and - // /status stay reachable, but /query needs the LLM. - if (!g_llm) { + // /status stay reachable, but /query needs the LLM. Load the pointer + // once here and use that local for the rest of the request (it cannot + // revert to null), so there is no check-then-use race with the loader. + LLMGenerator* llm = g_llm.load(std::memory_order_acquire); + if (!llm) { + // Point at the *resolved* model directory (honours JIC_GGUF_DIR), the + // same value /status reports, rather than a hardcoded path. send_error(res, 503, - "LLM model not loaded. Place the GGUF models in " - "gguf_models/ and restart the container."); + "Language model not loaded yet — it loads automatically once " + "the GGUF files are present in " + resolved_gguf_dir() + + ". The document library stays browsable in the meantime."); return; } @@ -165,26 +177,32 @@ static void handle_query(const httplib::Request& req, httplib::Response& res) { std::string context; json matches = json::array(); - if (use_context && g_embeddings && g_chunk_count.load() > 0) { - auto q_emb = g_embeddings->get_embedding(query); - - auto results = g_index->hybrid_search( - q_emb, query, MAX_CONTEXT_CHUNKS, SEARCH_CANDIDATES); - - std::set seen_files; - for (int i = 0; i < static_cast(results.size()); i++) { - auto& r = results[i]; - context += "[REFERENCE " + std::to_string(i + 1) - + " from " + r.filename + "]\n" - + r.text + "\n" - + "[END REFERENCE " + std::to_string(i + 1) + "]\n\n"; - - if (seen_files.insert(r.filename).second) { - matches.push_back({ - {"filename", r.filename}, - {"text", r.text.substr(0, 250) + "..."}, - {"score", r.score} - }); + EmbeddingGenerator* emb = g_embeddings.load(std::memory_order_acquire); + if (use_context && emb && g_chunk_count.load() > 0) { + auto q_emb = emb->get_embedding(query); + + // Empty = the query could not be embedded; skip vector search + // rather than feeding a bad vector into the index (answer the + // question without retrieved context). + if (!q_emb.empty()) { + auto results = g_index->hybrid_search( + q_emb, query, MAX_CONTEXT_CHUNKS, SEARCH_CANDIDATES); + + std::set seen_files; + for (int i = 0; i < static_cast(results.size()); i++) { + auto& r = results[i]; + context += "[REFERENCE " + std::to_string(i + 1) + + " from " + r.filename + "]\n" + + r.text + "\n" + + "[END REFERENCE " + std::to_string(i + 1) + "]\n\n"; + + if (seen_files.insert(r.filename).second) { + matches.push_back({ + {"filename", r.filename}, + {"text", r.text.substr(0, 250) + "..."}, + {"score", r.score} + }); + } } } } @@ -212,7 +230,7 @@ static void handle_query(const httplib::Request& req, httplib::Response& res) { prompt += "User: " + query + "\n\nAssistant:"; // ── Generate ──────────────────────────────────────────────── - std::string answer = g_llm->generate(prompt); + std::string answer = llm->generate(prompt); // ── Update conversation history ───────────────────────────── { @@ -240,6 +258,13 @@ static void handle_query(const httplib::Request& req, httplib::Response& res) { // /status handler (uses cached counts) // ═════════════════════════════════════════════════════════════════════ +// Cheap, never-throwing existence check for /status (the file may sit on a +// missing or read-only mount; error_code overload keeps this safe to poll). +static bool model_file_present(const std::string& path) { + std::error_code ec; + return fs::exists(path, ec) && !ec; +} + static void handle_status(const httplib::Request&, httplib::Response& res) { auto uptime = std::chrono::duration_cast( std::chrono::steady_clock::now() - g_start_time).count(); @@ -249,10 +274,16 @@ static void handle_status(const httplib::Request&, httplib::Response& res) { status["uptime_seconds"] = uptime; status["documents_indexed"] = g_chunk_count.load(); status["files_processed"] = g_file_count.load(); - status["llm_loaded"] = (g_llm != nullptr); - status["embeddings_loaded"] = (g_embeddings != nullptr); + status["llm_loaded"] = (g_llm.load() != nullptr); + status["embeddings_loaded"] = (g_embeddings.load() != nullptr); status["llm_model"] = get_llm_model_name(); status["embedding_model"] = get_embedding_model_name(); + // Diagnostics: where the server is actually looking for models, and + // whether the GGUF files are present there — so "LLM missing" is + // self-explanatory (wrong mount path vs. files not yet provisioned). + status["gguf_dir"] = resolved_gguf_dir(); + status["llm_gguf_present"] = model_file_present(get_llm_model_path()); + status["embedding_gguf_present"] = model_file_present(get_embedding_model_path()); res.set_content(status.dump(), "application/json"); } @@ -308,6 +339,108 @@ static void count_refresher() { } } +// ═════════════════════════════════════════════════════════════════════ +// Background thread: load the GGUF models as soon as they are available +// +// Models are large, host-provisioned files that may not be present when the +// container starts (fresh install, the CI publish gate's empty gguf_models/, +// or files dropped into the volume later). Rather than fail or block startup, +// the server comes up immediately in degraded mode and this thread keeps +// trying to load each model until it succeeds — so provisioning the GGUF +// files "just works" without a restart, exactly as the ingestion service +// auto-indexes new documents. +// ═════════════════════════════════════════════════════════════════════ + +// A model is safe to load only once its file exists AND has stopped changing +// (mtime older than the settle window): this avoids mmap-ing a file that is +// still being copied in, which could later fault with SIGBUS on access. +static bool model_file_ready(const std::string& path) { + std::error_code ec; + if (!fs::exists(path, ec) || ec) return false; + auto mtime = fs::last_write_time(path, ec); + if (ec) return false; + return (fs::file_time_type::clock::now() - mtime) + > std::chrono::seconds(FILE_SETTLE_SECONDS); +} + +static void model_loader() { + bool warned_emb = false, warned_llm = false; // "waiting for file" + bool warned_emb_fail = false, warned_llm_fail = false; // "present but won't load" + + while (g_running.load()) { + // Load each model independently and only into an *empty* slot — never + // reload over an already-loaded pointer (that would leak a multi-GB + // model on every scan). Publish with a release store so a concurrent + // handler that acquire-loads a non-null pointer sees a fully-built + // object. On failure, delete the local and leave the slot null. + if (g_running.load() && + g_embeddings.load(std::memory_order_acquire) == nullptr) { + const std::string path = get_embedding_model_path(); + if (model_file_ready(path)) { + auto* e = new EmbeddingGenerator(); + if (e->init()) { + g_embeddings.store(e, std::memory_order_release); + std::cout << "Embedding model loaded — semantic search enabled (" + << describe_model_path(path) << ")" << std::endl; + } else { + delete e; // frees any partial state; retry next scan + if (!warned_emb_fail) { // don't re-log every scan + std::cerr << "Embedding model present but failed to load " + "(corrupt or incompatible?) — " + << describe_model_path(path) + << "; will keep retrying" << std::endl; + warned_emb_fail = true; + } + } + } else if (!warned_emb) { + std::cout << "Waiting for embedding model — " + << describe_model_path(path) << std::endl; + warned_emb = true; + } + } + + // Re-check g_running between the two loads: a model load can take + // seconds and cannot be interrupted mid-flight, so a SIGTERM that + // arrives during the embedding load must not then kick off the (much + // larger) LLM load and blow past Docker's stop grace. + if (g_running.load() && + g_llm.load(std::memory_order_acquire) == nullptr) { + const std::string path = get_llm_model_path(); + if (model_file_ready(path)) { + auto* l = new LLMGenerator(); + if (l->init()) { + g_llm.store(l, std::memory_order_release); + std::cout << "LLM loaded — /query now available (" + << describe_model_path(path) << ")" << std::endl; + } else { + delete l; + if (!warned_llm_fail) { // don't re-log every scan + std::cerr << "LLM present but failed to load " + "(corrupt or incompatible?) — " + << describe_model_path(path) + << "; will keep retrying" << std::endl; + warned_llm_fail = true; + } + } + } else if (!warned_llm) { + std::cout << "Waiting for LLM — " + << describe_model_path(path) << std::endl; + warned_llm = true; + } + } + + // Nothing left to do once both are up. + if (g_embeddings.load(std::memory_order_acquire) != nullptr && + g_llm.load(std::memory_order_acquire) != nullptr) + break; + + // Interruptible wait: poll g_running every 250 ms so shutdown is prompt. + const int wait = get_scan_interval_sec(); + for (int i = 0; i < wait * 4 && g_running.load(); i++) + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } +} + // ═════════════════════════════════════════════════════════════════════ // main // ═════════════════════════════════════════════════════════════════════ @@ -334,28 +467,17 @@ int main() { }, nullptr); ggml_backend_load_all(); - // ── Models ─────────────────────────────────────────────────────── - std::cout << "Loading models..." << std::endl; - - // Missing GGUF models must not kill the server: the CI publish gate (and - // a fresh install before models are provisioned) runs the container with - // an empty gguf_models/. Serve the UI and /status regardless; /query - // answers 503 until the models are in place. - g_embeddings = new EmbeddingGenerator(); - if (!g_embeddings->init()) { - std::cerr << "Embeddings init failed — continuing in degraded mode " - "(semantic search disabled)" << std::endl; - delete g_embeddings; - g_embeddings = nullptr; - } - - g_llm = new LLMGenerator(); - if (!g_llm->init()) { - std::cerr << "LLM init failed — continuing in degraded mode " - "(/query returns 503)" << std::endl; - delete g_llm; - g_llm = nullptr; - } + // ── Models (loaded in the background) ───────────────────────────── + // Missing GGUF models must not kill or stall the server: the CI publish + // gate (and a fresh install before models are provisioned) runs the + // container with an empty gguf_models/. The server serves the UI and + // /status immediately; the model_loader thread loads the models as soon + // as they appear — no restart required — and /query answers 503 until + // then. Reporting the resolved directory up front makes a mis-pointed + // bind mount obvious in the logs instead of a silent degraded run. + std::cout << "Model directory: " << resolved_gguf_dir() + << " — models load in the background as they become available" + << std::endl; // ── SQLite index ───────────────────────────────────────────────── const std::string db_path = get_db_path(); @@ -370,8 +492,12 @@ int main() { std::cout << "Index: " << g_chunk_count.load() << " chunks, " << g_file_count.load() << " files" << std::endl; - // Background count refresher (joined on shutdown) + // Background workers (joined on shutdown): count refresher + model loader. + // The loader is started before listen() so the UI/degraded mode is served + // immediately while models load; its first iteration attempts a load right + // away, so models already present at boot come up promptly. std::thread refresher(count_refresher); + std::thread loader(model_loader); // ── HTTP server ────────────────────────────────────────────────── httplib::Server svr; @@ -431,9 +557,12 @@ int main() { std::cout << "Shutting down..." << std::endl; g_running.store(false); refresher.join(); + loader.join(); // finishes any in-flight load, then exits (see model_loader) - delete g_embeddings; - delete g_llm; + // Safe to delete now: listen() has joined every request-handler worker, + // and the loader thread is joined, so nothing else touches these pointers. + delete g_embeddings.load(); + delete g_llm.load(); delete g_index; llama_backend_free(); std::cout << "Bye." << std::endl;