From 4a2a85a2939ef766f01cb8b7a6cbffbe9b6bdab6 Mon Sep 17 00:00:00 2001 From: "Aaron K. Clark (CryptoJones)" Date: Mon, 27 Jul 2026 02:11:39 -0500 Subject: [PATCH] feat: complete the hybrid retrieval backlog --- BACKLOG.md | 119 ++++---- CHANGELOG.md | 48 +++- docs/retrieval.md | 30 +- src/omind/access.py | 69 +++++ src/omind/bench.py | 62 +++++ src/omind/cli.py | 171 +++++++++++- src/omind/consolidate.py | 267 ++++++++++++++++++ src/omind/hooks.py | 37 +++ src/omind/lint.py | 84 +++++- src/omind/merge.py | 2 + src/omind/paths.py | 10 + src/omind/recall.py | 8 +- src/omind/searchindex.py | 502 ++++++++++++++++++++++++++++++---- src/omind/seeds.py | 3 + src/omind/server.py | 85 +++++- src/omind/store.py | 96 ++++++- tests/test_access.py | 36 +++ tests/test_bench.py | 28 ++ tests/test_cli.py | 9 + tests/test_cli_integration.py | 25 +- tests/test_consolidate.py | 113 ++++++++ tests/test_hooks.py | 19 ++ tests/test_lint.py | 9 +- tests/test_searchindex.py | 153 ++++++++++- tests/test_server.py | 22 +- tests/test_store.py | 2 + 26 files changed, 1845 insertions(+), 164 deletions(-) create mode 100644 src/omind/access.py create mode 100644 src/omind/consolidate.py create mode 100644 tests/test_access.py create mode 100644 tests/test_consolidate.py diff --git a/BACKLOG.md b/BACKLOG.md index 0206a35..736b1ff 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -15,73 +15,17 @@ to what omind actually needs. The foundation — a derived SQLite hybrid index ( RRF), excerpt-returning search, and paged MCP payloads — shipped first; see `docs/retrieval.md`. These are the next tier, ranked by leverage._ -### Retrieval quality - -- [ ] **Rerank the fused top-k so the result tail is clean** ([#167](https://github.com/CryptoJones/omind/issues/167)) — _enhancement (retrieval)_ — RRF over - an OR-matched keyword leg gets rank 1 right but admits weakly-related notes at ranks 4–10 (a - query for "how do I handle colorblindness" pulls in notes matching only "handle"). Reranking is - the one technique universal across the surveyed systems. Do it locally and cheaply — score the - top ~20 fused candidates against the query with the existing embedding model over the *whole - matched chunk* (not the note metadata), or a small cross-encoder behind the `[embed]` extra — - never an API call on the query path. -- [ ] **A retrieval-quality eval harness** ([#168](https://github.com/CryptoJones/omind/issues/168)) — _chore (testing)_ — ranking changes are currently - judged by eye on one vault. Add a small labelled query set (query → the note that should win, - LoCoMo/LongMemEval in miniature) plus a `omind bench --quality` mode reporting recall@1/@5 and - MRR, so a change to weights, fusion, or chunking is measurable instead of a vibe. Prerequisite - for tuning `_W_*`, `RRF_K`, or chunk size with any confidence. -- [ ] **Temporal validity on facts, so superseded memories stop being retrieved as current** ([#169](https://github.com/CryptoJones/omind/issues/169)) — - _enhancement (retrieval)_ — Zep/Graphiti's central idea: a fact has a validity interval, not just - a creation date. omind has many notes that supersede earlier ones ("v1.2.0 released" → "v1.3.0 - released") and search happily returns the stale one. Model supersession explicitly (a - `Supersedes:`/`Superseded by:` metadata line, indexed), and de-rank superseded chunks instead of - deleting them. -- [ ] **Weight auto-generated notes below hand-curated ones** ([#170](https://github.com/CryptoJones/omind/issues/170)) — _enhancement (retrieval)_ — session - journals, worklogs, and checkpoint rollups are numerous, dense, and topically broad, so they - crowd genuine memories out of BM25 results. The OKF `type` and the `Session Journal` naming - convention already distinguish them; make that a ranking signal. -- [ ] **Adaptive retrieval scope by query complexity** ([#171](https://github.com/CryptoJones/omind/issues/171)) — _enhancement (efficiency)_ — SimpleMem's - third stage: a one-word lookup and a multi-hop question should not retrieve the same `k`. Vary - depth and returned excerpt budget by query shape, so simple recalls get cheaper rather than - everything getting more expensive. - ### Memory shape -- [ ] **Consolidate near-duplicate notes instead of only listing them** ([#172](https://github.com/CryptoJones/omind/issues/172)) — _enhancement (memory)_ — - SimpleMem/RecMem both show recursive consolidation is where lifelong memory stops degrading. - `omind lint` reports near-duplicates and stops; the index now has the chunk vectors needed to - detect real semantic duplicates (not just similar titles). Add an `omind consolidate` that - proposes merges and applies them under review — never silently. -- [ ] **Tiered memory: a small always-loaded core, a large searched archive** ([#173](https://github.com/CryptoJones/omind/issues/173)) — _enhancement - (memory)_ — Letta/MemGPT's core/archival split. omind approximates it with `index.md` + - `Playbook.md` priming, but tier membership is hand-maintained. Promote/demote by access - frequency and recency, so the always-injected set stays small and earns its tokens. -- [ ] **Wire `lint` to the index (one scan, vector-based near-duplicate detection)** ([#174](https://github.com/CryptoJones/omind/issues/174)) — _chore - (perf)_ — `lint_vault` is now the last independent full-vault scanner, and its near-duplicate - pass is O(n²) over titles (~553k comparisons on 744 notes). Deliberately left alone during the - index work: lint strips code fences before extracting links, and the index does not, so a naive - swap would resurrect the false broken-link errors that fix exists to prevent. Do it properly — - index a fence-stripped link set alongside the raw one. +_No open items._ ### Efficiency -- [ ] **Quantize stored embeddings (int8) and shrink the index** ([#175](https://github.com/CryptoJones/omind/issues/175)) — _enhancement (efficiency)_ — the - index is ~19 MB for 5,678 chunks, almost all of it `float32` vectors. int8 with a per-vector - scale is standard practice (and what the low-storage retrievers in the survey do), cutting the - file ~4× and speeding the matmul, with negligible ranking loss at this scale. Add a residual-norm - tiebreaker for near-equal cosines. -- [ ] **Throttle the per-query index refresh** ([#176](https://github.com/CryptoJones/omind/issues/176)) — _enhancement (perf)_ — every `search()` calls - `refresh()`, which stats every note (~5 ms of a ~18 ms query on 762 notes). Skip the refresh when - the vault's cheap signature is unchanged since the last one within the same process, and let the - existing write-signal file invalidate it, so a burst of queries pays the stat sweep once. -- [ ] **Consolidate the four `graph-*` audit tools into one paged `graph` tool** ([#177](https://github.com/CryptoJones/omind/issues/177)) — _chore (tokens)_ - — 16 always-resident tool definitions cost ~1.2–1.8k tokens of every context. `graph-path`, - `graph-orphans`, `graph-dangling` and `graph-stats` are one tool with an `op` argument. **Needs a - decision first:** it renames tools that the OMI vault's `Playbook.md`, the managed `omind` skill, - and other fleet machines reference, so it is a coordinated change, not a local one. -- [ ] **`omind doctor`: report search-index health** ([#178](https://github.com/CryptoJones/omind/issues/178)) — _chore_ — doctor knows nothing about the - index. Report FTS5 availability, whether the semantic leg is on, index size/age, and any stale - or corrupt file (with the one-line `omind reindex --rebuild` fix), since a silently-degraded - index shows up only as worse answers. +- [ ] **Remove the deprecated `graph-*` MCP compatibility aliases after one release** + ([#181](https://github.com/CryptoJones/omind/issues/181)) — _chore (tokens)_ — + `graph-path`, `graph-orphans`, `graph-dangling`, and `graph-stats` remain for + one bridge release while fleet clients refresh; remove them in the following + release. `graph-neighbors` stays. ## Not planned @@ -90,9 +34,58 @@ These are the next tier, ranked by leverage._ ## Done +- [x] **Consolidate near-duplicate notes instead of only listing them** ([#172](https://github.com/CryptoJones/omind/issues/172)) — _enhancement (memory)_ — + `omind consolidate` creates machine-local JSON plans and editable Markdown + drafts without changing the vault. Explicit `--apply PLAN_ID` revalidates + both source versions, creates the reviewed merged note through OmiStore, and + archives rather than destroys the originals. +- [x] **Consolidate the four `graph-*` audit tools into one paged `graph` tool** ([#177](https://github.com/CryptoJones/omind/issues/177)) — _chore (tokens)_ — + `graph(op=path|orphans|dangling|stats)` is the new surface; list operations + remain paged. The old names are deprecated compatibility aliases for one + release, with their removal tracked in #181. +- [x] **Tiered memory: a small always-loaded core, a large searched archive** ([#173](https://github.com/CryptoJones/omind/issues/173)) — _enhancement + (memory)_ — actual note reads update machine-local frequency/recency state; + SessionStart promotes at most three earned notes and ages them out after 90 + days. Generated, credential-looking, archived, missing, and unsafe targets are + excluded; the fixed operational/persona core stays pinned. +- [x] **Wire `lint` to the index (one scan, vector-based near-duplicate detection)** ([#174](https://github.com/CryptoJones/omind/issues/174)) — _chore + (perf)_ — the index stores a fence-stripped lint link view alongside the raw + graph and exposes title-presence/archive state. Lint uses quantized chunk + centroids for semantic duplicate pairs, falling back to title Jaccard when + embeddings are off. Indexed and fallback live-vault issue counts match. +- [x] **Temporal validity on facts, so superseded memories stop being retrieved as current** ([#169](https://github.com/CryptoJones/omind/issues/169)) — + _enhancement (retrieval)_ — `Supersedes:` / `Superseded by:` metadata + round-trips through Markdown, CLI, MCP, and mesh merges. The index resolves + those relationships and de-ranks obsolete notes without deleting history. +- [x] **Quantize stored embeddings (int8) and shrink the index** ([#175](https://github.com/CryptoJones/omind/issues/175)) — _enhancement (efficiency)_ — + vectors use symmetric int8 storage with a per-vector scale and residual-error + tie-breaker. The live 5,691-vector index rebuilt at 13 MiB versus roughly + 19 MiB before, with unchanged quality metrics. +- [x] **Adaptive retrieval scope by query complexity** ([#171](https://github.com/CryptoJones/omind/issues/171)) — _enhancement (efficiency)_ — simple, + normal, and multi-hop queries now use progressively larger candidate depths, + result caps, and excerpt budgets (20/5/120, 60/10/180, 90/25/240). +- [x] **Rerank the fused top-k so the result tail is clean** ([#167](https://github.com/CryptoJones/omind/issues/167)) — _enhancement (retrieval)_ — the + fused top 20 receive one bounded, local embedding pass over each whole matched + chunk body. Weak candidates are rescaled without an API call; an unavailable + or malformed embedding backend fails open to the original RRF order. +- [x] **Weight auto-generated notes below hand-curated ones** ([#170](https://github.com/CryptoJones/omind/issues/170)) — _enhancement (retrieval)_ — + journal/worklog/checkpoint OKF types and the established Session Journal / + Worklog filename conventions receive a modest score penalty, keeping them + retrievable while comparable curated notes win. +- [x] **A retrieval-quality eval harness** ([#168](https://github.com/CryptoJones/omind/issues/168)) — _chore (testing)_ — `omind bench --quality` + evaluates a version-controlled labelled query set against the live vault and + reports recall@1, recall@5, MRR, skipped targets, and the worst misses. +- [x] **Throttle the per-query index refresh** ([#176](https://github.com/CryptoJones/omind/issues/176)) — _enhancement (perf)_ — a burst of indexed reads + pays the full-vault stat sweep once. OmiStore writes invalidate the process + cache immediately through the existing signal; direct external edits are + discovered after a one-second bound. - [x] **Adversarial review hardening: MCP/web transport deadlocks and API fallback** — _bug (availability/security)_ — `omind node` no longer depends on the SDK's AnyIO file-wrapper stdio path; it uses fd readiness and still feeds the normal MCP session streams, so stdin handshakes and EOF shutdown cannot wedge. The web API no longer relies on Starlette's thread-backed sync handlers or `StaticFiles` fallback; malformed encoded `/api/...` traversal paths return API 404/400 instead of falling into static serving, and packaged assets are served by a direct path-resolved responder that rejects escapes before reading bytes. - [x] **Adversarial review hardening: transfer archives + subprocess error redaction** — _bug (security/perf)_ — tar.gz export now excludes VCS control directories (`.git`, `.hg`, `.svn`) so mesh vault exports do not leak git history or produce giant bundles; tar.gz import now rejects control-directory members so crafted bundles cannot plant git config/hooks. Shared subprocess failures now redact URL userinfo, GitHub tokens, and Authorization headers before surfacing command/error text. -- [x] **Hybrid search index + MCP token budgets** — _enhancement_ — `omind.searchindex`: FTS5/BM25 over heading-split chunks, packed `float32` chunk vectors, RRF fusion with a weak recency leg, excerpt-returning hits, and the link graph, all in one disposable state-dir SQLite file. Retired `omind.vectorindex` (metadata-only embeddings, JSON float storage, per-query refresh, pure-Python cosine). Paged every list-shaped MCP tool; `read-note` stopped returning the body twice. Added `omind bench`, `omind search --explain`, `omind reindex --index-only/--rebuild`, `docs/retrieval.md`. Measured on a 744-note vault: search 268 ms → 18 ms, natural-language queries 0 hits → ranked answers, `list-notes` ~90,800 → 3,136 tokens. +- [x] **`omind doctor`: report search-index health** ([#178](https://github.com/CryptoJones/omind/issues/178)) — _chore_ — reports FTS5 availability, + semantic-leg status and its disabled reason, index size/age/note/vector counts, + and stale, corrupt, or incompatible files with the one-line + `omind reindex --rebuild` fix. +- [x] **Hybrid search index + MCP token budgets** — _enhancement_ — `omind.searchindex`: FTS5/BM25 over heading-split chunks, quantized chunk vectors, RRF fusion with a weak recency leg, excerpt-returning hits, and the link graph, all in one disposable state-dir SQLite file. Retired `omind.vectorindex` (metadata-only embeddings, JSON float storage, per-query refresh, pure-Python cosine). Paged every list-shaped MCP tool; `read-note` stopped returning the body twice. Added `omind bench`, `omind search --explain`, `omind reindex --index-only/--rebuild`, `docs/retrieval.md`. Measured on a 744-note vault: search 268 ms → 18 ms, natural-language queries 0 hits → ranked answers, `list-notes` ~90,800 → 3,136 tokens. - [x] **Deferred adversarial-review batch: #125–#131** — _meta_ — all seven shipped and closed upstream: web XSS/Host allowlist ([#125](https://github.com/CryptoJones/omind/issues/125)), macOS CI + wheel smoke ([#126](https://github.com/CryptoJones/omind/issues/126)), tombstone GC ([#127](https://github.com/CryptoJones/omind/issues/127)), per-session loopguard ([#128](https://github.com/CryptoJones/omind/issues/128)), web graph O(n²) ([#129](https://github.com/CryptoJones/omind/issues/129)), vault I/O off the event loop + store lock ([#130](https://github.com/CryptoJones/omind/issues/130)), dependency pinning ([#131](https://github.com/CryptoJones/omind/issues/131)). - [x] **Hardening batch from adversarial code review** ([#132](https://github.com/CryptoJones/omind/issues/132), [PR #124](https://github.com/CryptoJones/omind/pull/124)) — _meta_ — v3.7.6: note data-integrity (frontmatter/lead + fence-aware parse, symmetric mesh-merge convergence), one-bad-byte read hardening, guard false-positive fixes (freshness `-C`/compound forms, command-anchored forge rules, bare `>` side-effect, project-vs-global `.claude/`, negation-aware auth), guard crash-hardening, enforcement fail-open holes (adapter fail-closed, contentless-consult gate-dodge, secret-output `2>/dev/null` leak), atomic config/hook/backup writes, checkpoint/mesh/update/lint availability, and the migrate-hook data-loss. 714 tests + ruff + mypy green. Deferred items → #125–#131. Shipped in 3.7.6. - [x] **Codex CLI: `omind setup` only wired the guard, not the `omi` MCP server** ([GitHub #114](https://github.com/CryptoJones/omind/issues/114)) — _enhancement_ — `omind setup --agent codex` now also merges `[mcp_servers.omi]` into `~/.codex/config.toml` (via `tomlkit`, TOML round-trip preserved) so Codex can call the OMI memory tools directly, not just get blocked by the guard. `doctor --agent codex` reports `codex_mcp_registration` alongside `codex_guard`. Shipped in 3.7.0. diff --git a/CHANGELOG.md b/CHANGELOG.md index bdc7114..46b911e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **A derived hybrid search index (`omind.searchindex`).** One SQLite file per - vault, in the state dir, holding FTS5/BM25 over heading-split chunks, packed - `float32` chunk embeddings, and the resolved `[[wikilink]]` graph. Queries fuse + vault, in the state dir, holding FTS5/BM25 over heading-split chunks, quantized + int8 chunk embeddings, and the resolved `[[wikilink]]` graph. Queries fuse a keyword leg, a semantic leg, and a weak recency leg with Reciprocal Rank Fusion. Notes remain the source of truth; the index is disposable, machine-local, never committed and never mesh-synced, and refreshes only the notes whose bytes @@ -19,11 +19,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `omind bench` — measures index build/refresh, search latency (indexed *and* pre-index scan), capsule size, and the token cost of the listing payload, so retrieval performance is observed rather than asserted. +- `omind bench --quality` evaluates a labelled query set and reports recall@1, + recall@5, and mean reciprocal rank (MRR), including skipped targets and misses. - `omind search --explain` prints the fused score and per-leg ranks behind each hit; `omind reindex` gained `--index-only` and `--rebuild`. - `OMI_INDEX_DISABLE=1` turns the index off and restores the scanning search path. +- `omind doctor` reports FTS5 and semantic-search availability, the search + index's size and age, and stale, corrupt, or incompatible index files with the + `omind reindex --rebuild` repair command. +- `omind consolidate` writes near-duplicate merge proposals and editable drafts + to machine-local state without changing the vault. `--apply PLAN_ID` creates + the reviewed note through OmiStore and archives both sources only after their + versions are revalidated. +- The unified MCP `graph` tool selects `path`, `orphans`, `dangling`, or `stats` + with an `op` argument; list operations retain bounded pagination. ### Changed +- `graph-path`, `graph-orphans`, `graph-dangling`, and `graph-stats` remain as + deprecated compatibility aliases for one release while fleet clients migrate + to `graph`; their removal is tracked in #181. +- Hybrid-index reads within the same one-second burst reuse the last successful + refresh instead of statting every note again. OmiStore writes invalidate the + throttle immediately through the vault's write signal. +- Auto-generated journal, worklog, checkpoint, and rollup notes receive a modest + retrieval-score penalty, so equally relevant hand-curated memories rank first + without hiding the generated record. +- The fused retrieval top 20 are locally reranked against the whole matched + chunk body, reducing weak metadata/one-word matches in the result tail. The + bounded pass uses the existing embedding backend and fails open to RRF. +- Retrieval scope now follows query complexity: one-word lookups use fewer + candidates, hits, and excerpt characters; multi-hop questions receive a + larger search and context budget. +- Stored chunk embeddings are symmetrically quantized to int8 with per-vector + scales, shrinking the derived index; residual quantization error breaks ties + between effectively equal cosine scores. +- Notes can declare `Supersedes:` and `Superseded by:` metadata through + Markdown, CLI, or MCP. Superseded facts remain searchable history but rank + below current notes; mesh merges preserve the validity relationship. +- `omind lint` now reads top-level note/link state from the search index, + including a fence-stripped link view that preserves the no-false-code-link + rule. Near-duplicate detection uses mean chunk-vector similarity, with the + original title-Jaccard pass retained as the no-embedding fallback. +- Actual note reads update machine-local access frequency/recency state. + SessionStart promotes at most three recently/frequently recalled notes into a + bounded dynamic core and ages them out after 90 days. Generated, archived, + credential-looking, missing, and unsafe notes are excluded. - **Search is relevance-ranked, not substring-filtered.** `store.search` (and so `search-vault`, the web UI, and `omind search`) previously read and parsed every note on every query to run `needle in haystack`, then sorted the hits by *date*. @@ -48,6 +88,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 query, and pure-Python cosine loop are superseded by the chunk vectors in `omind.searchindex`; `nearest()` (the create-note dedup hint) moved across. +### Fixed +- Restore note parsing and strict typing in the hybrid search index after the + cyclic-import remediation removed required runtime references. + ## [4.2.3] - 2026-07-21 ### Fixed diff --git a/docs/retrieval.md b/docs/retrieval.md index 7b764e1..3d4a893 100644 --- a/docs/retrieval.md +++ b/docs/retrieval.md @@ -11,7 +11,7 @@ a disposable cache you can delete at any time. ▼ $XDG_STATE_HOME/omind/searchindex-.sqlite3 ├── chunks_fts FTS5 / BM25 over title, heading, tags, text, stems - ├── vectors float32 embeddings, one per chunk (optional) + ├── vectors int8 embeddings + per-vector scale (optional) ├── links resolved [[wikilinks]] └── notes identity, tags, created, archived flag ``` @@ -41,6 +41,9 @@ Two rules keep the result honest: - **Credential notes are de-prioritised** unless your query is itself about credentials — the same rule the consult gate applies. Search must never steer an agent toward the secrets notes. +- **Superseded facts remain history, not current truth.** A note carrying + `Superseded by:`—or targeted by another note's `Supersedes:` metadata—stays + searchable but receives a strong ranking penalty. The keyword leg is graded: chunks matching *every* word of your query rank above chunks matching only some. That keeps a filler word ("how do I **handle**…") @@ -57,6 +60,12 @@ Every hit carries a bounded **excerpt** — the matched text, snipped by FTS5 around your terms. That excerpt is usually enough to answer the question without opening the note at all, which is where the token savings come from. +An actual `read-note` or `recall-note` access updates a separate machine-local +frequency/recency counter. SessionStart uses that derived signal to promote at +most three earned notes into a bounded dynamic core; notes untouched for 90 days +age out. This state lives beside the index, never in the vault, and credential +or generated notes are never promoted. + ## Semantic search is optional Without the `[embed]` extra, the vector leg is simply skipped: BM25, recency, @@ -71,6 +80,23 @@ The model is `minishlab/potion-base-8M` (a ~30 MB static embedding — no GPU, n API, no network at query time). Override with `OMI_EMBED_MODEL`; changing it invalidates every stored vector and rebuilds. +## Reviewing near-duplicates + +The same chunk vectors power a guarded consolidation workflow: + +```bash +omind consolidate --limit 3 +# edit the reported machine-local .md draft, then: +omind consolidate --apply 0123456789abcdef +``` + +The first command does not edit the vault. It writes a JSON plan and an editable +Markdown draft under omind's machine-local state directory. Applying a plan +rechecks opaque content versions for both source notes, creates the reviewed +draft through `OmiStore`, then archives the originals with `Disabled: true`. +If either source changed during review, apply refuses the stale plan. It never +silently merges or hard-deletes memory. + ## Operating it ```bash @@ -95,7 +121,7 @@ lock — a broken index degrades search, it never breaks it. | --- | --- | --- | | `search "nebraska"` | 268 ms | 18 ms | | a natural-language question | 276 ms, **0 hits** | 45 ms, 10 ranked hits | -| full index build | — | 1.5 s (5,678 chunks, 19 MB) | +| full index build | — | 1.5 s (5,691 chunks, 13 MiB) | | incremental refresh | — | 5 ms | | `list-notes` tool payload | ~90,800 tokens | 3,136 tokens (one page) | diff --git a/src/omind/access.py b/src/omind/access.py new file mode 100644 index 0000000..16b6c47 --- /dev/null +++ b/src/omind/access.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Aaron K. Clark +"""Machine-local access statistics for dynamic core-memory selection.""" + +from __future__ import annotations + +import math +import sqlite3 +import time +from pathlib import Path + +from omind import paths + +_MAX_AGE_DAYS = 90 + + +def record(omi_dir: Path | str, filename: str, *, now: float | None = None) -> None: + """Record one actual note read. Advisory and fail-open.""" + path = paths.access_state_path(Path(omi_dir)) + try: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path, timeout=2.0) as db: + db.execute("PRAGMA busy_timeout=2000") + db.execute( + "CREATE TABLE IF NOT EXISTS access (" + "filename TEXT PRIMARY KEY, count INTEGER NOT NULL, last_access REAL NOT NULL)" + ) + db.execute( + "INSERT INTO access(filename, count, last_access) VALUES (?, 1, ?)" + " ON CONFLICT(filename) DO UPDATE SET" + " count = count + 1, last_access = excluded.last_access", + (filename, now if now is not None else time.time()), + ) + except (OSError, sqlite3.Error): + return + + +def core_members( + omi_dir: Path | str, + *, + limit: int = 3, + now: float | None = None, + max_age_days: int = _MAX_AGE_DAYS, +) -> list[str]: + """Best recently/frequently read notes, newest score first.""" + path = paths.access_state_path(Path(omi_dir)) + if limit <= 0 or not path.is_file(): + return [] + current = now if now is not None else time.time() + cutoff = current - max_age_days * 86_400 + try: + with sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True, timeout=1.0) as db: + rows = db.execute( + "SELECT filename, count, last_access FROM access WHERE last_access >= ?", + (cutoff,), + ) + scored = [ + ( + math.log1p(int(count)) + + 1.0 / (1.0 + max(0.0, current - float(last_access)) / 2_592_000), + float(last_access), + str(filename), + ) + for filename, count, last_access in rows + ] + except (OSError, sqlite3.Error, TypeError, ValueError): + return [] + scored.sort(key=lambda item: (-item[0], -item[1], item[2])) + return [filename for _score, _last_access, filename in scored[:limit]] diff --git a/src/omind/bench.py b/src/omind/bench.py index 8f0f65c..9430b08 100644 --- a/src/omind/bench.py +++ b/src/omind/bench.py @@ -34,6 +34,23 @@ "mesh sync conflict", ) +#: Small labelled set drawn from the durable notes present in CryptoJones's +#: reference vault. It is intentionally human-readable and version-controlled: +#: ranking changes should move these metrics, not merely "look better." +QUALITY_CASES = ( + ("where does long-term assistant memory live", "Omi Is The Memory.md"), + ( + "how does CryptoJones want the assistant to work", + "Working Preferences - How CryptoJones Wants Me to Operate.md", + ), + ("what voice and persona should Dix use", "Voice and Persona - Dix and Shelly.md"), + ( + "rules for working in git repos and handling secrets", + "Operational Rules - Git Repos and Secrets.md", + ), + ("how should durable memory notes be created", "Memory Workflow.md"), +) + @dataclass class Measurement: @@ -153,6 +170,51 @@ def run(omi_dir: Path | str, *, queries: tuple[str, ...] = SAMPLE_QUERIES) -> Re return report +def run_quality( + omi_dir: Path | str, + *, + cases: tuple[tuple[str, str], ...] = QUALITY_CASES, +) -> Report: + """Evaluate labelled query→expected-note pairs with recall@k and MRR.""" + from omind import searchindex + + omi = Path(omi_dir).expanduser() + report = Report(vault=str(omi)) + index = searchindex.SearchIndex(omi) + available = {path.name for path in omi.glob("*.md")} + evaluable = [(query, expected) for query, expected in cases if expected in available] + report.add("quality cases", len(evaluable), "count", f"{len(cases) - len(evaluable)} skipped") + if not evaluable: + report.add("recall@1", 0.0, "%", "no labelled target notes found") + report.add("recall@5", 0.0, "%", "no labelled target notes found") + report.add("MRR", 0.0, "score", "no labelled target notes found") + return report + + at_one = 0 + at_five = 0 + reciprocal = 0.0 + misses: list[str] = [] + for query, expected in evaluable: + hits = index.search(query, limit=50) or [] + ranked = [hit.filename for hit in hits] + try: + rank = ranked.index(expected) + 1 + except ValueError: + rank = 0 + at_one += rank == 1 + at_five += 0 < rank <= 5 + reciprocal += 1.0 / rank if rank else 0.0 + if not rank or rank > 5: + misses.append(f"{query!r}→{rank or 'miss'}") + + total = len(evaluable) + detail = "; ".join(misses[:3]) + report.add("recall@1", at_one * 100.0 / total, "%") + report.add("recall@5", at_five * 100.0 / total, "%", detail) + report.add("MRR", reciprocal / total, "score") + return report + + def _listing_tokens(notes: list[Any]) -> tuple[int, int]: """``(paged, unpaged)`` token estimate for the listing payload — the tool result that was ~87k tokens on a 744-note vault before it was paged.""" diff --git a/src/omind/cli.py b/src/omind/cli.py index 3457d09..b440850 100644 --- a/src/omind/cli.py +++ b/src/omind/cli.py @@ -15,6 +15,7 @@ * ``omind graph`` — query the [[wikilink]] knowledge graph (neighbors, path, orphans, dangling links, stats, export). * ``omind note`` — safely create/update one OMI note through OmiStore. + * ``omind consolidate`` — propose and explicitly apply reviewed note merges. * ``omind rollup`` — compact weeks of daily session journals into summaries. * ``omind backup`` — encrypted off-machine backup of the OMI folder (restic). """ @@ -23,6 +24,7 @@ import argparse import os +import shlex import sys from pathlib import Path @@ -320,6 +322,8 @@ def build_parser() -> argparse.ArgumentParser: ) note.add_argument("--tags", default="", help="comma-separated tags (no '#' needed)") note.add_argument("--related-to", default="", help="free-text 'related to' line") + note.add_argument("--supersedes", default="", help="older note this fact supersedes") + note.add_argument("--superseded-by", default="", help="newer note that supersedes this fact") note.add_argument("--connections", default="", help="comma-separated note titles to [[link]]") note.add_argument( "--connection", @@ -332,6 +336,25 @@ def build_parser() -> argparse.ArgumentParser: note.add_argument("--references", default="", help="comma-separated references") _add_vault_args(note) + consolidate = sub.add_parser( + "consolidate", + help="propose reviewed near-duplicate merges without changing the vault, " + "or explicitly apply one edited proposal", + ) + consolidate.add_argument( + "--limit", + type=int, + default=5, + help="maximum non-overlapping proposals to create (default: 5)", + ) + consolidate.add_argument( + "--apply", + metavar="PLAN_ID", + default=None, + help="apply one reviewed proposal, creating its draft and archiving its sources", + ) + _add_vault_args(consolidate) + search = sub.add_parser("search", help="search OMI notes from the terminal") search.add_argument( "query", help="free text; ranked by keyword (BM25) + semantic + recency relevance" @@ -355,6 +378,11 @@ def build_parser() -> argparse.ArgumentParser: bench.add_argument( "--query", default="", help="query to time (default: a built-in sample set)" ) + bench.add_argument( + "--quality", + action="store_true", + help="run the labelled retrieval-quality set and report recall@1, recall@5, and MRR", + ) bench.add_argument("--json", action="store_true", help="emit measurements as JSON") _add_vault_args(bench) @@ -641,11 +669,106 @@ def _run_quickstart(args: argparse.Namespace) -> int: def _diagnose_with_backup(config: SetupConfig) -> list[CheckResult]: - """The agent's wiring checks plus the backup and mesh checks.""" + """The agent's wiring checks plus index, backup, and mesh checks.""" from omind.backup import diagnose_backup from omind.mesh import diagnose_mesh - return diagnose_for(config) + diagnose_mesh(config) + diagnose_backup(config) + return ( + diagnose_for(config) + + _diagnose_search_index(config) + + diagnose_mesh(config) + + diagnose_backup(config) + ) + + +def _diagnose_search_index(config: SetupConfig) -> list[CheckResult]: + """Report whether hybrid retrieval is healthy, without mutating its cache.""" + from omind import embed, searchindex + + omi_dir = (config.vault / config.folder).expanduser() + health = searchindex.health(omi_dir) + fix = "run `omind reindex --rebuild`" + results: list[CheckResult] = [] + + if health.fts5: + state = "disabled by OMI_INDEX_DISABLE" if health.disabled else "available" + results.append(CheckResult("search_fts5", "ok", f"search index: FTS5 {state}")) + else: + results.append( + CheckResult( + "search_fts5", + "warn", + "search index: FTS5 unavailable; retrieval uses the scanning fallback", + ) + ) + + semantic = embed.status() + if semantic["available"]: + results.append( + CheckResult( + "search_semantic", + "ok", + f"semantic search: on (model {semantic['model']})", + ) + ) + else: + results.append( + CheckResult( + "search_semantic", + "ok", + f"semantic search: off (keyword path) — {semantic['reason']}", + ) + ) + + if not health.exists: + results.append( + CheckResult("search_index_file", "warn", f"search index has not been built; {fix}") + ) + elif health.corrupt: + results.append( + CheckResult( + "search_index_file", + "warn", + f"search index is corrupt or incompatible ({health.corrupt}); {fix}", + ) + ) + else: + age = _human_duration(health.age_seconds or 0.0) + size = _human_bytes(health.size_bytes) + detail = ( + f"search index: {size}, {age} old, {health.notes} notes, " + f"{health.vectors} vectors" + ) + if health.stale: + results.append( + CheckResult( + "search_index_file", + "warn", + f"{detail}; {health.stale} stale note(s); {fix}", + ) + ) + else: + results.append(CheckResult("search_index_file", "ok", detail)) + return results + + +def _human_bytes(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB"): + if value < 1024 or unit == "GiB": + return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} B" + value /= 1024 + raise AssertionError("unreachable") + + +def _human_duration(seconds: float) -> str: + if seconds < 60: + return f"{int(seconds)}s" + if seconds < 3600: + return f"{int(seconds // 60)}m" + if seconds < 86400: + return f"{int(seconds // 3600)}h" + return f"{int(seconds // 86400)}d" def _run_doctor(args: argparse.Namespace) -> int: @@ -956,8 +1079,11 @@ def _run_bench(args: argparse.Namespace) -> int: from omind import bench omi_dir = (args.vault / args.folder).expanduser() - queries = (args.query,) if args.query else bench.SAMPLE_QUERIES - report = bench.run(omi_dir, queries=queries) + if args.quality: + report = bench.run_quality(omi_dir) + else: + queries = (args.query,) if args.query else bench.SAMPLE_QUERIES + report = bench.run(omi_dir, queries=queries) print(json.dumps(report.to_dict(), indent=2) if args.json else report.format()) return 0 @@ -1139,6 +1265,8 @@ def _run_note(args: argparse.Namespace) -> int: details=details.strip(), tags=_split_csv(args.tags), related_to=args.related_to.strip(), + supersedes=args.supersedes.strip(), + superseded_by=args.superseded_by.strip(), # CSV titles plus any repeatable --connection (exact titles, comma-safe). connections=( _split_csv(args.connections) + [c.strip() for c in args.connection if c.strip()] @@ -1221,6 +1349,39 @@ def _run_rollup(args: argparse.Namespace) -> int: return 0 +def _run_consolidate(args: argparse.Namespace) -> int: + from omind.consolidate import ConsolidationError, apply, propose + + omi_dir = (args.vault / args.folder).expanduser() + try: + if args.apply is not None: + result = apply(omi_dir, args.apply) + print(f"created {result.filename}") + print("archived " + ", ".join(result.archived)) + return 0 + proposals = propose(omi_dir, limit=args.limit) + except ConsolidationError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if not proposals: + print("no near-duplicate notes found") + return 0 + print(f"created {len(proposals)} review proposal(s); the vault is unchanged") + for proposal in proposals: + left, right = proposal.sources + print( + f"{proposal.plan_id} {left.filename} + {right.filename} " + f"({proposal.similarity:.0%} similar)" + ) + print(f" draft: {proposal.draft_path}") + print( + f" apply: omind consolidate --apply {proposal.plan_id} " + f"--vault {shlex.quote(str(args.vault))} " + f"--folder {shlex.quote(args.folder)}" + ) + return 0 + + def _run_hook(args: argparse.Namespace) -> int: omi_dir = (args.vault / args.folder).expanduser() return run_hook(args.event, omi_dir) # always 0; must never block the agent @@ -1322,6 +1483,8 @@ def main(argv: list[str] | None = None) -> int: return _run_convert(args) if args.command == "note": return _run_note(args) + if args.command == "consolidate": + return _run_consolidate(args) if args.command == "rollup": return _run_rollup(args) if args.command == "hook": diff --git a/src/omind/consolidate.py b/src/omind/consolidate.py new file mode 100644 index 0000000..41f1ec0 --- /dev/null +++ b/src/omind/consolidate.py @@ -0,0 +1,267 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Aaron K. Clark +"""Reviewed consolidation of near-duplicate OMI notes. + +Proposals and editable drafts are machine-local derived state. The Markdown +vault changes only when an operator explicitly applies a proposal. +""" + +from __future__ import annotations + +import json +import re +import secrets +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from omind.lint import lint_vault +from omind.paths import atomic_write_text, consolidation_dir +from omind.store import ( + ActionItem, + NoteConflictError, + NoteError, + NoteFields, + OmiStore, + parse_note, + render_fields, +) + +_PLAN_ID_RE = re.compile(r"^[0-9a-f]{16}$") +_PERCENT_RE = re.compile(r"(\d+)%") + + +class ConsolidationError(Exception): + """A proposal cannot be safely created or applied.""" + + +@dataclass(frozen=True) +class Source: + filename: str + version: str + + +@dataclass(frozen=True) +class Proposal: + plan_id: str + sources: tuple[Source, Source] + similarity: float + plan_path: Path + draft_path: Path + + +@dataclass(frozen=True) +class ApplyResult: + filename: str + archived: tuple[str, str] + + +def _candidate_pairs(omi_dir: Path) -> list[tuple[str, str, float]]: + """Use lint's semantic detector and its title/periodic-series safeguards.""" + pairs: list[tuple[str, str, float]] = [] + for issue in lint_vault(omi_dir): + if issue.kind != "near-duplicate": + continue + names = issue.note.split(" | ", 1) + if len(names) != 2: + continue + match = _PERCENT_RE.search(issue.detail) + score = int(match.group(1)) / 100 if match else 0.0 + pairs.append((names[0], names[1], score)) + return sorted(pairs, key=lambda row: (-row[2], row[0].lower(), row[1].lower())) + + +def _unique(values: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for value in values: + clean = value.strip() + key = clean.casefold() + if clean and key not in seen: + seen.add(key) + result.append(clean) + return result + + +def _draft_fields( + left_name: str, + left: NoteFields, + right_name: str, + right: NoteFields, +) -> NoteFields: + """Build a lossless-by-default editable draft from two parsed notes.""" + left_title = left.title.strip() or Path(left_name).stem + right_title = right.title.strip() or Path(right_name).stem + summaries = _unique([left.summary, right.summary]) + + def source_body(fields: NoteFields) -> str: + parts = [fields.lead.strip(), fields.details.strip()] + if fields.supersedes: + parts.append(f"Supersedes: {fields.supersedes}") + if fields.superseded_by: + parts.append(f"Superseded by: {fields.superseded_by}") + return "\n\n".join(part for part in parts if part) or "_No detail body._" + + details = "\n\n".join( + ( + f"### From [[{Path(left_name).stem}]]\n\n" + source_body(left), + f"### From [[{Path(right_name).stem}]]\n\n" + source_body(right), + ) + ) + extras: dict[str, list[str]] = {} + for filename, fields in ((left_name, left), (right_name, right)): + label = Path(filename).stem + for heading, lines in fields.extras.items(): + extras[f"{heading} — from {label}"] = list(lines) + if fields.frontmatter.strip(): + extras[f"Source metadata — from {label}"] = [ + "```yaml", + *fields.frontmatter.strip().splitlines(), + "```", + ] + actions = [ + ActionItem(text=item.text, done=item.done) + for item in [*left.action_items, *right.action_items] + ] + return NoteFields( + title=f"Consolidated — {left_title} + {right_title}", + summary=" ".join(summaries), + details=details, + tags=_unique([*left.tags, *right.tags, "consolidated"]), + related_to="; ".join(_unique([left.related_to, right.related_to])), + connections=_unique( + [ + *left.connections, + *right.connections, + Path(left_name).stem, + Path(right_name).stem, + ] + ), + action_items=actions, + references=_unique([*left.references, *right.references]), + extras=extras, + okf_type=left.okf_type or right.okf_type, + ) + + +def _proposal_paths(omi_dir: Path, plan_id: str) -> tuple[Path, Path]: + root = consolidation_dir(omi_dir) + return root / f"{plan_id}.json", root / f"{plan_id}.md" + + +def propose(omi_dir: Path | str, *, limit: int = 5) -> list[Proposal]: + """Write at most ``limit`` non-overlapping review plans; never touch notes.""" + if limit < 1: + raise ConsolidationError("limit must be at least 1") + omi = Path(omi_dir).expanduser().resolve() + store = OmiStore(omi) + selected: list[tuple[str, str, float]] = [] + used: set[str] = set() + for left, right, score in _candidate_pairs(omi): + # Route every candidate through safe_name and skip stale/archived rows. + left_path = store.safe_name(left) + right_path = store.safe_name(right) + if left in used or right in used or not left_path.is_file() or not right_path.is_file(): + continue + left_fields = store.read_fields(left) + right_fields = store.read_fields(right) + if left_fields.disabled or right_fields.disabled: + continue + selected.append((left, right, score)) + used.update((left, right)) + if len(selected) >= limit: + break + + proposals: list[Proposal] = [] + for left, right, score in selected: + plan_id = secrets.token_hex(8) + plan_path, draft_path = _proposal_paths(omi, plan_id) + left_fields = store.read_fields(left) + right_fields = store.read_fields(right) + sources = ( + Source(left, store.note_version(left)), + Source(right, store.note_version(right)), + ) + draft = _draft_fields(left, left_fields, right, right_fields) + atomic_write_text(draft_path, render_fields(draft)) + payload: dict[str, Any] = { + "version": 1, + "plan_id": plan_id, + "vault": str(omi), + "similarity": score, + "draft": draft_path.name, + "sources": [asdict(source) for source in sources], + } + atomic_write_text(plan_path, json.dumps(payload, indent=2, sort_keys=True) + "\n") + proposals.append(Proposal(plan_id, sources, score, plan_path, draft_path)) + return proposals + + +def _load_plan(omi: Path, plan_id: str) -> tuple[dict[str, Any], Path]: + if not _PLAN_ID_RE.fullmatch(plan_id): + raise ConsolidationError("plan id must be exactly 16 lowercase hexadecimal characters") + plan_path, expected_draft = _proposal_paths(omi, plan_id) + try: + payload = json.loads(plan_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ConsolidationError(f"proposal not found: {plan_id}") from exc + except (OSError, json.JSONDecodeError) as exc: + raise ConsolidationError(f"cannot read proposal {plan_id}: {exc}") from exc + if not isinstance(payload, dict): + raise ConsolidationError("proposal metadata is invalid") + if payload.get("version") != 1 or payload.get("plan_id") != plan_id: + raise ConsolidationError("proposal metadata is invalid") + if payload.get("vault") != str(omi): + raise ConsolidationError("proposal belongs to a different vault") + if payload.get("draft") != expected_draft.name: + raise ConsolidationError("proposal draft path is invalid") + sources = payload.get("sources") + if not isinstance(sources, list) or len(sources) != 2: + raise ConsolidationError("proposal must name exactly two sources") + return payload, expected_draft + + +def apply(omi_dir: Path | str, plan_id: str) -> ApplyResult: + """Create the reviewed draft, then archive both unchanged source notes.""" + omi = Path(omi_dir).expanduser().resolve() + payload, draft_path = _load_plan(omi, plan_id) + store = OmiStore(omi) + sources: list[Source] = [] + for raw in payload["sources"]: + if not isinstance(raw, dict): + raise ConsolidationError("proposal source metadata is invalid") + source = Source(str(raw.get("filename", "")), str(raw.get("version", ""))) + try: + store.safe_name(source.filename) + except NoteError as exc: + raise ConsolidationError(f"proposal source is invalid: {exc}") from exc + if not source.version or store.note_version(source.filename) != source.version: + raise ConsolidationError( + f"source changed since review: {source.filename}; generate a new proposal" + ) + sources.append(source) + if sources[0].filename.casefold() == sources[1].filename.casefold(): + raise ConsolidationError("proposal sources must be two different notes") + try: + draft = parse_note(draft_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ConsolidationError(f"proposal draft is missing: {draft_path}") from exc + except OSError as exc: + raise ConsolidationError(f"cannot read proposal draft: {exc}") from exc + if not draft.title.strip(): + raise ConsolidationError("reviewed draft requires a title") + draft.rev = "" + draft.disabled = False + + try: + filename = store.create_and_disable_sources( + draft, + [(source.filename, source.version) for source in sources], + ) + except NoteConflictError as exc: + raise ConsolidationError( + f"{exc}; generate a new proposal before applying" + ) from exc + except NoteError as exc: + raise ConsolidationError(str(exc)) from exc + return ApplyResult(filename, (sources[0].filename, sources[1].filename)) diff --git a/src/omind/hooks.py b/src/omind/hooks.py index 5c46636..7c00fd1 100644 --- a/src/omind/hooks.py +++ b/src/omind/hooks.py @@ -57,6 +57,7 @@ # OMI and are recalled just-in-time from the user prompt instead of being copied # wholesale into every conversation. PRIMING_FILES = ("index.md", "Playbook.md", "Memory Workflow.md", "CLAUDE CODE PERSONALITY.md") +_DYNAMIC_CORE_LIMIT = 3 _PRIMING_FILE_CHAR_CAP = 16_000 # per-file guard so a runaway note can't flood context # Dynamic priming: include a session handoff only when its name matches the @@ -470,6 +471,42 @@ def build_session_start_context( if digest: sections.append(f"===== OMI capsule: {name} =====\n{digest}") + # A tiny earned core: actual recent/frequent recalls, never search results. + # Fixed operational/persona notes above remain pinned; generated and + # credential-looking notes are never promoted into always-on context. + try: + from omind import access, retrieve + from omind.store import OmiStore, parse_note + + pinned = {name.casefold() for name in PRIMING_FILES} + store = OmiStore(directory) + added = 0 + for name in access.core_members(directory, limit=_DYNAMIC_CORE_LIMIT * 4): + if added >= _DYNAMIC_CORE_LIMIT: + break + if name.casefold() in pinned or name.casefold().startswith( + ("session journal", "worklog ") + ): + continue + try: + path = store.safe_name(name) + except Exception: + continue + body = _read_priming_note(path) + if body is None: + continue + fields = parse_note(body) + if fields.disabled or retrieve._looks_credential(fields.title, " ".join(fields.tags)): + continue + digest = _note_digest(body, 800) + if digest: + sections.append( + f"===== OMI capsule: {name} (dynamic core) =====\n{digest}" + ) + added += 1 + except Exception: + pass + state_path = _select_session_state(directory, cwd) if state_path is not None: body = _read_priming_note(state_path) diff --git a/src/omind/lint.py b/src/omind/lint.py index 00850a7..aa17dd5 100644 --- a/src/omind/lint.py +++ b/src/omind/lint.py @@ -30,6 +30,7 @@ import re from dataclasses import dataclass from pathlib import Path +from typing import Any from omind.paths import RESERVED_FILENAMES from omind.store import _FENCE_RE, _WIKILINK_RE, NoteFields, parse_note @@ -168,9 +169,53 @@ def _load(omi_dir: Path | str) -> tuple[list[_Note], set[str]]: return notes, known_extra +def _load_indexed(omi_dir: Path | str) -> tuple[list[_Note], set[str], Any] | None: + """Use the derived index for top-level notes/links; scan only subfolders.""" + from omind import searchindex + + index = searchindex.shared(omi_dir) + if index is None: + return None + payload = index.lint_rows() + if payload is None: + return None + rows, links = payload + outbound: dict[str, set[str]] = {} + for source, target in links: + outbound.setdefault(source, set()).add(target) + omi = Path(omi_dir) + notes: list[_Note] = [] + known_extra: set[str] = set() + for row in rows: + fields = NoteFields( + title=row.title if row.has_title else "", + disabled=row.disabled, + ) + path = omi / row.filename + ids = _note_ids(path, fields) + if row.disabled: + known_extra |= ids + continue + notes.append(_Note(path, fields, outbound.get(row.filename, set()), frozenset(ids))) + for path in sorted(omi.rglob("*.md")): + if path.parent == omi or path.name.startswith("."): + continue + try: + fields = parse_note(path.read_text(encoding="utf-8", errors="replace")) + except OSError: + continue + known_extra |= _note_ids(path, fields) + return notes, known_extra, index + + def lint_vault(omi_dir: Path | str) -> list[LintIssue]: """Every problem found in the vault, ordered error → warn → info then by note.""" - notes, known_extra = _load(omi_dir) + indexed = _load_indexed(omi_dir) + if indexed is None: + notes, known_extra = _load(omi_dir) + index = None + else: + notes, known_extra, index = indexed # Every identifier any note can be linked by (+ reserved stems, archived # notes, and subfolder notes, which are legitimate link targets). known = {stem for path in RESERVED_FILENAMES for stem in (Path(path).stem.lower(),)} @@ -199,20 +244,35 @@ def lint_vault(omi_dir: Path | str) -> list[LintIssue]: LintIssue("isolated", "info", n.path.name, "no inbound or outbound links") ) - # Near-duplicate titles — each unordered pair reported once. - toks = [(_title_tokens(n.fields.title or n.path.stem), n.fields.title or n.path.stem, n) - for n in notes] - for i in range(len(toks)): - for j in range(i + 1, len(toks)): - if _jaccard(toks[i][0], toks[j][0]) < _NEAR_DUP: + # Semantic duplicate candidates from the index; title-Jaccard is the + # fail-open path when the optional embedding backend is absent. + semantic = index.duplicate_pairs() if index is not None else None + titles = {n.path.name: n.fields.title or n.path.stem for n in notes} + if semantic is not None: + for a, b, score in semantic: + if a not in titles or b not in titles: + continue + if _is_periodic_series(titles[a], titles[b]): continue - if _is_periodic_series(toks[i][1], toks[j][1]): - continue # "Worklog 2026-06-29" vs "…-30": a dated series, not a dupe - score = _jaccard(toks[i][0], toks[j][0]) - a, b = sorted((toks[i][2].path.name, toks[j][2].path.name)) issues.append( - LintIssue("near-duplicate", "info", f"{a} | {b}", f"titles {score:.0%} similar") + LintIssue("near-duplicate", "info", f"{a} | {b}", f"content {score:.0%} similar") ) + else: + toks = [ + (_title_tokens(n.fields.title or n.path.stem), n.fields.title or n.path.stem, n) + for n in notes + ] + for i in range(len(toks)): + for j in range(i + 1, len(toks)): + score = _jaccard(toks[i][0], toks[j][0]) + if score < _NEAR_DUP or _is_periodic_series(toks[i][1], toks[j][1]): + continue + a, b = sorted((toks[i][2].path.name, toks[j][2].path.name)) + issues.append( + LintIssue( + "near-duplicate", "info", f"{a} | {b}", f"titles {score:.0%} similar" + ) + ) rank = {"error": 0, "warn": 1, "info": 2} issues.sort(key=lambda x: (rank.get(x.severity, 9), x.kind, x.note)) diff --git a/src/omind/merge.py b/src/omind/merge.py index c902122..21e447d 100644 --- a/src/omind/merge.py +++ b/src/omind/merge.py @@ -266,6 +266,8 @@ def scalar(name: str) -> str | bool: created=str(scalar("created")), tags=_union3(base.tags, ours.tags, theirs.tags), related_to=str(scalar("related_to")), + supersedes=str(scalar("supersedes")), + superseded_by=str(scalar("superseded_by")), connections=_union3(base.connections, ours.connections, theirs.connections), action_items=_merge_actions(base.action_items, ours.action_items, theirs.action_items), references=_union3(base.references, ours.references, theirs.references), diff --git a/src/omind/paths.py b/src/omind/paths.py index a92aee1..0d9cf0a 100644 --- a/src/omind/paths.py +++ b/src/omind/paths.py @@ -101,3 +101,13 @@ def sync_signal_path(omi_dir: Path) -> Path: def sync_state_path(omi_dir: Path) -> Path: """Where `omind mesh sync` records its last outcome (read by doctor).""" return state_dir() / f"mesh-sync-{_omi_dir_digest(omi_dir)}.json" + + +def access_state_path(omi_dir: Path) -> Path: + """Machine-local access-frequency state for dynamic core-memory selection.""" + return state_dir() / f"access-{_omi_dir_digest(omi_dir)}.sqlite3" + + +def consolidation_dir(omi_dir: Path) -> Path: + """Machine-local proposal/draft storage for one vault's reviewed merges.""" + return state_dir() / f"consolidate-{_omi_dir_digest(omi_dir)}" diff --git a/src/omind/recall.py b/src/omind/recall.py index ba3c40f..65ee01b 100644 --- a/src/omind/recall.py +++ b/src/omind/recall.py @@ -69,6 +69,10 @@ def compact_recall( """Read one note without returning raw/parsed duplicate representations.""" store = OmiStore(omi_dir) raw = store.read_note(name) + filename = store.safe_name(name).name + from omind import access + + access.record(store.omi_dir, filename) fields = parse_note(raw) selected = _section(raw, section) if section else "" content = selected or _memory_text(fields) @@ -78,8 +82,8 @@ def compact_recall( marker = "\n…[truncated; request a section or a larger max_chars value]" content = content[: max(0, limit - len(marker))].rstrip() + marker return { - "filename": store.safe_name(name).name, - "title": fields.title or store.safe_name(name).stem, + "filename": filename, + "title": fields.title or Path(filename).stem, "summary": fields.summary, "content": content, "section": section if selected else "", diff --git a/src/omind/searchindex.py b/src/omind/searchindex.py index 2beb92f..8827824 100644 --- a/src/omind/searchindex.py +++ b/src/omind/searchindex.py @@ -19,7 +19,7 @@ * **BM25** through SQLite's stdlib FTS5 module, over title/heading/tags/text plus a *stems* column fed by :func:`omind.retrieve._stem`, so the existing stemmer still earns its keep and ``scoring``/``scored`` match ``score``. -* **Dense vectors** — packed ``float32`` BLOBs scored with one numpy matmul, +* **Dense vectors** — quantized ``int8`` BLOBs scored with one numpy matmul, replacing the old JSON-float-list store and its pure-Python ``sum(x*y)`` loop over every entry. Written only when :func:`omind.embed.available`. * **Reciprocal Rank Fusion** of the keyword, vector, and recency rankings. @@ -47,6 +47,7 @@ import functools import hashlib import os +import re import sqlite3 import threading import time @@ -58,7 +59,7 @@ from omind import paths #: Bumped whenever the schema below changes shape; a mismatch rebuilds from scratch. -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 4 #: RRF constant. 60 is the value from the original TREC paper and what every @@ -71,6 +72,13 @@ _W_KEYWORD = 1.0 _W_VECTOR = 0.9 _W_RECENCY = 0.25 +#: Auto-generated journals/worklogs are useful evidence but broad and numerous. +#: A modest multiplier lets a hand-curated note with comparable content win. +_GENERATED_WEIGHT = 0.75 +#: Superseded notes remain searchable history, but should not present as current. +_SUPERSEDED_WEIGHT = 0.35 +#: Only the fused head pays the extra local embedding pass. +_RERANK_DEPTH = 20 #: How many candidates each leg contributes before fusion. _LEG_DEPTH = 60 #: Soft cap on one indexed chunk. Long ``## Details`` bodies are split further so @@ -80,6 +88,10 @@ #: reads. Bounded here because "return excerpts, not documents" is where the #: token savings in the literature (Memori, SimpleMem) come from. EXCERPT_CHARS = 180 +#: A burst of reads should pay the full-vault stat sweep once. Direct writes +#: invalidate immediately through ``sync_signal_path``; edits made outside +#: OmiStore are still discovered after this short bound. +_REFRESH_THROTTLE_SECONDS = 1.0 _MODEL_ENV = "OMI_EMBED_MODEL" #: Set to disable the index entirely and fall back to the scanning search paths. @@ -132,6 +144,29 @@ class Refresh: seconds: float = 0.0 +@dataclass +class Health: + """Read-only diagnostic snapshot for one derived search index.""" + + fts5: bool + disabled: bool + path: Path + exists: bool = False + size_bytes: int = 0 + age_seconds: float | None = None + notes: int = 0 + vectors: int = 0 + stale: int = 0 + corrupt: str = "" + + +@dataclass(frozen=True) +class _Scope: + depth: int + limit: int + excerpt_chars: int + + @dataclass class _Chunk: heading: str @@ -147,6 +182,9 @@ class _NoteRow: title: str created: str okf_type: str = "" + supersedes: str = "" + superseded_by: str = "" + has_title: bool = True tags: list[str] = field(default_factory=list) disabled: bool = False @@ -157,6 +195,9 @@ class _NoteRow: title TEXT NOT NULL DEFAULT '', created TEXT NOT NULL DEFAULT '', okf_type TEXT NOT NULL DEFAULT '', + supersedes TEXT NOT NULL DEFAULT '', + superseded_by TEXT NOT NULL DEFAULT '', + has_title INTEGER NOT NULL DEFAULT 1, disabled INTEGER NOT NULL DEFAULT 0, mtime_ns INTEGER NOT NULL DEFAULT 0, size INTEGER NOT NULL DEFAULT 0, @@ -183,12 +224,15 @@ class _NoteRow: ); CREATE TABLE IF NOT EXISTS vectors ( chunk_id INTEGER PRIMARY KEY, - vec BLOB NOT NULL + vec BLOB NOT NULL, + scale REAL NOT NULL, + residual REAL NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS links ( src TEXT NOT NULL, target TEXT NOT NULL, -- lowercased, for resolution - raw TEXT NOT NULL -- as written, for reporting a dangling link + raw TEXT NOT NULL, -- as written, for reporting a dangling link + lint INTEGER NOT NULL DEFAULT 1 -- false when the link occurs only in code ); CREATE INDEX IF NOT EXISTS links_src ON links(src); CREATE INDEX IF NOT EXISTS links_target ON links(target); @@ -232,6 +276,80 @@ def index_path(omi_dir: Path | str) -> Path: return paths.state_dir() / f"searchindex-{_vault_id(omi_dir)}.sqlite3" +def health(omi_dir: Path | str) -> Health: + """Inspect an index without creating, refreshing, or repairing it. + + Doctor must be able to diagnose a corrupt cache without going through + :meth:`SearchIndex._connect`, whose normal query-path behaviour deliberately + hides errors and fails open. + """ + omi = Path(omi_dir) + path = index_path(omi) + result = Health( + fts5=_fts5_available(), + disabled=bool(os.environ.get(DISABLE_ENV)), + path=path, + exists=path.is_file(), + ) + if not result.exists: + return result + + files = [candidate for candidate in (path, Path(f"{path}-wal")) if candidate.is_file()] + try: + stats = [candidate.stat() for candidate in files] + result.size_bytes = sum(item.st_size for item in stats) + result.age_seconds = max(0.0, time.time() - max(item.st_mtime for item in stats)) + except OSError as exc: + result.corrupt = f"cannot stat index: {exc}" + return result + + try: + uri = f"{path.resolve().as_uri()}?mode=ro" + with contextlib.closing(sqlite3.connect(uri, uri=True, timeout=1.0)) as db: + db.row_factory = sqlite3.Row + check = db.execute("PRAGMA quick_check").fetchone() + if check is None or str(check[0]).lower() != "ok": + result.corrupt = f"SQLite quick_check: {check[0] if check else 'no result'}" + return result + schema = db.execute("SELECT value FROM meta WHERE key = 'schema'").fetchone() + model = db.execute("SELECT value FROM meta WHERE key = 'model'").fetchone() + if schema is None or str(schema[0]) != str(SCHEMA_VERSION): + result.corrupt = "search-index schema is missing or obsolete" + return result + from omind import embed + + expected_model = os.environ.get(_MODEL_ENV) or embed._DEFAULT_MODEL + if model is None or str(model[0]) != expected_model: + result.corrupt = "search-index embedding model does not match configuration" + return result + rows = { + str(row["filename"]): (int(row["mtime_ns"]), int(row["size"])) + for row in db.execute("SELECT filename, mtime_ns, size FROM notes") + } + result.notes = len(rows) + result.vectors = int(db.execute("SELECT count(*) FROM vectors").fetchone()[0]) + except (sqlite3.Error, OSError, ValueError) as exc: + result.corrupt = str(exc) + return result + + seen: set[str] = set() + if omi.is_dir(): + from omind.store import _is_reserved + + for note in omi.glob("*.md"): + if _is_reserved(note.name) or note.name.startswith("."): + continue + try: + stat = note.stat() + except OSError: + continue + seen.add(note.name) + if rows.get(note.name) != (stat.st_mtime_ns, stat.st_size): + result.stale += 1 + result.stale += len(set(rows) - seen) + return result + + def _stems_of(text: str) -> str: """The stem bag indexed alongside raw text, from omind's own stemmer.""" from omind import retrieve @@ -282,10 +400,31 @@ def _fts_query(query: str, *, require_all: bool = False) -> str: return " AND ".join(f"({part})" for part in joined) -def _pack(vec: Sequence[float]) -> bytes: +def _scope(query: str, requested_limit: int) -> _Scope: + """Retrieval depth/output budget derived from query complexity.""" + word_count = len(_query_words(query)) + low = query.lower() + multi_hop = word_count >= 3 and any( + marker in low for marker in ("why ", "how ", "compare ", "relationship", "because") + ) + if word_count <= 1: + return _Scope(depth=20, limit=min(requested_limit, 5), excerpt_chars=120) + if word_count >= 5 or multi_hop: + return _Scope(depth=90, limit=min(requested_limit, 25), excerpt_chars=240) + return _Scope(depth=_LEG_DEPTH, limit=min(requested_limit, 10), excerpt_chars=EXCERPT_CHARS) + + +def _quantize(vec: Sequence[float]) -> tuple[bytes, float, float]: + """Symmetric int8 vector plus per-row scale and quantization residual.""" import numpy as np - return bytes(np.asarray(vec, dtype="float32").tobytes()) + source = np.asarray(vec, dtype="float32") + peak = float(np.max(np.abs(source))) if source.size else 0.0 + scale = peak / 127.0 if peak else 1.0 + packed = np.clip(np.rint(source / scale), -127, 127).astype("int8") + restored = packed.astype("float32") * scale + residual = float(np.linalg.norm(source - restored)) + return bytes(packed.tobytes()), scale, residual def _query_vector(text: str) -> Any: @@ -321,7 +460,9 @@ def __init__(self, omi_dir: Path | str, *, model: str | None = None) -> None: self._db: sqlite3.Connection | None = None # Per-process cache of the packed vector matrix, keyed by the index # generation so a refresh (here or in another process) invalidates it. - self._matrix: tuple[str, list[int], Any] | None = None + self._matrix: tuple[str, list[int], Any, Any] | None = None + self._last_refresh_at = 0.0 + self._last_signature: tuple[int, int] | None = None # One connection shared across threads (the web app serves requests on a # thread pool), so every public entry point serializes on this lock — # sqlite3 connections are not safe to use concurrently. @@ -408,6 +549,28 @@ def _note_paths(self) -> Iterator[Path]: continue yield path + def _cheap_signature(self) -> tuple[int, int]: + """Directory/write-signal mtimes that cheaply catch normal vault writes.""" + signal = paths.sync_signal_path(self.omi_dir) + values: list[int] = [] + for path in (self.omi_dir, signal): + try: + values.append(path.stat().st_mtime_ns) + except OSError: + values.append(-1) + return values[0], values[1] + + def _refresh_if_needed(self) -> Refresh | None: + """Refresh once per burst, or immediately after an OmiStore write.""" + now = time.monotonic() + signature = self._cheap_signature() + if ( + self._last_signature == signature + and now - self._last_refresh_at < _REFRESH_THROTTLE_SECONDS + ): + return Refresh() + return self.refresh() + @_locked def refresh(self, *, vectors: bool = True) -> Refresh | None: """Bring the index in line with the notes on disk. @@ -454,6 +617,8 @@ def refresh(self, *, vectors: bool = True) -> Refresh | None: db.execute("COMMIT") stats.seconds = time.perf_counter() - started self._matrix = None + self._last_signature = self._cheap_signature() + self._last_refresh_at = time.monotonic() return stats except sqlite3.Error: with contextlib.suppress(sqlite3.Error): @@ -487,18 +652,24 @@ def _ingest( # Derived when undeclared, the same rule render_fields applies, so a # graph node built from the index always carries a non-empty type. okf_type=fields.okf_type.strip() or derive_okf_type(fields.tags), + supersedes=fields.supersedes, + superseded_by=fields.superseded_by, + has_title=bool(fields.title), tags=fields.tags, disabled=fields.disabled, ) self._forget(db, path.name) db.execute( - "INSERT INTO notes(filename, title, created, okf_type, disabled, mtime_ns, size, sha)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO notes(filename, title, created, okf_type, supersedes, superseded_by," + " has_title, disabled, mtime_ns, size, sha) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( row.filename, row.title, row.created, row.okf_type, + row.supersedes, + row.superseded_by, + int(row.has_title), int(row.disabled), st.st_mtime_ns, st.st_size, @@ -509,9 +680,13 @@ def _ingest( "INSERT INTO note_tags(filename, tag) VALUES (?, ?)", [(row.filename, tag.lower()) for tag in row.tags], ) + lint_targets = {target.lower() for target in lint_link_targets(text)} db.executemany( - "INSERT INTO links(src, target, raw) VALUES (?, ?, ?)", - [(row.filename, raw.lower(), raw) for raw in link_targets(text)], + "INSERT INTO links(src, target, raw, lint) VALUES (?, ?, ?, ?)", + [ + (row.filename, raw.lower(), raw, int(raw.lower() in lint_targets)) + for raw in link_targets(text) + ], ) tag_text = " ".join(row.tags) pending: list[tuple[int, str]] = [] @@ -558,9 +733,14 @@ def _embed_chunks(self, db: sqlite3.Connection, pending: list[tuple[int, str]]) if vecs is None: return 0 try: + rows = [ + (cid, *_quantize(row)) + for (cid, _text), row in zip(pending, vecs, strict=True) + ] db.executemany( - "INSERT OR REPLACE INTO vectors(chunk_id, vec) VALUES (?, ?)", - [(cid, _pack(row)) for (cid, _text), row in zip(pending, vecs, strict=True)], + "INSERT OR REPLACE INTO vectors(chunk_id, vec, scale, residual)" + " VALUES (?, ?, ?, ?)", + rows, ) except (sqlite3.Error, ValueError, ImportError): return 0 @@ -585,7 +765,7 @@ def search( db = self._connect() if db is None: return None - if self.refresh() is None and not self._has_rows(db): + if self._refresh_if_needed() is None and not self._has_rows(db): return None # nothing indexed and we couldn't build it try: allowed = self._candidates(db, tag=tag, include_disabled=include_disabled) @@ -593,26 +773,34 @@ def search( return [] if not query.strip(): return self._listing(db, allowed, limit) + scope = _scope(query, limit) expr = _fts_query(query) # the OR form; also snips the excerpts - keyword = self._keyword_leg(db, query, allowed) - vector = self._vector_leg(db, query, allowed) + keyword = self._keyword_leg(db, query, allowed, scope.depth) + vector = self._vector_leg(db, query, allowed, scope.depth) matched = set(keyword) | set(vector) if not matched: return [] # Recency re-ranks what the content legs matched; it never *adds* a # note. Left unrestricted it made every query return the whole vault # newest-first — the exact failure this index exists to end. - recency = [cid for cid in self._recency_leg(db, allowed) if cid in matched] + recency = [ + cid for cid in self._recency_leg(db, allowed, scope.depth) if cid in matched + ] fused = _fuse( [(_W_KEYWORD, keyword), (_W_VECTOR, vector), (_W_RECENCY, recency)] ) + fused = self._rerank(db, query, fused) + fused = self._weight_generated(db, fused) + fused = self._weight_superseded(db, fused) if not fused: return [] ranks = { "keyword": {cid: i + 1 for i, cid in enumerate(keyword)}, "vector": {cid: i + 1 for i, cid in enumerate(vector)}, } - return self._materialize(db, query, expr, fused, ranks, limit) + return self._materialize( + db, query, expr, fused, ranks, scope.limit, scope.excerpt_chars + ) except (sqlite3.Error, ValueError): return None @@ -638,7 +826,7 @@ def _candidates( return {str(r["filename"]) for r in db.execute(sql, args)} def _keyword_leg( - self, db: sqlite3.Connection, query: str, allowed: set[str] | None + self, db: sqlite3.Connection, query: str, allowed: set[str] | None, depth: int ) -> list[int]: """BM25 over the FTS index, graded: chunks matching every query word first, then chunks matching any of them.""" @@ -648,16 +836,18 @@ def _keyword_leg( expr = _fts_query(query, require_all=require_all) if not expr: break - for chunk_id in self._bm25(db, expr, allowed): + for chunk_id in self._bm25(db, expr, allowed, depth): if chunk_id not in seen: seen.add(chunk_id) ranked.append(chunk_id) - if len(ranked) >= _LEG_DEPTH: + if len(ranked) >= depth: break - return ranked[:_LEG_DEPTH] + return ranked[:depth] @staticmethod - def _bm25(db: sqlite3.Connection, expr: str, allowed: set[str] | None) -> list[int]: + def _bm25( + db: sqlite3.Connection, expr: str, allowed: set[str] | None, depth: int + ) -> list[int]: """Chunk ids for one MATCH expression, best BM25 first. Column weights put a title/tag match well above a body mention of the same word.""" try: @@ -667,7 +857,7 @@ def _bm25(db: sqlite3.Connection, expr: str, allowed: set[str] | None) -> list[i " WHERE chunks_fts MATCH ?" " ORDER BY bm25(chunks_fts, 5.0, 2.0, 4.0, 1.0, 1.0)" " LIMIT ?", - (expr, _LEG_DEPTH * 3), + (expr, depth * 3), ) except sqlite3.Error: return [] # an unparseable expression is no matches, never a crash @@ -676,7 +866,7 @@ def _bm25(db: sqlite3.Connection, expr: str, allowed: set[str] | None) -> list[i ] def _vector_leg( - self, db: sqlite3.Connection, query: str, allowed: set[str] | None + self, db: sqlite3.Connection, query: str, allowed: set[str] | None, depth: int ) -> list[int]: from omind import embed @@ -688,24 +878,25 @@ def _vector_leg( packed = self._vector_matrix(db) if packed is None: return [] - ids, matrix = packed + ids, matrix, residuals = packed qv = _query_vector(query) if qv is None or int(qv.shape[0]) != int(matrix.shape[1]): return [] scores = matrix @ qv - order = np.argsort(-scores)[: _LEG_DEPTH * 3] + # Residual error only decides effectively-equal cosine scores. + order = np.lexsort((residuals, -np.round(scores, 6)))[: depth * 3] owner = self._owners(db) ranked = [ids[int(i)] for i in order] return [ cid for cid in ranked if allowed is None or owner.get(cid, "") in allowed - ][:_LEG_DEPTH] + ][:depth] except (ImportError, ValueError, sqlite3.Error): return [] - def _vector_matrix(self, db: sqlite3.Connection) -> tuple[list[int], Any] | None: - """``(chunk_ids, (n, dim) float32 matrix)`` for every stored vector. + def _vector_matrix(self, db: sqlite3.Connection) -> tuple[list[int], Any, Any] | None: + """Chunk ids, dequantized float32 matrix, and per-row residuals. One matmul beats the per-entry Python dot product the old vector index did; the matrix is cached per process and invalidated by the generation. @@ -714,37 +905,154 @@ def _vector_matrix(self, db: sqlite3.Connection) -> tuple[list[int], Any] | None generation = self._meta(db, "generation") if self._matrix is not None and self._matrix[0] == generation: - return self._matrix[1], self._matrix[2] + return self._matrix[1], self._matrix[2], self._matrix[3] ids: list[int] = [] rows: list[Any] = [] + residuals: list[float] = [] width = 0 - for row in db.execute("SELECT chunk_id, vec FROM vectors ORDER BY chunk_id"): - vec = np.frombuffer(row["vec"], dtype="float32") + for row in db.execute( + "SELECT chunk_id, vec, scale, residual FROM vectors ORDER BY chunk_id" + ): + vec = np.frombuffer(row["vec"], dtype="int8").astype("float32") + vec *= float(row["scale"]) if width and vec.size != width: continue # a stale/corrupt row rather than a wrong score width = width or int(vec.size) ids.append(int(row["chunk_id"])) rows.append(vec) + residuals.append(float(row["residual"])) if not rows: return None matrix = np.vstack(rows) - self._matrix = (generation, ids, matrix) - return ids, matrix + residual_array = np.asarray(residuals, dtype="float32") + self._matrix = (generation, ids, matrix, residual_array) + return ids, matrix, residual_array + + @staticmethod + def _rerank( + db: sqlite3.Connection, query: str, fused: list[tuple[int, float]] + ) -> list[tuple[int, float]]: + """Locally rerank the fused head against each whole matched chunk. + + The stored vectors include title/tags to improve broad recall. This + second, bounded pass deliberately embeds only the chunk body so a weak + metadata or one-word match cannot dominate the result tail. + """ + from omind import embed + + if not embed.available() or not fused: + return fused + head = fused[:_RERANK_DEPTH] + placeholders = ", ".join("?" for _ in head) + try: + rows = db.execute( + f"SELECT rowid, text FROM chunks_fts WHERE rowid IN ({placeholders})", + [chunk_id for chunk_id, _score in head], + ) + texts = {int(row["rowid"]): str(row["text"]) for row in rows} + ordered = [(chunk_id, score, texts.get(chunk_id, "")) for chunk_id, score in head] + vectors = embed.encode([query, *(text for _chunk_id, _score, text in ordered)]) + if vectors is None: + return fused + import numpy as np + + matrix = np.asarray(vectors, dtype="float32") + if matrix.ndim != 2 or matrix.shape[0] != len(ordered) + 1: + return fused + query_vector = matrix[0] + query_norm = float(np.linalg.norm(query_vector)) + if not query_norm: + return fused + rescored: list[tuple[int, float]] = [] + for (chunk_id, score, _text), vector in zip(ordered, matrix[1:], strict=True): + denominator = query_norm * float(np.linalg.norm(vector)) + cosine = float(np.dot(query_vector, vector) / denominator) if denominator else 0.0 + rescored.append((chunk_id, score * (0.5 + max(0.0, cosine)))) + rescored.sort(key=lambda item: (-item[1], item[0])) + return [*rescored, *fused[_RERANK_DEPTH:]] + except (ImportError, sqlite3.Error, TypeError, ValueError): + return fused def _owners(self, db: sqlite3.Connection) -> dict[int, str]: rows = db.execute("SELECT id, filename FROM chunks") return {int(r["id"]): str(r["filename"]) for r in rows} - def _recency_leg(self, db: sqlite3.Connection, allowed: set[str] | None) -> list[int]: + def _recency_leg( + self, db: sqlite3.Connection, allowed: set[str] | None, depth: int + ) -> list[int]: rows = db.execute( "SELECT c.id AS id, c.filename AS filename FROM chunks c" " JOIN notes n ON n.filename = c.filename" " WHERE c.ord = 0 ORDER BY n.created DESC, n.filename LIMIT ?", - (_LEG_DEPTH * 3,), + (depth * 3,), ) return [ int(r["id"]) for r in rows if allowed is None or str(r["filename"]) in allowed - ][:_LEG_DEPTH] + ][:depth] + + @staticmethod + def _weight_generated( + db: sqlite3.Connection, fused: list[tuple[int, float]] + ) -> list[tuple[int, float]]: + """De-prioritise broad machine-written journals without excluding them.""" + rows = db.execute( + "SELECT c.id, c.filename, n.okf_type FROM chunks c" + " JOIN notes n ON n.filename = c.filename" + ) + generated: set[int] = set() + for row in rows: + name = Path(str(row["filename"])).stem.lower() + okf_type = str(row["okf_type"]).strip().lower() + if ( + okf_type in {"journal", "worklog", "checkpoint", "rollup"} + or name.startswith("session journal") + or name.startswith("worklog ") + ): + generated.add(int(row["id"])) + weighted = [ + (chunk_id, score * _GENERATED_WEIGHT if chunk_id in generated else score) + for chunk_id, score in fused + ] + return sorted(weighted, key=lambda item: (-item[1], item[0])) + + @staticmethod + def _weight_superseded( + db: sqlite3.Connection, fused: list[tuple[int, float]] + ) -> list[tuple[int, float]]: + """De-rank invalidated facts while preserving their searchable history.""" + notes = list( + db.execute("SELECT filename, title, supersedes, superseded_by FROM notes") + ) + aliases: dict[str, str] = {} + superseded: set[str] = set() + for row in notes: + filename = str(row["filename"]) + aliases[filename.lower()] = filename + aliases[Path(filename).stem.lower()] = filename + aliases[str(row["title"]).strip().lower()] = filename + if str(row["superseded_by"]).strip(): + superseded.add(filename) + for row in notes: + target = str(row["supersedes"]).strip() + if not target: + continue + clean = target.strip("[]").split("|", 1)[0].split("#", 1)[0].strip().lower() + if resolved := aliases.get(clean): + superseded.add(resolved) + chunk_owner = { + int(row["id"]): str(row["filename"]) + for row in db.execute("SELECT id, filename FROM chunks") + } + weighted = [ + ( + chunk_id, + score * _SUPERSEDED_WEIGHT + if chunk_owner.get(chunk_id) in superseded + else score, + ) + for chunk_id, score in fused + ] + return sorted(weighted, key=lambda item: (-item[1], item[0])) def _listing( self, db: sqlite3.Connection, allowed: set[str] | None, limit: int @@ -771,6 +1079,7 @@ def _materialize( fused: list[tuple[int, float]], ranks: dict[str, dict[int, int]], limit: int, + excerpt_chars: int, ) -> list[Hit]: """Best chunk per note, credential-penalised, with a bounded excerpt.""" from omind import retrieve @@ -799,7 +1108,9 @@ def _materialize( Hit( filename=name, heading=str(row["heading"]), - excerpt=self._excerpt(db, chunk_id, expr, filename=name), + excerpt=self._excerpt( + db, chunk_id, expr, filename=name, max_chars=excerpt_chars + ), score=score, keyword_rank=ranks["keyword"].get(chunk_id, 0), vector_rank=ranks["vector"].get(chunk_id, 0), @@ -809,7 +1120,15 @@ def _materialize( hits.sort(key=lambda h: (-h.score, h.filename)) return hits[:limit] - def _excerpt(self, db: sqlite3.Connection, chunk_id: int, expr: str, *, filename: str) -> str: + def _excerpt( + self, + db: sqlite3.Connection, + chunk_id: int, + expr: str, + *, + filename: str, + max_chars: int, + ) -> str: """FTS5's own snippet around the match, or the chunk's head for a vector-only hit (which by definition shares no literal term). @@ -831,8 +1150,8 @@ def _excerpt(self, db: sqlite3.Connection, chunk_id: int, expr: str, *, filename (filename,), ).fetchone() if summary is not None and str(summary["text"]).strip(): - return _collapse(str(summary["text"])) - return _collapse(str(row["text"])) + return _collapse(str(summary["text"]), max_chars) + return _collapse(str(row["text"]), max_chars) if expr: snippet = db.execute( "SELECT snippet(chunks_fts, 3, '', '', '…', 24) AS s FROM chunks_fts" @@ -840,8 +1159,8 @@ def _excerpt(self, db: sqlite3.Connection, chunk_id: int, expr: str, *, filename (chunk_id, expr), ).fetchone() if snippet is not None and str(snippet["s"] or "").strip(): - return _collapse(str(snippet["s"])) - return _collapse(str(row["text"])) + return _collapse(str(snippet["s"]), max_chars) + return _collapse(str(row["text"]), max_chars) # -- link graph --------------------------------------------------------- @@ -849,7 +1168,7 @@ def _excerpt(self, db: sqlite3.Connection, chunk_id: int, expr: str, *, filename def backlinks(self, filename: str, *, title: str = "") -> list[str] | None: """Filenames whose ``[[wikilinks]]`` resolve to this note, or ``None``.""" db = self._connect() - if db is None or self.refresh() is None: + if db is None or self._refresh_if_needed() is None: return None try: stem = filename[:-3] if filename.endswith(".md") else filename @@ -869,7 +1188,7 @@ def backlinks(self, filename: str, *, title: str = "") -> list[str] | None: def notes(self) -> list[_NoteRow] | None: """Every indexed note's identity row (filename, title, created, tags).""" db = self._connect() - if db is None or self.refresh() is None: + if db is None or self._refresh_if_needed() is None: return None return self._notes(db) @@ -884,11 +1203,15 @@ def _notes(db: sqlite3.Connection) -> list[_NoteRow]: title=str(r["title"]), created=str(r["created"]), okf_type=str(r["okf_type"]), + supersedes=str(r["supersedes"]), + superseded_by=str(r["superseded_by"]), + has_title=bool(r["has_title"]), tags=tags.get(str(r["filename"]), []), disabled=bool(r["disabled"]), ) for r in db.execute( - "SELECT filename, title, created, okf_type, disabled FROM notes" + "SELECT filename, title, created, okf_type, supersedes, superseded_by," + " has_title, disabled FROM notes" ) ] @@ -901,7 +1224,7 @@ def link_rows(self) -> tuple[list[_NoteRow], list[tuple[str, str]]] | None: the way the author typed it; resolution lowercases at comparison time. """ db = self._connect() - if db is None or self.refresh() is None: + if db is None or self._refresh_if_needed() is None: return None try: notes = self._notes(db) @@ -913,6 +1236,61 @@ def link_rows(self) -> tuple[list[_NoteRow], list[tuple[str, str]]] | None: except sqlite3.Error: return None + @_locked + def lint_rows(self) -> tuple[list[_NoteRow], list[tuple[str, str]]] | None: + """Top-level note identities and fence-stripped links for vault lint.""" + db = self._connect() + if db is None or self._refresh_if_needed() is None: + return None + try: + return ( + self._notes(db), + [ + (str(row["src"]), str(row["raw"])) + for row in db.execute("SELECT src, raw FROM links WHERE lint = 1") + ], + ) + except sqlite3.Error: + return None + + @_locked + def duplicate_pairs(self, *, threshold: float = 0.92) -> list[tuple[str, str, float]] | None: + """Semantically similar note pairs from mean chunk vectors.""" + from omind import embed + + if not embed.available(): + return None + db = self._connect() + if db is None or self._refresh_if_needed() is None: + return None + try: + import numpy as np + + packed = self._vector_matrix(db) + if packed is None: + return None + ids, matrix, _residuals = packed + owners = self._owners(db) + grouped: dict[str, list[Any]] = {} + for chunk_id, vector in zip(ids, matrix, strict=True): + if owner := owners.get(chunk_id): + grouped.setdefault(owner, []).append(vector) + names = sorted(grouped) + if len(names) < 2: + return [] + centroids = np.vstack([np.mean(grouped[name], axis=0) for name in names]) + norms = np.linalg.norm(centroids, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + centroids /= norms + similarity = centroids @ centroids.T + lefts, rights = np.where(np.triu(similarity, k=1) >= threshold) + return [ + (names[int(left)], names[int(right)], float(similarity[left, right])) + for left, right in zip(lefts, rights, strict=True) + ] + except (ImportError, sqlite3.Error, TypeError, ValueError): + return None + # -- dedup -------------------------------------------------------------- @_locked @@ -926,7 +1304,7 @@ def nearest( if not embed.available(): return None db = self._connect() - if db is None or self.refresh() is None: + if db is None or self._refresh_if_needed() is None: return None try: import numpy as np @@ -935,7 +1313,7 @@ def nearest( qv = _query_vector(text) if packed is None or qv is None: return None - ids, matrix = packed + ids, matrix, _residuals = packed if int(qv.shape[0]) != int(matrix.shape[1]): return None scores = matrix @ qv @@ -1090,6 +1468,26 @@ def link_targets(md: str) -> list[str]: return [targets[key] for key in sorted(targets)] +def lint_link_targets(md: str) -> list[str]: + """Wikilinks excluding fenced/inline code, for lint's graph view.""" + from omind.store import _FENCE_RE + + lines: list[str] = [] + in_fence = False + fence_ch = "" + for line in md.splitlines(): + fence = _FENCE_RE.match(line.lstrip()) + if fence: + ch = fence.group(1)[0] + if not in_fence: + in_fence, fence_ch = True, ch + elif ch == fence_ch: + in_fence = False + continue + lines.append("" if in_fence else re.sub(r"`[^`]*`", "", line)) + return link_targets("\n".join(lines)) + + def _fuse(legs: list[tuple[float, list[int]]]) -> list[tuple[int, float]]: """Reciprocal Rank Fusion over weighted ranked lists, best first.""" scores: dict[int, float] = {} @@ -1099,6 +1497,6 @@ def _fuse(legs: list[tuple[float, list[int]]]) -> list[tuple[int, float]]: return sorted(scores.items(), key=lambda kv: (-kv[1], kv[0])) -def _collapse(text: str) -> str: +def _collapse(text: str, limit: int = EXCERPT_CHARS) -> str: flat = " ".join(text.split()) - return flat if len(flat) <= EXCERPT_CHARS else flat[: EXCERPT_CHARS - 1].rstrip() + "…" + return flat if len(flat) <= limit else flat[: limit - 1].rstrip() + "…" diff --git a/src/omind/seeds.py b/src/omind/seeds.py index 01ceab3..5908fa5 100644 --- a/src/omind/seeds.py +++ b/src/omind/seeds.py @@ -115,6 +115,9 @@ If MCP is unavailable, run `omind help ` locally. - Search with `search-vault`, then retrieve selected memories with `recall-note`. Use `read-note` only for raw Markdown/editing fields. +- Query graph audits with `graph`: `op=path` (plus `source` and `target`), + `op=orphans`, `op=dangling`, or `op=stats`. `graph-neighbors` remains the + graph-aware recall tool. ## Writing memory — always through omind diff --git a/src/omind/server.py b/src/omind/server.py index 9ec3b25..06ecfee 100644 --- a/src/omind/server.py +++ b/src/omind/server.py @@ -220,12 +220,16 @@ def graph_for() -> graph.Graph: ) def read_note(name: str, representation: str = "fields") -> dict[str, object]: raw = store.read_note(name) + filename = store.safe_name(name).name + from omind import access + + access.record(store.omi_dir, filename) # One read + one parse: read_fields would re-read the file just read. # ONE representation, never both: returning `raw` and `fields` together # sent every note body through the context twice, and the editing caller # only ever uses one of them. payload: dict[str, object] = { - "filename": store.safe_name(name).name, + "filename": filename, "version": store.note_version(name), } if representation == "raw": @@ -262,6 +266,8 @@ def create_note( details: str = "", tags: list[str] | None = None, related_to: str = "", + supersedes: str = "", + superseded_by: str = "", connections: list[str] | None = None, action_items: list[str] | None = None, references: list[str] | None = None, @@ -272,6 +278,8 @@ def create_note( details=details, tags=tags or [], related_to=related_to, + supersedes=supersedes, + superseded_by=superseded_by, connections=connections or [], action_items=_parse_action_items(action_items or []), references=references or [], @@ -294,6 +302,8 @@ def edit_note( details: str | None = None, tags: list[str] | None = None, related_to: str | None = None, + supersedes: str | None = None, + superseded_by: str | None = None, connections: list[str] | None = None, action_items: list[str] | None = None, references: list[str] | None = None, @@ -310,6 +320,10 @@ def edit_note( fields.tags = tags if related_to is not None: fields.related_to = related_to + if supersedes is not None: + fields.supersedes = supersedes + if superseded_by is not None: + fields.superseded_by = superseded_by if connections is not None: fields.connections = connections if action_items is not None: @@ -413,46 +427,89 @@ def graph_neighbors( ] return _page(rows, limit, offset) + def _graph_query( + op: str, + source: str = "", + target: str = "", + limit: int = DEFAULT_PAGE, + offset: int = 0, + ) -> dict[str, object]: + operation = op.strip().lower() + if operation == "path": + if not source.strip() or not target.strip(): + raise ValueError("graph op=path requires source and target") + g = graph_for() + return {"path": graph.shortest_path(g, source, target)} + if operation == "orphans": + return _page(graph.orphans(graph_for()), limit, offset) + if operation == "dangling": + rows = [ + {"source": src, "target": raw_target} + for src, raw_target in graph.dangling_links(graph_for()) + ] + return _page(rows, limit, offset) + if operation == "stats": + return dict(graph.stats(graph_for())) + raise ValueError("graph op must be one of: path, orphans, dangling, stats") + + @mcp.tool( + name="graph", + description=( + "Graph audit/query selected by op: path (requires source + target), " + "orphans, dangling, or stats. Orphan/dangling results are paged." + ), + ) + def graph_tool( + op: str, + source: str = "", + target: str = "", + limit: int = DEFAULT_PAGE, + offset: int = 0, + ) -> dict[str, object]: + return _graph_query(op, source, target, limit, offset) + @mcp.tool( name="graph-path", description=( "Shortest [[wikilink]] path between two notes, as a list of filenames; " - "`path` is null when no path connects them." + "`path` is null when no path connects them. Deprecated: use graph " + "with op=path; this compatibility name will be removed next release." ), ) def graph_path(source: str, target: str) -> dict[str, object]: - g = graph_for() - return {"path": graph.shortest_path(g, source, target)} + return _graph_query("path", source, target) @mcp.tool( name="graph-orphans", description=( "One page of notes with no inbound or outbound [[wikilinks]]. " - "graph-stats gives the count without the list." + "Deprecated: use graph with op=orphans; this compatibility name " + "will be removed next release." ), ) def graph_orphans(limit: int = DEFAULT_PAGE, offset: int = 0) -> dict[str, object]: - return _page(graph.orphans(graph_for()), limit, offset) + return _graph_query("orphans", limit=limit, offset=offset) @mcp.tool( name="graph-dangling", description=( "One page of [[wikilinks]] resolving to no existing note, with their " - "source. graph-stats gives the count without the list." + "source. Deprecated: use graph with op=dangling; this compatibility " + "name will be removed next release." ), ) def graph_dangling(limit: int = DEFAULT_PAGE, offset: int = 0) -> dict[str, object]: - rows = [ - {"source": src, "target": target} for src, target in graph.dangling_links(graph_for()) - ] - return _page(rows, limit, offset) + return _graph_query("dangling", limit=limit, offset=offset) @mcp.tool( name="graph-stats", - description="Whole-graph counts: notes, links, orphans, and dangling links.", + description=( + "Whole-graph counts. Deprecated: use graph with op=stats; this " + "compatibility name will be removed next release." + ), ) - def graph_stats() -> dict[str, int]: - return graph.stats(graph_for()) + def graph_stats() -> dict[str, object]: + return _graph_query("stats") return mcp diff --git a/src/omind/store.py b/src/omind/store.py index 387dd62..9c168c7 100644 --- a/src/omind/store.py +++ b/src/omind/store.py @@ -178,6 +178,8 @@ class NoteFields: created: str = "" tags: list[str] = field(default_factory=list) related_to: str = "" + supersedes: str = "" + superseded_by: str = "" connections: list[str] = field(default_factory=list) action_items: list[ActionItem] = field(default_factory=list) references: list[str] = field(default_factory=list) @@ -223,6 +225,8 @@ def from_dict(cls, data: dict[str, Any]) -> NoteFields: created=str(data.get("created", "")).strip(), tags=[_clean_tag(t) for t in (data.get("tags") or []) if _clean_tag(t)], related_to=str(data.get("related_to", "")).strip(), + supersedes=str(data.get("supersedes", "")).strip(), + superseded_by=str(data.get("superseded_by", "")).strip(), connections=[str(c).strip() for c in (data.get("connections") or []) if str(c).strip()], action_items=items, references=[str(r).strip() for r in (data.get("references") or []) if str(r).strip()], @@ -459,6 +463,8 @@ def body(name: str) -> str: meta = sections.get("Metadata", []) created = "" related_to = "" + supersedes = "" + superseded_by = "" rev = "" disabled = False tags: list[str] = [] @@ -469,6 +475,10 @@ def body(name: str) -> str: tags = _TAG_RE.findall(m.group(1)) elif m := re.match(r"^\s*-\s*Related to:\s*(.*)$", line): related_to = m.group(1).strip() + elif m := re.match(r"^\s*-\s*Supersedes:\s*(.*)$", line): + supersedes = m.group(1).strip() + elif m := re.match(r"^\s*-\s*Superseded by:\s*(.*)$", line): + superseded_by = m.group(1).strip() elif m := _REV_LINE_RE.match(line): rev = m.group(1).strip() elif _DISABLED_LINE_RE.match(line): @@ -516,6 +526,8 @@ def body(name: str) -> str: created=created, tags=tags, related_to=related_to, + supersedes=supersedes, + superseded_by=superseded_by, connections=[c.strip() for c in connections if c.strip()], action_items=action_items, references=references, @@ -550,6 +562,10 @@ def render_fields(f: NoteFields) -> str: tag_str = " ".join(f"#{_clean_tag(t)}" for t in f.tags if _clean_tag(t)) out.append(f"- Tags: {tag_str}".rstrip()) out.append(f"- Related to: {f.related_to}".rstrip()) + if f.supersedes: + out.append(f"- Supersedes: {f.supersedes}") + if f.superseded_by: + out.append(f"- Superseded by: {f.superseded_by}") if f.rev: out.append(f"- Rev: {f.rev}") if f.disabled: @@ -788,15 +804,13 @@ def write_lock(self) -> Iterator[None]: _write_lock = write_lock def _signal_write(self) -> None: - """Advisory nudge for the mesh daemon's debounced sync; never raises. + """Advisory nudge for search-index invalidation and mesh sync; never raises. Lives in the store so *every* write surface (MCP server, web UI, - ``omind note``, import) triggers replication — previously only the MCP - server's tools remembered to, and edits made elsewhere sat - uncommitted for up to the full sync interval. + ``omind note``, import) invalidates the derived index immediately. In + mesh mode the same signal also triggers replication; previously only + the MCP server's tools remembered to signal writes. """ - if not self.mesh_mode(): - return try: signal = sync_signal_path(self.omi_dir) signal.parent.mkdir(parents=True, exist_ok=True) @@ -1199,6 +1213,58 @@ def create_note(self, fields: NoteFields) -> str: # concurrent-create race, not here. return self.write_note(filename, render_fields(fields), must_create=True) + def create_and_disable_sources( + self, + fields: NoteFields, + sources: list[tuple[str, str]], + ) -> str: + """Create one reviewed note and archive unchanged sources under one lock. + + The source tuples are ``(filename, expected_version)``. All versions and + the target's nonexistence are checked before the first write, closing + the gap where another OmiStore writer could change the second source + between a separate create and two archive calls. A process crash can + still leave extra recoverable copies, never a hard-deleted source. + """ + if not fields.title.strip(): + raise NoteError("a note requires a title") + if len(sources) < 2: + raise NoteError("consolidation requires at least two source notes") + if not fields.created: + fields.created = today() + target = self.safe_name(self.filename_for_title(fields.title)) + self._reject_reserved(target) + source_paths = [(self.safe_name(name), version) for name, version in sources] + for path, _version in source_paths: + self._reject_reserved(path) + _hoist_field_headings(fields) + + with self.write_lock(): + if target.exists(): + raise NoteError(f"a note named {target.name!r} already exists") + for path, expected in source_paths: + if not path.is_file(): + raise NoteNotFoundError(f"note not found: {path.name!r}") + current = self.note_version(path.name) + if current != expected: + raise NoteConflictError( + f"note {path.name!r} changed on disk (expected {expected!r}, " + f"found {current!r})" + ) + + content = render_fields(fields) + if self.node_id is not None: + content = self._stamped(target, content) + _atomic_write(target, content) + for path, _expected in source_paths: + archived = _with_disabled(_read_text(path), True) + if self.node_id is not None: + archived = self._stamped(path, archived) + _atomic_write(path, archived) + self._write_index() + self._signal_write() + return target.name + def update_note( self, name: str, fields: NoteFields, expected_version: str | None = None ) -> str: @@ -1233,6 +1299,10 @@ def transform(text: str) -> str: fields.tags = current.tags if not fields.okf_type: fields.okf_type = current.okf_type + if not fields.supersedes: + fields.supersedes = current.supersedes + if not fields.superseded_by: + fields.superseded_by = current.superseded_by # A multi-section body supplied through `details` (the only such # field the MCP/CLI API exposes) carries ## H2s that read back as # extras. Hoist them now so they REPLACE the same-named inherited @@ -1254,12 +1324,20 @@ def delete_note(self, name: str) -> None: else: self.purge_note(name) - def disable_note(self, name: str) -> str: - """Soft-delete: set ``Disabled: true``; hidden from listings, restorable.""" + def disable_note(self, name: str, expected_version: str | None = None) -> str: + """Soft-delete: set ``Disabled: true``; hidden from listings, restorable. + + ``expected_version`` lets reviewed multi-note operations refuse to + archive a source that changed after the operator inspected it. + """ path = self.safe_name(name) if _is_reserved(path.name): raise NoteError(f"refusing to disable reserved file: {path.name}") - return self._mutate_note(name, lambda md: _with_disabled(md, True)) + return self._mutate_note( + name, + lambda md: _with_disabled(md, True), + expected_version=expected_version, + ) def restore_note(self, name: str) -> str: """Clear a soft-deleted note's ``Disabled`` flag.""" diff --git a/tests/test_access.py b/tests/test_access.py new file mode 100644 index 0000000..a60fcd1 --- /dev/null +++ b/tests/test_access.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Aaron K. Clark +"""Tests for machine-local dynamic core membership.""" + +from __future__ import annotations + +from pathlib import Path + +from omind import access + + +def test_core_members_balance_frequency_and_recency(tmp_path: Path) -> None: + omi = tmp_path / "OMI" + omi.mkdir() + now = 2_000_000_000.0 + access.record(omi, "Frequent.md", now=now - 86_400) + access.record(omi, "Frequent.md", now=now - 86_400) + access.record(omi, "Recent.md", now=now) + assert access.core_members(omi, now=now) == ["Frequent.md", "Recent.md"] + + +def test_stale_accesses_demote_out_of_core(tmp_path: Path) -> None: + omi = tmp_path / "OMI" + omi.mkdir() + now = 2_000_000_000.0 + access.record(omi, "Old.md", now=now - 91 * 86_400) + access.record(omi, "Current.md", now=now) + assert access.core_members(omi, now=now) == ["Current.md"] + + +def test_access_state_is_bounded_by_requested_core_size(tmp_path: Path) -> None: + omi = tmp_path / "OMI" + omi.mkdir() + for number in range(10): + access.record(omi, f"Note {number}.md", now=float(number + 1)) + assert len(access.core_members(omi, limit=3, now=10.0)) == 3 diff --git a/tests/test_bench.py b/tests/test_bench.py index 08012b9..c5300d4 100644 --- a/tests/test_bench.py +++ b/tests/test_bench.py @@ -64,3 +64,31 @@ def test_report_serialises_to_json_and_text(tmp_path: Path) -> None: report = bench.run(_vault(tmp_path), queries=("nebraska",)) assert report.to_dict()["vault"].endswith("OMI") assert "omind bench" in report.format() + + +def test_quality_report_calculates_recall_and_mrr(tmp_path: Path) -> None: + omi = _vault(tmp_path) + report = bench.run_quality( + omi, + cases=( + ("release signing failure", "Signing Runbook.md"), + ("nebraska", "Nebraska.md"), + ("missing target is skipped", "Absent.md"), + ), + ) + by_name = {measurement.name: measurement for measurement in report.measurements} + assert by_name["quality cases"].value == 2 + assert by_name["quality cases"].detail == "1 skipped" + assert by_name["recall@1"].value == 100.0 + assert by_name["recall@5"].value == 100.0 + assert by_name["MRR"].value == 1.0 + + +def test_quality_report_handles_no_matching_labelled_notes(tmp_path: Path) -> None: + report = bench.run_quality( + _vault(tmp_path), + cases=(("unknown", "Absent.md"),), + ) + by_name = {measurement.name: measurement for measurement in report.measurements} + assert by_name["quality cases"].value == 0 + assert by_name["MRR"].value == 0 diff --git a/tests/test_cli.py b/tests/test_cli.py index 0eb0033..7c5e37b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,6 +61,15 @@ def test_reindex_subcommand_parses() -> None: assert args.folder == "OMI" +def test_consolidate_subcommand_parses() -> None: + args = build_parser().parse_args(["consolidate", "--limit", "2"]) + assert args.command == "consolidate" + assert args.limit == 2 + assert args.apply is None + applying = build_parser().parse_args(["consolidate", "--apply", "0123456789abcdef"]) + assert applying.apply == "0123456789abcdef" + + def test_reindex_regenerates_index_for_directly_written_note(tmp_path: Path) -> None: # Simulate a session that wrote a note file directly (bypassing the store), # then ran `omind reindex` to refresh index.md safely. diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 3ebfb59..aac1d0d 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -21,7 +21,7 @@ import pytest from fastapi import FastAPI -from omind import backup, provision +from omind import backup, embed, provision from omind.cli import main @@ -186,6 +186,29 @@ def test_doctor_reports_problems_with_exit_1_on_a_bare_machine( assert "backup" in out.lower() +def test_doctor_reports_search_index_health( + vault: Path, + isolate_config: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + embed, + "status", + lambda: { + "available": False, + "model": "test/model", + "reason": "test backend unavailable", + }, + ) + main(["doctor", "--vault", str(vault)]) + out = capsys.readouterr().out + assert "search index: FTS5 available" in out + assert "semantic search: off (keyword path) — test backend unavailable" in out + assert "search index has not been built" in out + assert "omind reindex --rebuild" in out + + # -- backup ------------------------------------------------------------------- diff --git a/tests/test_consolidate.py b/tests/test_consolidate.py new file mode 100644 index 0000000..31e564a --- /dev/null +++ b/tests/test_consolidate.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Aaron K. Clark +"""Reviewed near-duplicate consolidation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from omind import consolidate +from omind.store import NoteFields, OmiStore, parse_note, render_fields + + +@pytest.fixture +def vault(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + omi = tmp_path / "vault" / "OMI" + omi.mkdir(parents=True) + store = OmiStore(omi) + store.create_note( + NoteFields( + title="Alpha Memory", + summary="same durable fact", + details="alpha-only detail", + tags=["one"], + connections=["Alpha Link"], + ) + ) + store.create_note( + NoteFields( + title="Alpha Memory Copy", + summary="same durable fact restated", + details="beta-only detail", + tags=["two"], + connections=["Beta Link"], + ) + ) + monkeypatch.setattr( + consolidate, + "_candidate_pairs", + lambda _omi: [("Alpha Memory.md", "Alpha Memory Copy.md", 0.97)], + ) + return omi + + +def test_proposal_is_outside_vault_and_leaves_notes_byte_identical(vault: Path) -> None: + before = {path.name: path.read_bytes() for path in vault.glob("*.md")} + + proposals = consolidate.propose(vault, limit=1) + + assert len(proposals) == 1 + proposal = proposals[0] + assert vault not in proposal.plan_path.parents + assert vault not in proposal.draft_path.parents + assert proposal.plan_path.is_file() + draft = parse_note(proposal.draft_path.read_text(encoding="utf-8")) + assert "alpha-only detail" in draft.details + assert "beta-only detail" in draft.details + assert {"one", "two", "consolidated"} <= set(draft.tags) + assert before == {path.name: path.read_bytes() for path in vault.glob("*.md")} + + +def test_apply_edited_draft_creates_merge_and_archives_sources(vault: Path) -> None: + proposal = consolidate.propose(vault, limit=1)[0] + draft = parse_note(proposal.draft_path.read_text(encoding="utf-8")) + draft.title = "Reviewed Alpha Memory" + draft.summary = "human-approved summary" + proposal.draft_path.write_text(render_fields(draft), encoding="utf-8") + + result = consolidate.apply(vault, proposal.plan_id) + + assert result.filename == "Reviewed Alpha Memory.md" + store = OmiStore(vault) + merged = store.read_fields(result.filename) + assert merged.summary == "human-approved summary" + assert "alpha-only detail" in merged.details + assert "beta-only detail" in merged.details + assert store.read_fields("Alpha Memory.md").disabled is True + assert store.read_fields("Alpha Memory Copy.md").disabled is True + + +def test_apply_rejects_source_changed_after_review(vault: Path) -> None: + proposal = consolidate.propose(vault, limit=1)[0] + store = OmiStore(vault) + changed = store.read_fields("Alpha Memory.md") + changed.summary = "changed after the proposal" + store.update_note("Alpha Memory.md", changed) + + with pytest.raises(consolidate.ConsolidationError, match="source changed"): + consolidate.apply(vault, proposal.plan_id) + + expected = vault / "Consolidated — Alpha Memory + Alpha Memory Copy.md" + assert not expected.exists() + assert store.read_fields("Alpha Memory.md").disabled is False + assert store.read_fields("Alpha Memory Copy.md").disabled is False + + +@pytest.mark.parametrize("plan_id", ("../bad", "ABCDEF0123456789", "abcd")) +def test_apply_rejects_unsafe_plan_ids(vault: Path, plan_id: str) -> None: + with pytest.raises(consolidate.ConsolidationError, match="plan id"): + consolidate.apply(vault, plan_id) + + +def test_apply_rejects_tampered_source_path(vault: Path) -> None: + proposal = consolidate.propose(vault, limit=1)[0] + payload = json.loads(proposal.plan_path.read_text(encoding="utf-8")) + payload["sources"][0]["filename"] = "../outside.md" + proposal.plan_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(consolidate.ConsolidationError, match="source is invalid"): + consolidate.apply(vault, proposal.plan_id) diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 4302aa6..0ca1a13 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -261,6 +261,25 @@ def test_session_start_injects_priming_note_content(tmp_path: Path) -> None: assert "===== OMI capsule: index.md =====" in ctx +def test_session_start_promotes_recently_recalled_notes_into_a_small_core( + tmp_path: Path, +) -> None: + from omind import access + from omind.store import NoteFields, OmiStore + + store = OmiStore(tmp_path) + store.create_note(NoteFields(title="Earned Core", summary="frequently needed detail")) + store.create_note( + NoteFields(title="Forge Password", summary="never prime credentials", tags=["auth"]) + ) + access.record(tmp_path, "Earned Core.md") + access.record(tmp_path, "Forge Password.md") + context = hooks.build_session_start_context(tmp_path) + assert "Earned Core.md (dynamic core)" in context + assert "frequently needed detail" in context + assert "Forge Password.md (dynamic core)" not in context + + def test_session_start_caps_runaway_note(tmp_path: Path) -> None: (tmp_path / "Playbook.md").write_text( "x" * (hooks._PRIMING_FILE_CHAR_CAP + 500), encoding="utf-8" diff --git a/tests/test_lint.py b/tests/test_lint.py index 87da301..4ce20e7 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -169,7 +169,9 @@ def test_link_into_journal_subfolder_is_not_broken(tmp_path: Path) -> None: assert broken == [] -def test_wikilink_inside_code_fence_is_not_a_link(tmp_path: Path) -> None: +def test_wikilink_inside_code_fence_is_not_a_link( + tmp_path: Path, monkeypatch +) -> None: # type: ignore[no-untyped-def] """A [[wikilink]] quoted in a fenced code block is documentation, not a link.""" omi = _omi(tmp_path) _write( @@ -178,5 +180,10 @@ def test_wikilink_inside_code_fence_is_not_a_link(tmp_path: Path) -> None: "# Docs\n\n## Details\nExample:\n\n```\nUse [[Some Note]] to link.\n```\n\n" "## Connections\n- [[Docs]]\n", ) + monkeypatch.setattr( + lint, + "_load", + lambda _omi: (_ for _ in ()).throw(AssertionError("index path was not used")), + ) broken = [i for i in lint.lint_vault(omi) if i.kind == "broken-link"] assert broken == [] diff --git a/tests/test_searchindex.py b/tests/test_searchindex.py index fd4d0a0..66a1d94 100644 --- a/tests/test_searchindex.py +++ b/tests/test_searchindex.py @@ -89,6 +89,46 @@ def test_refresh_is_incremental(omi: Path) -> None: assert third is not None and third.reindexed == 1 # only the edited note +def test_query_burst_throttles_refresh_but_signal_invalidates( + omi: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _note(omi, "Release Guide", "how to cut a release", ["release"]) + idx = searchindex.SearchIndex(omi) + original = idx.refresh + calls = 0 + + def counted_refresh(*, vectors: bool = True) -> searchindex.Refresh | None: + nonlocal calls + calls += 1 + return original(vectors=vectors) + + monkeypatch.setattr(idx, "refresh", counted_refresh) + assert idx.search("release") + assert idx.search("release") + assert calls == 1 + + signal = searchindex.paths.sync_signal_path(omi) + signal.parent.mkdir(parents=True, exist_ok=True) + signal.touch() + assert idx.search("release") + assert calls == 2 + + +def test_refresh_throttle_expires_for_external_edits( + omi: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + note = _note(omi, "Release Guide", "how to cut a release", ["release"]) + idx = searchindex.SearchIndex(omi) + now = [100.0] + monkeypatch.setattr(searchindex.time, "monotonic", lambda: now[0]) + + assert idx.search("release") + note.write_text(note.read_text(encoding="utf-8") + "\nexternal edit", encoding="utf-8") + assert not idx.search("external") # still inside the bounded burst window + now[0] += searchindex._REFRESH_THROTTLE_SECONDS + assert idx.search("external") + + def test_touch_without_a_content_change_does_not_reindex(omi: Path) -> None: """An mtime bump with identical bytes (a mesh sync, a `touch`) is not a change.""" path = _note(omi, "Stable", "unchanged text", ["x"]) @@ -194,6 +234,35 @@ def test_credential_notes_are_deprioritised_for_unrelated_tasks(omi: Path) -> No assert hits[0].filename == "Forge Guide.md" +def test_generated_worklogs_rank_below_equivalent_curated_notes(omi: Path) -> None: + detail = "zebracorn deployment rollback procedure" + _note(omi, "Handbook", "curated operations", ["ops"], details=detail) + _note(omi, "Worklog 2026-07-27", "automatic activity", ["worklog"], details=detail) + hits = searchindex.SearchIndex(omi).search("zebracorn rollback") or [] + by_name = {hit.filename: hit for hit in hits} + assert set(by_name) == {"Handbook.md", "Worklog 2026-07-27.md"} + assert by_name["Handbook.md"].score > by_name["Worklog 2026-07-27.md"].score + assert hits[0].filename == "Handbook.md" + + +def test_superseded_notes_remain_searchable_but_rank_lower(omi: Path) -> None: + detail = "zebracorn release status" + _note(omi, "Release v1", "old release", ["release"], details=detail) + current = _note(omi, "Release v2", "current release", ["release"], details=detail) + current.write_text( + current.read_text(encoding="utf-8").replace( + "- Tags: #release", + "- Tags: #release\n- Supersedes: [[Release v1]]", + ), + encoding="utf-8", + ) + hits = searchindex.SearchIndex(omi).search("zebracorn status") or [] + by_name = {hit.filename: hit for hit in hits} + assert set(by_name) == {"Release v1.md", "Release v2.md"} + assert by_name["Release v2.md"].score > by_name["Release v1.md"].score + assert hits[0].filename == "Release v2.md" + + def test_query_punctuation_cannot_break_the_match_expression(omi: Path) -> None: """FTS5 operators in user text are data, not syntax (every term is quoted).""" _note(omi, "Quoted", "handling NEAR and OR in queries", ["x"]) @@ -206,7 +275,24 @@ def test_excerpt_is_bounded(omi: Path) -> None: _note(omi, "Wordy", "x", ["x"], details="haystack " * 500 + " needle") idx = searchindex.SearchIndex(omi) hits = idx.search("needle") or [] - assert hits and len(hits[0].excerpt) <= searchindex.EXCERPT_CHARS + assert hits and len(hits[0].excerpt) <= 120 + + +def test_query_complexity_adapts_result_and_excerpt_budgets(omi: Path) -> None: + detail = "needle alpha beta gamma relationship " + "context " * 100 + for number in range(12): + _note(omi, f"Item {number}", "matching record", ["x"], details=detail) + idx = searchindex.SearchIndex(omi) + simple = idx.search("needle", limit=50) or [] + complex_hits = idx.search( + "why needle alpha beta gamma relationship matters", + limit=50, + ) or [] + assert len(simple) == 5 + assert len(complex_hits) == 12 + assert all(len(hit.excerpt) <= 120 for hit in simple) + assert any(len(hit.excerpt) > 180 for hit in complex_hits) + assert all(len(hit.excerpt) <= 240 for hit in complex_hits) def test_identity_hit_shows_the_summary_not_the_title_echo(omi: Path) -> None: @@ -231,6 +317,42 @@ def test_vector_leg_finds_a_paraphrase_with_no_shared_term( assert hits[0].vector_rank > 0 # the semantic leg actually contributed +def test_vectors_are_quantized_to_int8_with_scale_and_residual( + omi: Path, semantic: None +) -> None: + _note(omi, "Release Guide", "release push forge version", ["release"]) + idx = searchindex.SearchIndex(omi) + assert idx.refresh() is not None + db = idx._connect() + assert db is not None + row = db.execute("SELECT vec, scale, residual FROM vectors LIMIT 1").fetchone() + assert row is not None + assert len(row["vec"]) == len(_VOCAB) + assert float(row["scale"]) > 0 + assert float(row["residual"]) >= 0 + + +def test_duplicate_pairs_use_mean_chunk_vector_similarity( + omi: Path, semantic: None +) -> None: + _note(omi, "First", "release push forge version", ["release"]) + _note(omi, "Second", "release push forge version", ["release"]) + _note(omi, "Different", "banana smoothie", ["food"]) + pairs = searchindex.SearchIndex(omi).duplicate_pairs(threshold=0.99) + assert pairs is not None + assert [(left, right) for left, right, _score in pairs] == [ + ("First.md", "Second.md") + ] + + +def test_reranker_scores_the_whole_chunk_body(omi: Path, semantic: None) -> None: + _note(omi, "Strong", "release push forge version", ["x"]) + _note(omi, "Weak", "release", ["x"]) + hits = searchindex.SearchIndex(omi).search("release push") or [] + assert [hit.filename for hit in hits[:2]] == ["Strong.md", "Weak.md"] + assert hits[0].score > hits[1].score + + def test_nearest_excludes_the_note_being_written(omi: Path, semantic: None) -> None: _note(omi, "Release Guide", "release push forge version", ["release"]) _note(omi, "Release Notes", "release push forge version", ["release"]) @@ -279,6 +401,35 @@ def test_a_corrupt_index_file_does_not_raise(omi: Path) -> None: assert broken.search("content") is None # fails open; caller scans instead +def test_health_reports_size_age_counts_and_staleness(omi: Path) -> None: + note = _note(omi, "Fine", "content here", ["x"]) + idx = searchindex.SearchIndex(omi) + assert idx.refresh() is not None + idx.close() + + healthy = searchindex.health(omi) + assert healthy.fts5 is True + assert healthy.exists is True + assert healthy.size_bytes > 0 + assert healthy.age_seconds is not None + assert healthy.notes == 1 + assert healthy.stale == 0 + assert healthy.corrupt == "" + + note.write_text(note.read_text(encoding="utf-8") + "\nchanged\n", encoding="utf-8") + assert searchindex.health(omi).stale == 1 + + +def test_health_explains_a_corrupt_index(omi: Path) -> None: + path = searchindex.index_path(omi) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"this is not a database") + + result = searchindex.health(omi) + assert result.exists is True + assert result.corrupt + + def test_store_search_falls_back_when_the_index_is_off( omi: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_server.py b/tests/test_server.py index dc82869..a702739 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -38,6 +38,7 @@ "backlinks", "list-tags", "graph-neighbors", + "graph", "graph-path", "graph-orphans", "graph-dangling", @@ -192,6 +193,8 @@ def test_every_list_tool_is_bounded(server: FastMCP) -> None: ("graph-orphans", {"limit": 1}), ("graph-dangling", {"limit": 1}), ("graph-neighbors", {"name": "Linker", "limit": 1}), + ("graph", {"op": "orphans", "limit": 1}), + ("graph", {"op": "dangling", "limit": 1}), ): page = call(server, tool, args) assert set(page) >= {"result", "count", "offset", "total", "has_more"}, tool @@ -283,9 +286,26 @@ def test_graph_tools(server: FastMCP) -> None: assert dangling == [{"source": "Lonely.md", "target": "Ghost"}] assert call(server, "graph-stats", {})["notes"] == 4 + # New unified surface; legacy names above remain for one compatibility release. + assert call( + server, + "graph", + {"op": "path", "source": "A", "target": "C"}, + )["path"] == ["A.md", "B.md", "C.md"] + assert call(server, "graph", {"op": "orphans"})["result"] == ["Lonely.md"] + assert call(server, "graph", {"op": "dangling"})["result"] == dangling + assert call(server, "graph", {"op": "stats"})["notes"] == 4 + + +def test_unified_graph_validates_operation_and_path_arguments(server: FastMCP) -> None: + with pytest.raises(ToolError, match="one of"): + call(server, "graph", {"op": "unknown"}) + with pytest.raises(ToolError, match="requires source and target"): + call(server, "graph", {"op": "path"}) + def test_graph_build_is_cached_and_busted_by_a_write(omi_dir: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] - """The 5 graph tools reuse one cached build; a write busts the cache (#130).""" + """All graph tools reuse one cached build; a write busts the cache (#130).""" from omind import graph as graph_mod calls = {"n": 0} diff --git a/tests/test_store.py b/tests/test_store.py index e5fe620..9b7ddda 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -39,6 +39,8 @@ def test_render_parse_round_trip() -> None: created="2026-06-03", tags=["omi", "memory", "thesis"], related_to="Some project", + supersedes="Older Memory", + superseded_by="Newer Memory", connections=["Concept One", "Concept Two"], action_items=[ ActionItem("do the thing", done=False),