From b61ab88c1672fd8a23cff8e7fcab60066691f53e Mon Sep 17 00:00:00 2001 From: Engineer Date: Sat, 1 Aug 2026 10:18:22 +0200 Subject: [PATCH 1/3] fix(sqlite): move the four long-lived stores off WAL to rollback journalling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of fix/wal-orphan-rollback-journal (651f2f8 + 00550a4) onto the deployed tree, which still opens one shared connection per store instead of the upstream thread-local one — the pragma call site differs, the bug and the remedy do not. WAL + POSIX locks owned per (process, inode) let an external opener unlink the -wal: any connection close drops the whole process's locks, the next opener believes it is alone, checkpoints and unlinks, and the daemon keeps writing into an orphaned inode — writes visible only in-process and lost at exit. Observed in production 2026-08-01 on conversations, tasks, agent_comms and research; conversations_tasks.db-wal went orphan again on the restarted daemon (pid 998969) before this deploy. --- src/pinky_daemon/agent_comms.py | 5 +- src/pinky_daemon/conversation_store.py | 5 +- src/pinky_daemon/research_store.py | 5 +- src/pinky_daemon/sqlite_journal.py | 103 ++++++++++++++ src/pinky_daemon/task_store.py | 5 +- tests/test_sqlite_journal.py | 190 +++++++++++++++++++++++++ 6 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 src/pinky_daemon/sqlite_journal.py create mode 100644 tests/test_sqlite_journal.py diff --git a/src/pinky_daemon/agent_comms.py b/src/pinky_daemon/agent_comms.py index db73304a..0322b75f 100644 --- a/src/pinky_daemon/agent_comms.py +++ b/src/pinky_daemon/agent_comms.py @@ -23,6 +23,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + @dataclass class AgentMessage: @@ -83,8 +85,7 @@ def _conn(self) -> sqlite3.Connection: if connection is None: connection = sqlite3.connect(self._db_path) connection.row_factory = sqlite3.Row - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="agent_comms.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/conversation_store.py b/src/pinky_daemon/conversation_store.py index a76dc88d..a3e3dfb5 100644 --- a/src/pinky_daemon/conversation_store.py +++ b/src/pinky_daemon/conversation_store.py @@ -18,6 +18,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _fts5_phrase(query: str) -> str: """Escape a user query as quoted FTS5 phrase tokens (no operator syntax).""" @@ -94,8 +96,7 @@ def _conn(self) -> sqlite3.Connection: if connection is None: connection = sqlite3.connect(self._db_path) connection.row_factory = sqlite3.Row - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="conversations.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/research_store.py b/src/pinky_daemon/research_store.py index 6f3b10c2..83c63be5 100644 --- a/src/pinky_daemon/research_store.py +++ b/src/pinky_daemon/research_store.py @@ -23,6 +23,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -147,8 +149,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="research.db") connection.execute("PRAGMA foreign_keys=ON") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/sqlite_journal.py b/src/pinky_daemon/sqlite_journal.py new file mode 100644 index 00000000..aa4c7b7a --- /dev/null +++ b/src/pinky_daemon/sqlite_journal.py @@ -0,0 +1,103 @@ +"""Shared SQLite journal-mode configuration for long-lived daemon stores. + +WAL mode is unsafe for the way this daemon holds SQLite open. Two properties +combine badly: + +1. Every store opens the same database file once **per thread** + (``self._thread_local.connection``), so a single process holds several + independent connections to one file. +2. POSIX record locks are owned per ``(process, inode)``, not per descriptor. + The moment *any* of those connections is closed — a worker thread ending, an + explicit ``_reset_connection()`` — the process drops **all** of its advisory + locks on that file, including the ones the surviving connections still rely + on. + +After that the daemon is invisible to other processes: the next external opener +(a backup, an operator running ``sqlite3``, a script) sees an unlocked database, +believes it is the sole connection, and on close checkpoints and **unlinks** the +``-wal``/``-shm``. The daemon's surviving connections keep writing happily into +the now-orphaned inode. Those writes are visible only through the daemon itself; +every external reader sees data frozen at the last checkpoint, and the whole tail +is lost when the process exits. + +That failure mode was observed in production on 2026-08-01: four stores +(conversations, tasks, agent_comms, research) were left holding +``...-wal (deleted)`` with 54 messages, 5 inbox rows and 2 tasks reachable only +from the daemon's address space, while ``/proc//locks`` showed the daemon +holding no locks at all. + +Rollback (TRUNCATE) journalling sidesteps the whole class: committed data lives +in the main database file, there is no ``-wal`` for anyone to unlink and no +``-shm`` to map. It is the same remedy already applied to +``conversations_agents.db`` (#797/#220), the skills DB and the dream runner — +this module exists so those hand-rolled copies can converge on one +implementation instead of drifting. + +Trade-off, deliberately accepted: rollback mode serialises readers against the +writer. These stores are low-throughput control-plane data, and correctness of +the write path matters more here than reader concurrency. +""" + +from __future__ import annotations + +import sqlite3 +import time + + +class SqliteJournalConfigError(RuntimeError): + """A store connection could not be moved off WAL.""" + + +def configure_rollback_journal( + conn: sqlite3.Connection, + *, + db_label: str, + busy_ms: int = 30_000, + retries: int = 6, +) -> str: + """Put ``conn`` in rollback (TRUNCATE) journal mode and confirm it took. + + Must run BEFORE table init and before anything can spawn stdio children that + hold the database, so an existing WAL file is converted in place while the + daemon is still the only writer. + + Any hot WAL content is checkpointed first, so nothing committed is stranded + when the wal-index is dropped. A busy database during that drain is + non-fatal — the mode switch below retries. + + Fails LOUD: raises :class:`SqliteJournalConfigError` rather than silently + running on WAL, because a silent fallback is exactly the state that loses + writes. + + Args: + conn: Connection to configure, before any table creation. + db_label: Human-readable database name, used in the error message. + busy_ms: ``busy_timeout`` applied to the connection. + retries: Bounded attempts before giving up. + + Returns: + The effective journal mode, always ``"truncate"``. + """ + conn.execute(f"PRAGMA busy_timeout={int(busy_ms)}") + last: str | None = None + for attempt in range(retries): + try: + cur = conn.execute("PRAGMA journal_mode").fetchone() + if cur and str(cur[0]).lower() == "wal": + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except sqlite3.OperationalError: + pass + try: + row = conn.execute("PRAGMA journal_mode=TRUNCATE").fetchone() + last = str(row[0]).lower() if row else None + if last == "truncate": + return last + except sqlite3.OperationalError as exc: + last = f"error:{exc}" + time.sleep(0.2 * (attempt + 1)) + + raise SqliteJournalConfigError( + f"{db_label} refused to leave WAL: journal_mode={last!r} after {retries} " + f"attempts — refusing to run on WAL, where an unlinked -wal silently " + f"strands committed writes in the daemon's address space (#797/#220)." + ) diff --git a/src/pinky_daemon/task_store.py b/src/pinky_daemon/task_store.py index a1f7197f..a99b33bb 100644 --- a/src/pinky_daemon/task_store.py +++ b/src/pinky_daemon/task_store.py @@ -22,6 +22,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -195,8 +197,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="tasks.db") connection.execute("PRAGMA foreign_keys=ON") self._thread_local.connection = connection return connection diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py new file mode 100644 index 00000000..5e63421b --- /dev/null +++ b/tests/test_sqlite_journal.py @@ -0,0 +1,190 @@ +"""Rollback-journal configuration for the long-lived daemon stores. + +Regression cover for the 2026-08-01 orphaned-WAL incident: WAL mode plus +per-thread connections let an external process unlink the ``-wal`` out from +under the daemon, stranding committed writes in its address space. +""" + +from __future__ import annotations + +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +from pinky_daemon.agent_comms import AgentComms +from pinky_daemon.conversation_store import ConversationStore +from pinky_daemon.research_store import ResearchStore +from pinky_daemon.sqlite_journal import ( + SqliteJournalConfigError, + configure_rollback_journal, +) +from pinky_daemon.task_store import TaskStore + + +def _journal_mode(db_path: Path) -> str: + """Journal mode seen by a *fresh* connection, i.e. what the file persists. + + SQLite only records "WAL vs rollback" in the database header — it does not + persist *which* rollback mode. A new connection therefore reports the + compile-time default ("delete") even when the writer set TRUNCATE, so + callers assert on the family (``!= "wal"``), not on the exact value. + """ + conn = sqlite3.connect(db_path) + try: + return str(conn.execute("PRAGMA journal_mode").fetchone()[0]).lower() + finally: + conn.close() + + +def test_configure_returns_truncate_on_fresh_db(tmp_path): + conn = sqlite3.connect(tmp_path / "fresh.db") + assert configure_rollback_journal(conn, db_label="fresh.db") == "truncate" + conn.close() + + +def test_converts_an_existing_wal_database_in_place(tmp_path): + db_path = tmp_path / "legacy.db" + seed = sqlite3.connect(db_path) + seed.execute("PRAGMA journal_mode=WAL") + seed.execute("CREATE TABLE t (v TEXT)") + seed.execute("INSERT INTO t VALUES ('committed-under-wal')") + seed.commit() + seed.close() + assert _journal_mode(db_path) == "wal" + + conn = sqlite3.connect(db_path) + assert configure_rollback_journal(conn, db_label="legacy.db") == "truncate" + + # Content written under WAL survives the conversion — the helper checkpoints + # before dropping the wal-index. + assert conn.execute("SELECT v FROM t").fetchone()[0] == "committed-under-wal" + conn.close() + assert _journal_mode(db_path) != "wal" + + +def test_leaves_no_wal_or_shm_sidecars(tmp_path): + """The whole point: no ``-wal``/``-shm`` for another process to unlink.""" + db_path = tmp_path / "sidecars.db" + conn = sqlite3.connect(db_path) + configure_rollback_journal(conn, db_label="sidecars.db") + conn.execute("CREATE TABLE t (v TEXT)") + conn.execute("INSERT INTO t VALUES ('x')") + conn.commit() + + assert not (tmp_path / "sidecars.db-wal").exists() + assert not (tmp_path / "sidecars.db-shm").exists() + conn.close() + + +def test_committed_writes_are_visible_to_a_separate_process_handle(tmp_path): + """The incident symptom, inverted: an outside reader must see the write. + + Under the orphaned WAL, ``sqlite3`` from outside saw data frozen at the last + checkpoint while the daemon reported success. + """ + db_path = tmp_path / "visible.db" + daemon_conn = sqlite3.connect(db_path) + configure_rollback_journal(daemon_conn, db_label="visible.db") + daemon_conn.execute("CREATE TABLE t (v TEXT)") + daemon_conn.execute("INSERT INTO t VALUES ('written-by-daemon')") + daemon_conn.commit() + + outsider = sqlite3.connect(db_path) + assert outsider.execute("SELECT v FROM t").fetchone()[0] == "written-by-daemon" + outsider.close() + daemon_conn.close() + + +class _RefusesModeSwitch(sqlite3.Connection): + """A connection that always reports the database as locked for the switch. + + ``sqlite3.Connection`` is a C type, so its ``execute`` cannot be + monkeypatched on an instance — subclassing via ``connect(factory=...)`` is + the way to simulate a database that will not leave WAL. + """ + + def execute(self, sql, *args): # noqa: D102 - inherited contract + if "journal_mode=TRUNCATE" in sql: + raise sqlite3.OperationalError("database is locked") + return super().execute(sql, *args) + + +def test_raises_rather_than_silently_staying_on_wal(tmp_path): + """A silent WAL fallback is the state that loses writes — it must fail loud.""" + conn = sqlite3.connect(tmp_path / "stuck.db", factory=_RefusesModeSwitch) + + with pytest.raises(SqliteJournalConfigError, match="refused to leave WAL"): + configure_rollback_journal(conn, db_label="stuck.db", retries=2) + conn.close() + + +# Seeds a conversations.db under WAL and exits WITHOUT closing the connection, +# leaving a hot -wal behind — the state conversation_store's checkpoint has to +# survive. os._exit skips interpreter cleanup, so SQLite never checkpoints. +_HOT_WAL_SEEDER = """ +import os, sys +sys.path.insert(0, {src!r}) +import sqlite3 +from pinky_daemon.conversation_store import ConversationStore + +store = ConversationStore(db_path={db!r}) +store._conn.execute("PRAGMA journal_mode=WAL") +store.append("s1", "user", "checkpointed under wal") +store.append("s1", "agent", "hot in the wal file") +assert store._conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" +os._exit(0) +""" + + +def test_fts5_shadow_tables_survive_the_wal_conversion(tmp_path): + """conversation_store.py warns that checkpointing with live writers corrupts + the FTS5 shadow tables. Our checkpoint runs at connection open, before table + init and with no concurrent writer — this pins that it is in fact safe, on a + genuinely hot WAL rather than on the reasoning alone. + """ + db_path = tmp_path / "conversations.db" + src = str(Path(__file__).resolve().parents[1] / "src") + seeded = subprocess.run( + [sys.executable, "-c", _HOT_WAL_SEEDER.format(src=src, db=str(db_path))], + capture_output=True, + text=True, + ) + assert seeded.returncode == 0, seeded.stderr + assert db_path.with_name("conversations.db-wal").exists(), "no hot WAL to convert" + + # Opening the store checkpoints the hot WAL and converts to rollback mode. + store = ConversationStore(db_path=str(db_path)) + + integrity = store._conn.execute( + "INSERT INTO messages_fts(messages_fts) VALUES('integrity-check')" + ) + assert integrity is not None # raises DatabaseError if the index is corrupt + + # Both the checkpointed and the WAL-resident row are searchable... + assert len(store.search("checkpointed")) == 1 + assert len(store.search("hot")) == 1 + # ...and the triggers keep indexing after the conversion. + store.append("s1", "user", "indexed after the conversion") + assert len(store.search("conversion")) == 1 + store.close() + + +@pytest.mark.parametrize( + ("factory", "filename", "conn_attr"), + [ + (ConversationStore, "conversations.db", "_conn"), + (TaskStore, "tasks.db", "_db"), + (AgentComms, "agent_comms.db", "_conn"), + (ResearchStore, "research.db", "_db"), + ], +) +def test_stores_open_in_rollback_mode(tmp_path, factory, filename, conn_attr): + db_path = tmp_path / filename + store = factory(db_path=str(db_path)) + # Touch the lazy per-thread connection so the store actually opens. + assert getattr(store, conn_attr) is not None + assert _journal_mode(db_path) != "wal" + assert not (tmp_path / f"{filename}-wal").exists() From aa4a11fb3c51926712cc8b912b0d4e4fad9b2b10 Mon Sep 17 00:00:00 2001 From: Fixer Date: Sat, 1 Aug 2026 10:50:32 +0200 Subject: [PATCH 2/3] fix(sqlite): move the remaining daemon stores off WAL to rollback journalling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to ba1bded, which converted the four stores caught by the 2026-08-01 orphaned-WAL incident (conversations, tasks, agent_comms, research). Every other daemon store had the same exposure: WAL plus several connections to one file inside one process, where the first close drops the whole process's POSIX locks and lets an outside opener checkpoint and unlink the -wal out from under the survivors. Converted via configure_rollback_journal(), which checkpoints any hot WAL first and raises rather than silently staying on WAL: activity, apps, audit (hooks), mesh, message_context, outreach_config, presentations, sessions (both SessionStore and SessionEventStore), triggers, user_profiles, voice_calls ...plus the stores that open a fresh connection per call — kb, librarian state, analytics and the plugin state/context DBs. Those are exposed too: two overlapping short connections in one process still share locks, so whichever closes first disarms the other. Notes: - message_context_store's explicit busy_timeout=5000 is dropped; the helper installs 30s, which is the right way round under rollback journalling (readers serialise against the writer). - analytics_store set the pragma inside its schema executescript; the call now runs on the init connection, before the script. TRUNCATE is not persisted in the header, so its later per-call connections report "delete" — still rollback, which is what matters. - Deliberately NOT ported: upstream's _reset_connection() healing. Closing a descriptor is precisely what drops the process's locks on that inode. - Out of scope, still on WAL: pinky_identity, pinky_hub, pinky_federation, pinky_memory. Tests: extends the existing rollback-mode parametrisation to the twelve long-lived stores, adds cover for the four per-call ones, and adds a source guard so a new daemon store cannot copy-paste its way back onto WAL. test_message_context_store's assertion updated (it pinned "wal"). 307 passed across the affected suites. Co-Authored-By: Claude Opus 5 --- src/pinky_daemon/activity_store.py | 5 +- src/pinky_daemon/analytics_store.py | 8 ++- src/pinky_daemon/app_store.py | 4 +- src/pinky_daemon/dream_runner.py | 4 +- src/pinky_daemon/hooks.py | 5 +- src/pinky_daemon/kb_store.py | 4 +- src/pinky_daemon/librarian_runner.py | 3 +- src/pinky_daemon/mesh_store.py | 4 +- src/pinky_daemon/message_context_store.py | 8 ++- src/pinky_daemon/outreach_config.py | 4 +- src/pinky_daemon/plugin_manager.py | 6 +- src/pinky_daemon/presentation_store.py | 5 +- src/pinky_daemon/session_store.py | 8 +-- src/pinky_daemon/skill_store.py | 5 +- src/pinky_daemon/trigger_store.py | 5 +- src/pinky_daemon/user_profile_store.py | 5 +- src/pinky_daemon/voice_store.py | 5 +- tests/test_message_context_store.py | 6 +- tests/test_sqlite_journal.py | 70 +++++++++++++++++++++++ 19 files changed, 131 insertions(+), 33 deletions(-) diff --git a/src/pinky_daemon/activity_store.py b/src/pinky_daemon/activity_store.py index 7545abbd..0fe470e7 100644 --- a/src/pinky_daemon/activity_store.py +++ b/src/pinky_daemon/activity_store.py @@ -13,6 +13,8 @@ import time from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + class ActivityStore: """SQLite-backed activity log.""" @@ -29,8 +31,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="activity.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/analytics_store.py b/src/pinky_daemon/analytics_store.py index 36469f31..38154e98 100644 --- a/src/pinky_daemon/analytics_store.py +++ b/src/pinky_daemon/analytics_store.py @@ -6,6 +6,8 @@ from contextlib import contextmanager from datetime import UTC, datetime, timedelta +from pinky_daemon.sqlite_journal import configure_rollback_journal + # Providers to include in analytics dashboards. # Only Anthropic/Claude usage — excludes Codex CLI (OpenAI) and other non-Anthropic providers. _ANTHROPIC_PROVIDERS = {"firstParty", "default", "anthropic", "bedrock", "vertex"} @@ -77,10 +79,12 @@ def _connect(self): def _init_db(self) -> None: with self._connect() as conn: + # Runs once per store construction, before any other connection is + # handed out. TRUNCATE is persisted in the database header, so the + # short-lived connections from _connect() inherit it. + configure_rollback_journal(conn, db_label="analytics.db") conn.executescript( """ - PRAGMA journal_mode=WAL; - CREATE TABLE IF NOT EXISTS analytics_session_facts ( session_id TEXT PRIMARY KEY, agent_name TEXT NOT NULL, diff --git a/src/pinky_daemon/app_store.py b/src/pinky_daemon/app_store.py index c0fa2bdb..194dc025 100644 --- a/src/pinky_daemon/app_store.py +++ b/src/pinky_daemon/app_store.py @@ -18,6 +18,8 @@ from dataclasses import dataclass from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -78,7 +80,7 @@ def __init__(self, db_path: str = "data/apps.db") -> None: Path(db_path).parent.mkdir(parents=True, exist_ok=True) self._db_path = db_path self._db = sqlite3.connect(db_path, check_same_thread=False) - self._db.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(self._db, db_label="apps.db") self._db.execute("PRAGMA foreign_keys=ON") self._init_tables() diff --git a/src/pinky_daemon/dream_runner.py b/src/pinky_daemon/dream_runner.py index 79b101c1..12007433 100644 --- a/src/pinky_daemon/dream_runner.py +++ b/src/pinky_daemon/dream_runner.py @@ -33,6 +33,7 @@ ) from pinky_daemon.dream_prompt import DREAM_SYSTEM_PROMPT from pinky_daemon.sdk_runner import SDKRunner, SDKRunnerConfig +from pinky_daemon.sqlite_journal import configure_rollback_journal from pinky_daemon.tmux_dream_runner import TmuxDreamConfig, TmuxDreamRunner @@ -114,8 +115,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="dreams.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/hooks.py b/src/pinky_daemon/hooks.py index 8545def2..119530d0 100644 --- a/src/pinky_daemon/hooks.py +++ b/src/pinky_daemon/hooks.py @@ -23,6 +23,8 @@ from enum import Enum from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -133,8 +135,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="audit.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/kb_store.py b/src/pinky_daemon/kb_store.py index f7536a4e..733d521f 100644 --- a/src/pinky_daemon/kb_store.py +++ b/src/pinky_daemon/kb_store.py @@ -34,6 +34,8 @@ import yaml +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -177,7 +179,7 @@ def __init__(self, data_dir: str | Path) -> None: def _conn(self) -> sqlite3.Connection: conn = sqlite3.connect(str(self.db_path)) - conn.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(conn, db_label="kb.db") conn.execute("PRAGMA foreign_keys=ON") conn.row_factory = sqlite3.Row return conn diff --git a/src/pinky_daemon/librarian_runner.py b/src/pinky_daemon/librarian_runner.py index 9c13a354..3cc26ca3 100644 --- a/src/pinky_daemon/librarian_runner.py +++ b/src/pinky_daemon/librarian_runner.py @@ -22,6 +22,7 @@ from pinky_daemon.kb_store import _FRONTMATTER_RE, KBStore, _content_hash from pinky_daemon.librarian_prompt import LIBRARIAN_SYSTEM_PROMPT from pinky_daemon.sdk_runner import SDKRunner, SDKRunnerConfig +from pinky_daemon.sqlite_journal import configure_rollback_journal def _log(msg: str) -> None: @@ -66,7 +67,7 @@ def __init__( def _conn(self) -> sqlite3.Connection: conn = sqlite3.connect(str(self._db_path)) - conn.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(conn, db_label="librarian_state.db") conn.row_factory = sqlite3.Row return conn diff --git a/src/pinky_daemon/mesh_store.py b/src/pinky_daemon/mesh_store.py index b553c333..8df19e7e 100644 --- a/src/pinky_daemon/mesh_store.py +++ b/src/pinky_daemon/mesh_store.py @@ -29,6 +29,8 @@ from pathlib import Path from typing import Any, Literal +from pinky_daemon.sqlite_journal import configure_rollback_journal + # -- Records ------------------------------------------------------------------- @@ -102,7 +104,7 @@ class MeshStore: def __init__(self, db_path: str = "data/mesh.db") -> None: Path(db_path).parent.mkdir(parents=True, exist_ok=True) self._db = sqlite3.connect(db_path, check_same_thread=False) - self._db.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(self._db, db_label="mesh.db") self._db.execute("PRAGMA foreign_keys=ON") self._lock = threading.RLock() self._init_tables() diff --git a/src/pinky_daemon/message_context_store.py b/src/pinky_daemon/message_context_store.py index 1166e745..53cf7349 100644 --- a/src/pinky_daemon/message_context_store.py +++ b/src/pinky_daemon/message_context_store.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import Any +from pinky_daemon.sqlite_journal import configure_rollback_journal + DEFAULT_RETENTION_DAYS = 30 DEFAULT_MAX_PER_AGENT = 1000 @@ -30,8 +32,10 @@ def __init__( Path(db_path).parent.mkdir(parents=True, exist_ok=True) self._db = sqlite3.connect(db_path, check_same_thread=False) self._db.row_factory = sqlite3.Row - self._db.execute("PRAGMA journal_mode=WAL") - self._db.execute("PRAGMA busy_timeout=5000") + # configure_rollback_journal also installs a 30s busy_timeout, which + # replaces the 5s this store used to set: rollback journalling + # serialises readers against the writer, so waiting longer is right. + configure_rollback_journal(self._db, db_label="message_context.db") self._create_schema() self._ensure_columns() self._ensure_identity_key() diff --git a/src/pinky_daemon/outreach_config.py b/src/pinky_daemon/outreach_config.py index 87b2e960..8b1c176f 100644 --- a/src/pinky_daemon/outreach_config.py +++ b/src/pinky_daemon/outreach_config.py @@ -17,6 +17,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -53,7 +55,7 @@ class OutreachConfigStore: def __init__(self, db_path: str = "data/outreach_config.db") -> None: Path(db_path).parent.mkdir(parents=True, exist_ok=True) self._db = sqlite3.connect(db_path, check_same_thread=False) - self._db.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(self._db, db_label="outreach_config.db") self._init_tables() def _init_tables(self) -> None: diff --git a/src/pinky_daemon/plugin_manager.py b/src/pinky_daemon/plugin_manager.py index d77b8509..a3bdcbfd 100644 --- a/src/pinky_daemon/plugin_manager.py +++ b/src/pinky_daemon/plugin_manager.py @@ -38,6 +38,8 @@ import yaml +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -139,7 +141,7 @@ def db(self) -> sqlite3.Connection: data_dir.mkdir(parents=True, exist_ok=True) self._db_path = str(data_dir / "plugins.db") self._db = sqlite3.connect(self._db_path, check_same_thread=False) - self._db.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(self._db, db_label="plugins.db") return self._db def create_table(self, table_name: str, schema: str) -> None: @@ -296,7 +298,7 @@ def _init_state_db(self) -> None: """Initialize the plugin state tracking table.""" Path(self._state_db_path).parent.mkdir(parents=True, exist_ok=True) db = sqlite3.connect(self._state_db_path, check_same_thread=False) - db.execute("PRAGMA journal_mode=WAL") + configure_rollback_journal(db, db_label="plugins.db") db.execute(""" CREATE TABLE IF NOT EXISTS plugin_state ( name TEXT PRIMARY KEY, diff --git a/src/pinky_daemon/presentation_store.py b/src/pinky_daemon/presentation_store.py index cae20ba2..da541a71 100644 --- a/src/pinky_daemon/presentation_store.py +++ b/src/pinky_daemon/presentation_store.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -137,8 +139,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="presentations.db") connection.execute("PRAGMA foreign_keys=ON") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/session_store.py b/src/pinky_daemon/session_store.py index 3e4568b6..ac34312f 100644 --- a/src/pinky_daemon/session_store.py +++ b/src/pinky_daemon/session_store.py @@ -20,6 +20,8 @@ from dataclasses import dataclass, field from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -71,8 +73,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="sessions.db") self._thread_local.connection = connection return connection @@ -314,8 +315,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="sessions.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/skill_store.py b/src/pinky_daemon/skill_store.py index c2c95cbe..5f8c6d56 100644 --- a/src/pinky_daemon/skill_store.py +++ b/src/pinky_daemon/skill_store.py @@ -29,6 +29,8 @@ from enum import Enum from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -153,8 +155,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="skills.db") connection.execute("PRAGMA foreign_keys=ON") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/trigger_store.py b/src/pinky_daemon/trigger_store.py index 53ff5293..622ccfc9 100644 --- a/src/pinky_daemon/trigger_store.py +++ b/src/pinky_daemon/trigger_store.py @@ -19,6 +19,8 @@ from dataclasses import dataclass from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -87,8 +89,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="triggers.db") connection.execute("PRAGMA foreign_keys=ON") connection.row_factory = sqlite3.Row self._thread_local.connection = connection diff --git a/src/pinky_daemon/user_profile_store.py b/src/pinky_daemon/user_profile_store.py index 4e4881be..e8985894 100644 --- a/src/pinky_daemon/user_profile_store.py +++ b/src/pinky_daemon/user_profile_store.py @@ -18,6 +18,8 @@ from dataclasses import asdict, dataclass from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + @dataclass class ProfileEntry: @@ -90,8 +92,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="user_profiles.db") self._thread_local.connection = connection return connection diff --git a/src/pinky_daemon/voice_store.py b/src/pinky_daemon/voice_store.py index a8a06c42..b8af4a43 100644 --- a/src/pinky_daemon/voice_store.py +++ b/src/pinky_daemon/voice_store.py @@ -21,6 +21,8 @@ from dataclasses import dataclass from pathlib import Path +from pinky_daemon.sqlite_journal import configure_rollback_journal + def _log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) @@ -229,8 +231,7 @@ def _db(self) -> sqlite3.Connection: connection = getattr(self._thread_local, "connection", None) if connection is None: connection = sqlite3.connect(self._db_path) - connection.execute("PRAGMA journal_mode=WAL") - connection.execute("PRAGMA busy_timeout=30000") + configure_rollback_journal(connection, db_label="voice_calls.db") connection.execute("PRAGMA foreign_keys=ON") connection.row_factory = sqlite3.Row self._thread_local.connection = connection diff --git a/tests/test_message_context_store.py b/tests/test_message_context_store.py index cf95f04b..783acd7d 100644 --- a/tests/test_message_context_store.py +++ b/tests/test_message_context_store.py @@ -71,7 +71,7 @@ def test_context_survives_broker_restart_via_load_through(tmp_path): second_store.close() -def test_store_uses_wal_and_self_heals_optional_columns(tmp_path): +def test_store_uses_rollback_journal_and_self_heals_optional_columns(tmp_path): db_path = tmp_path / "message-context.db" legacy = sqlite3.connect(db_path) legacy.execute( @@ -104,7 +104,9 @@ def test_store_uses_wal_and_self_heals_optional_columns(tmp_path): for row in store._db.execute("PRAGMA table_info(message_contexts)").fetchall() } - assert str(mode).lower() == "wal" + # Rollback journalling, not WAL: an unlinked -wal silently strands + # committed writes in the daemon's address space (#797/#220). + assert str(mode).lower() == "truncate" assert {"reply_to", "attachments_json", "metadata_json", "stored_at"} <= columns primary_key = tuple( row["name"] diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py index 5e63421b..5d284504 100644 --- a/tests/test_sqlite_journal.py +++ b/tests/test_sqlite_journal.py @@ -14,14 +14,29 @@ import pytest +from pinky_daemon.activity_store import ActivityStore from pinky_daemon.agent_comms import AgentComms +from pinky_daemon.analytics_store import AnalyticsStore +from pinky_daemon.app_store import AppStore from pinky_daemon.conversation_store import ConversationStore +from pinky_daemon.hooks import AuditStore +from pinky_daemon.kb_store import KBStore +from pinky_daemon.librarian_runner import LibrarianRunner +from pinky_daemon.mesh_store import MeshStore +from pinky_daemon.message_context_store import MessageContextStore +from pinky_daemon.outreach_config import OutreachConfigStore +from pinky_daemon.plugin_manager import PluginManager +from pinky_daemon.presentation_store import PresentationStore from pinky_daemon.research_store import ResearchStore +from pinky_daemon.session_store import SessionEventStore, SessionStore from pinky_daemon.sqlite_journal import ( SqliteJournalConfigError, configure_rollback_journal, ) from pinky_daemon.task_store import TaskStore +from pinky_daemon.trigger_store import TriggerStore +from pinky_daemon.user_profile_store import UserProfileStore +from pinky_daemon.voice_store import VoiceStore def _journal_mode(db_path: Path) -> str: @@ -179,6 +194,18 @@ def test_fts5_shadow_tables_survive_the_wal_conversion(tmp_path): (TaskStore, "tasks.db", "_db"), (AgentComms, "agent_comms.db", "_conn"), (ResearchStore, "research.db", "_db"), + (ActivityStore, "activity.db", "_db"), + (AppStore, "apps.db", "_db"), + (AuditStore, "audit.db", "_db"), + (MeshStore, "mesh.db", "_db"), + (MessageContextStore, "message_context.db", "_db"), + (OutreachConfigStore, "outreach_config.db", "_db"), + (PresentationStore, "presentations.db", "_db"), + (SessionStore, "sessions.db", "_db"), + (SessionEventStore, "session_events.db", "_db"), + (TriggerStore, "triggers.db", "_db"), + (UserProfileStore, "user_profiles.db", "_db"), + (VoiceStore, "voice_calls.db", "_db"), ], ) def test_stores_open_in_rollback_mode(tmp_path, factory, filename, conn_attr): @@ -188,3 +215,46 @@ def test_stores_open_in_rollback_mode(tmp_path, factory, filename, conn_attr): assert getattr(store, conn_attr) is not None assert _journal_mode(db_path) != "wal" assert not (tmp_path / f"{filename}-wal").exists() + + +def test_per_call_connection_stores_open_in_rollback_mode(tmp_path): + """Stores that open a fresh connection per call, not one long-lived handle. + + These are just as exposed: two overlapping short connections in one process + still share POSIX locks, so whichever closes first drops the locks the other + is relying on. + """ + kb = KBStore(data_dir=str(tmp_path / "kb")) + assert _journal_mode(Path(kb.db_path)) != "wal" + assert not Path(str(kb.db_path) + "-wal").exists() + + librarian_db = tmp_path / "librarian_state.db" + LibrarianRunner(kb, db_path=str(librarian_db)) + assert _journal_mode(librarian_db) != "wal" + + analytics_db = tmp_path / "analytics.db" + AnalyticsStore(db_path=str(analytics_db)) + assert _journal_mode(analytics_db) != "wal" + + plugins_db = tmp_path / "plugins.db" + PluginManager( + db_path=str(plugins_db), + api_url="http://localhost:8888", + working_dir=str(tmp_path / "wd"), + ) + assert _journal_mode(plugins_db) != "wal" + + +def test_no_daemon_module_still_asks_for_wal(): + """Guard against a new store landing back on WAL by copy-paste.""" + src = Path(__file__).resolve().parents[1] / "src" / "pinky_daemon" + offenders = [ + f"{path.relative_to(src)}:{n}" + for path in sorted(src.rglob("*.py")) + for n, line in enumerate(path.read_text().splitlines(), 1) + if "journal_mode=WAL" in line + ] + assert offenders == [], ( + "these daemon modules still open SQLite in WAL mode — use " + f"configure_rollback_journal() instead: {offenders}" + ) From 4b9bd7f7081ae55924864010f6bc1d423451b2ed Mon Sep 17 00:00:00 2001 From: Fixer Date: Sat, 1 Aug 2026 11:39:06 +0200 Subject: [PATCH 3/3] test(sqlite): assert rollback journalling in the per-store concurrency hammers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The thread-local hammer tests pinned `journal_mode == "wal"` on each connection they opened. Now that every long-lived store configures rollback journalling in its connection factory, they pin "truncate" — the same property, against the mode the stores actually run in. --- tests/test_activity_store.py | 2 +- tests/test_dream_runner.py | 2 +- tests/test_hooks.py | 2 +- tests/test_presentation_store.py | 2 +- tests/test_research_store.py | 2 +- tests/test_skill_store.py | 2 +- tests/test_trigger_store.py | 2 +- tests/test_user_profile_store.py | 2 +- tests/test_voice_store.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_activity_store.py b/tests/test_activity_store.py index 8664692a..9565971c 100644 --- a/tests/test_activity_store.py +++ b/tests/test_activity_store.py @@ -32,7 +32,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 0 agent_name = f"worker-{worker_index}" for round_index in range(rounds): diff --git a/tests/test_dream_runner.py b/tests/test_dream_runner.py index 5010360d..98795b29 100644 --- a/tests/test_dream_runner.py +++ b/tests/test_dream_runner.py @@ -46,7 +46,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 0 agent_name = f"worker-{worker_index}" for round_index in range(rounds): diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 7a12b1c9..8c496d23 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -33,7 +33,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 0 for round_index in range(rounds): marker = f"{worker_index}-{round_index}" diff --git a/tests/test_presentation_store.py b/tests/test_presentation_store.py index 55e83c5d..6f1877ac 100644 --- a/tests/test_presentation_store.py +++ b/tests/test_presentation_store.py @@ -39,7 +39,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 for round_index in range(rounds): marker = f"{worker_index}-{round_index}" diff --git a/tests/test_research_store.py b/tests/test_research_store.py index ca246099..b1224573 100644 --- a/tests/test_research_store.py +++ b/tests/test_research_store.py @@ -32,7 +32,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 for round_index in range(rounds): marker = f"{worker_index}-{round_index}" diff --git a/tests/test_skill_store.py b/tests/test_skill_store.py index 5b4c4ba6..9e23f2c1 100644 --- a/tests/test_skill_store.py +++ b/tests/test_skill_store.py @@ -136,7 +136,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 for round_index in range(rounds): marker = f"{worker_index}-{round_index}" diff --git a/tests/test_trigger_store.py b/tests/test_trigger_store.py index 2ba54c35..c7f4aaab 100644 --- a/tests/test_trigger_store.py +++ b/tests/test_trigger_store.py @@ -69,7 +69,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 for round_index in range(rounds): marker = f"{worker_index}-{round_index}" diff --git a/tests/test_user_profile_store.py b/tests/test_user_profile_store.py index 12a05d9d..490c15b3 100644 --- a/tests/test_user_profile_store.py +++ b/tests/test_user_profile_store.py @@ -36,7 +36,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 0 chat_id = f"worker-{worker_index}" for round_index in range(rounds): diff --git a/tests/test_voice_store.py b/tests/test_voice_store.py index 99284959..993051fb 100644 --- a/tests/test_voice_store.py +++ b/tests/test_voice_store.py @@ -33,7 +33,7 @@ def hammer(worker_index): connection_id = id(connection) assert connection.execute( "PRAGMA journal_mode" - ).fetchone()[0].lower() == "wal" + ).fetchone()[0].lower() == "truncate" assert connection.execute("PRAGMA foreign_keys").fetchone()[0] == 1 assert connection.row_factory is sqlite3.Row for round_index in range(rounds):