Harden launch readiness

.github/workflows/ci.yml

@@ -0,0 +1,42 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install package and dev tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e '.[all]' -r requirements-dev.txt ruff mypy bandit build twine coverage
+      - name: Ruff
+        run: python -m ruff check .
+      - name: Mypy
+        run: python -m mypy doxa --ignore-missing-imports
+      - name: Bandit
+        run: python -m bandit -q -r doxa
+      - name: Tests
+        run: python -m pytest -q
+      - name: Coverage
+        run: |
+          python -m coverage run -m pytest -q
+          python -m coverage report --fail-under=75
+      - name: Build package
+        run: python -m build --sdist --wheel
+      - name: Twine check
+        run: python -m twine check dist/*

.github/workflows/release.yml

@@ -0,0 +1,37 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+      - name: Build distributions
+        run: python -m build --sdist --wheel
+      - name: Check distributions
+        run: python -m twine check dist/*
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Create GitHub release
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*

doxa/cli.py

@@ -6,6 +6,7 @@
 from copy import deepcopy
 import json
 import os
+import shutil
 import sys
 from pathlib import Path
 from typing import Any
@@ -32,7 +33,7 @@
 from .schema import DoxaError, RetrievalResult
 from .sources import load_source
 from .sources.fetchers import INGEST_MODES, available_fetchers, build_fetch_prompt
-from .store import JsonlStore, index_postgres
+from .store import JsonlStore, index_postgres, postgres_table_prefix
 
 
 FIREWORKS_BASE_URL = "https://api.fireworks.ai/inference/v1"
@@ -225,8 +226,17 @@ def _init_answers(args: argparse.Namespace, *, interactive: bool) -> dict[str, s
     }
 
 
-def _build_init_config(answers: dict[str, str]) -> dict[str, Any]:
-    config = deepcopy(DEFAULT_CONFIG)
+def _build_init_config(answers: dict[str, str], *, full: bool = False) -> dict[str, Any]:
+    if full:
+        config: dict[str, Any] = deepcopy(DEFAULT_CONFIG)
+    else:
+        config = {
+            "project": {"name": "my-belief-base"},
+            "data": {"dir": "data"},
+            "lens": {},
+            "llm": {"provider": answers["provider"], "model": answers["model"], "temperature": 0},
+            "providers": {answers["provider"]: deepcopy(DEFAULT_CONFIG["providers"].get(answers["provider"], {}))},
+        }
     provider = answers["provider"]
     model = answers["model"]
     config["project"]["name"] = "my-belief-base"
@@ -269,7 +279,7 @@ def cmd_init(args: argparse.Namespace) -> int:
         print("doxa init: configure belief mining")
         print("Press Enter to accept any default.")
     answers = _init_answers(args, interactive=interactive)
-    _write_yaml_config(dest, _build_init_config(answers))
+    _write_yaml_config(dest, _build_init_config(answers, full=args.full))
     config = load_config(dest, allow_demo_default=False)
     data_dir(config).mkdir(parents=True, exist_ok=True)
     print(f"Created {dest}")
@@ -521,6 +531,56 @@ def cmd_status(args: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_doctor(args: argparse.Namespace) -> int:
+    """Check common local setup problems without mining or fetching anything."""
+
+    issues: list[str] = []
+    try:
+        config = load_config(getattr(args, "config", None), allow_demo_default=False)
+    except Exception as exc:
+        print(f"config: FAIL ({exc})")
+        return 1
+    print(f"config: OK ({config.get('_config_path') or 'defaults'})")
+    try:
+        postgres_table_prefix(config)
+        print("postgres table prefix: OK")
+    except Exception as exc:
+        issues.append(f"postgres table prefix: {exc}")
+        print(f"postgres table prefix: FAIL ({exc})")
+    store = JsonlStore(config)
+    try:
+        counts = (len(store.beliefs()), len(store.quotes()), len(store.sources()))
+        print(f"store: OK ({counts[0]} beliefs, {counts[1]} quotes, {counts[2]} sources)")
+    except Exception as exc:
+        issues.append(f"store: {exc}")
+        print(f"store: FAIL ({exc})")
+    provider = str((config.get("llm") or {}).get("provider") or "codex-cli")
+    provider_config = (config.get("providers") or {}).get(provider, {})
+    if provider in {"codex-cli", "claude-cli"}:
+        binary = str(provider_config.get("binary") or ("codex" if provider == "codex-cli" else "claude"))
+        if shutil.which(binary):
+            print(f"provider: OK ({provider}, binary {binary})")
+        else:
+            issues.append(f"provider binary not found: {binary}")
+            print(f"provider: FAIL ({provider} binary not found: {binary})")
+    else:
+        api_key_env = str(provider_config.get("api_key_env") or "")
+        if api_key_env and not os.environ.get(api_key_env):
+            issues.append(f"provider env var not set: {api_key_env}")
+            print(f"provider: FAIL ({provider}, set {api_key_env})")
+        else:
+            print(f"provider: OK ({provider})")
+    dsn_env = str((config.get("postgres") or {}).get("dsn_env") or "DOXA_POSTGRES_DSN")
+    print(f"semantic index: {'configured' if os.environ.get(dsn_env) else f'off (set {dsn_env} to enable)'}")
+    if issues:
+        print("\nDoctor found issues:")
+        for issue in issues:
+            print(f"  - {issue}")
+        return 1
+    print("\nDoctor found no blocking issues.")
+    return 0
+
+
 def cmd_sources(args: argparse.Namespace) -> int:
     command = getattr(args, "sources_command", None) or "list"
     config = load_config(getattr(args, "config", None))
@@ -537,16 +597,19 @@ def cmd_sources(args: argparse.Namespace) -> int:
         print("No sources ingested yet.")
         _hint("add one: doxa ingest <file|url|->")
         return 0
-    from collections import Counter
-
     beliefs = store.beliefs()
     quotes = store.quotes()
-    b_by = Counter((b.source.title, b.source.url) for b in beliefs)
-    q_by = Counter((q.source.title, q.source.url) for q in quotes)
+
+    def source_matches(ref, source) -> bool:
+        if getattr(ref, "id", ""):
+            return ref.id == source.id
+        return (ref.title, ref.url) == (source.title, source.url)
+
     print(f"{len(sources)} source(s):")
     for source in sources:
-        key = (source.title, source.url)
-        print(f"  {source.id}  {source.title}  [beliefs {b_by.get(key, 0)}, quotes {q_by.get(key, 0)}]")
+        belief_count = sum(1 for belief in beliefs if source_matches(belief.source, source))
+        quote_count = sum(1 for quote in quotes if source_matches(quote.source, source))
+        print(f"  {source.id}  {source.title}  [beliefs {belief_count}, quotes {quote_count}]")
         location = source.url or source.path
         if location:
             print(f"      {location}")
@@ -657,6 +720,7 @@ def build_parser() -> argparse.ArgumentParser:
     init.add_argument("--lens", help="Lens description.")
     init.add_argument("--lens-name", help="Lens name.")
     init.add_argument("--lens-question", help="Guiding question for the lens.")
+    init.add_argument("--full", action="store_true", help="Write the full documented config instead of the compact starter config.")
     init.set_defaults(func=cmd_init)
 
     ingest = subparsers.add_parser("ingest", help="Ingest one or more sources (text, PDF, URL, YouTube, or '-' from stdin).")
@@ -726,6 +790,10 @@ def build_parser() -> argparse.ArgumentParser:
     status.add_argument("--config")
     status.set_defaults(func=cmd_status)
 
+    doctor = subparsers.add_parser("doctor", help="Check config, storage, provider, and semantic-index readiness.")
+    doctor.add_argument("--config")
+    doctor.set_defaults(func=cmd_doctor)
+
     sources_p = subparsers.add_parser("sources", help="List or remove ingested sources.")
     sources_p.add_argument("--config")
     sources_p.set_defaults(func=cmd_sources)

doxa/config.py

@@ -37,12 +37,14 @@
     "providers": {
         "codex-cli": {
             "binary": "codex",
-            "flags": ["exec", "--dangerously-bypass-approvals-and-sandbox"],
+            "flags": ["exec"],
             "output_flag": "-o",
+            "timeout": 300,
         },
         "claude-cli": {
             "binary": "claude",
             "flags": ["-p", "{prompt}", "--output-format", "json"],
+            "timeout": 300,
         },
         "openai": {
             "base_url": "",
@@ -65,12 +67,12 @@
         # Override per-ingest with --via.
         "fetcher": "requests",
         "brightdata": {
-            "api_token_env": "BRIGHTDATA_API_TOKEN",
+            "api_token_env": "BRIGHTDATA_API_TOKEN",  # nosec B105
             "zone_env": "BRIGHTDATA_ZONE",
         },
         "jina": {"api_key_env": "JINA_API_KEY"},
         "firecrawl": {"api_key_env": "FIRECRAWL_API_KEY"},
-        # command: { argv: ["my-scraper", "{url}"] }  # or { shell: "..." }; prints text to stdout
+        # command: { argv: ["my-scraper", "{url}"] }  # shell requires allow_shell: true and is discouraged
     },
     "retrieval": {
         "default_search": "keyword",
@@ -223,6 +225,7 @@ def load_yaml(path: Path) -> dict[str, Any]:
 def load_config(path: str | Path | None = None, *, allow_demo_default: bool = True) -> dict[str, Any]:
     """Load config, falling back to bundled demo data when no config exists."""
 
+    config_path: Path | None
     if path:
         config_path = Path(path).expanduser().resolve()
         if not config_path.exists():

doxa/domains.py

@@ -6,7 +6,7 @@
 import os
 from pathlib import Path
 import re
-import subprocess
+import subprocess  # nosec B404
 import tempfile
 from typing import Any, Callable
 
@@ -434,7 +434,7 @@ def edit_domain_weights(path: str | Path | None) -> Path:
         handle.flush()
         temp_path = Path(handle.name)
     try:
-        proc = subprocess.run([editor, str(temp_path)], check=False)
+        proc = subprocess.run([editor, str(temp_path)], check=False)  # nosec B603 - explicit user editor from VISUAL/EDITOR.
         if proc.returncode != 0:
             raise DoxaError(f"Editor exited with status {proc.returncode}.")
         edited = load_yaml(temp_path)

doxa/embed.py

@@ -48,6 +48,6 @@ def embed_query(query: str, config: dict[str, Any]) -> list[float]:
         try:
             return [vector.tolist() for vector in query_embed([query])][0]
         except Exception:  # noqa: BLE001 - fall back to symmetric embedding
-            pass
+            return [vector.tolist() for vector in model.embed([query])][0]
     return [vector.tolist() for vector in model.embed([query])][0]
 

doxa/mine.py

@@ -105,6 +105,12 @@ def mine_source(
         raw = parse_provider_json(provider.complete(system, user))
         local_beliefs = [Belief.from_dict({**item, "source": item.get("source") or source_meta}) for item in raw.get("beliefs", [])]
         local_quotes = [Quote.from_dict({**item, "source": item.get("source") or source_meta}) for item in raw.get("quotes", [])]
+        for belief in local_beliefs:
+            if not belief.source.id:
+                belief.source.id = source.id
+        for quote in local_quotes:
+            if not quote.source.id:
+                quote.source.id = source.id
 
         id_map = {belief.id: _stable_id("b", source.id, chunk_index, belief.id) for belief in local_beliefs}
         for belief in local_beliefs:

doxa/providers/anthropic.py

@@ -34,7 +34,7 @@ def complete(self, system: str, user: str) -> str:
             system=system,
             messages=[{"role": "user", "content": user}],
         )
-        parts = [block.text for block in response.content if getattr(block, "type", "") == "text"]
+        parts = [str(text) for block in response.content if (text := getattr(block, "text", None)) is not None]
         text = "\n".join(parts).strip()
         if not text:
             raise DoxaError("Anthropic provider returned no text content.")

doxa/providers/claude_cli.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import shutil
-import subprocess
+import subprocess  # nosec B404
 from typing import Any
 
 from doxa.schema import DoxaError
@@ -16,6 +16,7 @@ def __init__(self, config: dict[str, Any]):
         provider_config = (config.get("providers") or {}).get("claude-cli", {})
         self.binary = str(provider_config.get("binary") or "claude")
         self.flags = [str(flag) for flag in provider_config.get("flags", ["-p", "{prompt}", "--output-format", "json"])]
+        self.timeout = int(provider_config.get("timeout", 300))
         if shutil.which(self.binary) is None:
             raise DoxaError(
                 "claude-cli provider needs the Claude Code CLI on PATH. Install Claude Code or set providers.claude-cli.binary."
@@ -24,13 +25,17 @@ def __init__(self, config: dict[str, Any]):
     def complete(self, system: str, user: str) -> str:
         prompt = system + "\n\n" + user
         cmd = [self.binary, *[flag.replace("{prompt}", prompt) for flag in self.flags]]
-        completed = subprocess.run(
-            cmd,
-            text=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=False,
-        )
+        try:
+            completed = subprocess.run(
+                cmd,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=False,
+                timeout=self.timeout,
+            )  # nosec B603 - configured argv-only provider CLI invocation.
+        except subprocess.TimeoutExpired:
+            raise DoxaError(f"Claude CLI provider timed out after {self.timeout}s.") from None
         if completed.returncode != 0:
             raise DoxaError(
                 "Claude CLI provider failed. Check your Claude Code auth or configure providers.claude-cli.flags. "
@@ -39,4 +44,3 @@ def complete(self, system: str, user: str) -> str:
         if not completed.stdout.strip():
             raise DoxaError("Claude CLI provider returned no output.")
         return completed.stdout
-