From bfa9c929edbfd42e709ac5782bea17e42b23ae63 Mon Sep 17 00:00:00 2001 From: Advait Jayant Date: Sat, 6 Jun 2026 21:24:40 +0100 Subject: [PATCH] Add examples gallery and safe config templates --- doxa.example.yaml | 51 ++++++++++---- examples/README.md | 63 +++++++++++++++++ examples/configs/agent-fetchers-safe.yaml | 50 ++++++++++++++ examples/configs/command-fetcher-argv.yaml | 44 ++++++++++++ .../configs/fireworks-openai-compatible.yaml | 36 ++++++++++ examples/configs/openai-minimal.yaml | 35 ++++++++++ examples/configs/semantic-postgres.yaml | 47 +++++++++++++ examples/scripts/demo_smoke.sh | 9 +++ tests/test_examples.py | 67 +++++++++++++++++++ 9 files changed, 389 insertions(+), 13 deletions(-) create mode 100644 examples/README.md create mode 100644 examples/configs/agent-fetchers-safe.yaml create mode 100644 examples/configs/command-fetcher-argv.yaml create mode 100644 examples/configs/fireworks-openai-compatible.yaml create mode 100644 examples/configs/openai-minimal.yaml create mode 100644 examples/configs/semantic-postgres.yaml create mode 100755 examples/scripts/demo_smoke.sh create mode 100644 tests/test_examples.py diff --git a/doxa.example.yaml b/doxa.example.yaml index 90d1273..f75159b 100644 --- a/doxa.example.yaml +++ b/doxa.example.yaml @@ -1,17 +1,16 @@ -# doxa configuration +# doxa configuration example # -# Copy this file to doxa.yaml with: +# For most projects, prefer: # doxa init # -# The default provider is codex-cli because it uses your existing Codex CLI auth -# and does not require a new API key. claude-cli behaves similarly for Claude -# Code users. API providers are available behind extras. +# This file is a fuller annotated reference you can copy to doxa.yaml when you +# want to tune providers, fetchers, retrieval, domains, or semantic search. project: name: my-belief-base data: - # JSONL is the source of truth. These paths are relative to this config file. + # JSONL is the source of truth. Paths are relative to this config file. dir: data beliefs: beliefs.jsonl quotes: quotes.jsonl @@ -41,21 +40,25 @@ llm: providers: codex-cli: - # Runs: codex exec --dangerously-bypass-approvals-and-sandbox -o - + # Runs `codex exec` through argv with a bounded timeout. + # Set unsafe_bypass: true only for trusted local workflows that explicitly + # accept unattended agent execution risk. binary: codex flags: - exec - - --dangerously-bypass-approvals-and-sandbox output_flag: -o + timeout: 300 + unsafe_bypass: false claude-cli: - # Runs: claude -p "" --output-format json + # Runs Claude Code through argv with a bounded timeout. binary: claude flags: - -p - "{prompt}" - --output-format - json + timeout: 300 openai: api_key_env: OPENAI_API_KEY @@ -79,16 +82,38 @@ providers: model: claude-3-5-sonnet-latest sources: - # URL fetcher options: requests, brightdata. - # For agents with a BrightData MCP, prefer fetching there and piping text: - # printf '%s' "$FETCHED_MARKDOWN" | doxa ingest - --title "Title" --url "https://..." + # URL fetcher options: requests, jina, firecrawl, brightdata, command, + # claude, codex, hermes. Override per ingest with `--via `. fetcher: requests + + jina: + api_key_env: JINA_API_KEY + + firecrawl: + api_key_env: FIRECRAWL_API_KEY + brightdata: api_token_env: BRIGHTDATA_API_TOKEN zone_env: BRIGHTDATA_ZONE + command: + # Prefer argv. It avoids a shell and can call any local scraper/MCP bridge. + # Example: + # argv: ["my-scraper", "{url}"] + timeout: 120 + # Shell mode is disabled unless allow_shell is true. + allow_shell: false + + codex: + timeout: 300 + unsafe_bypass: false + + hermes: + timeout: 300 + unsafe_yolo: false + retrieval: - # keyword works with zero infrastructure. semantic needs embeddings+postgres. + # Keyword works with zero infrastructure. semantic/hybrid need embeddings+postgres. default_search: keyword limit: 5 # Overfetch before final ranking so rare quote hits and domain boosts can surface. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..81e189c --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# Examples + +Concrete, copyable starting points for doxa. The examples avoid private data, +network-dependent tests, and unsafe defaults. + +## Zero-setup demo + +The bundled demo uses public-domain excerpts from Emerson, Plato, and Madison. +It requires no API key, database, embeddings, or network access. + +```bash +doxa demo +doxa query "self-reliance and conformity" --top 2 +doxa query "examined life" --answer +``` + +The same files are mirrored in [`examples/demo`](demo/) so you can inspect the +JSONL source of truth: + +```bash +doxa query "faction and liberty" --config examples/demo/doxa.yaml +``` + +## Config examples + +Copy one of these to `doxa.yaml`, then edit the project name, lens, and provider +settings for your corpus. + +| File | Use when | +| --- | --- | +| [`configs/openai-minimal.yaml`](configs/openai-minimal.yaml) | you want API-based mining with `OPENAI_API_KEY` | +| [`configs/fireworks-openai-compatible.yaml`](configs/fireworks-openai-compatible.yaml) | you want OpenAI-compatible mining through Fireworks or another compatible endpoint | +| [`configs/semantic-postgres.yaml`](configs/semantic-postgres.yaml) | you want optional pgvector-backed semantic/hybrid search | +| [`configs/agent-fetchers-safe.yaml`](configs/agent-fetchers-safe.yaml) | you want Claude/Codex/Hermes to fetch hard web pages without unattended bypass by default | +| [`configs/command-fetcher-argv.yaml`](configs/command-fetcher-argv.yaml) | you want to route URL fetching through a local scraper or MCP bridge | + +Example: + +```bash +cp examples/configs/openai-minimal.yaml doxa.yaml +export OPENAI_API_KEY=... +doxa ingest ./sources/my-essay.txt +doxa query "what does this source believe about agency?" --answer +``` + +## Public-domain source policy + +New checked-in corpora should be public domain, pre-1929, or explicitly licensed +for redistribution. For modern/private sources, keep the text outside the repo +and ingest locally: + +```bash +pbpaste | doxa ingest - --title "Private notes" --author "Me" +``` + +## Smoke script + +[`scripts/demo_smoke.sh`](scripts/demo_smoke.sh) exercises the offline demo and +is safe to run in CI or a clean local environment: + +```bash +bash examples/scripts/demo_smoke.sh +``` diff --git a/examples/configs/agent-fetchers-safe.yaml b/examples/configs/agent-fetchers-safe.yaml new file mode 100644 index 0000000..0751037 --- /dev/null +++ b/examples/configs/agent-fetchers-safe.yaml @@ -0,0 +1,50 @@ +# Safe agent fetcher examples. +# +# Agent fetchers let a local CLI agent fetch/render hard web pages before doxa +# mines them. They are safe by default: no unattended bypass/yolo flags unless +# you opt in explicitly for trusted sources. +# +# Usage examples: +# cp examples/configs/agent-fetchers-safe.yaml doxa.yaml +# doxa ingest https://example.com/article --via hermes --mode browser +# doxa ingest https://example.com/article --via codex --mode extract --prompt "return clean markdown only" + +project: + name: agent-fetched-belief-base + +data: + dir: data + +lens: + name: durable-beliefs + description: Extract durable claims, values, and stances that the source explicitly supports. + question: What does this source believe? + stances: [supports, questions, rejects, complicates] + tags: [web, research] + +llm: + provider: codex-cli + temperature: 0 + +providers: + codex-cli: + binary: codex + flags: [exec] + output_flag: -o + timeout: 300 + unsafe_bypass: false + +sources: + fetcher: requests + claude: + timeout: 300 + codex: + timeout: 300 + unsafe_bypass: false + hermes: + timeout: 300 + unsafe_yolo: false + +retrieval: + default_search: keyword + limit: 5 diff --git a/examples/configs/command-fetcher-argv.yaml b/examples/configs/command-fetcher-argv.yaml new file mode 100644 index 0000000..e6ef251 --- /dev/null +++ b/examples/configs/command-fetcher-argv.yaml @@ -0,0 +1,44 @@ +# Command fetcher through argv. +# +# Use this when you already have a local scraper, MCP bridge, or internal fetch +# command that prints markdown/text to stdout. argv is preferred because it avoids +# shell interpolation. +# +# Usage: +# cp examples/configs/command-fetcher-argv.yaml doxa.yaml +# doxa ingest https://example.com/article --via command + +project: + name: command-fetched-belief-base + +data: + dir: data + +lens: + name: durable-beliefs + description: Extract durable claims, values, and stances that the source explicitly supports. + question: What does this source believe? + stances: [supports, questions, rejects, complicates] + tags: [research] + +llm: + provider: openai + model: gpt-4.1-mini + temperature: 0 + +providers: + openai: + api_key_env: OPENAI_API_KEY + model: gpt-4.1-mini + +sources: + fetcher: command + command: + # Replace with your scraper. {url} and {prompt} placeholders are expanded. + argv: [python, scripts/fetch_url.py, "{url}"] + timeout: 120 + allow_shell: false + +retrieval: + default_search: keyword + limit: 5 diff --git a/examples/configs/fireworks-openai-compatible.yaml b/examples/configs/fireworks-openai-compatible.yaml new file mode 100644 index 0000000..14c9544 --- /dev/null +++ b/examples/configs/fireworks-openai-compatible.yaml @@ -0,0 +1,36 @@ +# OpenAI-compatible provider example, set up for Fireworks by default. +# +# Usage: +# cp examples/configs/fireworks-openai-compatible.yaml doxa.yaml +# export FIREWORKS_API_KEY=... +# doxa ingest ./source.txt +# +# Replace the model slug with the Fireworks model you want to use. + +project: + name: fireworks-belief-base + +data: + dir: data + +lens: + name: strategy-beliefs + description: Extract durable claims about strategy, tradeoffs, decisions, and operating principles. + question: What operating beliefs does this source explicitly support? + stances: [supports, questions, rejects, complicates] + tags: [strategy, decisions] + +llm: + provider: openai-compatible + model: accounts/fireworks/models/ + temperature: 0 + +providers: + openai-compatible: + base_url: https://api.fireworks.ai/inference/v1 + api_key_env: FIREWORKS_API_KEY + model: accounts/fireworks/models/ + +retrieval: + default_search: keyword + limit: 5 diff --git a/examples/configs/openai-minimal.yaml b/examples/configs/openai-minimal.yaml new file mode 100644 index 0000000..aed25f3 --- /dev/null +++ b/examples/configs/openai-minimal.yaml @@ -0,0 +1,35 @@ +# Minimal API-backed doxa project. +# +# Usage: +# cp examples/configs/openai-minimal.yaml doxa.yaml +# export OPENAI_API_KEY=... +# doxa ingest ./source.txt +# doxa query "what does this source believe?" --answer + +project: + name: openai-belief-base + +data: + dir: data + +lens: + name: durable-beliefs + description: Extract durable claims, values, and stances that the source explicitly supports. + question: What does this source believe about how people should think, decide, or act? + stances: [supports, questions, rejects, complicates] + tags: [agency, judgment] + +llm: + provider: openai + model: gpt-4.1-mini + temperature: 0 + +providers: + openai: + api_key_env: OPENAI_API_KEY + model: gpt-4.1-mini + base_url: "" + +retrieval: + default_search: keyword + limit: 5 diff --git a/examples/configs/semantic-postgres.yaml b/examples/configs/semantic-postgres.yaml new file mode 100644 index 0000000..0406ec4 --- /dev/null +++ b/examples/configs/semantic-postgres.yaml @@ -0,0 +1,47 @@ +# Optional semantic/hybrid search with Postgres + pgvector. +# +# Usage: +# cp examples/configs/semantic-postgres.yaml doxa.yaml +# export OPENAI_API_KEY=... +# export DOXA_POSTGRES_DSN=postgresql://user:pass@localhost:5432/doxa +# python -m pip install -e ".[embeddings,postgres,openai]" +# doxa ingest ./source.txt +# doxa index +# doxa query "political conflict" --search hybrid + +project: + name: semantic-belief-base + +data: + dir: data + +lens: + name: durable-beliefs + description: Extract durable claims, values, and stances that the source explicitly supports. + question: What does this source believe? + stances: [supports, questions, rejects, complicates] + tags: [research] + +llm: + provider: openai + model: gpt-4.1-mini + temperature: 0 + +providers: + openai: + api_key_env: OPENAI_API_KEY + model: gpt-4.1-mini + +retrieval: + default_search: hybrid + limit: 8 + candidate_limit: 80 + max_quotes_per_result: 3 + +embeddings: + model: BAAI/bge-small-en-v1.5 + dimension: 384 + +postgres: + dsn_env: DOXA_POSTGRES_DSN + table_prefix: doxa diff --git a/examples/scripts/demo_smoke.sh b/examples/scripts/demo_smoke.sh new file mode 100755 index 0000000..e8aa606 --- /dev/null +++ b/examples/scripts/demo_smoke.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +DOXA_BIN="${DOXA_BIN:-doxa}" + +"$DOXA_BIN" demo +"$DOXA_BIN" query "self-reliance and conformity" --top 2 +"$DOXA_BIN" query "examined life" --answer +"$DOXA_BIN" eval --config examples/demo/doxa.yaml diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..090c821 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +import yaml + +from doxa.config import load_config +from doxa.store import JsonlStore, postgres_table_prefix + +ROOT = Path(__file__).resolve().parents[1] +DANGEROUS_LITERALS = ( + "--dangerously-" + "bypass-approvals-and-sandbox", + "--yolo", +) +UNSAFE_BOOL_KEYS = {"allow_shell", "unsafe_bypass", "unsafe_yolo"} + + +def example_yaml_paths() -> list[Path]: + return [ + ROOT / "doxa.example.yaml", + ROOT / "examples" / "demo" / "doxa.yaml", + *sorted((ROOT / "examples" / "configs").glob("*.yaml")), + ] + + +def walk_mapping(value: Any) -> Iterable[tuple[str, Any]]: + if isinstance(value, dict): + for key, child in value.items(): + yield str(key), child + yield from walk_mapping(child) + elif isinstance(value, list): + for child in value: + yield from walk_mapping(child) + + +def test_example_yaml_files_parse_and_load() -> None: + paths = example_yaml_paths() + + assert paths + for path in paths: + with path.open(encoding="utf-8") as handle: + parsed = yaml.safe_load(handle) + assert isinstance(parsed, dict), path + config = load_config(path, allow_demo_default=False) + assert config["project"]["name"], path + postgres_table_prefix(config) + + +def test_checked_in_examples_do_not_enable_unsafe_execution_defaults() -> None: + for path in example_yaml_paths(): + text = path.read_text(encoding="utf-8") + for needle in DANGEROUS_LITERALS: + assert needle not in text, f"{path} contains {needle}" + parsed = yaml.safe_load(text) or {} + enabled = [key for key, value in walk_mapping(parsed) if key in UNSAFE_BOOL_KEYS and value is True] + assert not enabled, f"{path} enables unsafe execution knobs: {enabled}" + + +def test_demo_example_store_is_valid() -> None: + config = load_config(ROOT / "examples" / "demo" / "doxa.yaml", allow_demo_default=False) + store = JsonlStore(config) + + assert len(store.sources()) == 3 + assert len(store.beliefs()) == 8 + assert len(store.quotes()) == 8