diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..483200f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,21 @@ +name: tests + +on: + pull_request: + push: + branches: [main] + +jobs: + scrubber-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install deps + run: pip install pytest httpx presidio-analyzer presidio-anonymizer + - name: Download spaCy model + run: python -m spacy download en_core_web_md + - name: Run scrubber tests (incl. golden leak canary) + run: pytest tests/ -q diff --git a/README.md b/README.md index ed1a476..10ec19a 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,9 @@ path still runs. - `profile/` -- `config.yaml` (the full incognito profile), `SOUL.md` (the Ghost identity), `.env.example`, `pii_denylist.example.txt`, `uncensored_prefill.json` - `privacy/` - `scrubbing_proxy.py` -- the PII/secret scrubber + local model-catalog endpoint; forwards cleaned requests to og-veil - - `rotating_proxy.py` -- Webshare rotation + blocklist · `gen_searxng_settings.py` + - `scrub_patterns.py` -- single source of the secret + PII regexes (shared by both scrub paths) + - `presidio_scrub.py` -- NER PII detection (Presidio + spaCy) with reversible placeholders + stream de-anon + - `rotating_proxy.py` -- Webshare rotation (opt-in IP-masking) - `ensure_scrubber_route.py` -- self-heals the engine's hosted route back to the scrubber after a token refresh - _(the OHTTP/HPKE/registry/verification + Supabase auth that used to live here now comes from the `opengradient-veil` package -- run `og-veil`)_ - `scripts/` -- `fork-engine.sh` (copy + relocate venv + isolate skills) and `debrand.py` (scrub visible strings + the two ASCII-art logos) diff --git a/bin/ghost b/bin/ghost index 5004599..55a8263 100755 --- a/bin/ghost +++ b/bin/ghost @@ -69,10 +69,9 @@ elif [ "$HC" = "200" ]; then else STATUS="👻 ghost · ⚠️ scrubbing bridge DOWN (bridge=$HC) -- hosted unreachable, $FB_DOWN" fi -if scutil --dns 2>/dev/null | grep -q "103.86.9"; then VPN="✅ NordVPN"; else VPN="⚠️ no NordVPN"; fi [ -n "$PASS_PATHS" ] && STATUS="$STATUS · 🗂️ path-aware (real paths visible to hosted model)" [ -f "$NOSCRUB_MARK" ] && STATUS="$STATUS · 🔓 PII redaction OFF (secrets still scrubbed)" -echo "$STATUS · $VPN" >&2 +echo "$STATUS" >&2 HB="__ENG__/venv/bin/hermes" if [ -n "$PASS_PATHS" ]; then diff --git a/install.sh b/install.sh index c1cc539..0384a8e 100644 --- a/install.sh +++ b/install.sh @@ -26,6 +26,13 @@ # GHOST_CHAT_APP_URL= override the website used for `ghost-login` (default chat.opengradient.ai) set -euo pipefail +# macOS only: the privacy stack runs as launchd LaunchAgents and uses BSD tooling. Fail fast +# with a clear message rather than part-installing on Linux/WSL and erroring confusingly later. +if [ "$(uname -s)" != "Darwin" ]; then + echo "!! ghost's installer currently supports macOS only (it uses launchd). Detected: $(uname -s)." >&2 + exit 1 +fi + REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENGINE_HOME="${ENGINE_HOME:-$HOME/.hermes}" # where the Hermes engine installs (official installer default) GHOST_HOME="${GHOST_HOME:-$HOME/.ghost}" # ghost's ISOLATED state (profiles, privacy, auth) @@ -132,7 +139,7 @@ fi # ---------- 3. privacy stack (PII scrubber + og-veil always; rotating proxy only with GHOST_PROXY) ---------- say "Privacy stack (PII/secret scrubber -> og-veil${USE_PROXY:+ + rotating proxy})" -mkdir -p "$PRIV/searxng" +mkdir -p "$PRIV" cp "$REPO"/privacy/*.py "$PRIV/" # Enable the NER PII scrubber when Presidio + the spaCy model are present; else leave it off # (the bridge falls back to the regex scrubber). Toggle anytime: touch/rm $PRIV/.presidio. diff --git a/privacy/gen_searxng_settings.py b/privacy/gen_searxng_settings.py deleted file mode 100644 index 30ad480..0000000 --- a/privacy/gen_searxng_settings.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a SearXNG settings.yml for private, proxy-routed local search. - -SearXNG routes its OWN upstream engine requests through the rotating Webshare -proxies (round-robin over the list), so a private query never touches BrightData -or Nous -- only the search engines see it, and they see a Webshare IP. JSON API -is enabled (Hermes' searxng backend needs it) and the bot limiter is off (local -single-user). Re-run this whenever ~/.ghost/webshare_proxies.txt changes.""" -import os, secrets - -PROXIES = os.path.expanduser("~/.ghost/webshare_proxies.txt") -OUT_DIR = os.path.expanduser("~/.ghost/privacy/searxng") -OUT = os.path.join(OUT_DIR, "settings.yml") - -proxies = [] -try: - with open(PROXIES) as f: - for ln in f: - ln = ln.strip() - if ln and ln.count(":") >= 3: - ip, port, user, pwd = ln.split(":", 3) - proxies.append(f"http://{user}:{pwd}@{ip}:{port}") -except FileNotFoundError: - pass - -os.makedirs(OUT_DIR, exist_ok=True) -proxy_lines = "\n".join(f' - "{p}"' for p in proxies) - -settings = f"""# Auto-generated by gen_searxng_settings.py -- private proxy-routed search. -use_default_settings: true -server: - secret_key: "{secrets.token_hex(32)}" - limiter: false - image_proxy: false - method: "GET" -search: - safe_search: 0 - formats: - - html - - json -outgoing: - request_timeout: 30.0 - max_request_timeout: 40.0 - pool_connections: 100 - pool_maxsize: 50 - proxies: - "all://": -{proxy_lines} -""" - -open(OUT, "w").write(settings) -print(f"wrote {OUT} with {len(proxies)} rotating Webshare upstreams")