diff --git a/.gitignore b/.gitignore index 25cbe04..81866d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ __pycache__/ *.py[cod] *.egg-info/ +build/ +dist/ +*.whl .pytest_cache/ .mypy_cache/ .ruff_cache/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..60693bb --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +# sdist completeness: setup.py assembles framework/_bundled/ from fetchers/ and +# examples/ at build time, so an sdist (or the GitHub source tarball) must carry +# both, plus the core data files, or a wheel built from it comes out hollow. +graft fetchers +graft examples +graft framework/schemas +graft framework/reference +graft framework/tui/styles +graft uploaders +include requirements.txt + +prune fetchers/logos +global-exclude __pycache__ *.py[cod] .DS_Store diff --git a/docs/deferred/workspace_sync_design.md b/docs/deferred/workspace_sync_design.md new file mode 100644 index 0000000..bf70cb7 --- /dev/null +++ b/docs/deferred/workspace_sync_design.md @@ -0,0 +1,554 @@ +# [DEFERRED] Workspace materialize + sync reconcile — design + +**Status:** deferred, 2026-07-13 — superseded by the overlay model in +[`../distribution_design.md`](../distribution_design.md). This design solved +"users edit shipped files in place, upgrades must reconcile around the edits." +The expected customization turned out to be *creating* fetchers for +unsupported platforms, not modifying built-ins — which the overlay model +serves with a fraction of the machinery. **Revive this design only if +evidence shows customers routinely editing many built-ins in place** (at that +point copy-on-write overrides rot and reconcile earns its complexity). The +per-file decision table (§8.1) and migration framework (§9) here are fully +specified and reviewed — start from them, not from scratch. + +**Audience:** whoever builds the fork. +**Related:** [`../versioning.md`](../versioning.md) (the contract + bump policy), +[`../releasing.md`](../releasing.md) (how a release is cut), +[`../design.md`](../design.md) (framework overview). + +## 1. The problem + +Today the tool is **clone-only**. `api.find_repo_root()` walks up from the cwd +looking for sibling `fetchers/` + `framework/` dirs, schemas load from +`repo_root/framework/schemas/`, and fetchers run as subprocesses out of +`repo_root/fetchers/`. That makes it trivial to add a fetcher (drop a directory +in the writable tree) but ties the whole tool to a working checkout. + +Two consequences we want to fix: + +1. **No clean installed distribution.** `pip install`/`brew install` of the + current package ships only `framework*` — no fetchers, and `find_repo_root()` + throws unless you happen to be inside a clone. (`releasing.md` → *Artifacts* + already flags this as the blocker deferring wheel/PyPI.) +2. **`git pull` clobbers local edits.** Because the checkout *is* the + distribution, a customer's edits to a built-in `fetcher.yaml` collide with + upstream changes to the same tracked file — the pull conflicts or stomps + uncommitted work. Every built-in is a landmine. + +## 2. Goal + +Ship the tool as a normal installable package whose **content is user-owned and +self-updating**: + +- `brew install` / `pipx install` brings the **core** (engine + CLI + a bundled, + read-only snapshot of content). +- A first-run step **materializes** the content into a user-writable + **workspace** and records exactly what was shipped. +- On upgrade, a **reconcile** step updates only the files the user never touched, + preserves everything they did, and never silently discards a customization. +- Adding a fetcher stays trivial — you create a directory in the workspace. + +### Decision already made (logged, #224) + +Content is **bundled with each core release**, not shipped on a separate channel. +One version number for core+content ⇒ no core↔content compatibility matrix, and +`paramify sync` is a **purely local** reconcile (no network). The accepted +tradeoff: shipping a new/fixed fetcher requires cutting a release. + +### Non-goals + +- PyInstaller / single-file frozen binaries (blocked separately: Python fetchers + launch via `sys.executable`, which points at the frozen app, not Python). +- A network content registry / plugin marketplace. +- Changing what a fetcher *is* or the envelope format. + +## 3. Vocabulary + +| Term | Meaning | Owner | Writable | +|---|---|---|---| +| **Core** | `framework/` engine + CLI + JSON schemas | us (package manager) | no | +| **Pristine bundle** (`framework/_pristine/`) | read-only snapshot of shipped content (fetchers, example manifests; skills deferred — §11), baked into the release | us | no | +| **Workspace** | materialized, user-editable copy of the content | user | yes | +| **Manifest** (`.manifest.json`) | record of what we shipped into the workspace + hashes | tool | (tool-managed) | +| **User data** | run manifests, `EVIDENCE_DIR` output | user | yes | + +Three data classes, kept strictly separate: **tool assets** (core, read-only) · +**managed content** (workspace, reconciled by `sync`) · **user data** (cwd, never +touched by `sync`). + +## 4. On-disk layout + +``` +INSTALL (read-only, package-manager owned) ─────────────── +…/site-packages/ + framework/ + schemas/ fetcher_schema.json, category_schema.json, … + _pristine/ ← the shipped content snapshot + fetchers/ (inside the package, so importlib.resources + examples/ resolves it and nothing non-package lands + …engine… top-level in site-packages — see §10) + +WORKSPACE (writable, user owned) ───────────────────────── +$PARAMIFY_HOME (default: platformdirs user_data_dir — see §13) + fetchers/ ← materialized + your own; fetchers RUN from here + examples/ + .manifest.json ← what we shipped, with hashes + .backups// ← pre-migration snapshots + +USER DATA (writable, cwd) ───────────────────────────────── +/manifests/, /evidence/ ← never touched by sync +``` + +Claude skills are **deferred out of v1** (decision #234; see §11): `sync` +materializes only `fetchers/` and `examples/`, which is what buys the §12 +invariant that sync never writes outside `$PARAMIFY_HOME`. + +Note a *benefit* of running fetchers out of the writable workspace: fetchers that +write next to themselves (e.g. the checkov git-clone scanner) work again — they +were a known problem for a read-only install. + +## 5. Discovery: single root → search path + +The discovery seam is `config_loader.discover_fetchers(repo_root)` and +`discover_platforms(repo_root)`, plus schema resolution in +`config_loader._load_schema()`. The change: + +**Schema resolution** moves off the cwd walk. Schemas are core, so they resolve +from the installed `framework` package via `importlib.resources` — with the +cwd path kept as a dev fallback: + +```python +def load_schema(name="fetcher_schema.json") -> dict: + # 1. dev checkout (editable install): framework/schemas/ next to source + local = Path(__file__).parent / "schemas" / name + if local.exists(): + return json.loads(local.read_text()) + # 2. installed: packaged data via importlib.resources + return json.loads(resources.files("framework.schemas").joinpath(name).read_text()) +``` +(Requires shipping the schemas as package data — see §10. Once they ship, +branch 1 also resolves in site-packages — `framework/schemas/` is a real +directory there — so branch 2 effectively covers only zip imports. Keep both, +but the comment should say branch 1 is what serves installed users.) + +**Fetcher discovery** takes an ordered list of roots instead of one, and unions +results; earlier roots win a name collision so a user copy can shadow a built-in: + +```python +def fetcher_roots() -> list[Path]: + roots = [] + if env := os.environ.get("PARAMIFY_FETCHERS_PATH"): # 1 explicit override + roots += [Path(p) for p in env.split(os.pathsep)] + if co := _find_checkout_root(): # 2 dev clone (cwd walk) + roots.append(co / "fetchers") + roots.append(workspace_home() / "fetchers") # 3 workspace + return [r for r in roots if r.is_dir()] + +def discover_fetchers(roots: list[Path]) -> dict[str, Fetcher]: + schema = load_schema(); validator = Draft202012Validator(schema) + found = {} + for root in roots: # precedence: first root wins on name + for fetcher in _walk(root, validator): + found.setdefault(fetcher.name, fetcher) + return found +``` + +The checkout outranks the workspace **on purpose**: a developer inside a clone +who has ever run `sync` would otherwise have stale workspace copies silently +shadowing their in-tree edits — the most confusing possible failure of the §11 +"dev clone keeps working" promise. + +Two collision rules, kept distinct. **Within one root**, a duplicate name stays +a hard error, exactly as today (`config_loader.py:72`). **Across roots**, first +root wins — but the shadow is *reported*, never silent: `catalog` and `doctor` +list every name resolved by shadowing and which root lost. + +`discover_platforms()` gets the same treatment: each root's `_categories/*.yaml` +is loaded and unioned, first root winning **per category file**. + +`find_repo_root()` stays for the dev path but is no longer the only way to +locate content. Everything downstream (`catalog`, `run`, `doctor`) calls +`fetcher_roots()`. + +## 6. The manifest — what makes non-clobbering possible + +The single new asset. Without a record of *what we shipped*, "the user changed +this file" is indistinguishable from "we changed this file upstream." The +manifest is that record, written at materialize time and read at every sync. + +```json +{ + "schema": "paramify-workspace-manifest/v1", + "tool_version": "0.4.0", + "contract_version": 2, + "materialized_at": "2026-07-13T00:00:00Z", + "files": { + "fetchers/aws/storage_encryption_status/fetcher.yaml": { + "sha256": "9f2c…", "shipped_tool_version": "0.4.0", "contract_version": 2 + }, + "fetchers/aws/storage_encryption_status/fetch.py": { "sha256": "1a7b…" }, + "examples/demo.yaml": { "sha256": "c3d4…" } + } +} +``` + +Per file we store the content hash of exactly what we last put there. That is +the "base" in a 3-way comparison: **base = manifest hash**, **ours = workspace +file**, **theirs = new pristine file**. (`materialized_at` is stamped by the +caller, not inside any pure-function core — timestamps are a side input.) Hash +**newline-normalized** text (CRLF→LF), not raw bytes — otherwise a Windows/git +line-ending rewrite makes an untouched file look edited and fires a spurious +conflict on every sync (§13). Normalization applies to **text files only**: +`fetchers/` also carries PNGs/JPEGs/SVGs and fixture files, and CRLF-normalizing +a binary manufactures the same spurious-conflict bug in the opposite direction. +Sniff binary the way git does (NUL byte in the first 8 KB ⇒ binary ⇒ hash raw +bytes). (Worth deciding separately whether `fetchers/logos/` — README furniture, +unused at run time — belongs in the pristine bundle at all.) + +## 7. Lifecycle commands (CLI surface) + +New `@app.command`s in `framework/cli.py`, alongside `list`/`catalog`/`run`/`doctor`. + +### `paramify sync` +The workhorse. Idempotent. Runs materialize-or-reconcile against the currently +installed `_pristine/`. No network. + +| Flag | Effect | +|---|---| +| `--dry-run` | report the plan (per-file classification), touch nothing | +| `--json` | machine-readable plan/report (consistent with other commands) | +| `--force-theirs GLOB` | on conflict, take upstream for matching paths (backup first) | +| `--force-mine GLOB` | on conflict, keep local, discard the incoming `.new` | +| `--no-migrate` | reconcile files but skip schema migrations (escape hatch) | + +Output is a grouped report: **updated**, **added**, **kept (yours)**, +**kept deleted (yours)**, **conflicts (.new written)**, **still conflicted**, +**migrated**, **removed**, **quarantined**. + +### `paramify migrate` (may be folded into `sync`) +Runs pending contract migrations on workspace content only. Separated so it can +be run/tested in isolation and so `sync --no-migrate` has a counterpart. + +### `paramify doctor` (extend the existing command, `cli.py:335`) +Add a distribution section: install path + `tool_version`, workspace path + +manifest `tool_version`/`contract_version`, whether a sync is pending (installed +pristine newer than manifest), and any quarantined fetchers. + +## 8. The sync algorithm + +``` +1. locate INSTALL/_pristine and WORKSPACE; load .manifest.json + (first run = no manifest AND empty workspace; a populated workspace with + a missing/corrupt manifest is NOT a first run — it flows through the + §8.1 table like any other sync and is never blind-copied over) +2. if contract_version(pristine) > contract_version(manifest): + plan migrations (§9); snapshot workspace → .backups// +3. for each file in the NEW pristine set: + classify by (workspace hash, manifest hash, new pristine hash) → §8.1 + apply the action (respecting --dry-run / --force-*) +4. for each file in the manifest but absent from new pristine: → "removed" rule +5. run migrations on workspace files still at the old contract_version +6. re-validate every workspace fetcher against the installed schema; + any failure ⇒ quarantine (mark, keep, report — never delete, never run; + the mark is a "quarantined": true flag on the file's .manifest.json + entry, which discovery reads and skips) +7. rewrite .manifest.json to the new shipped set + hashes +8. print the grouped report +``` + +The manifest rewrite is deliberately last (step 7), so a sync can die at any +point without corrupting state — **provided** re-running is safe. The +**ws == new** short-circuit (first row of §8.1) is what makes it safe: on +re-run, files updated before the crash classify as *already current* instead of +falling through to a false "you edited it AND we shipped" conflict. Without +that row, one interrupted sync turns every already-updated built-in into a +spurious `.new`. + +### 8.1 Per-file decision table + +Rows are checked in order; the first match wins. "ws / manifest / new" are the +workspace file, the manifest entry, and the new pristine file; a missing file +or entry fails every hash equality. The two groups mirror steps 3 and 4 of the +algorithm. + +**Files in the new pristine set** (step 3): + +| State | Detected by | Action | +|---|---|---| +| **Already current** | ws == new | **No-op**; record in manifest. First check — this is what makes an interrupted sync safely re-runnable | +| **New built-in** | absent from ws and manifest | **Add** (materialize). On a true first run, every file takes this row | +| **You deleted it** | absent from ws; in manifest | **Stays deleted** — never resurrect; drop the manifest entry, report once | +| **You never touched it** | ws == manifest | **Update in place** to new pristine | +| **You edited it, we didn't** | ws ≠ manifest, new == manifest | **Keep yours** (nothing to apply) | +| **You edited it AND we shipped a new version** | in all three, all hashes differ | **Conflict:** keep yours, write ours as `.new`, report | +| **No record** | in ws, absent from manifest, ws ≠ new | **Conflict** (as above) — never blind-overwrite. Arises on first run over a populated workspace, a lost/corrupt `.manifest.json`, or upstream re-adding a path we earlier kept | + +**Manifest entries absent from the new pristine set** (step 4): + +| State | Detected by | Action | +|---|---|---| +| **We removed it, you never touched it** | ws == manifest | Remove the file, drop the entry | +| **We removed it, you edited it** | in ws, ws ≠ manifest | Keep + warn; drop the entry (the file becomes user content) | +| **Gone everywhere** | absent from ws | Drop the entry | + +Workspace files in neither the manifest nor the pristine set are pure user +content — neither pass ever visits them. + +This table *is* the answer to "will it overwrite my changes." Only the conflict +rows have real UX; the rest are mechanical. + +### 8.2 Conflict resolution + +Default = **`.new` file** (dpkg/pacman style): zero data loss, dead simple, and +non-interactive-safe for cron. `--force-mine`/`--force-theirs` and a future +interactive/TUI review are layered on top. A 3-way *content* merge is a possible +enhancement but not required for v1. + +`.new` handling is itself idempotent: rewrite `.new` only when its content +would differ, and report an unresolved carry-over as **still conflicted** (with +the version it first appeared in), separately from new conflicts — otherwise +every sync re-announces the same conflict as if it were news. + +## 9. Schema evolution & migrations + +Per `versioning.md`, the schemas are **the contract**. Two change classes: + +### Additive (non-breaking) — brew upgrade only +New **optional** field. The permissive schema (no top-level +`additionalProperties: false`) means old files still validate. After +`brew upgrade` the new validator accepts the field; existing workspace fetchers +keep working untouched. `sync` is only needed to receive *our* built-ins that use +the field. `versioning.md` bump: **minor**. + +### Breaking — needs a migration +Removing/renaming/retyping a field, or adding a **required** one. Existing files +become invalid on upgrade. `versioning.md` bump: **major** (pre-1.0: **minor**). + +**Enabler — a contract version marker.** `fetcher.yaml` needs to say which shape +it is, or migrations can't reliably target it. The envelope already does this +(`schema_version` in `framework/envelope.py`); mirror it: add a required +`schema_version` (or `contract_version`) to `fetcher_schema.json`. This is the +migration counterpart of the workspace manifest — the thing that makes safe +breaking changes *possible*. Bootstrap rule: **a file with no marker is +contract version 1** — adding the required marker is itself the first breaking +change, so it rides its own grace window (the loader treats an absent marker as +v1; the v1→v2 migration stamps it). + +**Migration framework.** Migrations ship in the core with the release that breaks +the format: + +``` +framework/migrations/ + __init__.py # registry: ordered [(1→2, fn), (2→3, fn), …] + v1_to_v2.py # def migrate(doc: dict) -> dict (pure, deterministic) +``` +Properties: **chained** (v1→v2→v3, so version-skippers catch up in order), +**version-gated + idempotent** (re-running is a no-op), **pure** (dict→dict, so +unit-testable with fixtures). `sync`/`migrate` apply pending migrations to each +workspace fetcher, bump its marker, and re-validate. + +**Recommended posture — a one-release grace window.** Avoid a hard cutover: for +one release the loader accepts *both* shapes and normalizes old→new in memory +with a deprecation warning; the *next* release drops the old shape. Turns a break +into announce → warn → remove (matches the post-1.0 deprecation policy in +`versioning.md`, applied early). + +**The hard case — user-edited AND breaking.** A built-in the user edited (row 3) +that *also* changed shape: +- migration is a pure structural transform that commutes with their edit ⇒ apply + it to their file, keep their customization, bump the marker; +- migration can't apply cleanly ⇒ **quarantine**: keep their file, mark it + invalid, write our migrated version as `.new`, surface it loudly with the + migration notes. Two invariants: never leave a silently-broken fetcher, never + silently discard a customization. + +## 10. Release integration + +Slots into the existing `releasing.md` steps: + +- **Build the pristine bundle into the artifact.** Add + `[tool.setuptools.package-data]` so the wheel/tarball ships + `framework/schemas/*.json`, `framework/tui/styles/*.tcss`, and a + `framework/_pristine/` tree assembled from `fetchers/` and `examples/` + (skills deferred — §11). `_pristine/` lives *inside* the package: resolvable + via `importlib.resources`, shippable by the existing `include = ["framework*"]`, + and nothing non-package lands top-level in site-packages. The assembly step + must run **inside the build backend** (a custom setuptools `build_py` hook), + not in a release workflow — the supported install is still `pip install` from + the GitHub source tarball, and a workflow-side copy would leave source + installs with no `_pristine/`. This is the change that unblocks the deferred + wheel/PyPI item in `releasing.md`. +- **Author a migration** if the release makes a breaking schema change (same PR). +- **Changelog:** a `BREAKING` subsection with migration notes when applicable; + keep calling out the three version axes. +- **Bump:** per `versioning.md` bump policy (breaking ⇒ minor pre-1.0). + +Everything stays manual/tag-driven; nothing here forces `release-please`. + +## 11. Backwards-compat & open questions + +- **Dev clone keeps working.** `pip install -e .` in a checkout: the cwd walk is + root #2 and **outranks the workspace** (§5), schemas resolve from the local + `framework/schemas/`, no `_pristine` needed. This is also how the existing + test suite keeps running. +- **RESOLVED — skills are deferred out of v1** (decision #234, 2026-07-13). + v1 `sync` materializes only `fetchers/` and `examples/`; skills stay in the + repo for clone users and out of the pristine bundle. This sidesteps the + user-global (`~/.claude/skills/`) vs project-scoped (`/.claude/skills/`) + placement question until there's demand, and buys the §12 invariant that + sync never writes outside `$PARAMIFY_HOME`. +- **Workspace root** — resolve with `platformdirs.user_data_dir("paramify")` so + each OS gets its native location (§13), with a `$PARAMIFY_HOME` override. +- **Multi-project.** One workspace per user, or per project? Start with one per + user; `$PARAMIFY_HOME` lets a project opt into its own. + +## 12. Safety & recoverability + +- **Sync writes only inside `$PARAMIFY_HOME`.** With skills deferred (§11), v1 + sync touches nothing outside the workspace — no cwd files, no `~/.claude`, no + install tree. One directory to back up, one to blow away. +- **Backup before migrating** → `.backups//` so a bad transform is + recoverable. +- **`--dry-run`** reports the full plan before touching anything. +- **Validate after** every sync/migrate; anything still invalid is **quarantined** + (kept + flagged), never run and never deleted. +- **Refuse, don't crash.** A fetcher that fails schema validation is skipped with + a visible error (candidate for the deferred `SETUP-ERR` run status), not a + whole-run abort. + +## 13. Platform nuances (macOS / Windows / Linux) + +The **framework** (discovery, sync, migrations, CLI) is pure Python + `pathlib` +and portable. The nuances live in two places: **how you install** and **how +fetchers run**. + +### Install path +- **Homebrew is macOS + Linux only** — there is no brew on Windows. The universal + path is **pipx** (Mac/Windows/Linux); brew is a Mac/Linux convenience layer + (another reason the prototype goes pipx-first, §15). A native Windows package + could later use Scoop or WinGet. +- **pipx PATH nuance.** Console scripts of extras (e.g. `checkov`) install into + the pipx venv's `bin/`, which pipx does **not** put on `PATH` — and the + executor passes children only the inherited `PATH` (`executor.py:32`). A bash + fetcher shelling out to `checkov` works in the clone-venv world and breaks + under pipx. Either the executor prepends `Path(sys.executable).parent` to the + child `PATH`, or checkov joins the external-CLI list alongside `aws`/`jq`. + +### Fetcher execution — the big one +- **90 of 109 fetchers are `bash`.** `runner/executor.py:248` launches them as + `["bash", entry]`, so they need `bash` on `PATH`. **Stock Windows has none** — + the bulk of the catalog is unusable on native Windows without WSL, Git Bash, or + Cygwin. The **19 python** fetchers use `sys.executable` (`executor.py:246`) and + are portable. +- The executor invokes the interpreter explicitly (no reliance on the shebang or + the executable bit — neither of which Windows honors), so bash fetchers *do* + run if a `bash` is on `PATH`. But the scripts assume POSIX and call + `aws`/`kubectl`/`jq`/`git`/`checkov`, which must be installed and on `PATH`. A + brew tap can declare those as deps on Mac/Linux; Windows has no single + declaration. + +### Paths & the workspace (affects `sync`) +- **Workspace location differs by OS.** `~/.paramify` is a Unix dotfile idiom; + Windows convention is `%LOCALAPPDATA%`. Use + `platformdirs.user_data_dir("paramify")` → `~/Library/Application Support/paramify` + (Mac), `%LOCALAPPDATA%\paramify` (Windows), `~/.local/share/paramify` (Linux). + Keep `$PARAMIFY_HOME` as an override. (Resolves the §11 open question.) +- **Search-path separator** for `$PARAMIFY_FETCHERS_PATH` is `os.pathsep` — `;` on + Windows, `:` on Unix. Never hardcode `:`. Build every path with `pathlib.Path`, + never string-join. + +### `sync` correctness gotchas +- **Line endings.** Hash newline-normalized text, not raw bytes (see §6). The + subtlest cross-platform bug in the model — design the hasher around it. +- **Case sensitivity.** macOS (case-insensitive default) and Windows differ from + Linux (case-sensitive). Store manifest keys as normalized POSIX forward-slash + relative paths; don't let name-collision/dedup logic behave differently per OS. +- **Atomic writes.** `os.replace()` is atomic cross-platform on one volume (good + for the manifest), but Windows can't replace an open file — close handles first. + +### The equalizer +- **Containers normalize all of the above** — the image is Linux, so `bash`, the + external CLIs, and POSIX paths all just work. For Windows-heavy customers the + container (Docker Desktop) is the recommended path; the native pip/brew install + is really a Mac/Linux operator convenience. + +### The container story + +The clone-based image build keeps working **verbatim**: `deploy/Dockerfile` +COPYs the repo into `/app` and `pip install -e .`s it, and the cwd walk (root +#2 in §5) finds `/app/fetchers`. Run manifests are user data — the existing +bake-from-`manifests/` flow (`deploy/README.md`) is untouched. + +The gap this design creates: post-rewrite, a customer's customizations live in +the **workspace — outside the Docker build context**. `docker build` from a +clone ships pristine built-ins, not their edited or custom fetchers; nothing +errors, the content is just missing at 2am when cron fires. And a pipx/brew +install has no clone (and no `deploy/`) to build from at all. + +The bridge is the search path itself: + +- **Near-term pattern** (document in `deploy/README.md`): `COPY` the user's + workspace fetchers into the image (e.g. `/overrides`) and set + `PARAMIFY_FETCHERS_PATH=/overrides` — root precedence makes their copies + shadow the baked-in built-ins, reusing §5's shadowing mechanism as-is. +- **End state:** the future `paramify package` command (`deploy/README.md` + already names this bundle as its template) generates the build context from + the *merged workspace* plus the cwd `manifests/` — making the workspace, not + a checkout, the canonical content source for images. + +## 14. Testing strategy + +The historical failure mode is "green in dev, broken installed" — every current +test runs from the clone. Add: + +- **Installed-layout tests** — build the wheel, install into a temp venv, run + `sync`/`catalog`/`run` with no checkout on disk. The thing the clone tests can't + catch. +- **Sync decision-matrix tests** — fabricate a workspace + manifest + new pristine + and assert every row of the §8.1 table produces the right action (incl. `.new`, + quarantine, removal, a deletion staying deleted, and the no-record conflict). +- **Migration round-trip tests** — fixture `fetcher.yaml` at vN → migrate → assert + vN+1 shape + schema-valid; assert idempotence and chaining (v1→v3). +- **Upgrade simulation** (the prototype acceptance test, §15). + +## 15. Prototype plan (build order for the fork) + +Prove the risky parts first; each phase is independently demonstrable. + +1. **Multi-root discovery** — `fetcher_roots()` + `discover_fetchers(roots)` + + `importlib.resources` schema loading. Prove: `catalog` finds fetchers from a + workspace dir with no cwd checkout. +2. **Package data + `_pristine` build** — ship schemas + a `_pristine/` tree in + the wheel; `pip install` into a clean venv and confirm the core resolves. +3. **Manifest + `sync` (materialize + reconcile, `.new` conflicts)** — no + migrations yet. Prove the §15 acceptance test. +4. **Contract version + migration framework** — add `schema_version` to the + schema, a trivial v1→v2 migration, and wire it into `sync`. Prove a breaking + change survives with a user edit preserved or cleanly quarantined. +5. **`doctor` extension, backups, `--dry-run`, quarantine** — the safety layer. +6. **`brew` tap / `pipx`** — package for real. (Per prior analysis, prove with + **pipx first** — cheapest, surfaces every install-layout issue — then a private + tap; the Proprietary/GPL license choice rules out homebrew-core, so a tap that + can also declare aws/kubectl/checkov/jq/git as deps.) + +### Acceptance test (the "does this work?" bar) + +``` +1. pip install the built wheel into a clean venv (no checkout present) +2. paramify sync → workspace materialized, manifest written +3. edit fetchers/aws//fetcher.yaml → a user customization +4. add fetchers/custom// → a user-created fetcher +5. delete fetchers/okta// → a deliberate user deletion +6. simulate an upgrade: bump _pristine → change an untouched built-in, + change the one you edited, add a new one +7. paramify sync — killed mid-run; then paramify sync again + ASSERT: untouched built-in updated + ASSERT: your edited fetcher preserved + /fetcher.yaml.new written + ASSERT: your custom fetcher untouched + ASSERT: the deleted fetcher stays deleted (not resurrected) + ASSERT: the new built-in added + ASSERT: the re-run reports no conflicts beyond — the kill is invisible + ASSERT: catalog lists all of them; run still works +``` + +If step 7 holds, the model works. diff --git a/docs/distribution_design.md b/docs/distribution_design.md index c7ff8ca..afeca4f 100644 --- a/docs/distribution_design.md +++ b/docs/distribution_design.md @@ -1,6 +1,9 @@ -# Distributable install & self-updating content — design +# Distributable install & user fetchers — design -**Status:** draft / prototype target. **Audience:** whoever builds the fork. +**Status:** draft v2 / prototype target. Replaces the workspace-reconcile +draft, preserved at +[`deferred/workspace_sync_design.md`](deferred/workspace_sync_design.md). +**Audience:** whoever builds the fork. **Related:** [`versioning.md`](versioning.md) (the contract + bump policy), [`releasing.md`](releasing.md) (how a release is cut), [`design.md`](design.md) (framework overview). @@ -10,91 +13,102 @@ Today the tool is **clone-only**. `api.find_repo_root()` walks up from the cwd looking for sibling `fetchers/` + `framework/` dirs, schemas load from `repo_root/framework/schemas/`, and fetchers run as subprocesses out of -`repo_root/fetchers/`. That makes it trivial to add a fetcher (drop a directory -in the writable tree) but ties the whole tool to a working checkout. - -Two consequences we want to fix: +`repo_root/fetchers/`. Two consequences we want to fix: 1. **No clean installed distribution.** `pip install`/`brew install` of the - current package ships only `framework*` — no fetchers, and `find_repo_root()` - throws unless you happen to be inside a clone. (`releasing.md` → *Artifacts* - already flags this as the blocker deferring wheel/PyPI.) -2. **`git pull` clobbers local edits.** Because the checkout *is* the - distribution, a customer's edits to a built-in `fetcher.yaml` collide with - upstream changes to the same tracked file — the pull conflicts or stomps - uncommitted work. Every built-in is a landmine. - -## 2. Goal - -Ship the tool as a normal installable package whose **content is user-owned and -self-updating**: - -- `brew install` / `pipx install` brings the **core** (engine + CLI + a bundled, - read-only snapshot of content). -- A first-run step **materializes** the content into a user-writable - **workspace** and records exactly what was shipped. -- On upgrade, a **reconcile** step updates only the files the user never touched, - preserves everything they did, and never silently discards a customization. -- Adding a fetcher stays trivial — you create a directory in the workspace. - -### Decision already made (logged, #224) - -Content is **bundled with each core release**, not shipped on a separate channel. -One version number for core+content ⇒ no core↔content compatibility matrix, and -`paramify sync` is a **purely local** reconcile (no network). The accepted -tradeoff: shipping a new/fixed fetcher requires cutting a release. + current package ships only `framework*` — no fetchers, no schemas as data — + and `find_repo_root()` throws unless you happen to be inside a clone. + (`releasing.md` → *Artifacts* already flags this as the blocker deferring + wheel/PyPI.) +2. **User fetchers have no home.** The customization we expect and plan for is + customers **writing their own fetchers** for platforms we don't support. + Today that means writing into the checkout — where `git pull` collides with + their work and every upgrade is a merge hazard. + +## 2. What drives the design + +Two facts scope this down hard: + +- **The expected customization is *creation*, not modification.** Customers + build fetchers for platforms we don't cover. They are not expected to edit + shipped files: `fetcher.yaml` is explicitly *"ships with the code, customers + never edit this"* (README), and per-customer intent — config, secrets, + targets — lives in the **run manifest**, by design. +- **We control discovery.** dpkg needs 3-way conffile merges because `/etc` + files must be edited in place — services read fixed paths. We don't have + that constraint: the loader can **overlay** a user directory on top of the + shipped content, so upstream never writes into — and never has to reconcile + with — anything the user owns. + +Hence the model: **read-only shipped content + a writable user overlay.** No +materialized workspace, no hash manifest, no sync engine, no per-file decision +table. The no-clobber guarantee is *structural* (upstream and user content +live in different directories), not algorithmic. + +### Decisions (logged) + +- **#224 stands unchanged:** content is **bundled with each core release**, + not shipped on a separate channel. One version number for core+content ⇒ no + compatibility matrix. Tradeoff: shipping a new/fixed fetcher requires + cutting a release. +- **#243 (supersedes #223):** no materialize/sync step. Built-ins are read + directly from the installed bundle; user customization is copy-on-write + shadowing. The update flow in #225 simplifies to *just the package upgrade*. +- **#234:** Claude skills are deferred out of v1 entirely. ### Non-goals -- PyInstaller / single-file frozen binaries (blocked separately: Python fetchers - launch via `sys.executable`, which points at the frozen app, not Python). +- PyInstaller / single-file frozen binaries (blocked separately: Python + fetchers launch via `sys.executable`, which points at the frozen app). - A network content registry / plugin marketplace. - Changing what a fetcher *is* or the envelope format. +- Reconciling user edits to shipped files — see + [`deferred/workspace_sync_design.md`](deferred/workspace_sync_design.md) + and §11 for what would revive that. ## 3. Vocabulary | Term | Meaning | Owner | Writable | |---|---|---|---| | **Core** | `framework/` engine + CLI + JSON schemas | us (package manager) | no | -| **Pristine bundle** (`_pristine/`) | read-only snapshot of shipped content (fetchers, skills, example manifests), baked into the release | us | no | -| **Workspace** | materialized, user-editable copy of the content | user | yes | -| **Manifest** (`.manifest.json`) | record of what we shipped into the workspace + hashes | tool | (tool-managed) | +| **Bundled content** (`framework/_bundled/`) | shipped fetchers + example manifests, read directly at run time | us | no | +| **User dir** (`$PARAMIFY_HOME/fetchers/`) | user-created fetchers + deliberate overrides of built-ins | user | yes | | **User data** | run manifests, `EVIDENCE_DIR` output | user | yes | -Three data classes, kept strictly separate: **tool assets** (core, read-only) · -**managed content** (workspace, reconciled by `sync`) · **user data** (cwd, never -touched by `sync`). +There is no pristine-vs-workspace distinction and no tool-managed manifest: +the shipped content **is** the live content. ## 4. On-disk layout ``` INSTALL (read-only, package-manager owned) ─────────────── -/opt/homebrew/.../paramify/ (or site-packages/) +…/site-packages/ framework/ schemas/ fetcher_schema.json, category_schema.json, … - …engine… - _pristine/ ← the shipped content snapshot - fetchers/ - skills/ - examples/ - -WORKSPACE (writable, user owned) ───────────────────────── -$PARAMIFY_HOME (default ~/.paramify/) - fetchers/ ← materialized + your own; fetchers RUN from here - examples/ - .manifest.json ← what we shipped, with hashes - .backups// ← pre-migration snapshots - -SKILLS (writable) ──────────────────────────────────────── -~/.claude/skills/ ← materialized here (see §11, OPEN question) + _bundled/ ← shipped content; built-ins RUN from here + fetchers/ (inside the package, so importlib.resources + examples/ resolves it and nothing non-package lands + …engine… top-level in site-packages — see §8) + +USER DIR (writable, user owned) ────────────────────────── +$PARAMIFY_HOME (default: platformdirs user_data_dir — see §9) + fetchers/ ← user-created fetchers + overrides; shadow built-ins by name + _categories/ ← user category files for NEW platforms (see §5) USER DATA (writable, cwd) ───────────────────────────────── -/manifests/, /evidence/ ← never touched by sync +/manifests/, /evidence/ ← never touched by the tool ``` -Note a *benefit* of running fetchers out of the writable workspace: fetchers that -write next to themselves (e.g. the checkov git-clone scanner) work again — they -were a known problem for a read-only install. +The tool writes into the user dir only on explicit `paramify create` / +`paramify customize` (§6), and never writes anywhere else outside `EVIDENCE_DIR`. +Upgrades replace the install atomically and *cannot* touch user files. + +**Prerequisite — fetchers must not write next to themselves.** Built-ins run +from a read-only install, so a fetcher that writes into its own directory +breaks. Verified satisfied: the checkov scanners (the one suspected offender) +clone via `mktemp -d` with trap cleanup and write only to `EVIDENCE_DIR` and +mktemp scratch (`fetchers/checkov/_shared/clone.sh`). Hold new fetchers to the +same rule — it's already the contract (*"writes only to `EVIDENCE_DIR`"*). ## 5. Discovery: single root → search path @@ -104,18 +118,18 @@ The discovery seam is `config_loader.discover_fetchers(repo_root)` and **Schema resolution** moves off the cwd walk. Schemas are core, so they resolve from the installed `framework` package via `importlib.resources` — with the -cwd path kept as a dev fallback: +source path kept as a dev fallback: ```python def load_schema(name="fetcher_schema.json") -> dict: - # 1. dev checkout (editable install): framework/schemas/ next to source + # 1. framework/schemas/ next to the source — serves BOTH the dev checkout + # and a normal install (package data lands here in site-packages) local = Path(__file__).parent / "schemas" / name if local.exists(): return json.loads(local.read_text()) - # 2. installed: packaged data via importlib.resources + # 2. zip-safe fallback via importlib.resources return json.loads(resources.files("framework.schemas").joinpath(name).read_text()) ``` -(Requires shipping the schemas as package data — see §10.) **Fetcher discovery** takes an ordered list of roots instead of one, and unions results; earlier roots win a name collision so a user copy can shadow a built-in: @@ -125,9 +139,10 @@ def fetcher_roots() -> list[Path]: roots = [] if env := os.environ.get("PARAMIFY_FETCHERS_PATH"): # 1 explicit override roots += [Path(p) for p in env.split(os.pathsep)] - roots.append(workspace_home() / "fetchers") # 2 workspace - if co := _find_checkout_root(): # 3 dev clone (cwd walk) + if co := _find_checkout_root(): # 2 dev clone (cwd walk) roots.append(co / "fetchers") + roots.append(user_home() / "fetchers") # 3 user dir + roots.append(_bundled_root() / "fetchers") # 4 installed bundle return [r for r in roots if r.is_dir()] def discover_fetchers(roots: list[Path]) -> dict[str, Fetcher]: @@ -139,298 +154,216 @@ def discover_fetchers(roots: list[Path]) -> dict[str, Fetcher]: return found ``` -`find_repo_root()` stays for the dev path but is no longer the only way to -locate content. Everything downstream (`catalog`, `run`, `doctor`) calls -`fetcher_roots()`. - -## 6. The manifest — what makes non-clobbering possible - -The single new asset. Without a record of *what we shipped*, "the user changed -this file" is indistinguishable from "we changed this file upstream." The -manifest is that record, written at materialize time and read at every sync. - -```json -{ - "schema": "paramify-workspace-manifest/v1", - "tool_version": "0.4.0", - "contract_version": 2, - "materialized_at": "2026-07-13T00:00:00Z", - "files": { - "fetchers/aws/storage_encryption_status/fetcher.yaml": { - "sha256": "9f2c…", "shipped_tool_version": "0.4.0", "contract_version": 2 - }, - "fetchers/aws/storage_encryption_status/fetch.py": { "sha256": "1a7b…" }, - "skills/create-fetcher/SKILL.md": { "sha256": "c3d4…" } - } -} -``` +The checkout outranks the user dir **on purpose**: a developer inside a clone +would otherwise have user-dir copies silently shadowing their in-tree edits. +For end users there is no checkout, so the user dir shadows the bundle — which +is exactly the override mechanism. -Per file we store the content hash of exactly what we last put there. That is -the "base" in a 3-way comparison: **base = manifest hash**, **ours = workspace -file**, **theirs = new pristine file**. (`materialized_at` is stamped by the -caller, not inside any pure-function core — timestamps are a side input.) Hash -**newline-normalized** text (CRLF→LF), not raw bytes — otherwise a Windows/git -line-ending rewrite makes an untouched file look edited and fires a spurious -conflict on every sync (§13). +Two collision rules, kept distinct. **Within one root**, a duplicate name stays +a hard error, exactly as today (`config_loader.py:72`). **Across roots**, first +root wins — but the shadow is *reported*, never silent: `catalog` and `doctor` +list every name resolved by shadowing and which root lost. -## 7. Lifecycle commands (CLI surface) +**`discover_platforms()` gets the same treatment** — and this matters more +here than it did in the sync design: a user fetcher for an *unsupported +platform* needs its own `_categories/.yaml` (platform config + auth +passthrough). Each root's `_categories/*.yaml` is loaded and unioned, first +root winning **per category file**, so users can define entirely new +categories in the user dir without touching the install. -New `@app.command`s in `framework/cli.py`, alongside `list`/`catalog`/`run`/`doctor`. +`find_repo_root()` stays for the dev path but is no longer the only way to +locate content. Everything downstream (`catalog`, `run`, `doctor`) calls +`fetcher_roots()`. -### `paramify sync` -The workhorse. Idempotent. Runs materialize-or-reconcile against the currently -installed `_pristine/`. No network. +## 6. CLI surface -| Flag | Effect | -|---|---| -| `--dry-run` | report the plan (per-file classification), touch nothing | -| `--json` | machine-readable plan/report (consistent with other commands) | -| `--force-theirs GLOB` | on conflict, take upstream for matching paths (backup first) | -| `--force-mine GLOB` | on conflict, keep local, discard the incoming `.new` | -| `--no-migrate` | reconcile files but skip schema migrations (escape hatch) | +No `sync`, no `migrate`. Three additions to `framework/cli.py`: -Output is a grouped report: **updated**, **added**, **kept (yours)**, -**conflicts (.new written)**, **migrated**, **removed**, **quarantined**. +### `paramify create /` +Scaffolds a new fetcher in the **user dir** from the bundled `_template/` — +the front door for the expected use case. Installed users have no checkout to +copy `fetchers/_template/` out of; this command is how they start a fetcher +for a platform we don't support. `--category-file` also scaffolds a +`_categories/.yaml` when the category is new. Refuses to overwrite. -### `paramify migrate` (may be folded into `sync`) -Runs pending contract migrations on workspace content only. Separated so it can -be run/tested in isolation and so `sync --no-migrate` has a counterpart. +### `paramify customize ` +Copy-on-write override: copies a built-in from the bundle into the user dir, +where it shadows by name. Writes a `.customized.json` sidecar recording the +source `tool_version` and per-file hashes, so `doctor` can later say *"your +override of X was copied from 0.4.0; the built-in has since changed"*. +Refuses to overwrite an existing override. (Discovery ignores the sidecar — +it only looks for `fetcher.yaml`.) ### `paramify doctor` (extend the existing command, `cli.py:335`) -Add a distribution section: install path + `tool_version`, workspace path + -manifest `tool_version`/`contract_version`, whether a sync is pending (installed -pristine newer than manifest), and any quarantined fetchers. - -## 8. The sync algorithm - -``` -1. locate INSTALL/_pristine and WORKSPACE; load .manifest.json (empty ⇒ first run) -2. if contract_version(pristine) > contract_version(manifest): - plan migrations (§9); snapshot workspace → .backups// -3. for each file in the NEW pristine set: - classify by (workspace hash, manifest hash, new pristine hash) → §8.1 - apply the action (respecting --dry-run / --force-*) -4. for each file in the manifest but absent from new pristine: → "removed" rule -5. run migrations on workspace files still at the old contract_version -6. re-validate every workspace fetcher against the installed schema; - any failure ⇒ quarantine (mark, keep, report — never delete, never run) -7. rewrite .manifest.json to the new shipped set + hashes -8. print the grouped report -``` - -### 8.1 Per-file decision table - -| State | Detected by | Action | -|---|---|---| -| **You never touched it** | ws == manifest | **Update in place** to new pristine | -| **You edited it, we didn't** | ws ≠ manifest, new == manifest | **Keep yours** (nothing to apply) | -| **You edited it AND we shipped a new version** | ws ≠ manifest, new ≠ manifest | **Conflict:** keep yours, write ours as `.new`, report | -| **You created it** | in ws, absent from manifest | **Never touched** (pure user content) | -| **We removed it** | in manifest, absent from new pristine | Remove **iff** ws == manifest; else keep + warn | +Add a distribution section: install path + `tool_version`; user dir path; +every cross-root shadow and which root lost; stale overrides (sidecar hash ≠ +current bundled hash); user-dir fetchers that fail schema validation. -This table *is* the answer to "will it overwrite my changes." Only row 3 has real -UX; the rest are mechanical. +Example manifests ship in the bundle; `paramify manifest new --from-example +` copies one into the cwd (run manifests are user data — the tool never +resolves them from the bundle implicitly). -### 8.2 Conflict resolution +## 7. Upgrades & schema evolution -Default = **`.new` file** (dpkg/pacman style): zero data loss, dead simple, and -non-interactive-safe for cron. `--force-mine`/`--force-theirs` and a future -interactive/TUI review are layered on top. A 3-way *content* merge is a possible -enhancement but not required for v1. - -## 9. Schema evolution & migrations +**Upgrade = `pipx upgrade` / `brew upgrade`. That's the whole flow.** +Built-ins update atomically with the package; user files are untouched by +construction. There is no step 2. Per `versioning.md`, the schemas are **the contract**. Two change classes: -### Additive (non-breaking) — brew upgrade only -New **optional** field. The permissive schema (no top-level -`additionalProperties: false`) means old files still validate. After -`brew upgrade` the new validator accepts the field; existing workspace fetchers -keep working untouched. `sync` is only needed to receive *our* built-ins that use -the field. `versioning.md` bump: **minor**. - -### Breaking — needs a migration -Removing/renaming/retyping a field, or adding a **required** one. Existing files -become invalid on upgrade. `versioning.md` bump: **major** (pre-1.0: **minor**). - -**Enabler — a contract version marker.** `fetcher.yaml` needs to say which shape -it is, or migrations can't reliably target it. The envelope already does this -(`schema_version` in `framework/envelope.py`); mirror it: add a required -`schema_version` (or `contract_version`) to `fetcher_schema.json`. This is the -migration counterpart of the workspace manifest — the thing that makes safe -breaking changes *possible*. - -**Migration framework.** Migrations ship in the core with the release that breaks -the format: - -``` -framework/migrations/ - __init__.py # registry: ordered [(1→2, fn), (2→3, fn), …] - v1_to_v2.py # def migrate(doc: dict) -> dict (pure, deterministic) -``` -Properties: **chained** (v1→v2→v3, so version-skippers catch up in order), -**version-gated + idempotent** (re-running is a no-op), **pure** (dict→dict, so -unit-testable with fixtures). `sync`/`migrate` apply pending migrations to each -workspace fetcher, bump its marker, and re-validate. - -**Recommended posture — a one-release grace window.** Avoid a hard cutover: for -one release the loader accepts *both* shapes and normalizes old→new in memory -with a deprecation warning; the *next* release drops the old shape. Turns a break -into announce → warn → remove (matches the post-1.0 deprecation policy in -`versioning.md`, applied early). - -**The hard case — user-edited AND breaking.** A built-in the user edited (row 3) -that *also* changed shape: -- migration is a pure structural transform that commutes with their edit ⇒ apply - it to their file, keep their customization, bump the marker; -- migration can't apply cleanly ⇒ **quarantine**: keep their file, mark it - invalid, write our migrated version as `.new`, surface it loudly with the - migration notes. Two invariants: never leave a silently-broken fetcher, never - silently discard a customization. - -## 10. Release integration +- **Additive (non-breaking).** New optional field; the permissive schema means + existing user fetchers keep validating. Nothing to do. Bump: **minor**. +- **Breaking.** Removing/renaming/retyping a field, or adding a required one — + existing *user-dir* fetchers become invalid on upgrade (built-ins ship + already-migrated). Posture: **a one-release grace window** — the loader + accepts both shapes and normalizes old→new in memory with a deprecation + warning; the next release drops the old shape (announce → warn → remove, + matching `versioning.md`). `doctor` lists user fetchers still on the old + shape. A fetcher that fails validation outright is **skipped with a visible + error** (refuse, don't crash), never a whole-run abort. + +**Deferred until the first real break:** the contract-version marker and the +chained migration framework (specified in the deferred doc, §9). Pre-1.0 with +a small install base, the grace-window loader covers the transition without +them. If we adopt the marker later, the bootstrap rule is: **a file with no +marker is contract version 1.** + +## 8. Release integration Slots into the existing `releasing.md` steps: -- **Build the pristine bundle into the artifact.** Add - `[tool.setuptools.package-data]` (or a `MANIFEST.in`) so the wheel/tarball ships - `framework/schemas/*.json`, `framework/tui/styles/*.tcss`, and a `_pristine/` - tree assembled from `fetchers/`, `.claude/skills/`, and `examples/`. A small - build step copies those into `_pristine/` before packaging. This is the change - that unblocks the deferred wheel/PyPI item in `releasing.md`. -- **Author a migration** if the release makes a breaking schema change (same PR). -- **Changelog:** a `BREAKING` subsection with migration notes when applicable; - keep calling out the three version axes. -- **Bump:** per `versioning.md` bump policy (breaking ⇒ minor pre-1.0). +- **Build the bundle into the artifact.** Add `[tool.setuptools.package-data]` + so the wheel/tarball ships `framework/schemas/*.json`, + `framework/tui/styles/*.tcss`, and a `framework/_bundled/` tree assembled + from `fetchers/` and `examples/`. `_bundled/` lives *inside* the package: + resolvable via `importlib.resources`, shippable by the existing + `include = ["framework*"]`, and nothing non-package lands top-level in + site-packages. The assembly step must run **inside the build backend** (a + custom setuptools `build_py` hook), not in a release workflow — the + supported install is still `pip install` from the GitHub source tarball, + and a workflow-side copy would leave source installs with no `_bundled/`. + This is the change that unblocks the deferred wheel/PyPI item in + `releasing.md`. (Decide whether `fetchers/logos/` — README furniture, + unused at run time — belongs in the bundle at all.) +- **Changelog:** keep calling out the three version axes; a `BREAKING` + subsection with the grace-window notes when applicable. +- **Bump:** per `versioning.md` (breaking ⇒ minor pre-1.0). Everything stays manual/tag-driven; nothing here forces `release-please`. -## 11. Backwards-compat & open questions - -- **Dev clone keeps working.** `pip install -e .` in a checkout: the cwd walk is - still root #3, schemas resolve from the local `framework/schemas/`, no `_pristine` - needed. This is also how the existing test suite keeps running. -- **OPEN — where skills materialize.** `~/.claude/skills/` (user-global, - available in every project, but pollutes the global skill space) vs - `/.claude/skills/` (project-scoped, contained, but per-project). Lean - user-global by default with a `--skills-scope project` option. Decide before - building §7. -- **Workspace root** — resolve with `platformdirs.user_data_dir("paramify")` so - each OS gets its native location (§13), with a `$PARAMIFY_HOME` override. -- **Multi-project.** One workspace per user, or per project? Start with one per - user; `$PARAMIFY_HOME` lets a project opt into its own. - -## 12. Safety & recoverability - -- **Backup before migrating** → `.backups//` so a bad transform is - recoverable. -- **`--dry-run`** reports the full plan before touching anything. -- **Validate after** every sync/migrate; anything still invalid is **quarantined** - (kept + flagged), never run and never deleted. -- **Refuse, don't crash.** A fetcher that fails schema validation is skipped with - a visible error (candidate for the deferred `SETUP-ERR` run status), not a - whole-run abort. - -## 13. Platform nuances (macOS / Windows / Linux) - -The **framework** (discovery, sync, migrations, CLI) is pure Python + `pathlib` -and portable. The nuances live in two places: **how you install** and **how -fetchers run**. +## 9. Platform nuances (macOS / Windows / Linux) + +The framework (discovery, CLI) is pure Python + `pathlib` and portable. The +nuances live in how you install and how fetchers run. ### Install path -- **Homebrew is macOS + Linux only** — there is no brew on Windows. The universal - path is **pipx** (Mac/Windows/Linux); brew is a Mac/Linux convenience layer - (another reason the prototype goes pipx-first, §15). A native Windows package - could later use Scoop or WinGet. - -### Fetcher execution — the big one -- **90 of 109 fetchers are `bash`.** `runner/executor.py:248` launches them as - `["bash", entry]`, so they need `bash` on `PATH`. **Stock Windows has none** — - the bulk of the catalog is unusable on native Windows without WSL, Git Bash, or - Cygwin. The **19 python** fetchers use `sys.executable` (`executor.py:246`) and - are portable. -- The executor invokes the interpreter explicitly (no reliance on the shebang or - the executable bit — neither of which Windows honors), so bash fetchers *do* - run if a `bash` is on `PATH`. But the scripts assume POSIX and call - `aws`/`kubectl`/`jq`/`git`/`checkov`, which must be installed and on `PATH`. A - brew tap can declare those as deps on Mac/Linux; Windows has no single - declaration. - -### Paths & the workspace (affects `sync`) -- **Workspace location differs by OS.** `~/.paramify` is a Unix dotfile idiom; - Windows convention is `%LOCALAPPDATA%`. Use - `platformdirs.user_data_dir("paramify")` → `~/Library/Application Support/paramify` - (Mac), `%LOCALAPPDATA%\paramify` (Windows), `~/.local/share/paramify` (Linux). - Keep `$PARAMIFY_HOME` as an override. (Resolves the §11 open question.) -- **Search-path separator** for `$PARAMIFY_FETCHERS_PATH` is `os.pathsep` — `;` on - Windows, `:` on Unix. Never hardcode `:`. Build every path with `pathlib.Path`, - never string-join. - -### `sync` correctness gotchas -- **Line endings.** Hash newline-normalized text, not raw bytes (see §6). The - subtlest cross-platform bug in the model — design the hasher around it. -- **Case sensitivity.** macOS (case-insensitive default) and Windows differ from - Linux (case-sensitive). Store manifest keys as normalized POSIX forward-slash - relative paths; don't let name-collision/dedup logic behave differently per OS. -- **Atomic writes.** `os.replace()` is atomic cross-platform on one volume (good - for the manifest), but Windows can't replace an open file — close handles first. +- **Homebrew is macOS + Linux only.** The universal path is **pipx** + (Mac/Windows/Linux); brew is a Mac/Linux convenience layer (another reason + the prototype goes pipx-first, §12). A native Windows package could later + use Scoop or WinGet. +- **pipx PATH nuance — RESOLVED.** Console scripts of extras (e.g. `checkov`) + install into the pipx venv's `bin/`, which pipx does **not** put on `PATH`. + The executor now prepends `Path(sys.executable).parent` to every child's + `PATH` (`runner/executor.py`, `_build_env`), so venv-installed CLIs resolve + under pipx and unactivated venvs alike (`pipx inject paramify-fetchers + checkov` just works). + +### Fetcher execution +- **90 of 109 fetchers are `bash`** (`runner/executor.py:248` launches + `["bash", entry]`). Stock Windows has no bash — the bulk of the catalog + needs WSL, Git Bash, or the container on native Windows. The 19 python + fetchers use `sys.executable` (`executor.py:246`) and are portable. +- The executor invokes the interpreter explicitly (no reliance on shebangs or + the executable bit, neither of which Windows honors) — this also means + copying fetchers into the user dir needs no permission-bit handling. + +### Paths & the user dir +- **`$PARAMIFY_HOME` resolves via `platformdirs.user_data_dir("paramify")`** → + `~/Library/Application Support/paramify` (Mac), `%LOCALAPPDATA%\paramify` + (Windows), `~/.local/share/paramify` (Linux), with the env var as override. +- **Search-path separator** for `$PARAMIFY_FETCHERS_PATH` is `os.pathsep`. + Build every path with `pathlib.Path`, never string-join. ### The equalizer -- **Containers normalize all of the above** — the image is Linux, so `bash`, the - external CLIs, and POSIX paths all just work. For Windows-heavy customers the - container (Docker Desktop) is the recommended path; the native pip/brew install - is really a Mac/Linux operator convenience. +- **Containers normalize all of the above** — the image is Linux, so `bash`, + the external CLIs, and POSIX paths all just work. For Windows-heavy + customers the container (Docker Desktop) is the recommended path. -## 14. Testing strategy +## 10. The container story -The historical failure mode is "green in dev, broken installed" — every current -test runs from the clone. Add: +The clone-based image build keeps working **verbatim**: `deploy/Dockerfile` +COPYs the repo into `/app` and `pip install -e .`s it, and the cwd walk (root +#2 in §5) finds `/app/fetchers`. Run manifests are user data — the existing +bake-from-`manifests/` flow (`deploy/README.md`) is untouched. -- **Installed-layout tests** — build the wheel, install into a temp venv, run - `sync`/`catalog`/`run` with no checkout on disk. The thing the clone tests can't - catch. -- **Sync decision-matrix tests** — fabricate a workspace + manifest + new pristine - and assert each of the five states produces the right action (incl. `.new`, - quarantine, removal). -- **Migration round-trip tests** — fixture `fetcher.yaml` at vN → migrate → assert - vN+1 shape + schema-valid; assert idempotence and chaining (v1→v3). -- **Upgrade simulation** (the prototype acceptance test, §15). +For customers with user-dir fetchers, the bridge is the search path itself: + +- **Near-term pattern** (document in `deploy/README.md`): `COPY` the user + dir's fetchers into the image (e.g. `/overrides`) and set + `PARAMIFY_FETCHERS_PATH=/overrides` — root precedence makes their fetchers + and overrides shadow the baked-in built-ins, reusing §5's mechanism as-is. +- **End state:** the future `paramify package` command (`deploy/README.md` + already names this bundle as its template) generates the build context from + the *merged view* (bundle + user dir) plus the cwd `manifests/` — so an + installed, clone-less customer can produce a deployable image. + +## 11. What would revive the reconcile model + +The deferred design ([`deferred/workspace_sync_design.md`](deferred/workspace_sync_design.md)) +exists for one scenario: customers routinely making small in-place edits to +**many** built-ins, where copy-on-write overrides would rot en masse. Watch +for it via `doctor`'s stale-override count. If that pattern shows up, the +sync/manifest/migration machinery is fully specified there — start from it. +Until then: every line of it is complexity spent on edits we tell customers +not to make. -## 15. Prototype plan (build order for the fork) +## 12. Prototype plan (build order for the fork) Prove the risky parts first; each phase is independently demonstrable. 1. **Multi-root discovery** — `fetcher_roots()` + `discover_fetchers(roots)` + - `importlib.resources` schema loading. Prove: `catalog` finds fetchers from a - workspace dir with no cwd checkout. -2. **Package data + `_pristine` build** — ship schemas + a `_pristine/` tree in - the wheel; `pip install` into a clean venv and confirm the core resolves. -3. **Manifest + `sync` (materialize + reconcile, `.new` conflicts)** — no - migrations yet. Prove the §15 acceptance test. -4. **Contract version + migration framework** — add `schema_version` to the - schema, a trivial v1→v2 migration, and wire it into `sync`. Prove a breaking - change survives with a user edit preserved or cleanly quarantined. -5. **`doctor` extension, backups, `--dry-run`, quarantine** — the safety layer. -6. **`brew` tap / `pipx`** — package for real. (Per prior analysis, prove with - **pipx first** — cheapest, surfaces every install-layout issue — then a private - tap; the Proprietary/GPL license choice rules out homebrew-core, so a tap that - can also declare aws/kubectl/checkov/jq/git as deps.) + multi-root `discover_platforms()` + `importlib.resources` schema loading. + Prove: `catalog` finds fetchers from a user dir with no checkout on disk. +2. **Package data + `_bundled/` build hook** — ship schemas + the bundle in + the wheel; `pip install` into a clean venv and confirm `catalog`/`run` work + from the bundle alone. Fix the checkov write-next-to-itself violation. +3. **`create` / `customize` / `doctor`** — scaffolding, copy-on-write with the + staleness sidecar, shadow + stale reporting. +4. **pipx, then brew tap** — package for real. pipx first (cheapest, surfaces + every install-layout issue), then a private tap that can also declare + `aws`/`kubectl`/`checkov`/`jq`/`git` as deps. ### Acceptance test (the "does this work?" bar) ``` 1. pip install the built wheel into a clean venv (no checkout present) -2. paramify sync → workspace materialized, manifest written -3. edit fetchers/aws//fetcher.yaml → a user customization -4. add fetchers/custom// → a user-created fetcher -5. simulate an upgrade: bump _pristine → change an untouched built-in, - change the one you edited, add a new one -6. paramify sync - ASSERT: untouched built-in updated - ASSERT: your edited fetcher preserved + /fetcher.yaml.new written - ASSERT: your custom fetcher untouched - ASSERT: the new built-in added - ASSERT: catalog lists all of them; run still works +2. paramify catalog → all built-ins listed, from the bundle +3. paramify run examples-derived manifest → evidence written; run works installed +4. paramify create custom/ → user fetcher scaffolded in the user dir, + appears in catalog, runs +5. paramify customize aws/; edit it → override shadows the built-in; + catalog + doctor report the shadow +6. install a newer wheel (simulated upgrade) + ASSERT: built-ins updated (bundle is the new version — nothing to reconcile) + ASSERT: your custom fetcher untouched, still runs + ASSERT: your override still wins; doctor flags it stale if changed upstream + ASSERT: no file in the user dir was created, modified, or deleted by the upgrade ``` -If step 6 holds, the model works. +If step 6 holds, the model works — and note *why* it holds: the upgrade never +had write access to anything the user owns. + +## 13. Testing strategy + +The historical failure mode is "green in dev, broken installed" — every +current test runs from the clone. Add: + +- **Installed-layout tests** — build the wheel, install into a temp venv, run + `catalog`/`run`/`create`/`customize` with no checkout on disk. +- **Precedence tests** — fabricate multiple roots and assert shadowing order, + in-root duplicate errors, shadow reporting, and per-category-file platform + precedence. +- **Staleness tests** — customize, bump the bundle, assert `doctor` flags the + override; assert an unmodified bundle stays quiet. +- **Grace-window loader tests** (when the first breaking change lands) — old + shape accepted + warned, new shape accepted, next release rejects old. diff --git a/docs/releasing.md b/docs/releasing.md index 24a05ae..c6ab20b 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -57,12 +57,24 @@ Using `X.Y.Z` as the version (no `v` in files; the **tag** is `vX.Y.Z`). ``` GitHub automatically attaches the **source code** as `.tar.gz` and `.zip` to - every release — that is our release artifact for now (see - [Artifacts](#artifacts)). Add `--generate-notes` if you also want GitHub's + every release. Add `--generate-notes` if you also want GitHub's auto-generated commit/PR list appended. -7. **Verify:** `gh release view vX.Y.Z` — confirm the notes, the tag, and that - the source archives are attached. +7. **Build and attach the wheel** (the pipx-installable artifact — see + [Artifacts](#artifacts)). The `setup.py` build hook assembles the + `framework/_bundled/` content snapshot into it: + + ```bash + .venv/bin/pip wheel --no-deps --no-build-isolation -w dist . + gh release upload vX.Y.Z dist/*.whl + ``` + +8. **Bump the Homebrew tap** (if published): stamp the new tarball url/sha256 + and regenerate the python resources — see + [`packaging/brew/README.md`](../packaging/brew/README.md). + +9. **Verify:** `gh release view vX.Y.Z` — confirm the notes, the tag, the + source archives, and the wheel. ## Pre-releases @@ -80,13 +92,31 @@ Use `-rc.N` (release candidate) or `-beta.N`. These sort before the final ## Artifacts -- **Now:** the GitHub-attached **source tarball/zip**. The supported install is - from source (`pip install -e .`), so the source archive is the artifact. -- **Deferred:** wheel/sdist + PyPI publishing. Blocked on the editable-only / - cwd-repo-root packaging issue (schemas resolve via the discovered repo root, - and fetchers execute as subprocesses out of `fetchers/`, which a plain wheel - doesn't ship). Revisit when that's fixed; a build+publish step then slots into - step 6. +- **Source tarball/zip** — attached by GitHub automatically. Installs work from + source (`pip install .` — the build hook assembles the content bundle) and + editable in a clone (`pip install -e .`). +- **Wheel** — built and attached in step 7; carries the engine, schemas, the + `framework/_bundled/` content snapshot, and the uploader. This is the + **pipx** path (the universal install: Mac/Windows/Linux): + + ```bash + pipx install https://github.com/paramify/paramify-fetchers/releases/download/vX.Y.Z/paramify_fetchers-X.Y.Z-py3-none-any.whl + ``` + + Upgrading to a new release is `pipx install --force `. + Caveat (found in dist testing): pipx ≥ 1.15 with the uv backend can fail + `--force` into an existing venv (`uv venv` refuses to overwrite, exit 1, + old version left in place) — prepend `UV_VENV_CLEAR=1`, or + `pipx uninstall paramify-fetchers` first. + + (The former blocker — cwd-repo-root schemas, unshipped fetchers — was removed + by the overlay distribution work; see `docs/distribution_design.md`.) +- **Homebrew tap** — the Mac/Linux convenience layer on top; can also declare + the external CLIs (`aws`/`kubectl`/`jq`/`git`/`checkov`) as dependencies. + Formula template + bump procedure: [`packaging/brew/`](../packaging/brew/README.md). +- **Deferred: PyPI publishing.** Nothing blocks it anymore technically; it's a + distribution-channel decision (name squatting, support expectations), not a + packaging one. ## What a release does *not* touch diff --git a/framework/api.py b/framework/api.py index 9444a1b..d06de75 100644 --- a/framework/api.py +++ b/framework/api.py @@ -29,7 +29,8 @@ import yaml -from framework.config_loader import discover_fetchers, discover_platforms +from framework import roots as roots_mod +from framework.config_loader import discover, discover_platforms from framework.contract import ConfigField, Secret, TargetField from framework.envelope import is_enveloped, wrap_outputs from framework.runner import manifest_loader @@ -42,14 +43,29 @@ # --------------------------------------------------------------------------- # def find_repo_root(start: Optional[Path] = None) -> Path: - """Locate the repo root by walking up for sibling fetchers/ + framework/ dirs.""" - cur = (start or Path.cwd()).resolve() - for parent in [cur, *cur.parents]: - if (parent / "fetchers").is_dir() and (parent / "framework").is_dir(): - return parent - raise RuntimeError( - "Could not locate repo root (looking for sibling fetchers/ and framework/ dirs)" - ) + """Locate the repo root by walking up for sibling fetchers/ + framework/ dirs. + + Raises outside a checkout — kept for callers that require a source tree. + Front-ends should prefer locate_root(), which returns None instead: content + then resolves via the overlay search path (framework.roots) and user data + (manifests/) falls back to the cwd.""" + root = roots_mod.find_checkout_root(start) + if root is None: + raise RuntimeError( + "Could not locate repo root (looking for sibling fetchers/ and framework/ dirs)" + ) + return root + + +def locate_root(start: Optional[Path] = None) -> Optional[Path]: + """The dev checkout root when inside one, else None (installed mode).""" + return roots_mod.find_checkout_root(start) + + +def content_roots(root: Optional[Path] = None) -> List[Path]: + """The ordered fetchers/ search path (env override → checkout → user dir → + installed bundle). `root` pins the checkout explicitly when already known.""" + return roots_mod.fetcher_roots(checkout=root) # --------------------------------------------------------------------------- # @@ -106,12 +122,15 @@ def _fetcher_descriptor(f) -> dict: } -def catalog(root: Path) -> dict: - """Discover all fetchers, group them by category, and describe every editable - field. This single structure is both the UI form schema and the AI-readable - `catalog --json` output.""" - fetchers = discover_fetchers(root) - platforms = discover_platforms(root) +def catalog(root: Optional[Path] = None) -> dict: + """Discover all fetchers across the content roots, group them by category, + and describe every editable field. This single structure is both the UI + form schema and the AI-readable `catalog --json` output. Cross-root name + shadows are reported in `shadows`, never resolved silently.""" + roots = content_roots(root) + result = discover(roots) + fetchers = result.fetchers + platforms = discover_platforms(roots) by_category: Dict[str, List[Any]] = {} for f in fetchers.values(): @@ -136,19 +155,26 @@ def catalog(root: Path) -> dict: ], }) - return {"categories": categories, "fetcher_count": len(fetchers)} + return { + "categories": categories, + "fetcher_count": len(fetchers), + "roots": [str(r) for r in roots], + "shadows": result.shadows, + "invalid": result.invalid, + } # --------------------------------------------------------------------------- # # KSI coverage — join fetcher `ksis` against the FedRAMP 20x reference # --------------------------------------------------------------------------- # -def _load_ksi_reference(root: Path) -> dict: - """Load the canonical FedRAMP 20x KSI reference (the coverage denominator).""" - return yaml.safe_load((root / "framework" / "reference" / "ksis.yaml").read_text()) +def _load_ksi_reference() -> dict: + """Load the canonical FedRAMP 20x KSI reference (the coverage denominator). + A core asset — resolves from the framework package, not the content roots.""" + return yaml.safe_load((Path(__file__).parent / "reference" / "ksis.yaml").read_text()) -def ksi_coverage(root: Path) -> dict: +def ksi_coverage(root: Optional[Path] = None) -> dict: """Coverage of the FedRAMP 20x KSIs by discovered fetchers. Joins each fetcher's `ksis` against framework/reference/ksis.yaml and returns @@ -159,8 +185,8 @@ def ksi_coverage(root: Path) -> dict: config-evidenceable; else `organizational` (evidenced by HR/training/manual, not cloud config). coverage_pct is over the config-evidenceable set only. """ - ref = _load_ksi_reference(root) - fetchers = discover_fetchers(root) + ref = _load_ksi_reference() + fetchers = discover(content_roots(root)).fetchers by_ksi: Dict[str, List[str]] = {} for f in fetchers.values(): @@ -238,7 +264,7 @@ def ksi_coverage(root: Path) -> dict: _ENV_REF = re.compile(r"\$\{env:([^}]+)\}") -def doctor(root: Path, manifest_path: Optional[Path] = None) -> dict: +def doctor(root: Optional[Path] = None, manifest_path: Optional[Path] = None) -> dict: """Preflight check for running fetchers here. Reports the Python version, whether the external CLIs the relevant categories @@ -256,7 +282,9 @@ def doctor(root: Path, manifest_path: Optional[Path] = None) -> dict: "ok": (py.major, py.minor) >= (3, 10), } - fetchers = discover_fetchers(root) + roots = content_roots(root) + result = discover(roots) + fetchers = result.fetchers categories = sorted({f.category for f in fetchers.values() if f.category}) tools_required = manifest_path is not None @@ -325,10 +353,215 @@ def doctor(root: Path, manifest_path: Optional[Path] = None) -> dict: "tools": tools, "tools_required": tools_required, "manifest": manifest_report, + "distribution": { + "tool_version": _tool_version(), + "install_path": str(Path(__file__).resolve().parent), + "user_dir": str(roots_mod.user_home()), + "roots": [str(r) for r in roots], + "shadows": result.shadows, + "invalid": result.invalid, + "stale_overrides": _stale_overrides(result), + }, "ok": ok, } +# --------------------------------------------------------------------------- # +# User content — scaffold new fetchers + copy-on-write overrides +# (docs/distribution_design.md §6: the tool writes into the user dir only here) +# --------------------------------------------------------------------------- # + +_SIDECAR = ".customized.json" +_NAME_PART = re.compile(r"^[a-z0-9][a-z0-9_]*$") + + +def _tool_version() -> str: + try: + from importlib.metadata import version + return version("paramify-fetchers") + except Exception: + return "unknown" + + +def _sha256(path: Path) -> str: + import hashlib + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _copy_fetcher_dir(src: Path, dest: Path) -> Dict[str, str]: + """Copy a fetcher dir (skipping caches and the sidecar) and return + {relative posix path: sha256} for every file copied.""" + hashes: Dict[str, str] = {} + for f in sorted(src.rglob("*")): + rel = f.relative_to(src) + if "__pycache__" in rel.parts or f.name == _SIDECAR or f.name.endswith((".pyc", ".pyo")): + continue + target = dest / rel + if f.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(f.read_bytes()) + hashes[rel.as_posix()] = _sha256(target) + return hashes + + +def _find_template(root: Optional[Path]) -> Path: + """The scaffold source: the first content root that ships a _template/.""" + for r in content_roots(root): + candidate = r / "_template" + if candidate.is_dir(): + return candidate + raise RuntimeError("No fetcher template found in any content root (_template/)") + + +def create_fetcher(spec: str, root: Optional[Path] = None, category_file: bool = False) -> dict: + """Scaffold a new fetcher at /fetchers/// from the + shipped _template, substituting the name placeholders so it is discoverable + immediately. `spec` is "/". Refuses to overwrite, and + refuses a name that already resolves (shadowing an existing fetcher by + accident is a footgun — that's what customize is for).""" + category, sep, short = spec.partition("/") + if not sep or not _NAME_PART.match(category) or not _NAME_PART.match(short): + raise ValueError( + f"invalid spec {spec!r}: expected /, " + "lowercase [a-z0-9_] each" + ) + name = f"{category}_{short}" + + result = discover(content_roots(root)) + if name in result.fetchers: + raise FileExistsError( + f"a fetcher named '{name}' already exists at " + f"{result.fetchers[name].path} — to override a built-in, use customize" + ) + + dest = roots_mod.user_home() / "fetchers" / category / short + if dest.exists(): + raise FileExistsError(f"refusing to overwrite {dest}") + + template = _find_template(root) + files = _copy_fetcher_dir(template, dest) + for fname in ("fetcher.yaml", "README.md"): + p = dest / fname + if p.exists(): + text = p.read_text().replace("", category).replace("", short) + p.write_text(text) + + cat_file = None + if category_file: + platforms = discover_platforms(content_roots(root)) + if category not in platforms: + cat_dir = roots_mod.user_home() / "fetchers" / "_categories" + cat_dir.mkdir(parents=True, exist_ok=True) + cat_path = cat_dir / f"{category}.yaml" + if not cat_path.exists(): + cat_path.write_text( + f"description: \n" + "# auth:\n" + "# passthrough_env:\n" + "# - \n" + ) + cat_file = str(cat_path) + + return { + "name": name, + "path": str(dest), + "files": sorted(files), + "category_file": cat_file, + } + + +def customize_fetcher(name: str, root: Optional[Path] = None) -> dict: + """Copy-on-write override: copy the resolved fetcher into the user dir, + where it shadows the original by name. Writes a .customized.json sidecar + (source, tool_version, per-file hashes) so doctor can flag the override as + stale when the original moves on. Refuses to overwrite.""" + result = discover(content_roots(root)) + fetcher = result.fetchers.get(name) + if fetcher is None: + raise ValueError(f"unknown fetcher: {name}") + + user_fetchers = roots_mod.user_home() / "fetchers" + src = Path(fetcher.path) + try: + src.relative_to(user_fetchers.resolve()) + raise FileExistsError(f"'{name}' is already in your user dir ({src}) — edit it directly") + except ValueError: + pass # not under the user dir: it's a built-in, proceed + + dest = user_fetchers / src.parent.name / src.name + if dest.exists(): + raise FileExistsError(f"refusing to overwrite existing override at {dest}") + + hashes = _copy_fetcher_dir(src, dest) + sidecar = { + "schema": "paramify-customized/v1", + "name": name, + "source": str(src), + "tool_version": _tool_version(), + "copied_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "files": hashes, + } + (dest / _SIDECAR).write_text(json.dumps(sidecar, indent=2) + "\n") + + # Whether the copy actually wins discovery now: in a dev checkout it does + # NOT (the checkout outranks the user dir) — say so instead of lying. + resolved = discover(content_roots(root)).fetchers.get(name) + active = resolved is not None and Path(resolved.path) == dest.resolve() + + return { + "name": name, + "path": str(dest), + "source": str(src), + "files": sorted(hashes), + "active": active, + } + + +def _stale_overrides(result) -> List[dict]: + """Overrides whose original has changed since the copy: compare the current + counterpart's files against the hashes recorded in each sidecar. The + counterpart is the fetcher this override currently shadows (falling back to + the recorded source path); an override whose original vanished upstream is + reported as orphaned.""" + by_winner = {s["winner"]: s["shadowed"] for s in result.shadows} + stale: List[dict] = [] + user_fetchers = roots_mod.user_home() / "fetchers" + if not user_fetchers.is_dir(): + return stale + for sidecar_path in sorted(user_fetchers.glob(f"*/*/{_SIDECAR}")): + try: + sidecar = json.loads(sidecar_path.read_text()) + except (OSError, json.JSONDecodeError): + continue + override_dir = sidecar_path.parent + counterpart = by_winner.get(str(override_dir.resolve())) or sidecar.get("source") + if not counterpart or not Path(counterpart).is_dir(): + stale.append({ + "name": sidecar.get("name") or override_dir.name, + "path": str(override_dir), + "status": "orphaned", + "copied_tool_version": sidecar.get("tool_version"), + "changed": [], + }) + continue + changed = [] + for rel, recorded in (sidecar.get("files") or {}).items(): + current = Path(counterpart) / rel + if not current.is_file() or _sha256(current) != recorded: + changed.append(rel) + if changed: + stale.append({ + "name": sidecar.get("name") or override_dir.name, + "path": str(override_dir), + "status": "stale", + "copied_tool_version": sidecar.get("tool_version"), + "changed": sorted(changed), + }) + return stale + + # --------------------------------------------------------------------------- # # Manifest read / write # --------------------------------------------------------------------------- # @@ -343,7 +576,7 @@ def read_manifest(path: Path) -> dict: return data if isinstance(data, dict) else init_manifest() -def dump_manifest(manifest: dict, path: Path, root: Path) -> None: +def dump_manifest(manifest: dict, path: Path, root: Optional[Path] = None) -> None: """Write a manifest dict to YAML. Refuses to write a structurally invalid (schema-invalid) manifest; semantic gaps (e.g. a not-yet-filled secret) are allowed so work-in-progress can be saved.""" @@ -456,7 +689,9 @@ def set_passthrough_env(m: dict, category: str, env_vars: List[str]) -> dict: # on a merely-incomplete manifest) # --------------------------------------------------------------------------- # -def validate(manifest: dict, root: Path, fetchers=None, platforms=None) -> List[str]: +def validate( + manifest: dict, root: Optional[Path] = None, fetchers=None, platforms=None +) -> List[str]: """Validate a manifest dict against the schema and the discovered fetchers. Returns a list of human-readable error strings (empty == valid+runnable). @@ -468,10 +703,12 @@ def validate(manifest: dict, root: Path, fetchers=None, platforms=None) -> List[ if errors: return errors # can't do semantic checks on a structurally-broken manifest - if fetchers is None: - fetchers = discover_fetchers(root) - if platforms is None: - platforms = discover_platforms(root) + if fetchers is None or platforms is None: + roots = content_roots(root) + if fetchers is None: + fetchers = discover(roots).fetchers + if platforms is None: + platforms = discover_platforms(roots) parsed = manifest_loader.parse_manifest(manifest) for i, entry in enumerate(parsed.entries): @@ -547,19 +784,21 @@ def _invocation_record(r) -> dict: return record -def _manifest_id(path, root: Path) -> str: +def _manifest_id(path, root: Optional[Path]) -> str: """Stable identity for a manifest in run metadata: its path relative to the - repo root (so attribution survives the repo moving), absolute otherwise.""" + project root (checkout if present, else cwd — so attribution survives the + project moving), absolute otherwise.""" p = Path(path).resolve() + base = Path(root) if root is not None else Path.cwd() try: - return str(p.relative_to(Path(root).resolve())) + return str(p.relative_to(base.resolve())) except ValueError: return str(p) def run( manifest: dict, - root: Path, + root: Optional[Path] = None, on_event: Optional[Callable[[dict], None]] = None, manifest_path: Optional[Path] = None, ) -> dict: @@ -579,8 +818,9 @@ def run( if errs: raise ValueError("manifest schema invalid:\n " + "\n ".join(errs)) - fetchers = discover_fetchers(root) - platforms = discover_platforms(root) + roots = content_roots(root) + fetchers = discover(roots).fetchers + platforms = discover_platforms(roots) parsed = manifest_loader.parse_manifest(manifest) def emit(event: dict) -> None: @@ -676,9 +916,23 @@ def on_line(line: str, _use=entry.use) -> None: # Upload — Paramify evidence uploader facade (powers CLI + TUI) # --------------------------------------------------------------------------- # -def _load_paramify_uploader(root: Path): - """Load the source-tree uploader without requiring uploaders/ to be packaged.""" - path = Path(root) / "uploaders" / "paramify_evidence" / "uploader.py" +def _load_paramify_uploader(root: Optional[Path] = None): + """Load the Paramify evidence uploader. + + In a dev checkout, load uploader.py by path from the tree (in-tree edits + win, matching content-root precedence). Installed, import the packaged + uploaders.paramify_evidence module shipped in the wheel.""" + base = root or roots_mod.find_checkout_root() + if base is None: + try: + from uploaders.paramify_evidence import uploader as module + return module + except ImportError as exc: + raise RuntimeError( + "Paramify evidence uploader not found: not inside a source " + "checkout and the packaged uploader is not installed." + ) from exc + path = Path(base) / "uploaders" / "paramify_evidence" / "uploader.py" if not path.exists(): raise RuntimeError(f"Paramify evidence uploader not found at {path}") spec = importlib.util.spec_from_file_location("paramify_evidence_uploader", path) @@ -691,7 +945,7 @@ def _load_paramify_uploader(root: Path): def upload_preflight( run_dir, - root: Path, + root: Optional[Path] = None, config_path: Optional[Path] = None, *, dry_run: bool = False, @@ -738,7 +992,7 @@ def upload_preflight( def upload_run( run_dir, - root: Path, + root: Optional[Path] = None, config_path: Optional[Path] = None, *, dry_run: bool = False, @@ -861,7 +1115,9 @@ def read_evidence(path) -> dict: # Manifests — discover selectable run manifests (powers the welcome screen) # --------------------------------------------------------------------------- # -def _manifest_summary(path: Path, root: Path, fetchers=None, platforms=None) -> Optional[dict]: +def _manifest_summary( + path: Path, root: Optional[Path], fetchers=None, platforms=None +) -> Optional[dict]: """Summarize a manifest file, or return None if it isn't a run manifest (its top level must be a mapping with a `run` key).""" summary = { @@ -908,17 +1164,19 @@ def _manifest_summary(path: Path, root: Path, fetchers=None, platforms=None) -> return summary -def list_manifests(root) -> List[dict]: - """Discover selectable run manifests: /manifests/*.yaml, plus a legacy - /manifest.yaml if present (listed first). Each is summarized with its - fetcher count, validity (issues), and last-run info for the welcome picker. - Non-manifest YAML files (no top-level `run`) are skipped.""" - root = Path(root) +def list_manifests(root=None) -> List[dict]: + """Discover selectable run manifests: /manifests/*.yaml, plus a + legacy /manifest.yaml if present (listed first). Manifests are + user data: the project dir is the checkout when inside one, else the cwd. + Each is summarized with its fetcher count, validity (issues), and last-run + info for the welcome picker. Non-manifest YAML files (no top-level `run`) + are skipped.""" + project = Path(root) if root is not None else Path.cwd() paths: List[Path] = [] - mdir = root / "manifests" + mdir = project / "manifests" if mdir.is_dir(): paths += sorted(mdir.glob("*.yaml")) - legacy = root / "manifest.yaml" + legacy = project / "manifest.yaml" if legacy.exists() and legacy.resolve() not in {p.resolve() for p in paths}: paths.insert(0, legacy) if not paths: @@ -928,24 +1186,28 @@ def list_manifests(root) -> List[dict]: fetchers: Optional[dict] = None platforms: Optional[dict] = None try: - fetchers = discover_fetchers(root) - platforms = discover_platforms(root) + roots = content_roots(root) + fetchers = discover(roots).fetchers + platforms = discover_platforms(roots) except Exception: pass summaries = [_manifest_summary(p, root, fetchers, platforms) for p in paths] return [s for s in summaries if s is not None] -def new_manifest_path(root, name: str, output_dir: str = "./evidence") -> Path: - """Create a fresh manifest file at /manifests/.yaml and return - its path. Raises FileExistsError if it already exists, ValueError on a bad - name.""" +def new_manifest_path( + root: Optional[Path], name: str, output_dir: str = "./evidence" +) -> Path: + """Create a fresh manifest file at /manifests/.yaml and + return its path (project = checkout when inside one, else cwd). Raises + FileExistsError if it already exists, ValueError on a bad name.""" safe = name.strip() if not safe or "/" in safe or safe.startswith("."): raise ValueError(f"invalid manifest name: {name!r}") if not safe.endswith(".yaml"): safe += ".yaml" - mdir = Path(root) / "manifests" + project = Path(root) if root is not None else Path.cwd() + mdir = project / "manifests" mdir.mkdir(parents=True, exist_ok=True) path = mdir / safe if path.exists(): diff --git a/framework/cli.py b/framework/cli.py index 0adca46..a6e732f 100644 --- a/framework/cli.py +++ b/framework/cli.py @@ -18,6 +18,10 @@ paramify evidence [--json] # read one evidence file paramify upload [run-dir] [--dry-run] [--json] +User content (the only commands that write into your user dir): + paramify create / [--category-file] # scaffold a new fetcher + paramify customize # override a built-in (copy-on-write) + Manifest editing (writes the manifest file; -f/--file, default ./manifest.yaml; every subcommand accepts --json, emitting {"ok", "path", "errors"}): paramify manifest init [--output-dir DIR] @@ -128,7 +132,7 @@ def _find_fetcher(cat: dict, name: str): return None -def _config_type(root: Path, fetcher_name: str, key: str) -> str: +def _config_type(root: Optional[Path], fetcher_name: str, key: str) -> str: """Look up a config field's declared type (fetcher then platform), else string.""" cat = api.catalog(root) f = _find_fetcher(cat, fetcher_name) @@ -145,7 +149,7 @@ def _config_type(root: Path, fetcher_name: str, key: str) -> str: return "string" -def _platform_config_type(root: Path, category: str, key: str) -> str: +def _platform_config_type(root: Optional[Path], category: str, key: str) -> str: cat = api.catalog(root) for c in cat["categories"]: if c["name"] == category and c.get("platform"): @@ -155,7 +159,7 @@ def _platform_config_type(root: Path, category: str, key: str) -> str: return "string" -def _target_field_type(root: Path, fetcher_name: str, key: str) -> str: +def _target_field_type(root: Optional[Path], fetcher_name: str, key: str) -> str: cat = api.catalog(root) f = _find_fetcher(cat, fetcher_name) if f: @@ -231,7 +235,7 @@ def on_event(ev: dict) -> None: @app.command("list") def list_cmd(json_out: bool = typer.Option(False, "--json", help="Emit JSON")): """List discovered fetchers (flat).""" - root = api.find_repo_root() + root = api.locate_root() cat = api.catalog(root) fetchers = sorted( (f for c in cat["categories"] for f in c["fetchers"]), @@ -252,7 +256,7 @@ def list_cmd(json_out: bool = typer.Option(False, "--json", help="Emit JSON")): @app.command("catalog") def catalog_cmd(json_out: bool = typer.Option(False, "--json", help="Emit JSON")): """Show categories -> fetchers -> editable fields.""" - root = api.find_repo_root() + root = api.locate_root() cat = api.catalog(root) if json_out: typer.echo(json.dumps(cat, indent=2)) @@ -274,7 +278,7 @@ def describe_cmd( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Describe one fetcher's config / secrets / target fields.""" - root = api.find_repo_root() + root = api.locate_root() cat = api.catalog(root) f = _find_fetcher(cat, fetcher) if f is None: @@ -299,7 +303,7 @@ def describe_cmd( @app.command("ksi") def ksi_cmd(json_out: bool = typer.Option(False, "--json", help="Emit JSON")): """Show FedRAMP 20x KSI coverage across discovered fetchers.""" - root = api.find_repo_root() + root = api.locate_root() cov = api.ksi_coverage(root) if json_out: typer.echo(json.dumps(cov, indent=2)) @@ -340,7 +344,7 @@ def doctor_cmd( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Preflight: Python version, required CLIs on PATH, and (with a manifest) secrets.""" - root = api.find_repo_root() + root = api.locate_root() rep = api.doctor(root, Path(manifest) if manifest else None) if json_out: typer.echo(json.dumps(rep, indent=2)) @@ -371,14 +375,100 @@ def doctor_cmd( else: typer.echo(f" {bad_mark} {fr['use']} missing: {', '.join(fr['missing'])}") + dist = rep.get("distribution") or {} + if dist: + typer.echo( + f"\nDistribution: paramify-fetchers {dist['tool_version']}" + f" (install: {dist['install_path']})" + ) + typer.echo(f" user dir: {dist['user_dir']}") + typer.echo(" content roots (first wins):") + for i, r in enumerate(dist["roots"], 1): + typer.echo(f" {i}. {r}") + for s in dist["shadows"]: + typer.echo(f" ⤷ shadow: {s['name']} {s['winner']} (hides {s['shadowed']})") + for o in dist["stale_overrides"]: + if o["status"] == "orphaned": + typer.echo(f" {bad_mark} override {o['name']} is orphaned — its original is gone ({o['path']})") + else: + typer.echo( + f" ⚠️ override {o['name']} is stale — the original changed since you " + f"copied it (v{o['copied_tool_version']}): {', '.join(o['changed'])}" + ) + for inv in dist["invalid"]: + typer.echo(f" {bad_mark} invalid fetcher.yaml skipped: {inv['path']}") + typer.echo(f"\n{'All good.' if rep['ok'] else 'Issues found — see above.'}") raise typer.Exit(0 if rep["ok"] else 1) +# --------------------------------------------------------------------------- # +# User content — scaffold + copy-on-write overrides +# --------------------------------------------------------------------------- # + +@app.command("create") +def create_cmd( + spec: str = typer.Argument(..., help="/, e.g. datadog/monitors"), + category_file: bool = typer.Option( + False, "--category-file", + help="Also scaffold _categories/.yaml when the category is new", + ), + json_out: bool = typer.Option(False, "--json", help="Emit JSON"), +): + """Scaffold a new fetcher in your user dir from the shipped template.""" + root = api.locate_root() + try: + res = api.create_fetcher(spec, root, category_file=category_file) + except (ValueError, FileExistsError, RuntimeError) as exc: + if json_out: + typer.echo(json.dumps({"ok": False, "errors": [str(exc)]})) + else: + typer.echo(f"Create failed: {exc}", err=True) + raise typer.Exit(1) + if json_out: + typer.echo(json.dumps({"ok": True, **res}, indent=2)) + return + typer.echo(f"Scaffolded {res['name']} at {res['path']}") + for f in res["files"]: + typer.echo(f" {f}") + if res["category_file"]: + typer.echo(f" + category file: {res['category_file']}") + typer.echo("Fill in the placeholders, then wire it into a manifest " + f"(`paramify manifest add {res['name']}`).") + + +@app.command("customize") +def customize_cmd( + fetcher: str = typer.Argument(..., help="Name of the built-in fetcher to override"), + json_out: bool = typer.Option(False, "--json", help="Emit JSON"), +): + """Copy a built-in fetcher into your user dir, where it overrides the original.""" + root = api.locate_root() + try: + res = api.customize_fetcher(fetcher, root) + except (ValueError, FileExistsError, RuntimeError) as exc: + if json_out: + typer.echo(json.dumps({"ok": False, "errors": [str(exc)]})) + else: + typer.echo(f"Customize failed: {exc}", err=True) + raise typer.Exit(1) + if json_out: + typer.echo(json.dumps({"ok": True, **res}, indent=2)) + return + typer.echo(f"Copied {res['name']} → {res['path']}") + if res["active"]: + typer.echo(f" (from {res['source']}; your copy now wins — `paramify doctor` " + "flags it if the original changes)") + else: + typer.echo(f" (from {res['source']}; a higher-priority root still wins here — " + "inside a checkout the in-tree copy is used. `paramify doctor` shows " + "the shadow.)") + + @app.command("manifests") def manifests_cmd(json_out: bool = typer.Option(False, "--json", help="Emit JSON")): """List discovered run manifests (manifests/*.yaml + legacy manifest.yaml).""" - root = api.find_repo_root() + root = api.locate_root() items = api.list_manifests(root) if json_out: typer.echo(json.dumps(items, indent=2)) @@ -464,7 +554,7 @@ def validate_cmd( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Validate a manifest against the schema + discovered fetchers.""" - root = api.find_repo_root() + root = api.locate_root() mpath = Path(manifest).resolve() if not mpath.is_file(): msg = f"no such manifest: {manifest}" @@ -499,7 +589,7 @@ def run_cmd( json_out: bool = typer.Option(False, "--json", help="Emit JSON summary"), ): """Run a manifest; streams per-fetcher results (or a JSON summary with --json).""" - root = api.find_repo_root() + root = api.locate_root() mpath = Path(manifest).resolve() if not mpath.is_file(): msg = f"no such manifest: {manifest}" @@ -542,7 +632,7 @@ def upload_cmd( json_out: bool = typer.Option(False, "--json", help="Emit JSON summary"), ): """Upload one evidence run to Paramify.""" - root = api.find_repo_root() + root = api.locate_root() if run_dir: resolved_run_dir = Path(run_dir).resolve() else: @@ -607,7 +697,9 @@ def _read_for_edit(path: Path, json_out: bool) -> dict: raise typer.Exit(1) -def _save_and_report(manifest: dict, path: Path, root: Path, json_out: bool, *, verb: str = "Wrote") -> None: +def _save_and_report( + manifest: dict, path: Path, root: Optional[Path], json_out: bool, *, verb: str = "Wrote" +) -> None: """Dump + validate, then report. Exit 0 even when not-yet-runnable (errors are surfaced) so a manifest can be built incrementally; exit 1 only if the dump is rejected as structurally invalid.""" @@ -637,7 +729,7 @@ def manifest_init( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Create a new empty manifest file.""" - root = api.find_repo_root() + root = api.locate_root() m = api.init_manifest(output_dir) _save_and_report(m, Path(file).resolve(), root, json_out) @@ -649,7 +741,7 @@ def manifest_new( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Create a fresh manifest at manifests/.yaml (the picker convention).""" - root = api.find_repo_root() + root = api.locate_root() try: path = api.new_manifest_path(root, name, output_dir) except (FileExistsError, ValueError) as e: @@ -674,7 +766,7 @@ def manifest_add( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Add a fetcher entry to the manifest.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.add_entry(m, fetcher) @@ -688,7 +780,7 @@ def manifest_remove( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Remove a fetcher entry from the manifest.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.remove_entry(m, fetcher) @@ -703,7 +795,7 @@ def manifest_set_config( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Set a config key on a fetcher entry.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) key, raw = _parse_kv(kv, path, json_out) @@ -721,7 +813,7 @@ def manifest_set_secret( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Set a secret reference (${env:VAR}) on a fetcher entry.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.set_secret(m, fetcher, secret_name, env_var) @@ -737,7 +829,7 @@ def manifest_add_target( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Append a fanout target (with optional per-target secrets) to a fetcher.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) vals = {} @@ -760,7 +852,7 @@ def manifest_remove_target( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Remove the fanout target at the given index from a fetcher entry.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.remove_target(m, fetcher, index) @@ -775,7 +867,7 @@ def manifest_set_platform_config( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Set a platform-wide config key for a category.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) key, raw = _parse_kv(kv, path, json_out) @@ -792,7 +884,7 @@ def manifest_set_passthrough( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Set the ambient passthrough env vars for a platform category.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.set_passthrough_env(m, category, env_vars) @@ -806,7 +898,7 @@ def manifest_set_output_dir( json_out: bool = typer.Option(False, "--json", help="Emit JSON"), ): """Set the manifest's run output directory.""" - root = api.find_repo_root() + root = api.locate_root() path = Path(file).resolve() m = _read_for_edit(path, json_out) api.set_output_dir(m, output_dir) @@ -845,7 +937,13 @@ def tui_cmd( try: from framework.tui.__main__ import launch except ImportError as e: - _err(f"The TUI requires the 'tui' extra (textual). Install it: pip install 'paramify[tui]'\n ({e})") + _err( + "The TUI requires the 'tui' extra (textual). Install it:\n" + " pipx: pipx install --force 'paramify-fetchers[tui]' " + "(or: pipx inject paramify-fetchers textual)\n" + " pip: pip install 'paramify-fetchers[tui]'\n" + f" ({e})" + ) raise typer.Exit(1) launch(manifest, at) diff --git a/framework/config_loader.py b/framework/config_loader.py index 8957297..36f418e 100644 --- a/framework/config_loader.py +++ b/framework/config_loader.py @@ -1,12 +1,15 @@ -"""Discover fetchers in the repo and validate their fetcher.yaml files. +"""Discover fetchers across the content roots and validate their fetcher.yaml files. -Walks `fetchers///fetcher.yaml`, skipping any directory -that starts with `_` (e.g. _shared, _categories, _template). +Walks `///fetcher.yaml` for each root, skipping any +directory that starts with `_` (e.g. _shared, _categories, _template). Roots +come from framework.roots (overlay search path: env override → dev checkout → +user dir → installed bundle); within one root a duplicate fetcher name is a +hard error, across roots the earlier root wins and the shadow is reported. """ import json from pathlib import Path -from typing import Dict, Optional +from typing import Dict, List, NamedTuple, Optional, Sequence, Union import yaml from jsonschema import Draft202012Validator @@ -20,9 +23,29 @@ TargetField, ) +Roots = Union[str, Path, Sequence[Path]] -def _load_schema(repo_root: Path, name: str = "fetcher_schema.json") -> dict: - return json.loads((repo_root / "framework" / "schemas" / name).read_text()) + +def load_schema(name: str = "fetcher_schema.json") -> dict: + """Load a contract schema. Schemas are core, not content: they resolve from + the framework package itself (source tree and normal installs alike), with + an importlib.resources fallback for zip imports.""" + local = Path(__file__).parent / "schemas" / name + if local.exists(): + return json.loads(local.read_text()) + from importlib import resources + return json.loads( + resources.files("framework").joinpath("schemas").joinpath(name).read_text() + ) + + +def _as_fetchers_roots(roots: Roots) -> List[Path]: + """Normalize the roots argument. A single path is a legacy repo root (its + fetchers/ subdir is the root); a sequence is an ordered list of fetchers/ + dirs, as produced by framework.roots.fetcher_roots().""" + if isinstance(roots, (str, Path)): + return [Path(roots) / "fetchers"] + return [Path(r) for r in roots] def _parse_config_schema(raw: Optional[dict]) -> dict: @@ -41,18 +64,22 @@ def _parse_config_schema(raw: Optional[dict]) -> dict: return out -def discover_fetchers(repo_root: Path) -> Dict[str, Fetcher]: - """Walk fetchers/*/*/fetcher.yaml. Returns {fetcher_name: Fetcher}. +class DiscoveryResult(NamedTuple): + fetchers: Dict[str, Fetcher] + shadows: List[dict] # cross-root name collisions: {name, winner, shadowed} + invalid: List[dict] # unloadable fetcher.yaml files: {path, detail} - Raises ValueError on schema-invalid yaml or duplicate fetcher names. - """ - schema = _load_schema(repo_root) - validator = Draft202012Validator(schema) +def _walk_root( + root: Path, validator: Draft202012Validator, invalid: List[dict] +) -> Dict[str, Fetcher]: + """Walk one fetchers/ root. A schema-invalid or unparseable fetcher.yaml is + recorded in `invalid` and skipped — refuse, don't crash: one broken fetcher + (typically a user's work in progress) must not take down discovery. A + duplicate fetcher name within this root stays a hard error (duplicates + across roots are shadowing, handled by the caller).""" fetchers: Dict[str, Fetcher] = {} - fetchers_root = repo_root / "fetchers" - - for category_dir in sorted(fetchers_root.iterdir()): + for category_dir in sorted(root.iterdir()): if not category_dir.is_dir() or category_dir.name.startswith("_"): continue for fetcher_dir in sorted(category_dir.iterdir()): @@ -62,11 +89,16 @@ def discover_fetchers(repo_root: Path) -> Dict[str, Fetcher]: if not yaml_path.exists(): continue - data = yaml.safe_load(yaml_path.read_text()) + try: + data = yaml.safe_load(yaml_path.read_text()) + except yaml.YAMLError as exc: + invalid.append({"path": str(yaml_path), "detail": f" {exc}"}) + continue errors = list(validator.iter_errors(data)) if errors: detail = "\n".join(f" {e.message}" for e in errors) - raise ValueError(f"{yaml_path}: schema validation failed:\n{detail}") + invalid.append({"path": str(yaml_path), "detail": detail}) + continue fetcher = _parse_fetcher(data, fetcher_dir) if fetcher.name in fetchers: @@ -75,10 +107,50 @@ def discover_fetchers(repo_root: Path) -> Dict[str, Fetcher]: f"{fetchers[fetcher.name].path} and {fetcher_dir}" ) fetchers[fetcher.name] = fetcher - return fetchers +def discover(roots: Roots) -> DiscoveryResult: + """Discover fetchers across the ordered roots. + + Earlier roots win a name collision — that's the override mechanism — and + every collision is reported in `shadows` so catalog/doctor can surface it + rather than resolve it silently. Broken fetcher.yaml files land in + `invalid` (skipped, reported, never fatal). + """ + validator = Draft202012Validator(load_schema()) + fetchers: Dict[str, Fetcher] = {} + shadows: List[dict] = [] + invalid: List[dict] = [] + for root in _as_fetchers_roots(roots): + if not root.is_dir(): + continue + for name, fetcher in _walk_root(root, validator, invalid).items(): + if name in fetchers: + shadows.append({ + "name": name, + "winner": str(fetchers[name].path), + "shadowed": str(fetcher.path), + }) + else: + fetchers[name] = fetcher + return DiscoveryResult(fetchers, shadows, invalid) + + +def discover_fetchers(roots: Roots) -> Dict[str, Fetcher]: + """Discover fetchers across the roots (see discover()). Accepts a legacy + single repo root or an ordered list of fetchers/ dirs. + + Strict variant: raises ValueError on the first schema-invalid fetcher.yaml + (as the single-root loader always did) or on duplicate names within a root. + """ + result = discover(roots) + if result.invalid: + first = result.invalid[0] + raise ValueError(f"{first['path']}: schema validation failed:\n{first['detail']}") + return result.fetchers + + def _parse_fetcher(data: dict, path: Path) -> Fetcher: secrets = [ Secret(name=s["name"], env=s["env"], per_target=s.get("per_target", False)) @@ -130,34 +202,40 @@ def _parse_fetcher(data: dict, path: Path) -> Fetcher: ) -def discover_platforms(repo_root: Path) -> Dict[str, PlatformSpec]: - """Load fetchers/_categories/.yaml into {category: PlatformSpec}. +def discover_platforms(roots: Roots) -> Dict[str, PlatformSpec]: + """Load /_categories/.yaml across the ordered roots into + {category: PlatformSpec}. The first root wins per category file — so a + user-dir category file (a new platform, or a deliberate override) shadows + the shipped one as a whole. Empty or absent files yield an empty spec for that category. Raises ValueError on schema-invalid category files. """ - schema = _load_schema(repo_root, "category_schema.json") + schema = load_schema("category_schema.json") validator = Draft202012Validator(schema) platforms: Dict[str, PlatformSpec] = {} - categories_dir = repo_root / "fetchers" / "_categories" - if not categories_dir.is_dir(): - return platforms - - for yaml_path in sorted(categories_dir.glob("*.yaml")): - category = yaml_path.stem - data = yaml.safe_load(yaml_path.read_text()) or {} - errors = list(validator.iter_errors(data)) - if errors: - detail = "\n".join(f" {e.message}" for e in errors) - raise ValueError(f"{yaml_path}: schema validation failed:\n{detail}") - - auth = data.get("auth") or {} - platforms[category] = PlatformSpec( - category=category, - config_schema=_parse_config_schema(data.get("config_schema")), - passthrough_env=list(auth.get("passthrough_env") or []), - description=data.get("description"), - ) + for root in _as_fetchers_roots(roots): + categories_dir = root / "_categories" + if not categories_dir.is_dir(): + continue + + for yaml_path in sorted(categories_dir.glob("*.yaml")): + category = yaml_path.stem + if category in platforms: # an earlier root already defined it + continue + data = yaml.safe_load(yaml_path.read_text()) or {} + errors = list(validator.iter_errors(data)) + if errors: + detail = "\n".join(f" {e.message}" for e in errors) + raise ValueError(f"{yaml_path}: schema validation failed:\n{detail}") + + auth = data.get("auth") or {} + platforms[category] = PlatformSpec( + category=category, + config_schema=_parse_config_schema(data.get("config_schema")), + passthrough_env=list(auth.get("passthrough_env") or []), + description=data.get("description"), + ) return platforms diff --git a/framework/roots.py b/framework/roots.py new file mode 100644 index 0000000..cac24b8 --- /dev/null +++ b/framework/roots.py @@ -0,0 +1,84 @@ +"""Content-root resolution for the overlay distribution model. + +Three kinds of location, kept strictly separate (docs/distribution_design.md): +- core assets (schemas, ksis.yaml) — package-relative, always inside framework/ +- content roots (fetchers, categories) — the ordered overlay search path below +- user data (manifests/, evidence/) — the project dir (checkout or cwd); the + tool never resolves content from it and never writes into it + +Search-path precedence (first root wins a fetcher-name collision, so a user +copy shadows a built-in): + 1. $PARAMIFY_FETCHERS_PATH — explicit override, os.pathsep-separated + 2. the dev checkout's fetchers/ (cwd walk) — in-tree edits always win for devs + 3. $PARAMIFY_HOME/fetchers/ — user-created fetchers and overrides + 4. framework/_bundled/fetchers/ — built-ins shipped inside the package +""" + +import os +from pathlib import Path +from typing import List, Optional + +ENV_HOME = "PARAMIFY_HOME" +ENV_FETCHERS_PATH = "PARAMIFY_FETCHERS_PATH" + + +def user_home() -> Path: + """The user's writable paramify dir: $PARAMIFY_HOME, else the platform's + native user-data location (~/Library/Application Support/paramify on macOS, + %LOCALAPPDATA%\\paramify on Windows, ~/.local/share/paramify on Linux).""" + env = os.environ.get(ENV_HOME) + if env: + return Path(env).expanduser() + from platformdirs import user_data_dir # lazy: only needed without the env var + return Path(user_data_dir("paramify")) + + +def find_checkout_root(start: Optional[Path] = None) -> Optional[Path]: + """Locate a dev checkout by walking up for sibling fetchers/ + framework/ + dirs. Returns None outside a checkout (unlike api.find_repo_root, which + raises).""" + cur = (start or Path.cwd()).resolve() + for parent in [cur, *cur.parents]: + if (parent / "fetchers").is_dir() and (parent / "framework").is_dir(): + return parent + return None + + +def bundled_content_root() -> Optional[Path]: + """framework/_bundled — the shipped content snapshot, assembled into the + wheel at build time. Absent in a dev tree, where the checkout's own + fetchers/ serves instead.""" + p = Path(__file__).parent / "_bundled" + return p if p.is_dir() else None + + +def fetcher_roots( + checkout: Optional[Path] = None, start: Optional[Path] = None +) -> List[Path]: + """The ordered fetchers/ search path; only existing dirs are returned. + + Pass `checkout` to pin the dev root explicitly (tests, `paramify tui --at`); + otherwise it is located by walking up from `start` (default: cwd). + """ + roots: List[Path] = [] + env = os.environ.get(ENV_FETCHERS_PATH) + if env: + roots += [Path(p).expanduser() for p in env.split(os.pathsep) if p] + co = checkout or find_checkout_root(start) + if co is not None: + roots.append(Path(co) / "fetchers") + roots.append(user_home() / "fetchers") + bundled = bundled_content_root() + if bundled is not None: + roots.append(bundled / "fetchers") + + seen, ordered = set(), [] + for r in roots: + if not r.is_dir(): + continue + key = r.resolve() + if key in seen: + continue + seen.add(key) + ordered.append(r) + return ordered diff --git a/framework/runner/executor.py b/framework/runner/executor.py index b20f618..8abb960 100644 --- a/framework/runner/executor.py +++ b/framework/runner/executor.py @@ -163,6 +163,13 @@ def _build_env( manifest + fetcher.yaml + category platform spec. """ env = {k: os.environ[k] for k in _INHERITED_ENV_VARS if k in os.environ} + # Console scripts installed next to the running interpreter (a pipx venv, + # an unactivated .venv) must be visible to fetchers — e.g. the `checkov` + # extra, whose CLI lands in a bin/ that pipx never puts on PATH. + interp_bin = str(Path(sys.executable).parent) + parts = env.get("PATH", "").split(os.pathsep) if env.get("PATH") else [] + if interp_bin not in parts: + env["PATH"] = os.pathsep.join([interp_bin, *parts]) env["PYTHONUNBUFFERED"] = "1" env["EVIDENCE_DIR"] = str(output_dir.resolve()) diff --git a/framework/runner/manifest_loader.py b/framework/runner/manifest_loader.py index a4f882e..15dc07c 100644 --- a/framework/runner/manifest_loader.py +++ b/framework/runner/manifest_loader.py @@ -1,23 +1,22 @@ """Load and validate run manifests.""" -import json from pathlib import Path -from typing import List +from typing import List, Optional import yaml from jsonschema import Draft202012Validator +from framework.config_loader import load_schema from framework.contract import Manifest, ManifestEntry, PlatformConfig, TargetInstance -def _load_schema(repo_root: Path) -> dict: - schema_path = repo_root / "framework" / "schemas" / "run_manifest_schema.json" - return json.loads(schema_path.read_text()) +def schema_errors(data: dict, repo_root: Optional[Path] = None) -> List[str]: + """Return manifest schema-validation errors as readable strings (empty if valid). - -def schema_errors(data: dict, repo_root: Path) -> List[str]: - """Return manifest schema-validation errors as readable strings (empty if valid).""" - validator = Draft202012Validator(_load_schema(repo_root)) + The schema is core and resolves from the framework package itself; + repo_root is accepted (and ignored) for backwards compatibility. + """ + validator = Draft202012Validator(load_schema("run_manifest_schema.json")) return [ f"{'.'.join(str(p) for p in e.absolute_path) or ''}: {e.message}" for e in validator.iter_errors(data) @@ -61,7 +60,7 @@ def parse_manifest(data: dict) -> Manifest: return Manifest(output_dir=output_dir, entries=entries, platforms=platforms) -def load_manifest(path: Path, repo_root: Path) -> Manifest: +def load_manifest(path: Path, repo_root: Optional[Path] = None) -> Manifest: """Load a manifest yaml, validate against the manifest schema, return a Manifest. Raises ValueError if the file is schema-invalid. diff --git a/framework/tui/app.py b/framework/tui/app.py index e9cf424..9553a93 100644 --- a/framework/tui/app.py +++ b/framework/tui/app.py @@ -52,7 +52,7 @@ def on_mount(self) -> None: def _discover(self) -> None: try: - self.root_path = api.find_repo_root(self._root_override) + self.root_path = api.locate_root(self._root_override) self.catalog_data = api.catalog(self.root_path) except Exception as exc: # repo-root discovery / fetcher load failures self.catalog_data = None @@ -88,8 +88,6 @@ def _on_manifest_selected(self, event: ManifestSelected) -> None: def open_manifest_picker(self) -> None: """Quick-picker overlay to swap the active manifest without leaving the workspace.""" - if self.root_path is None: - return options = [] for m in api.list_manifests(self.root_path): flag = f" ⚠ {m['issues']}" if m["issues"] else "" diff --git a/framework/tui/screens/run.py b/framework/tui/screens/run.py index 53d0d9a..5ac0691 100644 --- a/framework/tui/screens/run.py +++ b/framework/tui/screens/run.py @@ -103,7 +103,7 @@ def action_run_manifest(self) -> None: if self._running: self.notify("A run is already in progress.") return - if self.app.manifest is None or self.app.root_path is None: + if self.app.manifest is None: return errors = api.validate(self.app.manifest, self.app.root_path) if errors: diff --git a/framework/tui/screens/welcome.py b/framework/tui/screens/welcome.py index e891733..5e45ee1 100644 --- a/framework/tui/screens/welcome.py +++ b/framework/tui/screens/welcome.py @@ -148,7 +148,7 @@ def on_mount(self) -> None: dt.cursor_type = "row" dt.add_columns("manifest", "fetchers", "status", "last run") if self._manifests is None: - self._manifests = api.list_manifests(self.app.root_path) if self.app.root_path else [] + self._manifests = api.list_manifests(self.app.root_path) for m in self._manifests: dt.add_row( Text(m["name"], style="#7DCFFF"), @@ -410,7 +410,7 @@ def done(ok: bool) -> None: self.app.push_screen(ConfirmModal(f"Delete '{Path(path).name}' (the file on disk)?"), done) def _reload_table(self) -> None: - self._manifests = api.list_manifests(self.app.root_path) if self.app.root_path else [] + self._manifests = api.list_manifests(self.app.root_path) dt = self.query_one("#welcome-manifests", DataTable) dt.clear() for m in self._manifests: diff --git a/framework/tui/welcome_demo.py b/framework/tui/welcome_demo.py index 1a2fc77..e802241 100644 --- a/framework/tui/welcome_demo.py +++ b/framework/tui/welcome_demo.py @@ -17,7 +17,7 @@ class WelcomeDemo(App): def on_mount(self) -> None: self.theme = "tokyo-night" try: - self.root_path = api.find_repo_root() + self.root_path = api.locate_root() self.catalog_data = api.catalog(self.root_path) except Exception: self.root_path = None diff --git a/packaging/brew/Formula/paramify-fetchers.rb b/packaging/brew/Formula/paramify-fetchers.rb new file mode 100644 index 0000000..a04670b --- /dev/null +++ b/packaging/brew/Formula/paramify-fetchers.rb @@ -0,0 +1,92 @@ +# Homebrew formula for the paramify fetcher CLI. +# +# TEMPLATE — two things get stamped per release (see ../README.md): +# 1. url / sha256 of the release tarball +# 2. the python resource stanzas (`brew update-python-resources`) +# +# GPL licensing rules out homebrew-core, so this lives in the paramify tap +# (github.com/paramify/homebrew-tap → `brew tap paramify/tap`). +class ParamifyFetchers < Formula + include Language::Python::Virtualenv + + desc "Collect compliance evidence from your infrastructure and upload it to Paramify" + homepage "https://github.com/paramify/paramify-fetchers" + url "https://github.com/paramify/paramify-fetchers/archive/refs/tags/vX.Y.Z.tar.gz" + sha256 "FILL_ME_ON_RELEASE" # shasum -a 256 + license "GPL-3.0-only" + + depends_on "python@3.13" + + # External CLIs the fetcher catalog shells out to — the one thing pipx can't + # declare (docs/distribution_design.md §9). Heavy but complete; trim to + # jq + git if you'd rather let operators bring the cloud CLIs themselves. + depends_on "awscli" + depends_on "checkov" + depends_on "git" + depends_on "jq" + depends_on "kubectl" + + # ---------------------------------------------------------------------- # + # Python resources. Regenerate the whole block after stamping url/sha256: + # brew update-python-resources Formula/paramify-fetchers.rb + # It expands the transitive tree (requests → certifi/idna/urllib3/…, + # typer → click/rich/…, jsonschema → referencing/rpds-py/…) automatically. + # + # setuptools + wheel are BUILD deps: brew builds offline (no build + # isolation), and the setup.py build_py hook that assembles the + # framework/_bundled content snapshot needs setuptools>=64 present. + # ---------------------------------------------------------------------- # + + resource "setuptools" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "wheel" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "platformdirs" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "python-dotenv" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "pyyaml" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "requests" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "jsonschema" do + url "FILL_ME" + sha256 "FILL_ME" + end + + resource "typer" do + url "FILL_ME" + sha256 "FILL_ME" + end + + def install + # pip install of the source tree runs the build_py hook, so the venv gets + # framework/_bundled (fetchers + examples) — the content built-ins run from. + virtualenv_install_with_resources + end + + test do + assert_match(/Discovered \d+ fetchers/, shell_output("#{bin}/paramify list")) + # no checkout, no user dir: catalog must resolve from the installed bundle + output = shell_output("#{bin}/paramify catalog --json") + assert_match "_bundled", output + end +end diff --git a/packaging/brew/README.md b/packaging/brew/README.md new file mode 100644 index 0000000..a046a64 --- /dev/null +++ b/packaging/brew/README.md @@ -0,0 +1,48 @@ +# Homebrew tap — publishing & bumping + +`Formula/paramify-fetchers.rb` here is the **template**; the live copy belongs +in a tap repo. brew is the Mac/Linux convenience layer — **pipx is the +universal install path** (`pipx install` the release wheel; works on Windows +too). See `docs/distribution_design.md` §9. + +## One-time: create the tap + +1. Create the repo `paramify/homebrew-tap` with a `Formula/` directory. +2. Copy `Formula/paramify-fetchers.rb` into it (stamped — see below). +3. Users then install with: + + ```bash + brew tap paramify/tap + brew install paramify-fetchers + ``` + +## Per release: stamp the formula + +After `vX.Y.Z` is tagged and the GitHub Release exists (docs/releasing.md): + +```bash +# 1. point at the release tarball and record its checksum +curl -fsSL -o /tmp/pf.tar.gz \ + https://github.com/paramify/paramify-fetchers/archive/refs/tags/vX.Y.Z.tar.gz +shasum -a 256 /tmp/pf.tar.gz # → stamp url + sha256 in the formula + +# 2. regenerate every python resource stanza (expands the transitive tree) +brew update-python-resources Formula/paramify-fetchers.rb + +# 3. verify locally before pushing the tap +brew install --build-from-source Formula/paramify-fetchers.rb +brew test paramify-fetchers +brew audit --strict paramify-fetchers +``` + +## Notes + +- **Why the external CLIs are deps:** declaring `awscli`/`kubectl`/`jq`/`git`/ + `checkov` is the one thing brew can do that pipx can't — a single install + that can run the whole catalog. Trim the list in the tap if the closure is + too heavy for your users. +- **Why setuptools/wheel are resources:** brew builds offline with no build + isolation, and the `setup.py` `build_py` hook that assembles + `framework/_bundled/` needs setuptools>=64 at build time. +- **The formula's `test do`** asserts the catalog resolves from `_bundled` — + the installed-layout guarantee, same as `tests/test_installed.py`. diff --git a/pyproject.toml b/pyproject.toml index 45e388a..a4346e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,14 +21,15 @@ dependencies = [ "pyyaml", "jsonschema", "typer>=0.12", + "platformdirs", ] [project.optional-dependencies] tui = ["textual>=1.0,<2.0"] checkov = ["checkov"] -dev = ["pytest", "ruff", "mypy"] +dev = ["pytest", "ruff", "mypy", "setuptools>=64", "wheel"] # Convenience: every front-end + dev tooling in one install. -all = ["textual>=1.0,<2.0", "checkov", "pytest", "ruff", "mypy"] +all = ["textual>=1.0,<2.0", "checkov", "pytest", "ruff", "mypy", "setuptools>=64", "wheel"] # The single entry point. `paramify` steers every front-end: the headless # commands, plus `paramify tui`. (Renaming later is a one-line change here; add @@ -36,18 +37,18 @@ all = ["textual>=1.0,<2.0", "checkov", "pytest", "ruff", "mypy"] [project.scripts] paramify = "framework.cli:app" -# Supported install is editable/source: `pip install -e .` in the clone. Only -# the importable `framework` package's .py modules are shipped here. The data -# files (framework/schemas/*.json, framework/tui/styles/*.tcss) are NOT -# packaged: schemas resolve via the cwd-discovered repo root, while the TUI -# stylesheet is read relative to its own module file — both present in an -# editable/source tree. A pure built wheel therefore can't serve the TUI -# (missing data files) or run fetchers (they execute as subprocesses out of -# fetchers/, never shipped). If a non-editable wheel is ever needed, add -# [tool.setuptools.package-data] for those dirs. +# Installable distribution (docs/distribution_design.md): the wheel ships the +# framework engine, its data files (schemas, KSI reference, TUI styles), the +# uploader package, and a framework/_bundled/ content snapshot (fetchers + +# examples) assembled at build time by setup.py's build_py hook. A dev checkout +# still works editable: framework.roots prefers the checkout's fetchers/ and +# treats a missing _bundled/ as "dev tree". [tool.setuptools.packages.find] where = ["."] -include = ["framework*"] +include = ["framework*", "uploaders*"] + +[tool.setuptools.package-data] +framework = ["schemas/*.json", "reference/*.yaml", "tui/styles/*.tcss"] # pytest: scope collection to the framework test suite. Per-fetcher tests/ dirs # (e.g. the _template scaffold) are placeholders with no settled convention yet; @@ -82,3 +83,10 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = "framework.tui.*" ignore_errors = true + +# The uploader is held to the same bar as fetchers (working v0.x code, not the +# gated framework core); it entered mypy's import graph when api.py gained the +# packaged-module fallback. Gate it when it gets its own typing pass. +[[tool.mypy.overrides]] +module = "uploaders.*" +ignore_errors = true diff --git a/requirements.txt b/requirements.txt index ec5f044..e369c0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ python-dotenv requests pyyaml jsonschema +platformdirs # Unified CLI (`paramify` / framework.cli) — the single command surface that # steers every front-end (headless commands + `paramify tui`). diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..072673a --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +"""Build-time assembly of the shipped content bundle (docs/distribution_design.md §8). + +The wheel carries the content built-ins run from: framework/_bundled/, copied +from fetchers/ and examples/ during build_py. Assembling inside the build +backend — not a release workflow — keeps `pip install` from a source tarball +working: every build path produces the bundle. The bundle is written straight +into build_lib, so the source tree is never touched and a dev checkout never +grows a stale _bundled/ (framework.roots treats its absence as "dev tree"). +""" + +import shutil +from pathlib import Path + +from setuptools import setup +from setuptools.command.build_py import build_py + +HERE = Path(__file__).resolve().parent + +# Content dirs shipped as framework/_bundled/. fetchers/logos/ stays out: +# README furniture, unused at run time. +BUNDLED = ("fetchers", "examples") +_EXCLUDED_NAMES = {"__pycache__", "logos", ".DS_Store"} + + +def _ignore(_src, names): + return {n for n in names if n in _EXCLUDED_NAMES or n.endswith((".pyc", ".pyo"))} + + +class build_py_with_bundle(build_py): + def run(self): + super().run() + dest = Path(self.build_lib) / "framework" / "_bundled" + if dest.exists(): + shutil.rmtree(dest) + for name in BUNDLED: + src = HERE / name + if not src.is_dir(): + raise RuntimeError( + f"cannot assemble framework/_bundled: {src} missing from the source tree" + ) + shutil.copytree(src, dest / name, ignore=_ignore) + + +setup(cmdclass={"build_py": build_py_with_bundle}) diff --git a/tests/test_cli.py b/tests/test_cli.py index d7b26c8..bde0ca8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,7 +108,7 @@ def _tui_api_calls() -> set[str]: # by every command (repo discovery / manifest read+write plumbing) rather than # its own command. Keep this in sync with the TUI; the test below enforces it. API_TO_CLI = { - "find_repo_root": "", + "locate_root": "", "catalog": "list / catalog / describe", "list_manifests": "manifests", "read_manifest": "manifest show", diff --git a/tests/test_executor.py b/tests/test_executor.py index eb0861c..4e19102 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -165,8 +165,27 @@ def test_build_env_strips_undeclared_ambient_vars(tmp_path, monkeypatch): assert "SNEAKY_AMBIENT_SECRET" not in env # stripped assert env["EVIDENCE_DIR"] == str(tmp_path.resolve()) - if "PATH" in os.environ: # whitelist still passes PATH - assert env["PATH"] == os.environ["PATH"] + if "PATH" in os.environ: # whitelist still passes PATH (interp bin may be prepended) + assert env["PATH"].endswith(os.environ["PATH"]) + + +def test_build_env_prepends_interpreter_bin_to_path(tmp_path, monkeypatch): + """Console scripts living next to the running interpreter (a pipx venv, an + unactivated .venv) must be resolvable by fetchers — e.g. the checkov CLI, + which pipx installs into a bin/ it never puts on PATH.""" + import sys + from pathlib import Path as _P + interp_bin = str(_P(sys.executable).parent) + + monkeypatch.setenv("PATH", "/usr/bin") + fetcher = make_fetcher(tmp_path) + env = _build_env(fetcher, ManifestEntry(use="x"), None, tmp_path) + assert env["PATH"].split(os.pathsep) == [interp_bin, "/usr/bin"] + + # already on PATH → not duplicated + monkeypatch.setenv("PATH", os.pathsep.join([interp_bin, "/usr/bin"])) + env2 = _build_env(fetcher, ManifestEntry(use="x"), None, tmp_path) + assert env2["PATH"].split(os.pathsep).count(interp_bin) == 1 def test_build_env_injects_resolved_secret_under_declared_name(tmp_path, monkeypatch): diff --git a/tests/test_installed.py b/tests/test_installed.py new file mode 100644 index 0000000..f692a39 --- /dev/null +++ b/tests/test_installed.py @@ -0,0 +1,107 @@ +"""Installed-layout tests (phase 2 of docs/distribution_design.md §12). + +Build the wheel, install it into an isolated target dir, and drive the CLI +from an empty cwd with no checkout on disk — the "green in dev, broken +installed" gap that every clone-based test misses. The installed copy is put +first on PYTHONPATH so it, not the editable dev install, is what runs; the +run-time deps (typer, yaml, …) still come from the test venv. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(scope="session") +def installed_site(tmp_path_factory) -> Path: + """Build the wheel and install it (no deps) into a bare target dir.""" + wheel_dir = tmp_path_factory.mktemp("wheel") + subprocess.run( + [sys.executable, "-m", "pip", "wheel", "--no-deps", "--no-build-isolation", + "--quiet", "-w", str(wheel_dir), str(REPO_ROOT)], + check=True, capture_output=True, text=True, + ) + [wheel] = wheel_dir.glob("*.whl") + site = tmp_path_factory.mktemp("site") + subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "--quiet", + "--target", str(site), str(wheel)], + check=True, capture_output=True, text=True, + ) + return site + + +def paramify(site: Path, cwd: Path, *args: str, env: dict | None = None): + """Run the installed CLI: `python -c` shim so no console script is needed.""" + e = {k: v for k, v in os.environ.items() + if k not in ("PARAMIFY_HOME", "PARAMIFY_FETCHERS_PATH", "PYTHONPATH")} + e["PYTHONPATH"] = str(site) + if env: + e.update(env) + code = "import sys; from framework.cli import app; sys.argv = ['paramify'] + sys.argv[1:]; app()" + return subprocess.run( + [sys.executable, "-c", code, *args], + cwd=cwd, env=e, capture_output=True, text=True, + ) + + +def test_installed_copy_is_the_one_running(installed_site, tmp_path): + out = subprocess.run( + [sys.executable, "-c", "import framework; print(framework.__file__)"], + cwd=tmp_path, capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": str(installed_site)}, + ) + assert str(installed_site) in out.stdout + + +def test_catalog_from_bundle_no_checkout(installed_site, tmp_path): + res = paramify(installed_site, tmp_path, "catalog", "--json") + assert res.returncode == 0, res.stderr + cat = json.loads(res.stdout) + assert cat["fetcher_count"] == 109 + assert cat["roots"] == [str(installed_site / "framework" / "_bundled" / "fetchers")] + assert cat["shadows"] == [] + categories = {c["name"] for c in cat["categories"]} + assert {"aws", "okta", "demo", "checkov"} <= categories + + +def test_run_bundled_demo_fetcher_no_checkout(installed_site, tmp_path): + (tmp_path / "manifest.yaml").write_text( + "run:\n output_dir: ./evidence\n fetchers:\n - use: demo_hello\n" + ) + res = paramify(installed_site, tmp_path, "run", "manifest.yaml") + assert res.returncode == 0, res.stderr + + [run_dir] = (tmp_path / "evidence").glob("run-*") + evidence = json.loads((run_dir / "demo_hello.json").read_text()) + assert evidence["metadata"]["fetcher_name"] == "demo_hello" + assert evidence["metadata"]["status"] == "success" + assert evidence["payload"] + + +def test_user_dir_shadows_bundled_builtin(installed_site, tmp_path): + """The override mechanism, installed: a user copy of a built-in wins over + the bundle (no checkout → user dir is the highest content root).""" + home = tmp_path / "home" + src = installed_site / "framework" / "_bundled" / "fetchers" / "demo" / "hello" + dst = home / "fetchers" / "demo" / "hello" + dst.mkdir(parents=True) + for f in src.iterdir(): + if f.is_file(): # skip __pycache__ from pip's install-time compile + (dst / f.name).write_bytes(f.read_bytes()) + + res = paramify(installed_site, tmp_path, "catalog", "--json", + env={"PARAMIFY_HOME": str(home)}) + assert res.returncode == 0, res.stderr + cat = json.loads(res.stdout) + assert cat["fetcher_count"] == 109 # shadowed, not duplicated + [shadow] = cat["shadows"] + assert shadow["name"] == "demo_hello" + assert str(home) in shadow["winner"] + assert "_bundled" in shadow["shadowed"] diff --git a/tests/test_roots.py b/tests/test_roots.py new file mode 100644 index 0000000..b6ff428 --- /dev/null +++ b/tests/test_roots.py @@ -0,0 +1,217 @@ +"""Overlay discovery: root resolution, precedence, shadowing (phase 1 of +docs/distribution_design.md §12). + +The phase-1 acceptance bar: `catalog` finds fetchers from a user dir with no +checkout on disk. +""" + +import os +from pathlib import Path + +import pytest + +from framework import api +from framework.config_loader import ( + discover, + discover_fetchers, + discover_platforms, + load_schema, +) +from framework.roots import fetcher_roots, find_checkout_root, user_home + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +FETCHER_YAML = """\ +name: {name} +version: 0.1.0 +description: test fetcher +category: {category} +runtime: + type: python + entry: fetcher.py +output: + type: json + path: out.json +secrets: [] +""" + +CATEGORY_YAML = """\ +description: {description} +auth: + passthrough_env: + - {env_var} +""" + + +def make_fetcher(root: Path, category: str, short: str, name: str) -> Path: + d = root / category / short + d.mkdir(parents=True) + (d / "fetcher.yaml").write_text(FETCHER_YAML.format(name=name, category=category)) + (d / "fetcher.py").write_text("print('hi')\n") + return d + + +def make_category(root: Path, category: str, description: str, env_var: str) -> Path: + d = root / "_categories" + d.mkdir(parents=True, exist_ok=True) + p = d / f"{category}.yaml" + p.write_text(CATEGORY_YAML.format(description=description, env_var=env_var)) + return p + + +# --------------------------------------------------------------------------- # +# Root resolution +# --------------------------------------------------------------------------- # + +def test_user_home_honors_env(monkeypatch, tmp_path): + monkeypatch.setenv("PARAMIFY_HOME", str(tmp_path / "home")) + assert user_home() == tmp_path / "home" + + +def test_find_checkout_root_walks_up(): + assert find_checkout_root(REPO_ROOT / "framework" / "tui") == REPO_ROOT + + +def test_find_checkout_root_returns_none_outside(tmp_path): + assert find_checkout_root(tmp_path) is None + + +def test_fetcher_roots_order_and_existence(monkeypatch, tmp_path): + """env override → checkout → user dir; missing dirs are dropped.""" + override = tmp_path / "override" + override.mkdir() + home = tmp_path / "home" + (home / "fetchers").mkdir(parents=True) + monkeypatch.setenv("PARAMIFY_FETCHERS_PATH", os.pathsep.join([str(override), str(tmp_path / "missing")])) + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + + roots = fetcher_roots(checkout=REPO_ROOT) + assert roots == [override, REPO_ROOT / "fetchers", home / "fetchers"] + + +def test_fetcher_roots_no_checkout(monkeypatch, tmp_path): + home = tmp_path / "home" + (home / "fetchers").mkdir(parents=True) + monkeypatch.delenv("PARAMIFY_FETCHERS_PATH", raising=False) + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + + roots = fetcher_roots(start=tmp_path) # cwd walk from outside any checkout + assert roots == [home / "fetchers"] + + +# --------------------------------------------------------------------------- # +# Schema loading — core assets resolve without any repo root +# --------------------------------------------------------------------------- # + +def test_load_schema_package_relative(): + schema = load_schema() + assert schema.get("$schema") or schema.get("properties") + assert load_schema("run_manifest_schema.json") + + +# --------------------------------------------------------------------------- # +# Multi-root discovery: precedence, shadowing, duplicates +# --------------------------------------------------------------------------- # + +def test_discover_legacy_single_repo_root(): + """Back-compat: a single Path is a repo root (its fetchers/ subdir walks).""" + fetchers = discover_fetchers(REPO_ROOT) + assert "demo_hello" in fetchers + + +def test_first_root_wins_and_shadow_is_reported(tmp_path): + user_root = tmp_path / "user" + bundle_root = tmp_path / "bundle" + make_fetcher(user_root, "aws", "thing", "aws_thing") + make_fetcher(bundle_root, "aws", "thing", "aws_thing") + make_fetcher(bundle_root, "aws", "other", "aws_other") + + result = discover([user_root, bundle_root]) + assert result.fetchers["aws_thing"].path == (user_root / "aws" / "thing").resolve() + assert "aws_other" in result.fetchers + assert result.invalid == [] + assert result.shadows == [{ + "name": "aws_thing", + "winner": str((user_root / "aws" / "thing").resolve()), + "shadowed": str((bundle_root / "aws" / "thing").resolve()), + }] + + +def test_duplicate_within_one_root_still_raises(tmp_path): + root = tmp_path / "user" + make_fetcher(root, "aws", "a", "same_name") + make_fetcher(root, "aws", "b", "same_name") + with pytest.raises(ValueError, match="Duplicate fetcher name"): + discover_fetchers([root]) + + +def test_platforms_first_root_wins_per_category_file(tmp_path): + user_root = tmp_path / "user" + bundle_root = tmp_path / "bundle" + user_root.mkdir() + bundle_root.mkdir() + make_category(user_root, "aws", "user override", "USER_VAR") + make_category(bundle_root, "aws", "shipped", "SHIPPED_VAR") + make_category(bundle_root, "okta", "shipped okta", "OKTA_VAR") + make_category(user_root, "datadog", "brand-new user platform", "DD_API_KEY") + + platforms = discover_platforms([user_root, bundle_root]) + assert platforms["aws"].description == "user override" + assert platforms["aws"].passthrough_env == ["USER_VAR"] + assert platforms["okta"].description == "shipped okta" + assert platforms["datadog"].passthrough_env == ["DD_API_KEY"] + + +# --------------------------------------------------------------------------- # +# The phase-1 acceptance bar: catalog with no checkout on disk +# --------------------------------------------------------------------------- # + +def test_catalog_from_user_dir_without_checkout(monkeypatch, tmp_path): + home = tmp_path / "home" + make_fetcher(home / "fetchers", "datadog", "monitors", "datadog_monitors") + make_category(home / "fetchers", "datadog", "Datadog", "DD_API_KEY") + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + monkeypatch.delenv("PARAMIFY_FETCHERS_PATH", raising=False) + monkeypatch.chdir(tmp_path) # no fetchers/+framework/ siblings anywhere above + + assert api.locate_root(tmp_path) is None + cat = api.catalog() + names = [f["name"] for c in cat["categories"] for f in c["fetchers"]] + assert names == ["datadog_monitors"] + dd = next(c for c in cat["categories"] if c["name"] == "datadog") + assert dd["platform"]["passthrough_env"] == ["DD_API_KEY"] + assert cat["shadows"] == [] + assert cat["roots"] == [str(home / "fetchers")] + + +def test_user_fetcher_shadows_builtin_in_catalog(monkeypatch, tmp_path): + """Inside the checkout, a user copy of a built-in loses to the in-tree one + (dev clone outranks the user dir) — and the shadow is reported.""" + home = tmp_path / "home" + make_fetcher(home / "fetchers", "demo", "hello", "demo_hello") + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + monkeypatch.delenv("PARAMIFY_FETCHERS_PATH", raising=False) + + cat = api.catalog(REPO_ROOT) + assert [s["name"] for s in cat["shadows"]] == ["demo_hello"] + shadow = cat["shadows"][0] + assert str(REPO_ROOT / "fetchers") in shadow["winner"] + assert str(home) in shadow["shadowed"] + + +def test_validate_and_manifests_without_checkout(monkeypatch, tmp_path): + home = tmp_path / "home" + make_fetcher(home / "fetchers", "datadog", "monitors", "datadog_monitors") + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + monkeypatch.delenv("PARAMIFY_FETCHERS_PATH", raising=False) + monkeypatch.chdir(tmp_path) + + manifest = {"run": {"output_dir": "./evidence", + "fetchers": [{"use": "datadog_monitors"}]}} + assert api.validate(manifest) == [] + + path = api.new_manifest_path(None, "smoke") + assert path == tmp_path / "manifests" / "smoke.yaml" + listed = api.list_manifests(None) + assert [m["name"] for m in listed] == ["smoke.yaml"] diff --git a/tests/test_user_content.py b/tests/test_user_content.py new file mode 100644 index 0000000..e356dba --- /dev/null +++ b/tests/test_user_content.py @@ -0,0 +1,173 @@ +"""User-content commands (phase 3 of docs/distribution_design.md §12): create, +customize (copy-on-write + staleness sidecar), and doctor's distribution +section — exercised against an installed-like topology (fake bundle root as +the lowest-priority content root, user dir above it, no checkout). +""" + +import json +import shutil +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from framework import api +from framework import roots as roots_mod +from framework.config_loader import discover_fetchers + +REPO_ROOT = Path(__file__).resolve().parents[1] + +FETCHER_YAML = """\ +name: {name} +version: 0.1.0 +description: test fetcher +category: {category} +runtime: + type: python + entry: fetcher.py +output: + type: json + path: out.json +secrets: [] +""" + + +def make_fetcher(root: Path, category: str, short: str, name: str) -> Path: + d = root / category / short + d.mkdir(parents=True) + (d / "fetcher.yaml").write_text(FETCHER_YAML.format(name=name, category=category)) + (d / "fetcher.py").write_text("print('hi')\n") + return d + + +@pytest.fixture +def overlay(monkeypatch, tmp_path): + """Installed-like topology: user dir over a fake bundle, no checkout. + The bundle carries the real _template so create() can scaffold.""" + home = tmp_path / "home" + bundle = tmp_path / "bundle" + (bundle / "fetchers").mkdir(parents=True) + shutil.copytree( + REPO_ROOT / "fetchers" / "_template", bundle / "fetchers" / "_template", + ignore=shutil.ignore_patterns("__pycache__"), + ) + monkeypatch.setenv("PARAMIFY_HOME", str(home)) + monkeypatch.delenv("PARAMIFY_FETCHERS_PATH", raising=False) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(roots_mod, "bundled_content_root", lambda: bundle) + return SimpleNamespace(home=home, bundle=bundle / "fetchers") + + +# --------------------------------------------------------------------------- # +# create +# --------------------------------------------------------------------------- # + +def test_create_scaffolds_into_user_dir(overlay): + res = api.create_fetcher("datadog/monitors", category_file=True) + assert res["name"] == "datadog_monitors" + dest = Path(res["path"]) + assert dest == overlay.home / "fetchers" / "datadog" / "monitors" + + text = (dest / "fetcher.yaml").read_text() + assert "datadog_monitors" in text and "" not in text + assert res["category_file"] == str( + overlay.home / "fetchers" / "_categories" / "datadog.yaml" + ) + + cat = api.catalog() + names = [f["name"] for c in cat["categories"] for f in c["fetchers"]] + assert "datadog_monitors" in names # discoverable immediately + + +def test_create_refuses_overwrite_and_collisions(overlay): + make_fetcher(overlay.bundle, "aws", "thing", "aws_thing") + with pytest.raises(FileExistsError, match="use customize"): + api.create_fetcher("aws/thing") # collides with a "built-in" name + + api.create_fetcher("datadog/monitors") + with pytest.raises(FileExistsError, match="already exists"): + api.create_fetcher("datadog/monitors") + + +def test_create_rejects_bad_spec(overlay): + for bad in ("nocategory", "Bad/Name", "aws/", "/thing", "a b/c"): + with pytest.raises(ValueError, match="invalid spec"): + api.create_fetcher(bad) + + +# --------------------------------------------------------------------------- # +# customize — copy-on-write + sidecar +# --------------------------------------------------------------------------- # + +def test_customize_copies_and_shadows(overlay): + make_fetcher(overlay.bundle, "aws", "thing", "aws_thing") + + res = api.customize_fetcher("aws_thing") + dest = Path(res["path"]) + assert dest == overlay.home / "fetchers" / "aws" / "thing" + assert sorted(res["files"]) == ["fetcher.py", "fetcher.yaml"] + assert res["active"] # no checkout here, so the user copy wins discovery + + sidecar = json.loads((dest / ".customized.json").read_text()) + assert sidecar["name"] == "aws_thing" + assert sidecar["source"] == str((overlay.bundle / "aws" / "thing").resolve()) + assert set(sidecar["files"]) == {"fetcher.py", "fetcher.yaml"} + + cat = api.catalog() + assert cat["fetcher_count"] == 1 # shadowed, not duplicated + [shadow] = cat["shadows"] + assert shadow["name"] == "aws_thing" + assert str(overlay.home) in shadow["winner"] + + # re-customizing resolves to the user copy itself — directed to edit it + with pytest.raises(FileExistsError, match="already in your user dir"): + api.customize_fetcher("aws_thing") + + +def test_customize_rejects_unknown_and_user_own(overlay): + with pytest.raises(ValueError, match="unknown fetcher"): + api.customize_fetcher("nope") + + api.create_fetcher("datadog/monitors") + with pytest.raises(FileExistsError, match="already in your user dir"): + api.customize_fetcher("datadog_monitors") + + +# --------------------------------------------------------------------------- # +# doctor — staleness + invalid reporting +# --------------------------------------------------------------------------- # + +def test_doctor_flags_stale_and_orphaned_overrides(overlay): + src = make_fetcher(overlay.bundle, "aws", "thing", "aws_thing") + api.customize_fetcher("aws_thing") + + dist = api.doctor()["distribution"] + assert dist["stale_overrides"] == [] + assert [s["name"] for s in dist["shadows"]] == ["aws_thing"] + assert dist["user_dir"] == str(overlay.home) + + (src / "fetcher.py").write_text("print('upstream improved this')\n") + [stale] = api.doctor()["distribution"]["stale_overrides"] + assert stale["status"] == "stale" + assert stale["changed"] == ["fetcher.py"] + + shutil.rmtree(src) + [orphan] = api.doctor()["distribution"]["stale_overrides"] + assert orphan["status"] == "orphaned" + assert orphan["name"] == "aws_thing" + + +def test_invalid_fetcher_reported_not_fatal(overlay): + make_fetcher(overlay.bundle, "aws", "thing", "aws_thing") + broken = overlay.home / "fetchers" / "wip" / "half_done" + broken.mkdir(parents=True) + (broken / "fetcher.yaml").write_text("name: wip_half_done\n") # misses required keys + + cat = api.catalog() # must not raise + assert cat["fetcher_count"] == 1 + [inv] = cat["invalid"] + assert str(broken / "fetcher.yaml") == inv["path"] + assert api.doctor()["distribution"]["invalid"] == cat["invalid"] + + with pytest.raises(ValueError, match="schema validation failed"): + discover_fetchers([overlay.home / "fetchers"]) # strict path still raises diff --git a/uploaders/__init__.py b/uploaders/__init__.py new file mode 100644 index 0000000..3debb08 --- /dev/null +++ b/uploaders/__init__.py @@ -0,0 +1,5 @@ +"""Uploaders — the separate stage that pushes collected evidence to Paramify. + +Packaged so `paramify upload` works from an installed distribution; in a dev +checkout the api loads uploader.py by path from this same tree. +""" diff --git a/uploaders/paramify_evidence/__init__.py b/uploaders/paramify_evidence/__init__.py new file mode 100644 index 0000000..8baffd7 --- /dev/null +++ b/uploaders/paramify_evidence/__init__.py @@ -0,0 +1 @@ +"""Paramify evidence uploader package (see uploader.py)."""