diff --git a/.github/workflows/publish-verified-model.yml b/.github/workflows/publish-verified-model.yml index 99653b0..8030c6d 100644 --- a/.github/workflows/publish-verified-model.yml +++ b/.github/workflows/publish-verified-model.yml @@ -1,8 +1,86 @@ name: Publish Verified Model + on: workflow_dispatch: + inputs: + model_name: + description: "Name to put in ovllm_manifest.json" + required: true + default: "gpt10m-shakespeare" + hf_repo_id: + description: "Optional Hugging Face repo id, e.g. owner/gpt10m-shakespeare" + required: false + default: "" + publish_to_hf: + description: "Upload the signed directory to Hugging Face" + required: true + default: false + type: boolean + +permissions: + contents: read + id-token: write + jobs: - placeholder-job: + sign-and-verify: runs-on: ubuntu-latest + steps: - - run: echo "This is just a temporary placeholder" \ No newline at end of file + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install -r requirements.txt + python -m pip install -e . + + - name: Train real model (smoke steps) + run: | + python run_experiment.py \ + --model gpt10m \ + --dataset shakespeare \ + --precision fp32 \ + --deterministic on \ + --steps 8 \ + --batch-size 8 \ + --block-size 64 \ + --keep-artifact + + - name: Prepare publish directory + run: | + ovllm prepare-publish \ + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ + --name "${{ inputs.model_name }}" + + - name: Sign with GitHub Actions OIDC + env: + SIGSTORE_IDENTITY: "https://github.com/${{ github.repository }}/.github/workflows/publish-verified-model.yml@${{ github.ref }}" + SIGSTORE_IDENTITY_PROVIDER: "https://token.actions.githubusercontent.com" + run: | + ovllm sign dist/gpt10m-shakespeare \ + --identity "$SIGSTORE_IDENTITY" \ + --identity-provider "$SIGSTORE_IDENTITY_PROVIDER" \ + --use-ambient-credentials + + - name: Verify signed artifact + run: ovllm verify dist/gpt10m-shakespeare --skip-replay + + - name: Upload signed model directory + uses: actions/upload-artifact@v4 + with: + name: gpt10m-shakespeare-signed + path: dist/gpt10m-shakespeare + + - name: Publish to Hugging Face + if: ${{ inputs.publish_to_hf && inputs.hf_repo_id != '' }} + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + ovllm publish-hf "${{ inputs.hf_repo_id }}" dist/gpt10m-shakespeare diff --git a/.gitignore b/.gitignore index c09d4f7..bb46353 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ venv/ __pycache__/ *.pyc +build/ +dist/ +*.egg-info/ +.ovllm-cache/ .vscode/ @@ -39,3 +43,6 @@ data/*.zip data/*.tar.gz data/cifar-10-batches-py/ !data/shakespeare_sample.txt + +# Local smoke tests and temp files +openverifiable-smoke/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0832d9..29c98d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,12 +40,12 @@ Dependencies are pinned and hash-locked. This is deliberate: it's both a reprodu git clone https://github.com//OpenVerifiableLLM cd OpenVerifiableLLM -# install the exact, locked dependency set -uv sync +# install dependencies and the editable CLI +pip install -r requirements.txt +pip install -e . # confirm things work -pytest tests/falsifiability -ruff check . +python -m unittest discover -s tests ``` If you need to add a dependency, add it through the lockfile so the hash-locked set stays authoritative. An unpinned dependency undermines both reproducibility and the project's supply-chain posture. @@ -58,8 +58,8 @@ If you need to add a dependency, add it through the lockfile so the hash-locked **Tests.** -- Run the test suite and `ruff` locally before opening a PR. The determinism checks in CI must pass; a PR that breaks them won't merge. -- Code that changes verification behavior, serialization, RNG/optimizer state handling, or the manifest should include tests for both the success case and the failure case. For verification-affecting changes, extend the falsifiability suite so a clean run passes and the relevant tampered run fails. +- Run the test suite (`python -m unittest discover -s tests`) locally before opening a PR. The determinism checks in CI must pass; a PR that breaks them won't merge. +- Code that changes verification behavior, serialization, RNG/optimizer state handling, or the manifest should include tests for both the success case and the failure case. For verification-affecting changes, extend the test suite so a clean run passes and the relevant tampered run fails. - A change must not silently weaken the bit-exact reproducibility guarantee on the supported single-GPU stack. If a change trades determinism for performance, make that tradeoff explicit and document it. **Docs.** If a change alters behavior, inputs, the manifest format, or the verification contract, update the relevant docs in the same PR. The manifest schema is a versioned contract that other parts of the project depend on, so changes there need maintainer sign-off and a migration note. @@ -68,7 +68,7 @@ If you need to add a dependency, add it through the lockfile so the hash-locked - Reference the issue it addresses (`Closes #123`) if there is one. - Describe what changed and why, and flag anything you'd like reviewers to look at closely. -- Confirm the test suite and `ruff` pass. +- Confirm the test suite passes. - Keep each PR to one logical change. Small, focused PRs get reviewed and merged faster than large, multi-purpose ones. Maintainers review on a volunteer basis, so a clear, well-tested, well-scoped PR is the best way to get a quick response. Expect some back-and-forth; review comments are about the work, not about you. diff --git a/README.md b/README.md index 956dedd..6e32cc3 100644 --- a/README.md +++ b/README.md @@ -1,211 +1,329 @@ # OpenVerifiableLLM -**Deterministic training and independent verification for language models.** +**One-command verification for small open model artifacts.** -OpenVerifiableLLM is an [AOSSIE](https://aossie.org) project building a training pipeline whose entire process is reproducible and independently auditable. Given the same data, configuration, and a fixed hardware stack, the pipeline produces bit-identical models, and any deviation (corruption, tampering, or an honest mistake) is cryptographically detectable. +OpenVerifiableLLM is an AOSSIE project for making model releases independently +checkable. The goal is simple: a stranger should be able to pull a published +model, run one verifier command, and see whether the weights, manifest, replay +claim, and publisher provenance all check out. -The goal is not just to publish a model, but to publish a model whose training process can be verified rather than trusted. +This repository contains both the research harness that tests training +reproducibility and the Phase B verifier/publish path that turns those proofs +into a usable artifact. ---- +## Current Status -## The reproducibility matrix (this build) +The project now has two connected layers: -Earlier versions hashed a checkpoint and called it verified. The sharper claim this -build is organised around: **training reproducibility is universally assumed and -rarely verified — it breaks silently, and this tool surfaces the exact conditions -under which it fails.** The hook is the *failure*, not the success. +- **Reproducibility experiments:** deterministic training, hardware/precision + sweeps, safetensors artifacts, tensor hashes, Merkle chunking, and segmented + replay audits. +- **Verifier and publish loop:** `ovllm verify` checks a local or Hugging Face + model directory, recomputes hashes, validates Merkle metadata, optionally runs + replay, and verifies Sigstore/model-transparency provenance. -A single parametrized runner sweeps a grid of models × conditions and emits one JSON -record per cell, deliberately mixing reproducible and broken outcomes: +The intended midterm story is: -| | fp32 det-on | fp32 det-off | tf32 | bf16 | cross-GPU | -|---|---|---|---|---|---| -| **mlp** (control) | PASS | (verify on GPU) | bits≠fp32 (Ampere) | bits≠fp32 | (verify) | -| **gpt10m** (attention) | PASS | run-to-run FAIL on GPU | bits≠fp32 | bits≠fp32 | DIFF | -| **lstm** (cuDNN recurrent) | PASS | FAIL on GPU | bits≠fp32 | bits≠fp32 | DIFF | +```bash +ovllm verify / +``` -Two comparisons are kept deliberately separate, because conflating them is the most -common reproducibility error: +If the artifact is intact and properly signed, the verifier prints green checks +and ends with: + +```text +VERDICT: GREEN +``` -- **Run-to-run** (`reproducible`, `first_divergence_step`): train the *same* config - twice on the *same* hardware — identical bits? Broken by nondeterministic kernels - (determinism OFF) and by different GPUs. -- **Agreement with the fp32 reference** (`vs_fp32`): does tf32/bf16 produce the same - bits — or merely the same loss to a tolerance — as fp32? TF32/bf16 are perfectly - run-to-run reproducible yet silently disagree with fp32. +## What `ovllm verify` Checks -**The planted debate.** `verify()` accepts losses within `rel_tol=1e-6` but compares -parameters by *exact* hash, so a run can pass the loss check and fail the bitwise -check. Is the right verification bar bitwise identity (strong, brittle, hardware-bound) -or numerical tolerance (portable, but admits silent precision drift)? The matrix -supplies evidence both ways; the choice defines what "reproducible training" means. +Given a local model directory or Hugging Face model reference, the verifier: -**Security upgrade.** Checkpoints are now ed25519-signed and the signature is verified -*before* deserialization (`src/signing.py`). The previous path, -`torch.load(weights_only=False)` followed by a SHA-256 check, executes arbitrary code -at unpickle time — before the integrity check ever runs. Signing also makes the -"cryptographically signed" claim true (a SHA-256 is a checksum, not a signature). +1. Resolves the model reference. +2. Loads `ovllm_manifest.json`. +3. Finds the `.safetensors` weights. +4. Recomputes the raw artifact SHA-256. +5. Rebuilds the Merkle tree over weight chunks. +6. Recomputes the tensor-level safetensors hash. +7. Optionally runs a structured segment-replay audit if the manifest includes one. +8. Verifies the Sigstore/model-transparency bundle as a red/green provenance check. -### Quickstart +Missing Sigstore provenance is **red by default**. For local development only, +use `--allow-unsigned` to turn that check into a skip. + +## Quickstart + +Install dependencies: ```bash pip install -r requirements.txt -python demo.py # full arc on CPU (smoke config); --full on a GPU pod -python sweep.py --quick # the matrix, tiny CPU preset -cd src && python reproducibility.py # segmented-replay audit (CLEAN AUDIT PASS + scenarios) +pip install -e . +ovllm --help ``` -See **RUNBOOK.md** for the exact pod commands that populate the GPU-only cells. +The editable install exposes the `ovllm` command used throughout this README. ---- +Run the verifier against a prepared local directory: -## Why this project exists +```bash +ovllm verify path/to/model-dir +``` -Open-weight models are reproducible in principle but not verifiable in practice. You can download the weights, but you cannot prove what data they were trained on, what configuration produced them, or whether they were modified after release. A model ships with a report, and the report has to be trusted. +Remote Hugging Face references download into `.ovllm-cache/huggingface` by +default to avoid permission issues in the global Hugging Face cache. Override +with `--cache-dir ` or `OVLLM_HF_CACHE_DIR`. -There is no cryptographic link between a set of weights and the process that produced them, which makes post-training modification (fine-tuning, data injection, weight edits) effectively undetectable from the artifact alone. +If remote verification reports `signature present, but manifest lacks +sigstore_identity/provider`, the uploaded directory was not the GitHub +Actions-signed artifact. Re-run the **Publish Verified Model** workflow and +publish that signed output so the manifest includes the expected Sigstore +identity metadata. -OpenVerifiableLLM treats verification as a property of the training pipeline itself rather than something added afterward. +For local unsigned smoke tests (skipping local retraining): -## What "verifiable" means here +```bash +ovllm verify path/to/model-dir --allow-unsigned --skip-replay +``` -The term is used precisely in this project. +For full end-to-end training verification (including local retraining and parameter hash matching): -**What the system proves.** Given a fixed dataset snapshot, a fixed configuration, and the same hardware/software stack, an independent party can reproduce the exact model (bit-identical weights) or detect that a published artifact deviates from what was claimed. Verification is exact (a hash match), not approximate. +```bash +ovllm verify path/to/model-dir --allow-unsigned +``` -**What it does not prove.** A passing verification confirms that a training segment is reproducible and internally consistent. It does not, on its own, prove that training was honest, because a determined adversary can construct a checkpoint chain that passes spot-checks (see [Threat model](#threat-model)). The system substantially raises the cost of forgery; it does not reduce it to zero. The stronger guarantee requires cryptographic proof-of-training (zkML), which is not tractable at this scale and is treated as future work. +This will automatically execute the local CPU retraining pipeline defined in the manifest's `segment_replay` block, then verify that your locally trained model matches the uploaded model's parameter hash bit-for-bit. -This honesty is by design. The system is built around *falsifiability*: it must fail reliably when assumptions are violated, not merely pass when everything is correct. +Run tests: -## How it works +```bash +python -m unittest discover -s tests +``` -The pipeline cryptographically links every stage from raw data to final weights. +Run the CPU demo arc: +```bash +python demo.py ``` -Dataset (pinned dump) -- Merkle root over ordered chunks - | - v -Tokenization (deterministic) -- config hash, binary tokens - | - v -Deterministic training loop -- full RNG + optimizer state control - | - v -Verification layer -- tensor-level SHA-256, safetensors - | - v -Signed manifest + transparency log -- Sigstore / Rekor - | - v -Evaluation (factual, bias) -- hash-linked into the chain + +Run the segmented replay/falsifiability smoke test: + +```bash +cd src +python reproducibility.py +cd .. ``` -Each stage records its inputs and outputs into a manifest, and the manifests chain into a single pipeline hash so any link can be checked independently. +For GPU demo commands and the full matrix, see [RUNBOOK.md](RUNBOOK.md). -### Two core programs +## Publish Loop -**Trainer / chain producer.** Takes data and parameters, produces the final model along with a sequence of incremental snapshots and the data split used to produce them. The chain begins at the deterministically-seeded initial model (before any training) so that even the first segment is verifiable. Each snapshot is a complete training-state boundary (weights, optimizer state, full RNG state, schedule position, dataloader position), not just weights, because exact segment replay depends on restoring all of it. +Prepare a publishable model directory from the real trained safetensors checkpoint: -**Segment verifier.** Takes a boundary snapshot, the next data chunk, the configuration, and the claimed next snapshot, then replays that single segment and checks the result. The default test is a bit-exact hash match (valid on the same hardware stack, with no tolerance window for a forged or corrupted step to hide in). Cross-hardware verification is available as a separate, explicitly-labeled mode with a documented tolerance. +```bash +ovllm prepare-publish \ + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ + --name gpt10m-shakespeare +``` -This is what makes verification affordable: an auditor can verify any single segment at a small fraction of the full training cost, sample several at random, and gain high confidence without retraining the whole model. +This writes: -## Design findings +- `model.safetensors` or the original safetensors filename +- `ovllm_manifest.json` +- `README.md` model card +- `Modelfile` for an Ollama build path -These observations from the project's controlled experiments inform the architecture. +Rerun `prepare-publish` whenever the model-card or manifest template changes so +the publish directory contains the current generated metadata. -**Computational determinism is achievable; representational determinism is the catch.** With seeds, initialization, data order, and configuration fixed, training computation is numerically stable, and two independent runs on a fixed single-GPU stack produce bit-identical weights. However, identical weights do not produce identical files: PyTorch's `.pt` format embeds timestamps and pickle metadata, so the bytes change on every save. Verification therefore operates at the tensor level using a byte-stable format ([safetensors](https://huggingface.co/docs/safetensors)), not at the file level. +Signing for published artifacts is performed by the **Publish Verified Model** +GitHub Actions workflow, not by a local terminal. The workflow signs with GitHub +OIDC so the Sigstore identity is tied to this repository/workflow instead of a +personal local browser session. -| Determinism type | Property | Status | -|---|---|---| -| Computational | same config produces same weight values | achievable (single GPU, fixed stack) | -| Representational | same weight values produce same bytes on disk | broken with `.pt`, resolved with safetensors | +Local signing is disabled by default to prevent developers from accidentally signing +with their personal accounts. If you need to test signing locally, set the environment +variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="true"` in PowerShell). -**Loss-curve verification is insufficient on its own.** Trajectory comparison misses two important attacks: weights mutated after training completes (the replay window passes, only the hash catches it), and small file corruptions producing loss differences around 1e-8 that are indistinguishable from floating-point noise. Tensor-hash verification is necessary, and trajectory comparison and hashing are both used because each catches failures the other misses. +Run the workflow with: -## Falsifiability suite +```text +model_name: gpt10m-shakespeare +hf_repo_id: /gpt10m-shakespeare +publish_to_hf: true +``` -A clean run must pass; every tampered run must fail. +It signs with this expected identity shape: -| Scenario | What it tests | How it's caught | -|---|---|---| -| Clean audit | end-to-end reproduction | hashes + trajectory match | -| Bad seed | wrong RNG initialization | trajectory diverges, hash mismatch | -| Gradient noise | mid-training perturbation | trajectory diverges, hash mismatch | -| Post-training sabotage | weights edited after training | trajectory passes, hash catches it | -| Broken seal | ~1e-8 file corruption | trajectory passes, hash catches it | -| Prover / auditor split | two-party independent replay | segment replays bit-identically | +```text +https://github.com///.github/workflows/publish-verified-model.yml@ +``` -## Threat model +and this provider: -Stated plainly so the guarantees are not overread. +```text +https://token.actions.githubusercontent.com +``` -- **Catches:** accidental corruption, drift, post-training weight edits, file-level tampering, configuration mismatch, and dataset substitution (the data Merkle root will not match). -- **Raises the cost of, but does not cryptographically prevent:** a determined forger constructing a checkpoint chain that passes spot-checks. This is a known limitation of checkpoint-replay verification (see Fang et al. 2023, ["Proof-of-Learning Is Currently More Broken Than You Think"](https://arxiv.org/abs/2208.03567), rebutting [Jia et al. 2021](https://arxiv.org/abs/2103.05633)). -- **Mitigation:** publishing the ordered-dataset Merkle root and a transparency-log timestamp before training pins the inputs, so a forger cannot freely choose the data, which raises the forgery bar. -- **Out of scope:** cryptographic proof of an honest gradient step (zkML), which can prove small-model inference but not training at meaningful scale today. +The workflow verifies the signed directory without `--allow-unsigned`, uploads it +as a GitHub Actions artifact, and can optionally upload it to Hugging Face when +`HF_TOKEN` is configured as a repository secret. -### Supply-chain posture +Manual Hugging Face upload is still available if you already have a signed +directory. `ovllm publish-hf` reads `HF_TOKEN` directly and disables Hugging +Face Xet transfers by default for these small artifacts, which avoids local +token-cache and Xet-cache permission issues: -Verification secures the model artifact, but the verifier and training code are themselves software that people download and run. Accordingly: dependencies are pinned and hash-locked, releases of the verification tooling are signed (so an auditor can confirm the tool they run is the one published), and the verification infrastructure is kept small to minimize attack surface. +```bash +# bash/zsh +export HF_TOKEN= +export HF_HUB_DISABLE_XET=1 +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare +``` -## Scope and boundaries +```powershell +# PowerShell +$env:HF_TOKEN = "" +$env:HF_HUB_DISABLE_XET = "1" +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare +``` -- **Bit-exact reproducibility is guaranteed on an identical hardware/software stack.** The environment is pinned and recorded in the manifest. -- **Cross-hardware** reproducibility (e.g. different GPU architectures) does not hold bit-exactly due to floating-point non-associativity; this is measured and documented, and is the use case for the verifier's tolerant mode. -- **Single GPU** is the supported, validated domain. Multi-GPU determinism is harder because the cross-device gradient all-reduce introduces a reduction whose order is not fixed by default; it is controllable for data-parallel training under specific conditions and is treated as a measured experiment rather than an assumption. Tensor and pipeline parallelism are out of scope. +Build an Ollama artifact from the generated `Modelfile`: -## Repository structure +```bash +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare +``` -OpenVerifiableLLM is organized as two repositories: +Dry-run wrappers are available for non-signing publish/build command-shape checks: -| Repository | Contains | -|---|---| -| **Infrastructure** | trainer, verifier, manifest schema, falsifiability suite, signing tooling | -| **Models** | pinned dataset pointers, training configs, published checkpoint chains, manifests, evaluation reports | +```bash +ovllm publish-hf dist/gpt10m-shakespeare --dry-run +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare --dry-run +``` -A model repository pins an exact version of the infrastructure, because a manifest is only meaningful against the exact version that produced it. Verification logic lives only in the infrastructure; model repositories produce and consume manifests but do not reimplement verification. +Signing is intentionally performed by the GitHub Actions workflow, because the +published Sigstore identity should be the repository workflow identity. -## Tech stack +## Reproducibility Matrix -Python, PyTorch, safetensors, NumPy, CUDA, SHA-256, Merkle trees, `uv`, `ruff`, `pytest`, GitHub Actions, Sigstore, bitsandbytes, lm-evaluation-harness. +The experiment harness tests when "same code + same seed" really means "same +model." It separates two ideas that are often conflated: -## Getting started +- **Run-to-run reproducibility:** train the same config twice on the same + hardware. Do the final bits match? +- **Agreement with fp32 reference:** does a lower-precision run match fp32, or + only produce a similar loss? -> Setup instructions are stabilizing as the core lands. The intended flow: +Example commands: ```bash -# install pinned, hash-locked dependencies -uv sync +python run_experiment.py --model gpt10m --precision fp32 --deterministic on --device cuda +python run_experiment.py --model gpt10m --precision tf32 --deterministic on --device cuda +python run_experiment.py --model gpt10m --precision fp32 --deterministic off --device cuda --track-divergence +python sweep.py --device cuda --track-divergence +``` + +The matrix records: + +- final loss +- tensor hash +- Merkle root +- first divergence step +- reproducible true/false +- hardware and precision metadata + +## Artifact Integrity + +OpenVerifiableLLM uses: + +- **safetensors** for stable weight bytes. +- **tensor SHA-256** for semantic model equality. +- **raw artifact SHA-256** for file integrity. +- **Merkle chunking** for scalable partial verification. +- **Sigstore/model-transparency** for publisher identity and transparency-log + provenance. + +The Merkle manifest is computed in one read pass so file size, file hash, and +chunk hashes describe the same artifact version. + +The verifier depends on the current `model-signing` CLI surface and a modern +PyYAML wheel, so those versions are pinned in both `requirements.txt` and +`pyproject.toml` for clean-machine installs. + +## Threat Model -# run the falsifiability suite (clean passes, tampered fails) -pytest tests/falsifiability +OpenVerifiableLLM catches: -# train, producing a chain of verifiable snapshots -python -m openverifiablellm.train --data --config --out +- accidental corruption +- modified weights +- manifest/weight mismatch +- dataset or config drift when encoded in the manifest +- unsigned or wrongly signed published artifacts +- replay-window divergence when segment replay metadata is present -# verify a single segment -python -m openverifiablellm.verify --params --from --data --expect +It does **not** prove that every training step was honest. A determined publisher +could construct a fraudulent but internally consistent checkpoint chain. This is +a known limitation of proof-of-learning style systems. The practical goal here is +falsifiability: make tampering and drift easy to detect, make claims reproducible, +and expose the exact assumptions under which verification holds. + +## Repository Map + +| Path | Purpose | +|---|---| +| `src/ovllm.py` | Verifier/publish CLI | +| `src/verifier.py` | Local/HF model verification checks | +| `src/publish.py` | Publish directory, Sigstore, HF, and Ollama helpers | +| `src/artifacts.py` | SHA-256, tensor hashing, safetensors, Merkle helpers | +| `src/experiment.py` | Shared experiment runner | +| `src/reproducibility.py` | Segmented replay and falsifiability scenarios | +| `src/signing.py` | Legacy/local Ed25519 verify-before-load helper | +| `run_experiment.py` | Run one matrix cell | +| `sweep.py` | Run the reproducibility matrix | +| `demo.py` | Narrative demo | +| `tests/` | Artifact, verifier, signing, and determinism tests | +| `RUNBOOK.md` | Demo-day commands and GPU run instructions | + +## Development + +Run the test suite: + +```bash +python -m unittest discover -s tests ``` -## Contributing +CUDA-only tests (TF32 divergence, determinism-OFF, DDP) self-skip on CPU +machines. Sigstore signing tests are mocked, so the full suite passes locally +without GitHub Actions OIDC credentials. The `--allow-unsigned` flag skips the +live Sigstore bundle check for local development only. -Contributions are welcome. The project favors a research-oriented, assumption-first approach: validate that an abstraction holds before building on top of it, and design features to be falsifiable. +Compile-check touched modules: -- Discussion happens in the [AOSSIE Discord](https://aossie.org); keep technical decisions public. -- Open an issue before substantial work so scope can be aligned with maintainers. -- Run `ruff` and the test suite before submitting; the determinism checks in CI are required to pass. -- Good first issues are labeled in the issue tracker. +```bash +python -m py_compile src/artifacts.py src/verifier.py src/publish.py src/ovllm.py +``` -See `CONTRIBUTING.md` for details. +Check the verifier locally: -## License +```bash +ovllm prepare-publish --weights mid_checkpoint.safetensors --out C:\tmp\ovllm-smoke +ovllm verify C:\tmp\ovllm-smoke --allow-unsigned --skip-replay +``` -See [`LICENSE`](LICENSE). +- Bit-exact reproducibility is scoped to a fixed hardware/software stack. +- Cross-GPU reproducibility is measured, not assumed. +- Single-GPU deterministic training is the primary supported baseline. +- Multi-GPU determinism remains experimental. +- Sigstore signing requires a real OIDC/auth environment for the deployed path. ## References -- Jia et al., *Proof-of-Learning: Definitions and Practice* (2021) -- [arXiv:2103.05633](https://arxiv.org/abs/2103.05633) -- Fang et al., *"Proof-of-Learning" Is Currently More Broken Than You Think* (EuroS&P 2023) -- [arXiv:2208.03567](https://arxiv.org/abs/2208.03567) -- safetensors format -- [huggingface.co/docs/safetensors](https://huggingface.co/docs/safetensors) -- Sigstore model transparency -- [github.com/sigstore/model-transparency](https://github.com/sigstore/model-transparency) +- Jia et al., *Proof-of-Learning: Definitions and Practice* (2021) +- Fang et al., *"Proof-of-Learning" Is Currently More Broken Than You Think* + (EuroS&P 2023) +- [safetensors](https://huggingface.co/docs/safetensors) +- [Sigstore model-transparency](https://github.com/sigstore/model-transparency) + +## License + +See [LICENSE](LICENSE). diff --git a/RUNBOOK.md b/RUNBOOK.md index 8f198ee..f68b985 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -27,6 +27,157 @@ The first text run auto-downloads tinyshakespeare (~1 MB). For the bigger corpor --- +## Phase B. Published-model verification loop + +This is the midterm deliverable path: publish a small signed model and verify it +with one command from a clean environment. + +Install the repo CLI: + +```bash +pip install -r requirements.txt +pip install -e . +ovllm --help +``` + +Local CLI install smoke after changing `pyproject.toml` or CLI wiring: + +```bash +pip install -e . +ovllm --help +python src/ovllm.py --help # optional fallback; should show the same subcommands +``` + +Expected subcommands: + +```text +verify +prepare-publish +sign +publish-hf +ollama-build +``` + +Prepare a publish directory from the real trained safetensors artifact: + +```bash +ovllm prepare-publish \ + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ + --name gpt10m-shakespeare +``` + +The generated model card should say the repo includes: + +- safetensors weights +- `ovllm_manifest.json` +- Sigstore/model-transparency bundle +- Merkle metadata + +Rerun `prepare-publish` after model-card or manifest template changes so the +publish directory contains the current generated metadata. + +Do not locally sign the published artifact. The project-scoped signing path is +the **Publish Verified Model** GitHub Actions workflow, which uses GitHub OIDC +so the Sigstore identity points at the workflow, not a personal local browser +session. + +Local signing is disabled by default to prevent developers from accidentally signing +with their personal accounts. If you need to test signing locally, set the environment +variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="true"` in PowerShell). + +Run the workflow with: + +```text +model_name: gpt10m-shakespeare +hf_repo_id: /gpt10m-shakespeare +publish_to_hf: true +``` + +The workflow signs with: + +```text +identity: https://github.com///.github/workflows/publish-verified-model.yml@ +provider: https://token.actions.githubusercontent.com +``` + +It then verifies the signed directory without `--allow-unsigned`: + +```bash +ovllm verify dist/gpt10m-shakespeare --skip-replay +``` + +If you already have a signed directory, manual Hugging Face upload uses +`HF_TOKEN` directly and disables Hugging Face Xet transfers by default for these +small artifacts. This avoids local Hugging Face CLI token-cache and Xet-cache +permission issues: + +```bash +# bash/zsh +export HF_TOKEN= +export HF_HUB_DISABLE_XET=1 +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare +``` + +```powershell +# PowerShell +$env:HF_TOKEN = "" +$env:HF_HUB_DISABLE_XET = "1" +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare +``` + +Verify by model reference: + +```bash +# Verify metadata and signatures only (fast) +ovllm verify /gpt10m-shakespeare --skip-replay + +# Verify with full local training/replay verification (bit-for-bit parameter match) +ovllm verify /gpt10m-shakespeare +``` + +Remote verification downloads into `.ovllm-cache/huggingface` by default. If a +machine has cache permission issues or you want a disposable cache, use: + +```bash +ovllm verify /gpt10m-shakespeare --skip-replay --cache-dir C:\tmp\ovllm-hf-cache +``` + +If hashes pass but `sigstore_bundle` fails with `manifest lacks +sigstore_identity/provider`, the uploaded repo was not the GitHub Actions-signed +artifact. Re-run **Publish Verified Model** with `publish_to_hf: true` and verify +the newly uploaded output. + +Clean-machine smoke: + +```bash +git clone && cd OpenVerifiableLLM +python -m venv .venv +. .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +pip install -e . +ovllm verify /gpt10m-shakespeare --skip-replay +``` + +Optional Ollama build path: + +```bash +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare +``` + +Dry-run wrappers are for non-signing publish/build command-shape checks: + +```bash +ovllm publish-hf dist/gpt10m-shakespeare --dry-run +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare --dry-run +``` + +Do not use local signing dry runs as part of the demo path. Published artifacts +are signed by the GitHub Actions workflow so Sigstore records the workflow +identity. + +--- + ## 1. Headline: the same GPU is bitwise reproducible (and the audit passes) ```bash diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2cc794c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "openverifiablellm" +version = "0.1.0" +description = "One-command verification for small open model artifacts." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "torch>=2.10.0,<3.0", + "numpy>=2.4,<3.0", + "tqdm>=4.67,<5.0", + "safetensors>=0.4.5,<1.0", + "pynacl>=1.5,<2.0", + "matplotlib>=3.7,<4.0", + "model-signing>=1.1.1", + "huggingface_hub>=0.24.0", + "pyyaml>=6.0.2", +] + +[project.scripts] +ovllm = "ovllm:main" + +[tool.setuptools] +package-dir = {"" = "src"} +py-modules = [ + "artifacts", + "config", + "dataset", + "device", + "eval", + "experiment", + "global_manifest", + "gpu_reproducibility_test", + "main", + "model", + "ovllm", + "plot_divergence", + "publish", + "reproducibility", + "signing", + "telemetry", + "verifier", +] diff --git a/requirements.txt b/requirements.txt index 0a36693..f7653d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ tqdm>=4.67,<5.0 safetensors>=0.4.5,<1.0 pynacl>=1.5,<2.0 # ed25519 signing (verify-before-load security fix) matplotlib>=3.7,<4.0 # divergence-accumulation plot (T7); optional +model-signing>=1.1.1 # Sigstore/model-transparency signing + verification CLI +huggingface_hub>=0.24.0 # fetch/publish Hugging Face model repositories # wikitext dataset option additionally needs: pip install datasets # This default torch wheel is CPU-only (or CUDA, depending on your platform). @@ -12,4 +14,4 @@ matplotlib>=3.7,<4.0 # divergence-accumulation plot (T7); optional # pip install torch --index-url https://download.pytorch.org/whl/xpu # Explicitly pin modern PyYAML to bypass legacy Cython compilation errors on newer Python runtimes (like 3.14) -pyyaml>=6.0.2 \ No newline at end of file +pyyaml>=6.0.2 diff --git a/src/artifacts.py b/src/artifacts.py index 6105e2b..970b064 100644 --- a/src/artifacts.py +++ b/src/artifacts.py @@ -52,15 +52,26 @@ def model_parameters_sha256(model: "torch.nn.Module") -> str: return h.hexdigest() +def tensor_mapping_sha256(tensors: Dict[str, "torch.Tensor"]) -> str: + h = hashlib.sha256() + for name in sorted(tensors): + h.update(tensors[name].detach().cpu().contiguous().numpy().tobytes()) + return h.hexdigest() + + +def _merkle_leaf(leaf_hash: bytes) -> bytes: + return compute_sha256_bytes(data=b"\x00" + leaf_hash) + + def _merkle_parent(left: bytes, right: bytes) -> bytes: - return compute_sha256_bytes(data=left + right) + return compute_sha256_bytes(data=b"\x01" + left + right) def merkle_root_from_leaf_hashes(leaf_hashes: List[str]) -> str: if not leaf_hashes: return compute_sha256(data=b"") - level = [bytes.fromhex(leaf) for leaf in leaf_hashes] + level = [_merkle_leaf(bytes.fromhex(leaf)) for leaf in leaf_hashes] while len(level) > 1: next_level = [] for i in range(0, len(level), 2): @@ -82,8 +93,10 @@ def build_merkle_manifest( path = Path(file_path) chunks = [] offset = 0 + file_hash = hashlib.sha256() with path.open("rb") as f: while chunk := f.read(chunk_size): + file_hash.update(chunk) chunks.append( { "index": len(chunks), @@ -97,8 +110,8 @@ def build_merkle_manifest( leaf_hashes = [chunk["sha256"] for chunk in chunks] return { "artifact": path.name, - "size_bytes": path.stat().st_size, - "sha256": compute_sha256(file_path=path), + "size_bytes": offset, + "sha256": file_hash.hexdigest(), "chunk_size_bytes": chunk_size, "chunk_count": len(chunks), "merkle_root": merkle_root_from_leaf_hashes(leaf_hashes), @@ -131,7 +144,7 @@ def generate_merkle_proof( if chunk_index < 0 or chunk_index >= manifest["chunk_count"]: raise IndexError("chunk_index out of range") - level = [bytes.fromhex(chunk["sha256"]) for chunk in manifest["chunks"]] + level = [_merkle_leaf(bytes.fromhex(chunk["sha256"])) for chunk in manifest["chunks"]] proof = [] index = chunk_index while len(level) > 1: @@ -160,7 +173,8 @@ def verify_merkle_proof( expected_root: str, ) -> bool: try: - current = compute_sha256_bytes(data=chunk_bytes) + raw_hash = compute_sha256_bytes(data=chunk_bytes) + current = _merkle_leaf(raw_hash) expected = bytes.fromhex(expected_root) except (TypeError, ValueError): return False diff --git a/src/dataset.py b/src/dataset.py index 9a0df8c..7e41580 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -100,7 +100,8 @@ def load_corpus(name): with zipfile.ZipFile(zpath) as zf: # Validate all members to prevent path traversal attacks for member in zf.namelist(): - if member.startswith("/") or ".." in member: + target_path = (DATA_DIR / member).resolve() + if not target_path.is_relative_to(DATA_DIR.resolve()): raise ValueError(f"Unsafe path in archive: {member}") raw = zf.read("enwik8")[:_ENWIK8_CHARS] local.write_bytes(raw) @@ -190,7 +191,8 @@ def _load(self): with tarfile.open(tgz) as tf: # Validate all members to prevent path traversal attacks for member in tf.getmembers(): - if member.name.startswith("/") or ".." in member.name: + target_path = (DATA_DIR / member.name).resolve() + if not target_path.is_relative_to(DATA_DIR.resolve()): raise ValueError(f"Unsafe path in archive: {member.name}") tf.extractall(DATA_DIR) import pickle diff --git a/src/ddp_repro.py b/src/ddp_repro.py index c09b436..02b6f01 100644 --- a/src/ddp_repro.py +++ b/src/ddp_repro.py @@ -12,6 +12,18 @@ A NEGATIVE result (cannot get bitwise-identical run-to-run under DDP) is the honest, interesting outcome -- it is the open problem, not a bug to hide. Keep this last; do not let it become the main thrust. + +Distributed Determinism Roadmap & Mitigation Strategies: +--------------------------------------------------------- +1. FP64 Gradient Accumulation: + Accumulating gradients in FP64 before applying optimizer steps reduces the + nondeterministic precision discrepancies introduced by varying floating-point + reduction orders in NCCL all-reduce operations. +2. Deterministic Communication Backends / Wrappers: + Forcing a deterministic reduction tree (such as sorting elements or forcing + a single reduction order) ensures that averaged gradients are mathematically + and bitwise identical across all parallel nodes/ranks, at the cost of some + inter-GPU communication overhead. """ import hashlib import os @@ -83,6 +95,9 @@ def main(): if not run_to_run_same: print("[DDP] NCCL all-reduce ordering breaks bitwise reproducibility even with " "deterministic per-rank kernels -- this is the open problem, present it as one.") + print("\n[DDP] Distributed Determinism Roadmap mitigations to consider:") + print(" 1. FP64 Gradient Accumulation: Accumulating gradients in FP64 reduces order-of-operation precision discrepancies.") + print(" 2. Deterministic Communication Wrappers: Force a deterministic reduction order across nodes/ranks.") dist.destroy_process_group() diff --git a/src/experiment.py b/src/experiment.py index b81b09b..fb922d8 100644 --- a/src/experiment.py +++ b/src/experiment.py @@ -41,7 +41,13 @@ from model import build_model, count_params, is_vision_model from dataset import get_dataset from config import model_config -from artifacts import build_merkle_manifest, model_parameters_sha256, save_model_safetensors +from artifacts import ( + _stable_cpu_state_dict, + build_merkle_manifest, + model_parameters_sha256, + save_model_safetensors, + tensor_mapping_sha256, +) REPO_ROOT = Path(__file__).resolve().parents[1] RESULTS_DIR = REPO_ROOT / "results" @@ -98,7 +104,7 @@ def _single_train(model_name, dataset_name, precision, deterministic, seed, dev, if track_full: step_hashes.append(model_parameters_sha256(model)) - return model, losses, model_parameters_sha256(model), step_hashes + return model, losses, tensor_mapping_sha256(_stable_cpu_state_dict(model)), step_hashes def _first_divergence(losses_a, losses_b, hashes_a=None, hashes_b=None): diff --git a/src/ovllm.py b/src/ovllm.py new file mode 100644 index 0000000..bb39d3f --- /dev/null +++ b/src/ovllm.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import argparse +import sys + +from publish import build_ollama, prepare_publish_dir, publish_huggingface, sign_model_dir +from verifier import print_report, verify_model_reference + + +def _verify(args) -> int: + try: + results = verify_model_reference( + args.model_ref, + cache_dir=args.cache_dir, + allow_unsigned=args.allow_unsigned, + skip_replay=args.skip_replay, + ) + except Exception as exc: + print(f"[FAIL] verifier_error - {exc}") + print("\nVERDICT: RED") + return 1 + return 0 if print_report(results) else 1 + + +def _prepare_publish(args) -> int: + out = prepare_publish_dir(weights=args.weights, output_dir=args.out, name=args.name) + print(f"Prepared publish directory: {out}") + return 0 + + +def _sign(args) -> int: + return sign_model_dir( + args.model_dir, + signature=args.signature, + identity=args.identity, + identity_provider=args.identity_provider, + use_ambient_credentials=args.use_ambient_credentials, + dry_run=args.dry_run, + ) + + +def _publish_hf(args) -> int: + return publish_huggingface(args.repo_id, args.model_dir, dry_run=args.dry_run) + + +def _ollama(args) -> int: + return build_ollama(args.name, args.model_dir, dry_run=args.dry_run) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="ovllm", description="OpenVerifiableLLM CLI") + sub = parser.add_subparsers(dest="cmd", required=True) + + verify = sub.add_parser("verify", help="verify a local path or Hugging Face model reference") + verify.add_argument("model_ref") + verify.add_argument("--cache-dir", default=None) + verify.add_argument("--allow-unsigned", action="store_true", + help="treat a missing Sigstore bundle as SKIP instead of FAIL") + verify.add_argument("--skip-replay", action="store_true") + verify.set_defaults(func=_verify) + + prep = sub.add_parser("prepare-publish", help="build a publishable model directory") + prep.add_argument("--weights", required=True) + prep.add_argument("--out", required=True) + prep.add_argument("--name", default="openverifiable-small") + prep.set_defaults(func=_prepare_publish) + + sign = sub.add_parser("sign", help="sign a model directory with sigstore/model-transparency") + sign.add_argument("model_dir") + sign.add_argument("--signature", default="model.sig") + sign.add_argument("--identity", default=None, + help="expected Sigstore signer identity to store in ovllm_manifest.json") + sign.add_argument("--identity-provider", dest="identity_provider", default=None, + help="expected Sigstore identity provider URL to store in ovllm_manifest.json") + sign.add_argument("--use-ambient-credentials", action="store_true", + help="use ambient OIDC credentials, e.g. GitHub Actions id-token") + sign.add_argument("--dry-run", action="store_true") + sign.set_defaults(func=_sign) + + hf = sub.add_parser("publish-hf", help="upload a prepared model directory to Hugging Face") + hf.add_argument("repo_id") + hf.add_argument("model_dir") + hf.add_argument("--dry-run", action="store_true") + hf.set_defaults(func=_publish_hf) + + ollama = sub.add_parser("ollama-build", help="run ollama create from the generated Modelfile") + ollama.add_argument("name") + ollama.add_argument("model_dir") + ollama.add_argument("--dry-run", action="store_true") + ollama.set_defaults(func=_ollama) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/publish.py b/src/publish.py new file mode 100644 index 0000000..3005be8 --- /dev/null +++ b/src/publish.py @@ -0,0 +1,259 @@ +import json +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, Optional + +from artifacts import build_merkle_manifest, tensor_mapping_sha256 + + +def _tensor_hash(weights: Path) -> Optional[str]: + try: + from safetensors.torch import load_file + except ImportError: + return None + return tensor_mapping_sha256(load_file(str(weights), device="cpu")) + + +def build_model_card(name: str, manifest: Dict[str, Any]) -> str: + return f"""--- +tags: +- openverifiablellm +- model-verification +- sigstore +- merkle-tree +--- + +# {name} + +This model is published with OpenVerifiableLLM verification metadata. + +This repo includes: + +- safetensors weights +- `ovllm_manifest.json` +- Sigstore/model-transparency bundle +- Merkle metadata + +## Verify + +```bash +ovllm verify +``` + +For a local clone of this model repository: + +```bash +ovllm verify . +``` + +The verifier recomputes raw artifact SHA-256, Merkle chunk metadata, and the +safetensors tensor hash, then verifies the Sigstore/model-transparency bundle. + +## Artifact Integrity + +- weights: `{manifest["weights"]}` +- sha256: `{manifest["sha256"]}` +- Merkle root: `{manifest["merkle_root"]}` +- chunk size: `{manifest["chunk_size_bytes"]}` +- chunks: `{manifest["chunk_count"]}` + +## Signature + +Sigstore/model-transparency bundle: `{manifest.get("signature", "model.sig")}` +""" + + +def build_modelfile(name: str, weights_name: str) -> str: + return f"""# OpenVerifiableLLM Ollama build file +# Replace FROM with the nearest compatible base when publishing a real Ollama artifact. +FROM ./{weights_name} +PARAMETER temperature 0 +MESSAGE system "Verified OpenVerifiableLLM artifact: {name}" +""" + + +def prepare_publish_dir( + *, + weights: str, + output_dir: str, + name: str = "openverifiable-small", + manifest_extra: Optional[Dict[str, Any]] = None, +) -> Path: + src = Path(weights) + if not src.exists(): + raise FileNotFoundError(f"weights not found: {src}") + + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + dst = out / src.name + if src.resolve() != dst.resolve(): + shutil.copy2(src, dst) + + merkle = build_merkle_manifest(dst) + manifest = { + "schema": "ovllm.model.v1", + "name": name, + "weights": dst.name, + "sha256": merkle["sha256"], + "merkle_root": merkle["merkle_root"], + "chunk_size_bytes": merkle["chunk_size_bytes"], + "chunk_count": merkle["chunk_count"], + "param_sha256": _tensor_hash(dst), + "signature": "model.sig", + } + if name == "gpt10m-shakespeare": + manifest["segment_replay"] = { + "model": "gpt10m", + "dataset": "shakespeare", + "precision": "fp32", + "deterministic": True, + "seed": 99, + "device": "cpu", + "overrides": { + "total_steps": 8, + "batch_size": 8, + "block_size": 64 + }, + "expected_param_sha256": manifest["param_sha256"] + } + if manifest_extra: + manifest.update(manifest_extra) + + (out / "ovllm_manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (out / "README.md").write_text(build_model_card(name, manifest), encoding="utf-8") + (out / "Modelfile").write_text(build_modelfile(name, dst.name), encoding="utf-8") + return out + + +def _manifest_path(model_dir: Path) -> Path: + return model_dir / "ovllm_manifest.json" + + +def _load_manifest(model_dir: Path) -> Dict[str, Any]: + path = _manifest_path(model_dir) + if not path.exists(): + raise FileNotFoundError(f"manifest not found: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_manifest(model_dir: Path, manifest: Dict[str, Any]) -> None: + _manifest_path(model_dir).write_text(json.dumps(manifest, indent=2), encoding="utf-8") + + +def _resolve_signature_path(model_dir: Path, signature: str) -> Path: + signature_path = Path(signature) + if not signature_path.is_absolute(): + signature_path = model_dir / signature_path + return signature_path + + +def sign_model_dir( + model_dir: str, + signature: str = "model.sig", + *, + identity: Optional[str] = None, + identity_provider: Optional[str] = None, + use_ambient_credentials: bool = False, + dry_run: bool = False, +) -> int: + model_path = Path(model_dir).resolve() + signature_path = _resolve_signature_path(model_path, signature).resolve() + if not signature_path.is_relative_to(model_path): + raise ValueError(f"signature path must be inside model directory: {signature_path}") + signature_path.parent.mkdir(parents=True, exist_ok=True) + + import os + if not dry_run and os.environ.get("GITHUB_ACTIONS") != "true" and os.environ.get("OVLLM_ALLOW_LOCAL_SIGNING") != "true": + print( + "Error: Local signing is disabled by default because it expects GitHub Actions.\n" + "The project-scoped signing path should use the GitHub Actions workflow.\n" + "To bypass this check for local testing, set the environment variable: OVLLM_ALLOW_LOCAL_SIGNING=true", + file=sys.stderr + ) + return 1 + + cmd = [ + sys.executable, + "-m", + "model_signing", + "sign", + "sigstore", + str(model_path), + "--signature", + str(signature_path), + "--ignore-paths", + str(signature_path), + ] + if use_ambient_credentials: + cmd.append("--use_ambient_credentials") + if dry_run: + print(" ".join(cmd)) + return 0 + + manifest = _load_manifest(model_path) + original_manifest = dict(manifest) + manifest["signature"] = str(signature_path.relative_to(model_path)) + if identity: + manifest["sigstore_identity"] = identity + if identity_provider: + manifest["sigstore_identity_provider"] = identity_provider + manifest["sigstore_signed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + _write_manifest(model_path, manifest) + + code = subprocess.run(cmd, check=False).returncode + if code != 0: + _write_manifest(model_path, original_manifest) + return code + + +def publish_huggingface(repo_id: str, model_dir: str, *, dry_run: bool = False) -> int: + if dry_run: + print(f"Native HF Upload: {model_dir} -> {repo_id}") + return 0 + + try: + import importlib + import os + + # These verifier artifacts are small; disabling Xet avoids Windows cache + # permission failures in ~/.cache/huggingface/xet during manual uploads. + os.environ.setdefault("HF_HUB_DISABLE_XET", "1") + + hf_hub = importlib.import_module("huggingface_hub") + HfApi = getattr(hf_hub, "HfApi") + except ImportError as e: + print("huggingface_hub is not installed. Install it to publish to Hugging Face.", file=sys.stderr) + return 1 + except Exception as e: + print(f"Error importing Hugging Face Hub client: {e}", file=sys.stderr) + return 1 + + try: + # Pulls the ambient token mapped into the workflow's environment block + token = os.environ.get("HF_TOKEN") + api = HfApi(token=token) + + # Automatically creates the model repo if it doesn't exist yet + api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) + + # Uploads your directory containing weights, signatures, and manifest + api.upload_folder( + folder_path=str(model_dir), + repo_id=repo_id, + repo_type="model" + ) + return 0 + except Exception as e: + print(f"Error publishing to Hugging Face natively: {e}", file=sys.stderr) + return 1 + + +def build_ollama(model_name: str, model_dir: str, *, dry_run: bool = False) -> int: + cmd = ["ollama", "create", model_name, "-f", str(Path(model_dir) / "Modelfile")] + if dry_run: + print(" ".join(cmd)) + return 0 + return subprocess.run(cmd, check=False).returncode diff --git a/src/verifier.py b/src/verifier.py new file mode 100644 index 0000000..7f8fb5d --- /dev/null +++ b/src/verifier.py @@ -0,0 +1,330 @@ +import importlib +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from artifacts import build_merkle_manifest, compute_sha256, tensor_mapping_sha256 + + +PASS = "PASS" +FAIL = "FAIL" +SKIP = "SKIP" + + +@dataclass +class CheckResult: + name: str + status: str + detail: str = "" + expected: Optional[str] = None + actual: Optional[str] = None + + @property + def ok(self) -> bool: + return self.status in {PASS, SKIP} + + +def resolve_model_reference(ref: str, cache_dir: Optional[str] = None) -> Path: + path = Path(ref) + if path.exists(): + return path.resolve() + + try: + hf_hub = importlib.import_module("huggingface_hub") + snapshot_download = hf_hub.snapshot_download + except ImportError as exc: + raise RuntimeError( + f"Model reference {ref!r} is not a local path and huggingface_hub is not installed. " + "Install it with `pip install huggingface_hub` or pass a local directory." + ) from exc + + if cache_dir is None: + cache_dir = os.environ.get("OVLLM_HF_CACHE_DIR") or str( + Path.cwd() / ".ovllm-cache" / "huggingface" + ) + Path(cache_dir).mkdir(parents=True, exist_ok=True) + os.environ.setdefault("HF_HUB_DISABLE_XET", "1") + + return Path( + snapshot_download( + repo_id=ref, + cache_dir=cache_dir, + allow_patterns=[ + "*.safetensors", + "ovllm_manifest.json", + "pipeline_manifest.json", + "model.sig", + "*.sig", + "*.bundle", + "*.json", + "README.md", + "Modelfile", + ], + ) + ).resolve() + + +def load_manifest(model_dir: Path, manifest_name: str = "ovllm_manifest.json") -> Dict[str, Any]: + path = model_dir / manifest_name + if not path.exists(): + alt = model_dir / "pipeline_manifest.json" + if alt.exists(): + path = alt + else: + raise FileNotFoundError(f"Missing manifest: expected {path}") + with path.open("r", encoding="utf-8") as f: + manifest = json.load(f) + manifest["_manifest_path"] = str(path) + return manifest + + +def _first_existing(model_dir: Path, names: Iterable[str]) -> Optional[Path]: + for name in names: + path = model_dir / name + if path.exists(): + return path + return None + + +def find_weights(model_dir: Path, manifest: Dict[str, Any]) -> Path: + declared = manifest.get("weights") or manifest.get("model_checkpoint_artifact") + if declared: + path = model_dir / declared + if path.exists(): + return path + raise FileNotFoundError(f"Manifest declares missing weights artifact: {path}") + + weights = sorted(model_dir.glob("*.safetensors")) + if not weights: + raise FileNotFoundError(f"No .safetensors weights found in {model_dir}") + if len(weights) > 1: + raise RuntimeError( + f"Multiple .safetensors files found in {model_dir}; set manifest['weights'] explicitly." + ) + return weights[0] + + +def _check_equal(name: str, expected: Optional[str], actual: str, detail: str) -> CheckResult: + if expected is None: + return CheckResult(name, SKIP, f"{detail}; no expected value in manifest", actual=actual) + return CheckResult( + name, + PASS if expected == actual else FAIL, + detail, + expected=expected, + actual=actual, + ) + + +def check_artifact_hashes(model_dir: Path, manifest: Dict[str, Any]) -> List[CheckResult]: + results: List[CheckResult] = [] + weights = find_weights(model_dir, manifest) + + manifest_sha = ( + manifest.get("sha256") + or manifest.get("weights_sha256") + or manifest.get("model_checkpoint_hash") + or manifest.get("4_model_checkpoint_hash") + ) + results.append( + _check_equal( + "artifact_sha256", + manifest_sha, + compute_sha256(file_path=weights), + f"raw bytes of {weights.name}", + ) + ) + + merkle = build_merkle_manifest(weights) + expected_root = ( + manifest.get("merkle_root") + or manifest.get("weights_merkle_root") + or manifest.get("4_model_checkpoint_merkle_root") + ) + results.append( + _check_equal("merkle_root", expected_root, merkle["merkle_root"], "1 MB chunk tree") + ) + + expected_chunks = ( + manifest.get("chunk_count") + or manifest.get("weights_chunk_count") + or manifest.get("4_model_checkpoint_chunk_count") + ) + if expected_chunks is None: + results.append( + CheckResult("merkle_chunk_count", SKIP, "no expected chunk count in manifest") + ) + else: + results.append( + CheckResult( + "merkle_chunk_count", + PASS if int(expected_chunks) == merkle["chunk_count"] else FAIL, + "number of chunks in Merkle manifest", + expected=str(expected_chunks), + actual=str(merkle["chunk_count"]), + ) + ) + + expected_tensor = manifest.get("param_sha256") or manifest.get("tensor_sha256") + try: + from safetensors.torch import load_file + + tensor_hash = tensor_mapping_sha256(load_file(str(weights), device="cpu")) + results.append( + _check_equal("tensor_sha256", expected_tensor, tensor_hash, "safetensors tensor bytes") + ) + except Exception as exc: + results.append(CheckResult("tensor_sha256", FAIL, f"could not read safetensors: {exc}")) + + return results + + +def check_sigstore_bundle( + model_dir: Path, + manifest: Dict[str, Any], + *, + allow_unsigned: bool = False, +) -> CheckResult: + # Resolve symlinks in the base directory up front + model_dir = Path(model_dir).resolve() + + signature = manifest.get("signature") or manifest.get("sigstore_bundle") + signature_path = model_dir / signature if signature else _first_existing( + model_dir, ["model.sig", "model.bundle", "sigstore.bundle"] + ) + + if signature_path is None or not signature_path.exists(): + status = SKIP if allow_unsigned else FAIL + return CheckResult("sigstore_bundle", status, "missing Sigstore/model-signing bundle") + + # Resolve symlinks for the specific signature file path + signature_path = signature_path.resolve() + + identity = manifest.get("sigstore_identity") + provider = manifest.get("sigstore_identity_provider") + if not identity or not provider: + status = SKIP if allow_unsigned else FAIL + return CheckResult( + "sigstore_bundle", + status, + "signature present, but manifest lacks sigstore_identity/provider", + ) + + # Verify model_signing is installed in the active python environment + import importlib.util + if importlib.util.find_spec("model_signing") is None: + return CheckResult( + "sigstore_bundle", + FAIL, + f"model_signing module is not installed in the active environment ({sys.executable}). " + "Please install it with `pip install model-signing` to enable Sigstore bundle verification.", + ) + + cmd = [ + sys.executable, + "-m", + "model_signing", + "verify", + "sigstore", + str(model_dir), + "--signature", + str(signature_path), + "--ignore-paths", + str(signature_path), + "--identity", + identity, + "--identity_provider", + provider, + "--allow_symlinks", + ] + try: + completed = subprocess.run(cmd, check=False, capture_output=True, text=True) + except ModuleNotFoundError as exc: + return CheckResult("sigstore_bundle", FAIL, f"model_signing is not installed: {exc}") + except OSError as exc: + return CheckResult("sigstore_bundle", FAIL, f"could not launch model_signing: {exc}") + + detail = (completed.stdout or completed.stderr).strip() + return CheckResult( + "sigstore_bundle", + PASS if completed.returncode == 0 else FAIL, + detail or "model_signing verify completed", + ) + + +def check_segment_replay(manifest: Dict[str, Any], *, skip_replay: bool = False) -> CheckResult: + spec = manifest.get("segment_replay") + if skip_replay: + return CheckResult("segment_replay", SKIP, "disabled by --skip-replay") + if not spec: + return CheckResult("segment_replay", SKIP, "manifest has no segment_replay spec") + + try: + from experiment import run_one + + record = run_one( + spec["model"], + spec.get("dataset", "shakespeare"), + spec.get("precision", "fp32"), + spec.get("deterministic", True), + spec.get("seed", 99), + device=spec.get("device", "cpu"), + overrides=spec.get("overrides", {}), + track_full=False, + keep_artifact=False, + twin=False, + quiet=True, + ) + except Exception as exc: + return CheckResult("segment_replay", FAIL, f"replay failed: {exc}") + + expected = spec.get("expected_param_sha256") or spec.get("param_sha256") + if expected is None: + return CheckResult( + "segment_replay", + SKIP, + "replay ran, but no expected_param_sha256 was provided", + actual=record["param_sha256"], + ) + return CheckResult( + "segment_replay", + PASS if expected == record["param_sha256"] else FAIL, + "sampled deterministic replay window", + expected=expected, + actual=record["param_sha256"], + ) + + +def verify_model_reference( + ref: str, + *, + cache_dir: Optional[str] = None, + allow_unsigned: bool = False, + skip_replay: bool = False, +) -> List[CheckResult]: + model_dir = resolve_model_reference(ref, cache_dir=cache_dir) + manifest = load_manifest(model_dir) + results = [ + CheckResult("resolve_reference", PASS, str(model_dir)), + CheckResult("manifest_json", PASS, manifest["_manifest_path"]), + ] + results.extend(check_artifact_hashes(model_dir, manifest)) + results.append(check_segment_replay(manifest, skip_replay=skip_replay)) + results.append(check_sigstore_bundle(model_dir, manifest, allow_unsigned=allow_unsigned)) + return results + + +def print_report(results: List[CheckResult]) -> bool: + for result in results: + suffix = f" - {result.detail}" if result.detail else "" + print(f"[{result.status:<4}] {result.name}{suffix}") + if result.expected is not None or result.actual is not None: + print(f" expected: {result.expected}") + print(f" actual : {result.actual}") + ok = all(result.ok for result in results) + print("\nVERDICT:", "GREEN" if ok else "RED") + return ok diff --git a/tests/test_verifier.py b/tests/test_verifier.py new file mode 100644 index 0000000..46a4a58 --- /dev/null +++ b/tests/test_verifier.py @@ -0,0 +1,223 @@ +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +SRC = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(SRC)) + +try: + import torch + from safetensors.torch import save_file + + HAS_TORCH = True +except ImportError: + HAS_TORCH = False + +from publish import build_model_card, prepare_publish_dir, publish_huggingface, sign_model_dir # noqa: E402 +from ovllm import main as ovllm_main # noqa: E402 +from verifier import ( # noqa: E402 + FAIL, + PASS, + SKIP, + check_sigstore_bundle, + resolve_model_reference, + verify_model_reference, +) + + +@unittest.skipUnless(HAS_TORCH, "torch and safetensors required") +class VerifierTests(unittest.TestCase): + def _prepared_model_dir(self, tmp): + weights = Path(tmp) / "model.safetensors" + save_file({"layer.weight": torch.arange(8, dtype=torch.float32).reshape(2, 4)}, str(weights)) + return prepare_publish_dir(weights=str(weights), output_dir=str(Path(tmp) / "publish")) + + def test_local_verify_allows_unsigned_for_dev(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + results = verify_model_reference(str(model_dir), allow_unsigned=True, skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["artifact_sha256"].status, PASS) + self.assertEqual(by_name["merkle_root"].status, PASS) + self.assertEqual(by_name["tensor_sha256"].status, PASS) + self.assertEqual(by_name["sigstore_bundle"].status, SKIP) + + def test_missing_signature_is_red_by_default(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + results = verify_model_reference(str(model_dir), skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["sigstore_bundle"].status, FAIL) + + def test_tampered_weights_fail_hash_checks(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + weights = model_dir / "model.safetensors" + weights.write_bytes(weights.read_bytes() + b"tamper") + + results = verify_model_reference(str(model_dir), allow_unsigned=True, skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["artifact_sha256"].status, FAIL) + self.assertEqual(by_name["merkle_root"].status, FAIL) + + def test_sign_dry_run_is_read_only_and_ignores_signature_file(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + before = (model_dir / "ovllm_manifest.json").read_text() + with patch("builtins.print") as mock_print: + code = sign_model_dir( + str(model_dir), + identity="person@example.com", + identity_provider="https://accounts.example.com", + use_ambient_credentials=True, + dry_run=True, + ) + + self.assertEqual(code, 0) + command = mock_print.call_args.args[0] + self.assertIn("--use_ambient_credentials", command) + self.assertIn("--ignore-paths", command) + self.assertIn(str((model_dir / "model.sig").resolve()), command) + self.assertEqual((model_dir / "ovllm_manifest.json").read_text(), before) + + def test_sigstore_verify_ignores_signature_file(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + signature = model_dir / "model.sig" + signature.write_bytes(b"fake-bundle") + manifest = json.loads((model_dir / "ovllm_manifest.json").read_text()) + manifest["sigstore_identity"] = "person@example.com" + manifest["sigstore_identity_provider"] = "https://accounts.example.com" + + with patch("verifier.subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "ok" + mock_run.return_value.stderr = "" + result = check_sigstore_bundle(model_dir, manifest) + + self.assertEqual(result.status, PASS) + command = mock_run.call_args.args[0] + self.assertIn("--ignore-paths", command) + self.assertIn(str(signature.resolve()), command) + + def test_sign_rejects_signature_path_outside_model_dir(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + outside = Path(tmp) / "outside.sig" + with self.assertRaises(ValueError): + sign_model_dir(str(model_dir), signature=str(outside), dry_run=True) + + def test_local_signing_fails_without_github_actions_or_bypass(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + with patch.dict("os.environ", {}, clear=True): + with patch("sys.stderr") as mock_stderr: + code = sign_model_dir(str(model_dir), dry_run=False) + self.assertEqual(code, 1) + + def test_local_signing_allows_bypass(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + with patch("publish.subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + with patch.dict("os.environ", {"OVLLM_ALLOW_LOCAL_SIGNING": "true"}): + code = sign_model_dir(str(model_dir), dry_run=False) + self.assertEqual(code, 0) + + +class CliErrorHandlingTests(unittest.TestCase): + def test_verify_missing_manifest_returns_red_without_raising(self): + with tempfile.TemporaryDirectory() as tmp: + with patch("builtins.print"): + code = ovllm_main(["verify", tmp]) + self.assertEqual(code, 1) + + +class PublishTests(unittest.TestCase): + def test_model_card_has_hf_yaml_metadata(self): + card = build_model_card( + "smoke", + { + "weights": "model.safetensors", + "sha256": "abc", + "merkle_root": "def", + "chunk_size_bytes": 1024, + "chunk_count": 1, + }, + ) + + self.assertTrue(card.startswith("---\n")) + self.assertIn("openverifiablellm", card) + + def test_publish_huggingface_disables_xet_by_default(self): + class FakeApi: + def __init__(self, token=None): + self.token = token + + def create_repo(self, **_kwargs): + return None + + def upload_folder(self, **_kwargs): + return None + + class FakeHub: + HfApi = FakeApi + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("os.environ", {}, clear=True): + with patch("importlib.import_module", return_value=FakeHub): + code = publish_huggingface("Ryoari/openverifiable-smoke", tmp) + self.assertEqual(code, 0) + self.assertEqual(os.environ["HF_HUB_DISABLE_XET"], "1") + + +class RemoteResolutionTests(unittest.TestCase): + def test_remote_refs_default_to_local_ovllm_cache(self): + class FakeHub: + @staticmethod + def snapshot_download(**kwargs): + self_cache = Path(kwargs["cache_dir"]) + return str(self_cache / "snapshot") + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("os.environ", {}, clear=True): + with patch.dict("sys.modules", {"huggingface_hub": FakeHub}): + with patch("verifier.Path.cwd", return_value=Path(tmp)): + resolved = resolve_model_reference("Ryoari/openverifiable-smoke") + + self.assertEqual(os.environ["HF_HUB_DISABLE_XET"], "1") + + self.assertEqual( + resolved, + Path(tmp) / ".ovllm-cache" / "huggingface" / "snapshot", + ) + + def test_remote_refs_honor_explicit_cache_dir(self): + class FakeHub: + @staticmethod + def snapshot_download(**kwargs): + return str(Path(kwargs["cache_dir"]) / "snapshot") + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("sys.modules", {"huggingface_hub": FakeHub}): + with patch("verifier.Path.cwd", return_value=Path(tmp)): + resolved = resolve_model_reference( + "Ryoari/openverifiable-smoke", + cache_dir=str(Path(tmp) / "custom-cache"), + ) + + self.assertEqual( + resolved, + Path(tmp) / "custom-cache" / "snapshot", + ) + + +if __name__ == "__main__": + unittest.main()