From 5269328d3ebf73f63bb4f5e8fecbc0288f5247b9 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sat, 4 Jul 2026 19:11:51 +0530 Subject: [PATCH 01/10] feat: add verifier publish and GitHub OIDC signing loop --- .github/workflows/publish-verified-model.yml | 86 +++++ README.md | 338 +++++++++++-------- requirements.txt | 2 + src/artifacts.py | 13 +- src/ovllm.py | 91 +++++ src/publish.py | 167 +++++++++ src/verifier.py | 300 ++++++++++++++++ tests/test_verifier.py | 84 +++++ 8 files changed, 943 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/publish-verified-model.yml create mode 100644 src/ovllm.py create mode 100644 src/publish.py create mode 100644 src/verifier.py create mode 100644 tests/test_verifier.py diff --git a/.github/workflows/publish-verified-model.yml b/.github/workflows/publish-verified-model.yml new file mode 100644 index 0000000..8ca2224 --- /dev/null +++ b/.github/workflows/publish-verified-model.yml @@ -0,0 +1,86 @@ +name: Publish Verified Model + +on: + workflow_dispatch: + inputs: + model_name: + description: "Name to put in ovllm_manifest.json" + required: true + default: "openverifiable-smoke" + hf_repo_id: + description: "Optional Hugging Face repo id, e.g. owner/openverifiable-smoke" + required: false + default: "" + publish_to_hf: + description: "Upload the signed directory to Hugging Face" + required: true + default: false + type: boolean + +permissions: + contents: read + id-token: write + +jobs: + sign-and-verify: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: python -m pip install -r requirements.txt + + - name: Build tiny safetensors artifact + run: | + python - <<'PY' + import torch + from safetensors.torch import save_file + + weights = { + "embed.weight": torch.arange(32, dtype=torch.float32).reshape(8, 4), + "head.bias": torch.linspace(-1, 1, steps=8, dtype=torch.float32), + } + save_file(weights, "openverifiable-smoke.safetensors") + PY + + - name: Prepare publish directory + run: | + python src/ovllm.py prepare-publish \ + --weights openverifiable-smoke.safetensors \ + --out dist/openverifiable-smoke \ + --name "${{ inputs.model_name }}" + + - name: Sign with GitHub Actions OIDC + env: + SIGSTORE_IDENTITY: "https://github.com/${{ github.repository }}/.github/workflows/publish-verified-model.yml@${{ github.ref }}" + SIGSTORE_IDENTITY_PROVIDER: "https://token.actions.githubusercontent.com" + run: | + python src/ovllm.py sign dist/openverifiable-smoke \ + --identity "$SIGSTORE_IDENTITY" \ + --identity-provider "$SIGSTORE_IDENTITY_PROVIDER" \ + --use-ambient-credentials + + - name: Verify signed artifact + run: python src/ovllm.py verify dist/openverifiable-smoke --skip-replay + + - name: Upload signed model directory + uses: actions/upload-artifact@v4 + with: + name: openverifiable-smoke-signed + path: dist/openverifiable-smoke + + - name: Publish to Hugging Face + if: ${{ inputs.publish_to_hf && inputs.hf_repo_id != '' }} + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + huggingface-cli login --token "$HF_TOKEN" + python src/ovllm.py publish-hf "${{ inputs.hf_repo_id }}" dist/openverifiable-smoke diff --git a/README.md b/README.md index 956dedd..d6e9f5e 100644 --- a/README.md +++ b/README.md @@ -1,211 +1,277 @@ # OpenVerifiableLLM -**Deterministic training and independent verification for language models.** +**One-command verification for small open model artifacts.** -OpenVerifiableLLM is an [AOSSIE](https://aossie.org) project building a training pipeline whose entire process is reproducible and independently auditable. Given the same data, configuration, and a fixed hardware stack, the pipeline produces bit-identical models, and any deviation (corruption, tampering, or an honest mistake) is cryptographically detectable. +OpenVerifiableLLM is an AOSSIE project for making model releases independently +checkable. The goal is simple: a stranger should be able to pull a published +model, run one verifier command, and see whether the weights, manifest, replay +claim, and publisher provenance all check out. -The goal is not just to publish a model, but to publish a model whose training process can be verified rather than trusted. +This repository contains both the research harness that tests training +reproducibility and the Phase B verifier/publish path that turns those proofs +into a usable artifact. ---- +## Current Status -## The reproducibility matrix (this build) +The project now has two connected layers: -Earlier versions hashed a checkpoint and called it verified. The sharper claim this -build is organised around: **training reproducibility is universally assumed and -rarely verified — it breaks silently, and this tool surfaces the exact conditions -under which it fails.** The hook is the *failure*, not the success. +- **Reproducibility experiments:** deterministic training, hardware/precision + sweeps, safetensors artifacts, tensor hashes, Merkle chunking, and segmented + replay audits. +- **Verifier and publish loop:** `ovllm verify` checks a local or Hugging Face + model directory, recomputes hashes, validates Merkle metadata, optionally runs + replay, and verifies Sigstore/model-transparency provenance. -A single parametrized runner sweeps a grid of models × conditions and emits one JSON -record per cell, deliberately mixing reproducible and broken outcomes: +The intended midterm story is: -| | fp32 det-on | fp32 det-off | tf32 | bf16 | cross-GPU | -|---|---|---|---|---|---| -| **mlp** (control) | PASS | (verify on GPU) | bits≠fp32 (Ampere) | bits≠fp32 | (verify) | -| **gpt10m** (attention) | PASS | run-to-run FAIL on GPU | bits≠fp32 | bits≠fp32 | DIFF | -| **lstm** (cuDNN recurrent) | PASS | FAIL on GPU | bits≠fp32 | bits≠fp32 | DIFF | +```bash +python src/ovllm.py verify / +``` + +If the artifact is intact and properly signed, the verifier prints green checks +and ends with: + +```text +VERDICT: GREEN +``` + +## What `ovllm verify` Checks -Two comparisons are kept deliberately separate, because conflating them is the most -common reproducibility error: +Given a local model directory or Hugging Face model reference, the verifier: -- **Run-to-run** (`reproducible`, `first_divergence_step`): train the *same* config - twice on the *same* hardware — identical bits? Broken by nondeterministic kernels - (determinism OFF) and by different GPUs. -- **Agreement with the fp32 reference** (`vs_fp32`): does tf32/bf16 produce the same - bits — or merely the same loss to a tolerance — as fp32? TF32/bf16 are perfectly - run-to-run reproducible yet silently disagree with fp32. +1. Resolves the model reference. +2. Loads `ovllm_manifest.json`. +3. Finds the `.safetensors` weights. +4. Recomputes the raw artifact SHA-256. +5. Rebuilds the Merkle tree over weight chunks. +6. Recomputes the tensor-level safetensors hash. +7. Optionally runs a structured segment-replay audit if the manifest includes one. +8. Verifies the Sigstore/model-transparency bundle as a red/green provenance check. -**The planted debate.** `verify()` accepts losses within `rel_tol=1e-6` but compares -parameters by *exact* hash, so a run can pass the loss check and fail the bitwise -check. Is the right verification bar bitwise identity (strong, brittle, hardware-bound) -or numerical tolerance (portable, but admits silent precision drift)? The matrix -supplies evidence both ways; the choice defines what "reproducible training" means. +Missing Sigstore provenance is **red by default**. For local development only, +use `--allow-unsigned` to turn that check into a skip. -**Security upgrade.** Checkpoints are now ed25519-signed and the signature is verified -*before* deserialization (`src/signing.py`). The previous path, -`torch.load(weights_only=False)` followed by a SHA-256 check, executes arbitrary code -at unpickle time — before the integrity check ever runs. Signing also makes the -"cryptographically signed" claim true (a SHA-256 is a checksum, not a signature). +## Quickstart -### Quickstart +Install dependencies: ```bash pip install -r requirements.txt -python demo.py # full arc on CPU (smoke config); --full on a GPU pod -python sweep.py --quick # the matrix, tiny CPU preset -cd src && python reproducibility.py # segmented-replay audit (CLEAN AUDIT PASS + scenarios) ``` -See **RUNBOOK.md** for the exact pod commands that populate the GPU-only cells. +Run the verifier against a prepared local directory: ---- +```bash +python src/ovllm.py verify path/to/model-dir +``` -## Why this project exists +For local unsigned smoke tests: -Open-weight models are reproducible in principle but not verifiable in practice. You can download the weights, but you cannot prove what data they were trained on, what configuration produced them, or whether they were modified after release. A model ships with a report, and the report has to be trusted. +```bash +python src/ovllm.py verify path/to/model-dir --allow-unsigned --skip-replay +``` -There is no cryptographic link between a set of weights and the process that produced them, which makes post-training modification (fine-tuning, data injection, weight edits) effectively undetectable from the artifact alone. +Run tests: -OpenVerifiableLLM treats verification as a property of the training pipeline itself rather than something added afterward. +```bash +python -m unittest discover -s tests +``` -## What "verifiable" means here +Run the CPU demo arc: -The term is used precisely in this project. +```bash +python demo.py +``` -**What the system proves.** Given a fixed dataset snapshot, a fixed configuration, and the same hardware/software stack, an independent party can reproduce the exact model (bit-identical weights) or detect that a published artifact deviates from what was claimed. Verification is exact (a hash match), not approximate. +Run the segmented replay/falsifiability smoke test: -**What it does not prove.** A passing verification confirms that a training segment is reproducible and internally consistent. It does not, on its own, prove that training was honest, because a determined adversary can construct a checkpoint chain that passes spot-checks (see [Threat model](#threat-model)). The system substantially raises the cost of forgery; it does not reduce it to zero. The stronger guarantee requires cryptographic proof-of-training (zkML), which is not tractable at this scale and is treated as future work. +```bash +cd src +python reproducibility.py +cd .. +``` -This honesty is by design. The system is built around *falsifiability*: it must fail reliably when assumptions are violated, not merely pass when everything is correct. +For GPU demo commands and the full matrix, see [RUNBOOK.md](RUNBOOK.md). -## How it works +## Publish Loop -The pipeline cryptographically links every stage from raw data to final weights. +Prepare a publishable model directory from a safetensors checkpoint: -``` -Dataset (pinned dump) -- Merkle root over ordered chunks - | - v -Tokenization (deterministic) -- config hash, binary tokens - | - v -Deterministic training loop -- full RNG + optimizer state control - | - v -Verification layer -- tensor-level SHA-256, safetensors - | - v -Signed manifest + transparency log -- Sigstore / Rekor - | - v -Evaluation (factual, bias) -- hash-linked into the chain +```bash +python src/ovllm.py prepare-publish \ + --weights mid_checkpoint.safetensors \ + --out dist/openverifiable-smoke \ + --name openverifiable-smoke ``` -Each stage records its inputs and outputs into a manifest, and the manifests chain into a single pipeline hash so any link can be checked independently. +This writes: -### Two core programs +- `model.safetensors` or the original safetensors filename +- `ovllm_manifest.json` +- `README.md` model card +- `Modelfile` for an Ollama build path -**Trainer / chain producer.** Takes data and parameters, produces the final model along with a sequence of incremental snapshots and the data split used to produce them. The chain begins at the deterministically-seeded initial model (before any training) so that even the first segment is verifiable. Each snapshot is a complete training-state boundary (weights, optimizer state, full RNG state, schedule position, dataloader position), not just weights, because exact segment replay depends on restoring all of it. +Sign the directory with Sigstore/model-transparency. For local development you +can use browser/OIDC signing, but do not publish a personal email identity if +that identity should stay private. Prefer the GitHub Actions workflow below for +public releases. -**Segment verifier.** Takes a boundary snapshot, the next data chunk, the configuration, and the claimed next snapshot, then replays that single segment and checks the result. The default test is a bit-exact hash match (valid on the same hardware stack, with no tolerance window for a forged or corrupted step to hide in). Cross-hardware verification is available as a separate, explicitly-labeled mode with a documented tolerance. +```bash +python src/ovllm.py sign dist/openverifiable-smoke \ + --identity "" \ + --identity-provider "" +``` -This is what makes verification affordable: an auditor can verify any single segment at a small fraction of the full training cost, sample several at random, and gain high confidence without retraining the whole model. +For project-scoped signing, run the **Publish Verified Model** workflow in +GitHub Actions. It signs with GitHub OIDC using this expected identity shape: -## Design findings +```text +https://github.com///.github/workflows/publish-verified-model.yml@ +``` -These observations from the project's controlled experiments inform the architecture. +and this provider: -**Computational determinism is achievable; representational determinism is the catch.** With seeds, initialization, data order, and configuration fixed, training computation is numerically stable, and two independent runs on a fixed single-GPU stack produce bit-identical weights. However, identical weights do not produce identical files: PyTorch's `.pt` format embeds timestamps and pickle metadata, so the bytes change on every save. Verification therefore operates at the tensor level using a byte-stable format ([safetensors](https://huggingface.co/docs/safetensors)), not at the file level. +```text +https://token.actions.githubusercontent.com +``` -| Determinism type | Property | Status | -|---|---|---| -| Computational | same config produces same weight values | achievable (single GPU, fixed stack) | -| Representational | same weight values produce same bytes on disk | broken with `.pt`, resolved with safetensors | +The workflow verifies the signed directory without `--allow-unsigned`, uploads it +as a GitHub Actions artifact, and can optionally upload it to Hugging Face when +`HF_TOKEN` is configured as a repository secret. -**Loss-curve verification is insufficient on its own.** Trajectory comparison misses two important attacks: weights mutated after training completes (the replay window passes, only the hash catches it), and small file corruptions producing loss differences around 1e-8 that are indistinguishable from floating-point noise. Tensor-hash verification is necessary, and trajectory comparison and hashing are both used because each catches failures the other misses. +Upload to Hugging Face: -## Falsifiability suite +```bash +python src/ovllm.py publish-hf /openverifiable-smoke dist/openverifiable-smoke +``` -A clean run must pass; every tampered run must fail. +Build an Ollama artifact from the generated `Modelfile`: -| Scenario | What it tests | How it's caught | -|---|---|---| -| Clean audit | end-to-end reproduction | hashes + trajectory match | -| Bad seed | wrong RNG initialization | trajectory diverges, hash mismatch | -| Gradient noise | mid-training perturbation | trajectory diverges, hash mismatch | -| Post-training sabotage | weights edited after training | trajectory passes, hash catches it | -| Broken seal | ~1e-8 file corruption | trajectory passes, hash catches it | -| Prover / auditor split | two-party independent replay | segment replays bit-identically | +```bash +python src/ovllm.py ollama-build openverifiable-smoke dist/openverifiable-smoke +``` -## Threat model +Dry-run wrappers are available for publish/sign commands: -Stated plainly so the guarantees are not overread. +```bash +python src/ovllm.py sign dist/openverifiable-smoke --dry-run +python src/ovllm.py publish-hf dist/openverifiable-smoke --dry-run +python src/ovllm.py ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run +``` -- **Catches:** accidental corruption, drift, post-training weight edits, file-level tampering, configuration mismatch, and dataset substitution (the data Merkle root will not match). -- **Raises the cost of, but does not cryptographically prevent:** a determined forger constructing a checkpoint chain that passes spot-checks. This is a known limitation of checkpoint-replay verification (see Fang et al. 2023, ["Proof-of-Learning Is Currently More Broken Than You Think"](https://arxiv.org/abs/2208.03567), rebutting [Jia et al. 2021](https://arxiv.org/abs/2103.05633)). -- **Mitigation:** publishing the ordered-dataset Merkle root and a transparency-log timestamp before training pins the inputs, so a forger cannot freely choose the data, which raises the forgery bar. -- **Out of scope:** cryptographic proof of an honest gradient step (zkML), which can prove small-model inference but not training at meaningful scale today. +## Reproducibility Matrix -### Supply-chain posture +The experiment harness tests when "same code + same seed" really means "same +model." It separates two ideas that are often conflated: -Verification secures the model artifact, but the verifier and training code are themselves software that people download and run. Accordingly: dependencies are pinned and hash-locked, releases of the verification tooling are signed (so an auditor can confirm the tool they run is the one published), and the verification infrastructure is kept small to minimize attack surface. +- **Run-to-run reproducibility:** train the same config twice on the same + hardware. Do the final bits match? +- **Agreement with fp32 reference:** does a lower-precision run match fp32, or + only produce a similar loss? -## Scope and boundaries +Example commands: -- **Bit-exact reproducibility is guaranteed on an identical hardware/software stack.** The environment is pinned and recorded in the manifest. -- **Cross-hardware** reproducibility (e.g. different GPU architectures) does not hold bit-exactly due to floating-point non-associativity; this is measured and documented, and is the use case for the verifier's tolerant mode. -- **Single GPU** is the supported, validated domain. Multi-GPU determinism is harder because the cross-device gradient all-reduce introduces a reduction whose order is not fixed by default; it is controllable for data-parallel training under specific conditions and is treated as a measured experiment rather than an assumption. Tensor and pipeline parallelism are out of scope. +```bash +python run_experiment.py --model gpt10m --precision fp32 --deterministic on --device cuda +python run_experiment.py --model gpt10m --precision tf32 --deterministic on --device cuda +python run_experiment.py --model gpt10m --precision fp32 --deterministic off --device cuda --track-divergence +python sweep.py --device cuda --track-divergence +``` -## Repository structure +The matrix records: -OpenVerifiableLLM is organized as two repositories: +- final loss +- tensor hash +- Merkle root +- first divergence step +- reproducible true/false +- hardware and precision metadata -| Repository | Contains | -|---|---| -| **Infrastructure** | trainer, verifier, manifest schema, falsifiability suite, signing tooling | -| **Models** | pinned dataset pointers, training configs, published checkpoint chains, manifests, evaluation reports | +## Artifact Integrity -A model repository pins an exact version of the infrastructure, because a manifest is only meaningful against the exact version that produced it. Verification logic lives only in the infrastructure; model repositories produce and consume manifests but do not reimplement verification. +OpenVerifiableLLM uses: -## Tech stack +- **safetensors** for stable weight bytes. +- **tensor SHA-256** for semantic model equality. +- **raw artifact SHA-256** for file integrity. +- **Merkle chunking** for scalable partial verification. +- **Sigstore/model-transparency** for publisher identity and transparency-log + provenance. -Python, PyTorch, safetensors, NumPy, CUDA, SHA-256, Merkle trees, `uv`, `ruff`, `pytest`, GitHub Actions, Sigstore, bitsandbytes, lm-evaluation-harness. +The Merkle manifest is computed in one read pass so file size, file hash, and +chunk hashes describe the same artifact version. -## Getting started +## Threat Model -> Setup instructions are stabilizing as the core lands. The intended flow: +OpenVerifiableLLM catches: -```bash -# install pinned, hash-locked dependencies -uv sync +- accidental corruption +- modified weights +- manifest/weight mismatch +- dataset or config drift when encoded in the manifest +- unsigned or wrongly signed published artifacts +- replay-window divergence when segment replay metadata is present -# run the falsifiability suite (clean passes, tampered fails) -pytest tests/falsifiability +It does **not** prove that every training step was honest. A determined publisher +could construct a fraudulent but internally consistent checkpoint chain. This is +a known limitation of proof-of-learning style systems. The practical goal here is +falsifiability: make tampering and drift easy to detect, make claims reproducible, +and expose the exact assumptions under which verification holds. -# train, producing a chain of verifiable snapshots -python -m openverifiablellm.train --data --config --out +## Repository Map -# verify a single segment -python -m openverifiablellm.verify --params --from --data --expect +| Path | Purpose | +|---|---| +| `src/ovllm.py` | Verifier/publish CLI | +| `src/verifier.py` | Local/HF model verification checks | +| `src/publish.py` | Publish directory, Sigstore, HF, and Ollama helpers | +| `src/artifacts.py` | SHA-256, tensor hashing, safetensors, Merkle helpers | +| `src/experiment.py` | Shared experiment runner | +| `src/reproducibility.py` | Segmented replay and falsifiability scenarios | +| `src/signing.py` | Legacy/local Ed25519 verify-before-load helper | +| `run_experiment.py` | Run one matrix cell | +| `sweep.py` | Run the reproducibility matrix | +| `demo.py` | Narrative demo | +| `tests/` | Artifact, verifier, signing, and determinism tests | +| `RUNBOOK.md` | Demo-day commands and GPU run instructions | + +## Development + +Run the test suite: + +```bash +python -m unittest discover -s tests ``` -## Contributing +Compile-check touched modules: -Contributions are welcome. The project favors a research-oriented, assumption-first approach: validate that an abstraction holds before building on top of it, and design features to be falsifiable. +```bash +python -m py_compile src/artifacts.py src/verifier.py src/publish.py src/ovllm.py +``` -- Discussion happens in the [AOSSIE Discord](https://aossie.org); keep technical decisions public. -- Open an issue before substantial work so scope can be aligned with maintainers. -- Run `ruff` and the test suite before submitting; the determinism checks in CI are required to pass. -- Good first issues are labeled in the issue tracker. +Check the verifier locally: -See `CONTRIBUTING.md` for details. +```bash +python src/ovllm.py prepare-publish --weights mid_checkpoint.safetensors --out C:\tmp\ovllm-smoke +python src/ovllm.py verify C:\tmp\ovllm-smoke --allow-unsigned --skip-replay +``` -## License +## Scope -See [`LICENSE`](LICENSE). +- Bit-exact reproducibility is scoped to a fixed hardware/software stack. +- Cross-GPU reproducibility is measured, not assumed. +- Single-GPU deterministic training is the primary supported baseline. +- Multi-GPU determinism remains experimental. +- Sigstore signing requires a real OIDC/auth environment for the deployed path. ## References -- Jia et al., *Proof-of-Learning: Definitions and Practice* (2021) -- [arXiv:2103.05633](https://arxiv.org/abs/2103.05633) -- Fang et al., *"Proof-of-Learning" Is Currently More Broken Than You Think* (EuroS&P 2023) -- [arXiv:2208.03567](https://arxiv.org/abs/2208.03567) -- safetensors format -- [huggingface.co/docs/safetensors](https://huggingface.co/docs/safetensors) -- Sigstore model transparency -- [github.com/sigstore/model-transparency](https://github.com/sigstore/model-transparency) +- Jia et al., *Proof-of-Learning: Definitions and Practice* (2021) +- Fang et al., *"Proof-of-Learning" Is Currently More Broken Than You Think* + (EuroS&P 2023) +- [safetensors](https://huggingface.co/docs/safetensors) +- [Sigstore model-transparency](https://github.com/sigstore/model-transparency) + +## License + +See [LICENSE](LICENSE). diff --git a/requirements.txt b/requirements.txt index b1fbf9a..02370cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ tqdm>=4.67,<5.0 safetensors>=0.4.5,<1.0 pynacl>=1.5,<2.0 # ed25519 signing (verify-before-load security fix) matplotlib>=3.7,<4.0 # divergence-accumulation plot (T7); optional +model-signing>=0.2.0 # Sigstore/model-transparency signing + verification CLI +huggingface_hub>=0.24.0 # fetch/publish Hugging Face model repositories # wikitext dataset option additionally needs: pip install datasets # This default torch wheel is CPU-only (or CUDA, depending on your platform). diff --git a/src/artifacts.py b/src/artifacts.py index 6105e2b..085751a 100644 --- a/src/artifacts.py +++ b/src/artifacts.py @@ -52,6 +52,13 @@ def model_parameters_sha256(model: "torch.nn.Module") -> str: return h.hexdigest() +def tensor_mapping_sha256(tensors: Dict[str, "torch.Tensor"]) -> str: + h = hashlib.sha256() + for name in sorted(tensors): + h.update(tensors[name].detach().cpu().contiguous().numpy().tobytes()) + return h.hexdigest() + + def _merkle_parent(left: bytes, right: bytes) -> bytes: return compute_sha256_bytes(data=left + right) @@ -82,8 +89,10 @@ def build_merkle_manifest( path = Path(file_path) chunks = [] offset = 0 + file_hash = hashlib.sha256() with path.open("rb") as f: while chunk := f.read(chunk_size): + file_hash.update(chunk) chunks.append( { "index": len(chunks), @@ -97,8 +106,8 @@ def build_merkle_manifest( leaf_hashes = [chunk["sha256"] for chunk in chunks] return { "artifact": path.name, - "size_bytes": path.stat().st_size, - "sha256": compute_sha256(file_path=path), + "size_bytes": offset, + "sha256": file_hash.hexdigest(), "chunk_size_bytes": chunk_size, "chunk_count": len(chunks), "merkle_root": merkle_root_from_leaf_hashes(leaf_hashes), diff --git a/src/ovllm.py b/src/ovllm.py new file mode 100644 index 0000000..66dff64 --- /dev/null +++ b/src/ovllm.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import argparse +import sys + +from publish import build_ollama, prepare_publish_dir, publish_huggingface, sign_model_dir +from verifier import print_report, verify_model_reference + + +def _verify(args) -> int: + results = verify_model_reference( + args.model_ref, + cache_dir=args.cache_dir, + allow_unsigned=args.allow_unsigned, + skip_replay=args.skip_replay, + ) + return 0 if print_report(results) else 1 + + +def _prepare_publish(args) -> int: + out = prepare_publish_dir(weights=args.weights, output_dir=args.out, name=args.name) + print(f"Prepared publish directory: {out}") + return 0 + + +def _sign(args) -> int: + return sign_model_dir( + args.model_dir, + signature=args.signature, + identity=args.identity, + identity_provider=args.identity_provider, + use_ambient_credentials=args.use_ambient_credentials, + dry_run=args.dry_run, + ) + + +def _publish_hf(args) -> int: + return publish_huggingface(args.repo_id, args.model_dir, dry_run=args.dry_run) + + +def _ollama(args) -> int: + return build_ollama(args.name, args.model_dir, dry_run=args.dry_run) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="ovllm", description="OpenVerifiableLLM CLI") + sub = parser.add_subparsers(dest="cmd", required=True) + + verify = sub.add_parser("verify", help="verify a local path or Hugging Face model reference") + verify.add_argument("model_ref") + verify.add_argument("--cache-dir", default=None) + verify.add_argument("--allow-unsigned", action="store_true", + help="treat a missing Sigstore bundle as SKIP instead of FAIL") + verify.add_argument("--skip-replay", action="store_true") + verify.set_defaults(func=_verify) + + prep = sub.add_parser("prepare-publish", help="build a publishable model directory") + prep.add_argument("--weights", required=True) + prep.add_argument("--out", required=True) + prep.add_argument("--name", default="openverifiable-small") + prep.set_defaults(func=_prepare_publish) + + sign = sub.add_parser("sign", help="sign a model directory with sigstore/model-transparency") + sign.add_argument("model_dir") + sign.add_argument("--signature", default="model.sig") + sign.add_argument("--identity", default=None, + help="expected Sigstore signer identity to store in ovllm_manifest.json") + sign.add_argument("--identity-provider", dest="identity_provider", default=None, + help="expected Sigstore identity provider URL to store in ovllm_manifest.json") + sign.add_argument("--use-ambient-credentials", action="store_true", + help="use ambient OIDC credentials, e.g. GitHub Actions id-token") + sign.add_argument("--dry-run", action="store_true") + sign.set_defaults(func=_sign) + + hf = sub.add_parser("publish-hf", help="upload a prepared model directory to Hugging Face") + hf.add_argument("repo_id") + hf.add_argument("model_dir") + hf.add_argument("--dry-run", action="store_true") + hf.set_defaults(func=_publish_hf) + + ollama = sub.add_parser("ollama-build", help="run ollama create from the generated Modelfile") + ollama.add_argument("name") + ollama.add_argument("model_dir") + ollama.add_argument("--dry-run", action="store_true") + ollama.set_defaults(func=_ollama) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/publish.py b/src/publish.py new file mode 100644 index 0000000..85ebbf5 --- /dev/null +++ b/src/publish.py @@ -0,0 +1,167 @@ +import json +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, Optional + +from artifacts import build_merkle_manifest, tensor_mapping_sha256 + + +def _tensor_hash(weights: Path) -> Optional[str]: + try: + from safetensors.torch import load_file + except ImportError: + return None + return tensor_mapping_sha256(load_file(str(weights), device="cpu")) + + +def build_model_card(name: str, manifest: Dict[str, Any]) -> str: + return f"""# {name} + +This model is published with OpenVerifiableLLM verification metadata. + +## Verify + +```bash +ovllm verify . +``` + +## Artifact Integrity + +- weights: `{manifest["weights"]}` +- sha256: `{manifest["sha256"]}` +- Merkle root: `{manifest["merkle_root"]}` +- chunk size: `{manifest["chunk_size_bytes"]}` +- chunks: `{manifest["chunk_count"]}` + +## Signature + +Sigstore/model-transparency bundle: `{manifest.get("signature", "model.sig")}` +""" + + +def build_modelfile(name: str, weights_name: str) -> str: + return f"""# OpenVerifiableLLM Ollama build file +# Replace FROM with the nearest compatible base when publishing a real Ollama artifact. +FROM ./{weights_name} +PARAMETER temperature 0 +MESSAGE system "Verified OpenVerifiableLLM artifact: {name}" +""" + + +def prepare_publish_dir( + *, + weights: str, + output_dir: str, + name: str = "openverifiable-small", + manifest_extra: Optional[Dict[str, Any]] = None, +) -> Path: + src = Path(weights) + if not src.exists(): + raise FileNotFoundError(f"weights not found: {src}") + + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + dst = out / src.name + if src.resolve() != dst.resolve(): + shutil.copy2(src, dst) + + merkle = build_merkle_manifest(dst) + manifest = { + "schema": "ovllm.model.v1", + "name": name, + "weights": dst.name, + "sha256": merkle["sha256"], + "merkle_root": merkle["merkle_root"], + "chunk_size_bytes": merkle["chunk_size_bytes"], + "chunk_count": merkle["chunk_count"], + "param_sha256": _tensor_hash(dst), + "signature": "model.sig", + } + if manifest_extra: + manifest.update(manifest_extra) + + (out / "ovllm_manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") + (out / "README.md").write_text(build_model_card(name, manifest), encoding="utf-8") + (out / "Modelfile").write_text(build_modelfile(name, dst.name), encoding="utf-8") + return out + + +def _manifest_path(model_dir: Path) -> Path: + return model_dir / "ovllm_manifest.json" + + +def _load_manifest(model_dir: Path) -> Dict[str, Any]: + path = _manifest_path(model_dir) + if not path.exists(): + raise FileNotFoundError(f"manifest not found: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_manifest(model_dir: Path, manifest: Dict[str, Any]) -> None: + _manifest_path(model_dir).write_text(json.dumps(manifest, indent=2), encoding="utf-8") + + +def _resolve_signature_path(model_dir: Path, signature: str) -> Path: + signature_path = Path(signature) + if not signature_path.is_absolute(): + signature_path = model_dir / signature_path + return signature_path + + +def sign_model_dir( + model_dir: str, + signature: str = "model.sig", + *, + identity: Optional[str] = None, + identity_provider: Optional[str] = None, + use_ambient_credentials: bool = False, + dry_run: bool = False, +) -> int: + model_path = Path(model_dir) + signature_path = _resolve_signature_path(model_path, signature) + signature_path.parent.mkdir(parents=True, exist_ok=True) + + manifest = _load_manifest(model_path) + manifest["signature"] = signature_path.name if signature_path.parent == model_path else str(signature_path) + if identity: + manifest["sigstore_identity"] = identity + if identity_provider: + manifest["sigstore_identity_provider"] = identity_provider + manifest["sigstore_signed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + _write_manifest(model_path, manifest) + + cmd = [ + sys.executable, + "-m", + "model_signing", + "sign", + "sigstore", + str(model_path), + "--signature", + str(signature_path), + ] + if use_ambient_credentials: + cmd.append("--use_ambient_credentials") + if dry_run: + print(" ".join(cmd)) + return 0 + return subprocess.run(cmd, check=False).returncode + + +def publish_huggingface(repo_id: str, model_dir: str, *, dry_run: bool = False) -> int: + cmd = ["huggingface-cli", "upload", repo_id, str(model_dir), "."] + if dry_run: + print(" ".join(cmd)) + return 0 + return subprocess.run(cmd, check=False).returncode + + +def build_ollama(model_name: str, model_dir: str, *, dry_run: bool = False) -> int: + cmd = ["ollama", "create", model_name, "-f", str(Path(model_dir) / "Modelfile")] + if dry_run: + print(" ".join(cmd)) + return 0 + return subprocess.run(cmd, check=False).returncode diff --git a/src/verifier.py b/src/verifier.py new file mode 100644 index 0000000..81e9131 --- /dev/null +++ b/src/verifier.py @@ -0,0 +1,300 @@ +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from artifacts import build_merkle_manifest, compute_sha256, tensor_mapping_sha256 + + +PASS = "PASS" +FAIL = "FAIL" +SKIP = "SKIP" + + +@dataclass +class CheckResult: + name: str + status: str + detail: str = "" + expected: Optional[str] = None + actual: Optional[str] = None + + @property + def ok(self) -> bool: + return self.status in {PASS, SKIP} + + +def resolve_model_reference(ref: str, cache_dir: Optional[str] = None) -> Path: + path = Path(ref) + if path.exists(): + return path.resolve() + + try: + from huggingface_hub import snapshot_download + except ImportError as exc: + raise RuntimeError( + f"Model reference {ref!r} is not a local path and huggingface_hub is not installed. " + "Install it with `pip install huggingface_hub` or pass a local directory." + ) from exc + + return Path( + snapshot_download( + repo_id=ref, + cache_dir=cache_dir, + allow_patterns=[ + "*.safetensors", + "ovllm_manifest.json", + "pipeline_manifest.json", + "model.sig", + "*.sig", + "*.bundle", + "*.json", + "README.md", + "Modelfile", + ], + ) + ).resolve() + + +def load_manifest(model_dir: Path, manifest_name: str = "ovllm_manifest.json") -> Dict[str, Any]: + path = model_dir / manifest_name + if not path.exists(): + alt = model_dir / "pipeline_manifest.json" + if alt.exists(): + path = alt + else: + raise FileNotFoundError(f"Missing manifest: expected {path}") + with path.open("r", encoding="utf-8") as f: + manifest = json.load(f) + manifest["_manifest_path"] = str(path) + return manifest + + +def _first_existing(model_dir: Path, names: Iterable[str]) -> Optional[Path]: + for name in names: + path = model_dir / name + if path.exists(): + return path + return None + + +def find_weights(model_dir: Path, manifest: Dict[str, Any]) -> Path: + declared = manifest.get("weights") or manifest.get("model_checkpoint_artifact") + if declared: + path = model_dir / declared + if path.exists(): + return path + raise FileNotFoundError(f"Manifest declares missing weights artifact: {path}") + + weights = sorted(model_dir.glob("*.safetensors")) + if not weights: + raise FileNotFoundError(f"No .safetensors weights found in {model_dir}") + if len(weights) > 1: + raise RuntimeError( + f"Multiple .safetensors files found in {model_dir}; set manifest['weights'] explicitly." + ) + return weights[0] + + +def _check_equal(name: str, expected: Optional[str], actual: str, detail: str) -> CheckResult: + if expected is None: + return CheckResult(name, SKIP, f"{detail}; no expected value in manifest", actual=actual) + return CheckResult( + name, + PASS if expected == actual else FAIL, + detail, + expected=expected, + actual=actual, + ) + + +def check_artifact_hashes(model_dir: Path, manifest: Dict[str, Any]) -> List[CheckResult]: + results: List[CheckResult] = [] + weights = find_weights(model_dir, manifest) + + manifest_sha = ( + manifest.get("sha256") + or manifest.get("weights_sha256") + or manifest.get("model_checkpoint_hash") + or manifest.get("4_model_checkpoint_hash") + ) + results.append( + _check_equal( + "artifact_sha256", + manifest_sha, + compute_sha256(file_path=weights), + f"raw bytes of {weights.name}", + ) + ) + + merkle = build_merkle_manifest(weights) + expected_root = ( + manifest.get("merkle_root") + or manifest.get("weights_merkle_root") + or manifest.get("4_model_checkpoint_merkle_root") + ) + results.append( + _check_equal("merkle_root", expected_root, merkle["merkle_root"], "1 MB chunk tree") + ) + + expected_chunks = ( + manifest.get("chunk_count") + or manifest.get("weights_chunk_count") + or manifest.get("4_model_checkpoint_chunk_count") + ) + if expected_chunks is None: + results.append( + CheckResult("merkle_chunk_count", SKIP, "no expected chunk count in manifest") + ) + else: + results.append( + CheckResult( + "merkle_chunk_count", + PASS if int(expected_chunks) == merkle["chunk_count"] else FAIL, + "number of chunks in Merkle manifest", + expected=str(expected_chunks), + actual=str(merkle["chunk_count"]), + ) + ) + + expected_tensor = manifest.get("param_sha256") or manifest.get("tensor_sha256") + try: + from safetensors.torch import load_file + + tensor_hash = tensor_mapping_sha256(load_file(str(weights), device="cpu")) + results.append( + _check_equal("tensor_sha256", expected_tensor, tensor_hash, "safetensors tensor bytes") + ) + except Exception as exc: + results.append(CheckResult("tensor_sha256", FAIL, f"could not read safetensors: {exc}")) + + return results + + +def check_sigstore_bundle( + model_dir: Path, + manifest: Dict[str, Any], + *, + allow_unsigned: bool = False, +) -> CheckResult: + signature = manifest.get("signature") or manifest.get("sigstore_bundle") + signature_path = model_dir / signature if signature else _first_existing( + model_dir, ["model.sig", "model.bundle", "sigstore.bundle"] + ) + if signature_path is None or not signature_path.exists(): + status = SKIP if allow_unsigned else FAIL + return CheckResult("sigstore_bundle", status, "missing Sigstore/model-signing bundle") + + identity = manifest.get("sigstore_identity") + provider = manifest.get("sigstore_identity_provider") + if not identity or not provider: + status = SKIP if allow_unsigned else FAIL + return CheckResult( + "sigstore_bundle", + status, + "signature present, but manifest lacks sigstore_identity/provider", + ) + + cmd = [ + sys.executable, + "-m", + "model_signing", + "verify", + "sigstore", + str(model_dir), + "--signature", + str(signature_path), + "--identity", + identity, + "--identity_provider", + provider, + ] + try: + completed = subprocess.run(cmd, check=False, capture_output=True, text=True) + except ModuleNotFoundError as exc: + return CheckResult("sigstore_bundle", FAIL, f"model_signing is not installed: {exc}") + except OSError as exc: + return CheckResult("sigstore_bundle", FAIL, f"could not launch model_signing: {exc}") + + detail = (completed.stdout or completed.stderr).strip() + return CheckResult( + "sigstore_bundle", + PASS if completed.returncode == 0 else FAIL, + detail or "model_signing verify completed", + ) + + +def check_segment_replay(manifest: Dict[str, Any], *, skip_replay: bool = False) -> CheckResult: + spec = manifest.get("segment_replay") + if skip_replay: + return CheckResult("segment_replay", SKIP, "disabled by --skip-replay") + if not spec: + return CheckResult("segment_replay", SKIP, "manifest has no segment_replay spec") + + try: + from experiment import run_one + + record = run_one( + spec["model"], + spec.get("dataset", "shakespeare"), + spec.get("precision", "fp32"), + spec.get("deterministic", True), + spec.get("seed", 99), + device=spec.get("device", "cpu"), + overrides=spec.get("overrides", {}), + track_full=False, + keep_artifact=False, + twin=False, + quiet=True, + ) + except Exception as exc: + return CheckResult("segment_replay", FAIL, f"replay failed: {exc}") + + expected = spec.get("expected_param_sha256") or spec.get("param_sha256") + if expected is None: + return CheckResult( + "segment_replay", + SKIP, + "replay ran, but no expected_param_sha256 was provided", + actual=record["param_sha256"], + ) + return CheckResult( + "segment_replay", + PASS if expected == record["param_sha256"] else FAIL, + "sampled deterministic replay window", + expected=expected, + actual=record["param_sha256"], + ) + + +def verify_model_reference( + ref: str, + *, + cache_dir: Optional[str] = None, + allow_unsigned: bool = False, + skip_replay: bool = False, +) -> List[CheckResult]: + model_dir = resolve_model_reference(ref, cache_dir=cache_dir) + manifest = load_manifest(model_dir) + results = [ + CheckResult("resolve_reference", PASS, str(model_dir)), + CheckResult("manifest_json", PASS, manifest["_manifest_path"]), + ] + results.extend(check_artifact_hashes(model_dir, manifest)) + results.append(check_segment_replay(manifest, skip_replay=skip_replay)) + results.append(check_sigstore_bundle(model_dir, manifest, allow_unsigned=allow_unsigned)) + return results + + +def print_report(results: List[CheckResult]) -> bool: + for result in results: + suffix = f" - {result.detail}" if result.detail else "" + print(f"[{result.status:<4}] {result.name}{suffix}") + if result.expected is not None or result.actual is not None: + print(f" expected: {result.expected}") + print(f" actual : {result.actual}") + ok = all(result.ok for result in results) + print("\nVERDICT:", "GREEN" if ok else "RED") + return ok diff --git a/tests/test_verifier.py b/tests/test_verifier.py new file mode 100644 index 0000000..6d7aa51 --- /dev/null +++ b/tests/test_verifier.py @@ -0,0 +1,84 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +SRC = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(SRC)) + +try: + import torch + from safetensors.torch import save_file + + HAS_TORCH = True +except ImportError: + HAS_TORCH = False + +from publish import prepare_publish_dir, sign_model_dir # noqa: E402 +from verifier import FAIL, PASS, SKIP, verify_model_reference # noqa: E402 + + +@unittest.skipUnless(HAS_TORCH, "torch and safetensors required") +class VerifierTests(unittest.TestCase): + def _prepared_model_dir(self, tmp): + weights = Path(tmp) / "model.safetensors" + save_file({"layer.weight": torch.arange(8, dtype=torch.float32).reshape(2, 4)}, str(weights)) + return prepare_publish_dir(weights=str(weights), output_dir=str(Path(tmp) / "publish")) + + def test_local_verify_allows_unsigned_for_dev(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + results = verify_model_reference(str(model_dir), allow_unsigned=True, skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["artifact_sha256"].status, PASS) + self.assertEqual(by_name["merkle_root"].status, PASS) + self.assertEqual(by_name["tensor_sha256"].status, PASS) + self.assertEqual(by_name["sigstore_bundle"].status, SKIP) + + def test_missing_signature_is_red_by_default(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + results = verify_model_reference(str(model_dir), skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["sigstore_bundle"].status, FAIL) + + def test_tampered_weights_fail_hash_checks(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + weights = model_dir / "model.safetensors" + weights.write_bytes(weights.read_bytes() + b"tamper") + + results = verify_model_reference(str(model_dir), allow_unsigned=True, skip_replay=True) + by_name = {r.name: r for r in results} + + self.assertEqual(by_name["artifact_sha256"].status, FAIL) + self.assertEqual(by_name["merkle_root"].status, FAIL) + + def test_sign_dry_run_updates_manifest_before_signing(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + with patch("builtins.print") as mock_print: + code = sign_model_dir( + str(model_dir), + identity="person@example.com", + identity_provider="https://accounts.example.com", + use_ambient_credentials=True, + dry_run=True, + ) + + self.assertEqual(code, 0) + command = mock_print.call_args.args[0] + self.assertIn("--use_ambient_credentials", command) + self.assertIn(str(model_dir / "model.sig"), command) + manifest = __import__("json").loads((model_dir / "ovllm_manifest.json").read_text()) + self.assertEqual(manifest["signature"], "model.sig") + self.assertEqual(manifest["sigstore_identity"], "person@example.com") + self.assertEqual(manifest["sigstore_identity_provider"], "https://accounts.example.com") + self.assertIn("sigstore_signed_at", manifest) + + +if __name__ == "__main__": + unittest.main() From d39620504c478481d76198180806263143199d5a Mon Sep 17 00:00:00 2001 From: Rajat Date: Sat, 4 Jul 2026 21:09:23 +0530 Subject: [PATCH 02/10] ci: fix huggingface auth using ambient environment token --- .github/workflows/publish-verified-model.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-verified-model.yml b/.github/workflows/publish-verified-model.yml index 8ca2224..2216207 100644 --- a/.github/workflows/publish-verified-model.yml +++ b/.github/workflows/publish-verified-model.yml @@ -82,5 +82,4 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - huggingface-cli login --token "$HF_TOKEN" python src/ovllm.py publish-hf "${{ inputs.hf_repo_id }}" dist/openverifiable-smoke From 6829bfce4fc03793f273977727101717db32b933 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sat, 4 Jul 2026 21:18:29 +0530 Subject: [PATCH 03/10] refactor: replace deprecated huggingface-cli wrapper with native HfApi client --- src/publish.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/publish.py b/src/publish.py index 85ebbf5..21cb214 100644 --- a/src/publish.py +++ b/src/publish.py @@ -4,7 +4,7 @@ import sys import time from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, TYPE_CHECKING from artifacts import build_merkle_manifest, tensor_mapping_sha256 @@ -12,6 +12,9 @@ def _tensor_hash(weights: Path) -> Optional[str]: try: from safetensors.torch import load_file + if TYPE_CHECKING: + # Make mypy happy, as load_file is dynamically imported + from safetensors.torch import load_file as _load_file except ImportError: return None return tensor_mapping_sha256(load_file(str(weights), device="cpu")) @@ -152,11 +155,41 @@ def sign_model_dir( def publish_huggingface(repo_id: str, model_dir: str, *, dry_run: bool = False) -> int: - cmd = ["huggingface-cli", "upload", repo_id, str(model_dir), "."] if dry_run: - print(" ".join(cmd)) + print(f"Native HF Upload: {model_dir} -> {repo_id}") return 0 - return subprocess.run(cmd, check=False).returncode + + try: + import importlib + import os + + hf_hub = importlib.import_module("huggingface_hub") + HfApi = getattr(hf_hub, "HfApi") + except ImportError as e: + print("huggingface_hub is not installed. Install it to publish to Hugging Face.", file=sys.stderr) + return 1 + except Exception as e: + print(f"Error importing Hugging Face Hub client: {e}", file=sys.stderr) + return 1 + + try: + # Pulls the ambient token mapped into the workflow's environment block + token = os.environ.get("HF_TOKEN") + api = HfApi(token=token) + + # Automatically creates the model repo if it doesn't exist yet + api.create_repo(repo_id=repo_id, repo_type="model", exist_ok=True) + + # Uploads your directory containing weights, signatures, and manifest + api.upload_folder( + folder_path=str(model_dir), + repo_id=repo_id, + repo_type="model" + ) + return 0 + except Exception as e: + print(f"Error publishing to Hugging Face natively: {e}", file=sys.stderr) + return 1 def build_ollama(model_name: str, model_dir: str, *, dry_run: bool = False) -> int: From 425f571ee1d46864383bac9709ba9063ecb4c027 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sat, 4 Jul 2026 21:32:46 +0530 Subject: [PATCH 04/10] fix: resolve symlinks in sigstore path calculation to support hf hub cache layout --- src/verifier.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/verifier.py b/src/verifier.py index 81e9131..1339c6c 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -1,3 +1,4 @@ +import importlib import json import subprocess import sys @@ -32,7 +33,8 @@ def resolve_model_reference(ref: str, cache_dir: Optional[str] = None) -> Path: return path.resolve() try: - from huggingface_hub import snapshot_download + hf_hub = importlib.import_module("huggingface_hub") + snapshot_download = hf_hub.snapshot_download except ImportError as exc: raise RuntimeError( f"Model reference {ref!r} is not a local path and huggingface_hub is not installed. " @@ -179,14 +181,21 @@ def check_sigstore_bundle( *, allow_unsigned: bool = False, ) -> CheckResult: + # Resolve symlinks in the base directory up front + model_dir = Path(model_dir).resolve() + signature = manifest.get("signature") or manifest.get("sigstore_bundle") signature_path = model_dir / signature if signature else _first_existing( model_dir, ["model.sig", "model.bundle", "sigstore.bundle"] ) + if signature_path is None or not signature_path.exists(): status = SKIP if allow_unsigned else FAIL return CheckResult("sigstore_bundle", status, "missing Sigstore/model-signing bundle") + # Resolve symlinks for the specific signature file path + signature_path = signature_path.resolve() + identity = manifest.get("sigstore_identity") provider = manifest.get("sigstore_identity_provider") if not identity or not provider: @@ -210,6 +219,7 @@ def check_sigstore_bundle( identity, "--identity_provider", provider, + "--allow_symlinks", ] try: completed = subprocess.run(cmd, check=False, capture_output=True, text=True) @@ -225,7 +235,6 @@ def check_sigstore_bundle( detail or "model_signing verify completed", ) - def check_segment_replay(manifest: Dict[str, Any], *, skip_replay: bool = False) -> CheckResult: spec = manifest.get("segment_replay") if skip_replay: From 3febcd301fdff6cc520c8e7e5e75324267d76e44 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sun, 5 Jul 2026 00:33:36 +0530 Subject: [PATCH 05/10] Add installable ovllm verifier and publish loop --- .github/workflows/publish-verified-model.yml | 12 ++- .gitignore | 13 +++ README.md | 32 +++--- RUNBOOK.md | 102 +++++++++++++++++++ pyproject.toml | 46 +++++++++ requirements.txt | 4 +- src/ovllm.py | 17 ++-- src/publish.py | 54 +++++++--- src/verifier.py | 5 +- tests/test_verifier.py | 51 ++++++++-- 10 files changed, 286 insertions(+), 50 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/publish-verified-model.yml b/.github/workflows/publish-verified-model.yml index 2216207..695f609 100644 --- a/.github/workflows/publish-verified-model.yml +++ b/.github/workflows/publish-verified-model.yml @@ -36,7 +36,9 @@ jobs: cache: "pip" - name: Install dependencies - run: python -m pip install -r requirements.txt + run: | + python -m pip install -r requirements.txt + python -m pip install -e . - name: Build tiny safetensors artifact run: | @@ -53,7 +55,7 @@ jobs: - name: Prepare publish directory run: | - python src/ovllm.py prepare-publish \ + ovllm prepare-publish \ --weights openverifiable-smoke.safetensors \ --out dist/openverifiable-smoke \ --name "${{ inputs.model_name }}" @@ -63,13 +65,13 @@ jobs: SIGSTORE_IDENTITY: "https://github.com/${{ github.repository }}/.github/workflows/publish-verified-model.yml@${{ github.ref }}" SIGSTORE_IDENTITY_PROVIDER: "https://token.actions.githubusercontent.com" run: | - python src/ovllm.py sign dist/openverifiable-smoke \ + ovllm sign dist/openverifiable-smoke \ --identity "$SIGSTORE_IDENTITY" \ --identity-provider "$SIGSTORE_IDENTITY_PROVIDER" \ --use-ambient-credentials - name: Verify signed artifact - run: python src/ovllm.py verify dist/openverifiable-smoke --skip-replay + run: ovllm verify dist/openverifiable-smoke --skip-replay - name: Upload signed model directory uses: actions/upload-artifact@v4 @@ -82,4 +84,4 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python src/ovllm.py publish-hf "${{ inputs.hf_repo_id }}" dist/openverifiable-smoke + ovllm publish-hf "${{ inputs.hf_repo_id }}" dist/openverifiable-smoke diff --git a/.gitignore b/.gitignore index c09d4f7..cd6e8c9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ venv/ __pycache__/ *.pyc +build/ +dist/ +*.egg-info/ .vscode/ @@ -39,3 +42,13 @@ data/*.zip data/*.tar.gz data/cifar-10-batches-py/ !data/shakespeare_sample.txt + +# Local smoke tests and temp files +openverifiable-smoke/ +C:\tmp\* + +# Model weights, signatures, and datasets +*.safetensors +*.sig +*.pt +*.jsonl diff --git a/README.md b/README.md index d6e9f5e..c8b9e00 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The project now has two connected layers: The intended midterm story is: ```bash -python src/ovllm.py verify / +ovllm verify / ``` If the artifact is intact and properly signed, the verifier prints green checks @@ -57,18 +57,22 @@ Install dependencies: ```bash pip install -r requirements.txt +pip install -e . +ovllm --help ``` +The editable install exposes the `ovllm` command used throughout this README. + Run the verifier against a prepared local directory: ```bash -python src/ovllm.py verify path/to/model-dir +ovllm verify path/to/model-dir ``` For local unsigned smoke tests: ```bash -python src/ovllm.py verify path/to/model-dir --allow-unsigned --skip-replay +ovllm verify path/to/model-dir --allow-unsigned --skip-replay ``` Run tests: @@ -98,7 +102,7 @@ For GPU demo commands and the full matrix, see [RUNBOOK.md](RUNBOOK.md). Prepare a publishable model directory from a safetensors checkpoint: ```bash -python src/ovllm.py prepare-publish \ +ovllm prepare-publish \ --weights mid_checkpoint.safetensors \ --out dist/openverifiable-smoke \ --name openverifiable-smoke @@ -117,7 +121,7 @@ that identity should stay private. Prefer the GitHub Actions workflow below for public releases. ```bash -python src/ovllm.py sign dist/openverifiable-smoke \ +ovllm sign dist/openverifiable-smoke \ --identity "" \ --identity-provider "" ``` @@ -142,21 +146,21 @@ as a GitHub Actions artifact, and can optionally upload it to Hugging Face when Upload to Hugging Face: ```bash -python src/ovllm.py publish-hf /openverifiable-smoke dist/openverifiable-smoke +ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke ``` Build an Ollama artifact from the generated `Modelfile`: ```bash -python src/ovllm.py ollama-build openverifiable-smoke dist/openverifiable-smoke +ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke ``` Dry-run wrappers are available for publish/sign commands: ```bash -python src/ovllm.py sign dist/openverifiable-smoke --dry-run -python src/ovllm.py publish-hf dist/openverifiable-smoke --dry-run -python src/ovllm.py ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run +ovllm sign dist/openverifiable-smoke --dry-run +ovllm publish-hf dist/openverifiable-smoke --dry-run +ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run ``` ## Reproducibility Matrix @@ -201,6 +205,10 @@ OpenVerifiableLLM uses: The Merkle manifest is computed in one read pass so file size, file hash, and chunk hashes describe the same artifact version. +The verifier depends on the current `model-signing` CLI surface and a modern +PyYAML wheel, so those versions are pinned in both `requirements.txt` and +`pyproject.toml` for clean-machine installs. + ## Threat Model OpenVerifiableLLM catches: @@ -252,8 +260,8 @@ python -m py_compile src/artifacts.py src/verifier.py src/publish.py src/ovllm.p Check the verifier locally: ```bash -python src/ovllm.py prepare-publish --weights mid_checkpoint.safetensors --out C:\tmp\ovllm-smoke -python src/ovllm.py verify C:\tmp\ovllm-smoke --allow-unsigned --skip-replay +ovllm prepare-publish --weights mid_checkpoint.safetensors --out C:\tmp\ovllm-smoke +ovllm verify C:\tmp\ovllm-smoke --allow-unsigned --skip-replay ``` ## Scope diff --git a/RUNBOOK.md b/RUNBOOK.md index 8f198ee..7bd952c 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -27,6 +27,108 @@ The first text run auto-downloads tinyshakespeare (~1 MB). For the bigger corpor --- +## Phase B. Published-model verification loop + +This is the midterm deliverable path: publish a small signed model and verify it +with one command from a clean environment. + +Install the repo CLI: + +```bash +pip install -r requirements.txt +pip install -e . +ovllm --help +``` + +Local CLI install smoke after changing `pyproject.toml` or CLI wiring: + +```bash +pip install -e . +ovllm --help +python src/ovllm.py --help # optional fallback; should show the same subcommands +``` + +Expected subcommands: + +```text +verify +prepare-publish +sign +publish-hf +ollama-build +``` + +Prepare a publish directory from a safetensors artifact: + +```bash +ovllm prepare-publish \ + --weights mid_checkpoint.safetensors \ + --out dist/openverifiable-smoke \ + --name openverifiable-smoke +``` + +The generated model card should say the repo includes: + +- safetensors weights +- `ovllm_manifest.json` +- Sigstore/model-transparency bundle +- Merkle metadata + +Sign the directory with Sigstore/model-transparency: + +```bash +ovllm sign dist/openverifiable-smoke \ + --identity "" \ + --identity-provider "" +``` + +Verify the signed local directory. This should be green without +`--allow-unsigned`: + +```bash +ovllm verify dist/openverifiable-smoke --skip-replay +``` + +Publish to Hugging Face: + +```bash +huggingface-cli login +ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +``` + +Verify by model reference: + +```bash +ovllm verify /openverifiable-smoke --skip-replay +``` + +Clean-machine smoke: + +```bash +git clone && cd OpenVerifiableLLM +python -m venv .venv +. .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +pip install -e . +ovllm verify /openverifiable-smoke --skip-replay +``` + +Optional Ollama build path: + +```bash +ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke +``` + +Dry-run wrappers: + +```bash +ovllm sign dist/openverifiable-smoke --dry-run +ovllm publish-hf dist/openverifiable-smoke --dry-run +ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run +``` + +--- + ## 1. Headline: the same GPU is bitwise reproducible (and the audit passes) ```bash diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2cc794c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "openverifiablellm" +version = "0.1.0" +description = "One-command verification for small open model artifacts." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "torch>=2.10.0,<3.0", + "numpy>=2.4,<3.0", + "tqdm>=4.67,<5.0", + "safetensors>=0.4.5,<1.0", + "pynacl>=1.5,<2.0", + "matplotlib>=3.7,<4.0", + "model-signing>=1.1.1", + "huggingface_hub>=0.24.0", + "pyyaml>=6.0.2", +] + +[project.scripts] +ovllm = "ovllm:main" + +[tool.setuptools] +package-dir = {"" = "src"} +py-modules = [ + "artifacts", + "config", + "dataset", + "device", + "eval", + "experiment", + "global_manifest", + "gpu_reproducibility_test", + "main", + "model", + "ovllm", + "plot_divergence", + "publish", + "reproducibility", + "signing", + "telemetry", + "verifier", +] diff --git a/requirements.txt b/requirements.txt index b50b43c..f7653d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ tqdm>=4.67,<5.0 safetensors>=0.4.5,<1.0 pynacl>=1.5,<2.0 # ed25519 signing (verify-before-load security fix) matplotlib>=3.7,<4.0 # divergence-accumulation plot (T7); optional -model-signing>=0.2.0 # Sigstore/model-transparency signing + verification CLI +model-signing>=1.1.1 # Sigstore/model-transparency signing + verification CLI huggingface_hub>=0.24.0 # fetch/publish Hugging Face model repositories # wikitext dataset option additionally needs: pip install datasets @@ -14,4 +14,4 @@ huggingface_hub>=0.24.0 # fetch/publish Hugging Face model repositories # pip install torch --index-url https://download.pytorch.org/whl/xpu # Explicitly pin modern PyYAML to bypass legacy Cython compilation errors on newer Python runtimes (like 3.14) -pyyaml>=6.0.2 \ No newline at end of file +pyyaml>=6.0.2 diff --git a/src/ovllm.py b/src/ovllm.py index 66dff64..bb39d3f 100644 --- a/src/ovllm.py +++ b/src/ovllm.py @@ -7,12 +7,17 @@ def _verify(args) -> int: - results = verify_model_reference( - args.model_ref, - cache_dir=args.cache_dir, - allow_unsigned=args.allow_unsigned, - skip_replay=args.skip_replay, - ) + try: + results = verify_model_reference( + args.model_ref, + cache_dir=args.cache_dir, + allow_unsigned=args.allow_unsigned, + skip_replay=args.skip_replay, + ) + except Exception as exc: + print(f"[FAIL] verifier_error - {exc}") + print("\nVERDICT: RED") + return 1 return 0 if print_report(results) else 1 diff --git a/src/publish.py b/src/publish.py index 21cb214..b27d7f2 100644 --- a/src/publish.py +++ b/src/publish.py @@ -4,7 +4,7 @@ import sys import time from pathlib import Path -from typing import Any, Dict, Optional, TYPE_CHECKING +from typing import Any, Dict, Optional from artifacts import build_merkle_manifest, tensor_mapping_sha256 @@ -12,9 +12,6 @@ def _tensor_hash(weights: Path) -> Optional[str]: try: from safetensors.torch import load_file - if TYPE_CHECKING: - # Make mypy happy, as load_file is dynamically imported - from safetensors.torch import load_file as _load_file except ImportError: return None return tensor_mapping_sha256(load_file(str(weights), device="cpu")) @@ -25,12 +22,28 @@ def build_model_card(name: str, manifest: Dict[str, Any]) -> str: This model is published with OpenVerifiableLLM verification metadata. +This repo includes: + +- safetensors weights +- `ovllm_manifest.json` +- Sigstore/model-transparency bundle +- Merkle metadata + ## Verify +```bash +ovllm verify +``` + +For a local clone of this model repository: + ```bash ovllm verify . ``` +The verifier recomputes raw artifact SHA-256, Merkle chunk metadata, and the +safetensors tensor hash, then verifies the Sigstore/model-transparency bundle. + ## Artifact Integrity - weights: `{manifest["weights"]}` @@ -123,19 +136,12 @@ def sign_model_dir( use_ambient_credentials: bool = False, dry_run: bool = False, ) -> int: - model_path = Path(model_dir) - signature_path = _resolve_signature_path(model_path, signature) + model_path = Path(model_dir).resolve() + signature_path = _resolve_signature_path(model_path, signature).resolve() + if not signature_path.is_relative_to(model_path): + raise ValueError(f"signature path must be inside model directory: {signature_path}") signature_path.parent.mkdir(parents=True, exist_ok=True) - manifest = _load_manifest(model_path) - manifest["signature"] = signature_path.name if signature_path.parent == model_path else str(signature_path) - if identity: - manifest["sigstore_identity"] = identity - if identity_provider: - manifest["sigstore_identity_provider"] = identity_provider - manifest["sigstore_signed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - _write_manifest(model_path, manifest) - cmd = [ sys.executable, "-m", @@ -145,13 +151,29 @@ def sign_model_dir( str(model_path), "--signature", str(signature_path), + "--ignore-paths", + str(signature_path), ] if use_ambient_credentials: cmd.append("--use_ambient_credentials") if dry_run: print(" ".join(cmd)) return 0 - return subprocess.run(cmd, check=False).returncode + + manifest = _load_manifest(model_path) + original_manifest = dict(manifest) + manifest["signature"] = signature_path.name if signature_path.parent == model_path else str(signature_path) + if identity: + manifest["sigstore_identity"] = identity + if identity_provider: + manifest["sigstore_identity_provider"] = identity_provider + manifest["sigstore_signed_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + _write_manifest(model_path, manifest) + + code = subprocess.run(cmd, check=False).returncode + if code != 0: + _write_manifest(model_path, original_manifest) + return code def publish_huggingface(repo_id: str, model_dir: str, *, dry_run: bool = False) -> int: diff --git a/src/verifier.py b/src/verifier.py index 1339c6c..c000031 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -215,11 +215,13 @@ def check_sigstore_bundle( str(model_dir), "--signature", str(signature_path), + "--ignore-paths", + str(signature_path), "--identity", identity, "--identity_provider", provider, - "--allow_symlinks", + "--allow_symlinks", ] try: completed = subprocess.run(cmd, check=False, capture_output=True, text=True) @@ -235,6 +237,7 @@ def check_sigstore_bundle( detail or "model_signing verify completed", ) + def check_segment_replay(manifest: Dict[str, Any], *, skip_replay: bool = False) -> CheckResult: spec = manifest.get("segment_replay") if skip_replay: diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 6d7aa51..7ff7a7b 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -1,3 +1,4 @@ +import json import sys import tempfile import unittest @@ -16,7 +17,8 @@ HAS_TORCH = False from publish import prepare_publish_dir, sign_model_dir # noqa: E402 -from verifier import FAIL, PASS, SKIP, verify_model_reference # noqa: E402 +from ovllm import main as ovllm_main # noqa: E402 +from verifier import FAIL, PASS, SKIP, check_sigstore_bundle, verify_model_reference # noqa: E402 @unittest.skipUnless(HAS_TORCH, "torch and safetensors required") @@ -57,9 +59,10 @@ def test_tampered_weights_fail_hash_checks(self): self.assertEqual(by_name["artifact_sha256"].status, FAIL) self.assertEqual(by_name["merkle_root"].status, FAIL) - def test_sign_dry_run_updates_manifest_before_signing(self): + def test_sign_dry_run_is_read_only_and_ignores_signature_file(self): with tempfile.TemporaryDirectory() as tmp: model_dir = self._prepared_model_dir(tmp) + before = (model_dir / "ovllm_manifest.json").read_text() with patch("builtins.print") as mock_print: code = sign_model_dir( str(model_dir), @@ -72,12 +75,44 @@ def test_sign_dry_run_updates_manifest_before_signing(self): self.assertEqual(code, 0) command = mock_print.call_args.args[0] self.assertIn("--use_ambient_credentials", command) - self.assertIn(str(model_dir / "model.sig"), command) - manifest = __import__("json").loads((model_dir / "ovllm_manifest.json").read_text()) - self.assertEqual(manifest["signature"], "model.sig") - self.assertEqual(manifest["sigstore_identity"], "person@example.com") - self.assertEqual(manifest["sigstore_identity_provider"], "https://accounts.example.com") - self.assertIn("sigstore_signed_at", manifest) + self.assertIn("--ignore-paths", command) + self.assertIn(str((model_dir / "model.sig").resolve()), command) + self.assertEqual((model_dir / "ovllm_manifest.json").read_text(), before) + + def test_sigstore_verify_ignores_signature_file(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + signature = model_dir / "model.sig" + signature.write_bytes(b"fake-bundle") + manifest = json.loads((model_dir / "ovllm_manifest.json").read_text()) + manifest["sigstore_identity"] = "person@example.com" + manifest["sigstore_identity_provider"] = "https://accounts.example.com" + + with patch("verifier.subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "ok" + mock_run.return_value.stderr = "" + result = check_sigstore_bundle(model_dir, manifest) + + self.assertEqual(result.status, PASS) + command = mock_run.call_args.args[0] + self.assertIn("--ignore-paths", command) + self.assertIn(str(signature.resolve()), command) + + def test_sign_rejects_signature_path_outside_model_dir(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + outside = Path(tmp) / "outside.sig" + with self.assertRaises(ValueError): + sign_model_dir(str(model_dir), signature=str(outside), dry_run=True) + + +class CliErrorHandlingTests(unittest.TestCase): + def test_verify_missing_manifest_returns_red_without_raising(self): + with tempfile.TemporaryDirectory() as tmp: + with patch("builtins.print"): + code = ovllm_main(["verify", tmp]) + self.assertEqual(code, 1) if __name__ == "__main__": From 12b3eb1d712294ac32163ac631d3b1c13a6ae2e5 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sun, 5 Jul 2026 00:50:14 +0530 Subject: [PATCH 06/10] fix doc on signing local dry runs --- README.md | 30 +++++++++++++++++------------- RUNBOOK.md | 34 ++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c8b9e00..3572229 100644 --- a/README.md +++ b/README.md @@ -115,19 +115,20 @@ This writes: - `README.md` model card - `Modelfile` for an Ollama build path -Sign the directory with Sigstore/model-transparency. For local development you -can use browser/OIDC signing, but do not publish a personal email identity if -that identity should stay private. Prefer the GitHub Actions workflow below for -public releases. +Signing for published artifacts is performed by the **Publish Verified Model** +GitHub Actions workflow, not by a local terminal. The workflow signs with GitHub +OIDC so the Sigstore identity is tied to this repository/workflow instead of a +personal local browser session. -```bash -ovllm sign dist/openverifiable-smoke \ - --identity "" \ - --identity-provider "" +Run the workflow with: + +```text +model_name: openverifiable-smoke +hf_repo_id: /openverifiable-smoke +publish_to_hf: true ``` -For project-scoped signing, run the **Publish Verified Model** workflow in -GitHub Actions. It signs with GitHub OIDC using this expected identity shape: +It signs with this expected identity shape: ```text https://github.com///.github/workflows/publish-verified-model.yml@ @@ -143,7 +144,8 @@ The workflow verifies the signed directory without `--allow-unsigned`, uploads i as a GitHub Actions artifact, and can optionally upload it to Hugging Face when `HF_TOKEN` is configured as a repository secret. -Upload to Hugging Face: +Manual Hugging Face upload is still available if you already have a signed +directory: ```bash ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke @@ -155,14 +157,16 @@ Build an Ollama artifact from the generated `Modelfile`: ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke ``` -Dry-run wrappers are available for publish/sign commands: +Dry-run wrappers are available for non-signing publish/build command-shape checks: ```bash -ovllm sign dist/openverifiable-smoke --dry-run ovllm publish-hf dist/openverifiable-smoke --dry-run ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run ``` +Signing is intentionally performed by the GitHub Actions workflow, because the +published Sigstore identity should be the repository workflow identity. + ## Reproducibility Matrix The experiment harness tests when "same code + same seed" really means "same diff --git a/RUNBOOK.md b/RUNBOOK.md index 7bd952c..d721d44 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -74,22 +74,33 @@ The generated model card should say the repo includes: - Sigstore/model-transparency bundle - Merkle metadata -Sign the directory with Sigstore/model-transparency: +Do not locally sign the published artifact. The project-scoped signing path is +the **Publish Verified Model** GitHub Actions workflow, which uses GitHub OIDC +so the Sigstore identity points at the workflow, not a personal local browser +session. -```bash -ovllm sign dist/openverifiable-smoke \ - --identity "" \ - --identity-provider "" +Run the workflow with: + +```text +model_name: openverifiable-smoke +hf_repo_id: /openverifiable-smoke +publish_to_hf: true +``` + +The workflow signs with: + +```text +identity: https://github.com///.github/workflows/publish-verified-model.yml@ +provider: https://token.actions.githubusercontent.com ``` -Verify the signed local directory. This should be green without -`--allow-unsigned`: +It then verifies the signed directory without `--allow-unsigned`: ```bash ovllm verify dist/openverifiable-smoke --skip-replay ``` -Publish to Hugging Face: +If you already have a signed directory, manual Hugging Face upload is: ```bash huggingface-cli login @@ -119,14 +130,17 @@ Optional Ollama build path: ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke ``` -Dry-run wrappers: +Dry-run wrappers are for non-signing publish/build command-shape checks: ```bash -ovllm sign dist/openverifiable-smoke --dry-run ovllm publish-hf dist/openverifiable-smoke --dry-run ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run ``` +Do not use local signing dry runs as part of the demo path. Published artifacts +are signed by the GitHub Actions workflow so Sigstore records the workflow +identity. + --- ## 1. Headline: the same GPU is bitwise reproducible (and the audit passes) From 845d0d38057e1c444073658e9f450b5330f1d951 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sun, 5 Jul 2026 02:13:47 +0530 Subject: [PATCH 07/10] fix: updating docs and files for unexpected errors --- .gitignore | 8 +-- CONTRIBUTING.md | 14 +++--- README.md | 38 +++++++++++++-- RUNBOOK.md | 35 ++++++++++++- src/publish.py | 24 ++++++++- src/verifier.py | 8 +++ tests/test_verifier.py | 108 ++++++++++++++++++++++++++++++++++++++++- 7 files changed, 213 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index cd6e8c9..bb46353 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ build/ dist/ *.egg-info/ +.ovllm-cache/ .vscode/ @@ -45,10 +46,3 @@ data/cifar-10-batches-py/ # Local smoke tests and temp files openverifiable-smoke/ -C:\tmp\* - -# Model weights, signatures, and datasets -*.safetensors -*.sig -*.pt -*.jsonl diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0832d9..29c98d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,12 +40,12 @@ Dependencies are pinned and hash-locked. This is deliberate: it's both a reprodu git clone https://github.com//OpenVerifiableLLM cd OpenVerifiableLLM -# install the exact, locked dependency set -uv sync +# install dependencies and the editable CLI +pip install -r requirements.txt +pip install -e . # confirm things work -pytest tests/falsifiability -ruff check . +python -m unittest discover -s tests ``` If you need to add a dependency, add it through the lockfile so the hash-locked set stays authoritative. An unpinned dependency undermines both reproducibility and the project's supply-chain posture. @@ -58,8 +58,8 @@ If you need to add a dependency, add it through the lockfile so the hash-locked **Tests.** -- Run the test suite and `ruff` locally before opening a PR. The determinism checks in CI must pass; a PR that breaks them won't merge. -- Code that changes verification behavior, serialization, RNG/optimizer state handling, or the manifest should include tests for both the success case and the failure case. For verification-affecting changes, extend the falsifiability suite so a clean run passes and the relevant tampered run fails. +- Run the test suite (`python -m unittest discover -s tests`) locally before opening a PR. The determinism checks in CI must pass; a PR that breaks them won't merge. +- Code that changes verification behavior, serialization, RNG/optimizer state handling, or the manifest should include tests for both the success case and the failure case. For verification-affecting changes, extend the test suite so a clean run passes and the relevant tampered run fails. - A change must not silently weaken the bit-exact reproducibility guarantee on the supported single-GPU stack. If a change trades determinism for performance, make that tradeoff explicit and document it. **Docs.** If a change alters behavior, inputs, the manifest format, or the verification contract, update the relevant docs in the same PR. The manifest schema is a versioned contract that other parts of the project depend on, so changes there need maintainer sign-off and a migration note. @@ -68,7 +68,7 @@ If you need to add a dependency, add it through the lockfile so the hash-locked - Reference the issue it addresses (`Closes #123`) if there is one. - Describe what changed and why, and flag anything you'd like reviewers to look at closely. -- Confirm the test suite and `ruff` pass. +- Confirm the test suite passes. - Keep each PR to one logical change. Small, focused PRs get reviewed and merged faster than large, multi-purpose ones. Maintainers review on a volunteer basis, so a clear, well-tested, well-scoped PR is the best way to get a quick response. Expect some back-and-forth; review comments are about the work, not about you. diff --git a/README.md b/README.md index 3572229..9deb793 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,16 @@ Run the verifier against a prepared local directory: ovllm verify path/to/model-dir ``` +Remote Hugging Face references download into `.ovllm-cache/huggingface` by +default to avoid permission issues in the global Hugging Face cache. Override +with `--cache-dir ` or `OVLLM_HF_CACHE_DIR`. + +If remote verification reports `signature present, but manifest lacks +sigstore_identity/provider`, the uploaded directory was not the GitHub +Actions-signed artifact. Re-run the **Publish Verified Model** workflow and +publish that signed output so the manifest includes the expected Sigstore +identity metadata. + For local unsigned smoke tests: ```bash @@ -115,11 +125,18 @@ This writes: - `README.md` model card - `Modelfile` for an Ollama build path +Rerun `prepare-publish` whenever the model-card or manifest template changes so +the publish directory contains the current generated metadata. + Signing for published artifacts is performed by the **Publish Verified Model** GitHub Actions workflow, not by a local terminal. The workflow signs with GitHub OIDC so the Sigstore identity is tied to this repository/workflow instead of a personal local browser session. +Local signing is disabled by default to prevent developers from accidentally signing +with their personal accounts. If you need to test signing locally, set the environment +variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="true"` in PowerShell). + Run the workflow with: ```text @@ -145,9 +162,21 @@ as a GitHub Actions artifact, and can optionally upload it to Hugging Face when `HF_TOKEN` is configured as a repository secret. Manual Hugging Face upload is still available if you already have a signed -directory: +directory. `ovllm publish-hf` reads `HF_TOKEN` directly and disables Hugging +Face Xet transfers by default for these small artifacts, which avoids local +token-cache and Xet-cache permission issues: ```bash +# bash/zsh +export HF_TOKEN= +export HF_HUB_DISABLE_XET=1 +ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +``` + +```powershell +# PowerShell +$env:HF_TOKEN = "" +$env:HF_HUB_DISABLE_XET = "1" ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke ``` @@ -255,6 +284,11 @@ Run the test suite: python -m unittest discover -s tests ``` +CUDA-only tests (TF32 divergence, determinism-OFF, DDP) self-skip on CPU +machines. Sigstore signing tests are mocked, so the full suite passes locally +without GitHub Actions OIDC credentials. The `--allow-unsigned` flag skips the +live Sigstore bundle check for local development only. + Compile-check touched modules: ```bash @@ -268,8 +302,6 @@ ovllm prepare-publish --weights mid_checkpoint.safetensors --out C:\tmp\ovllm-sm ovllm verify C:\tmp\ovllm-smoke --allow-unsigned --skip-replay ``` -## Scope - - Bit-exact reproducibility is scoped to a fixed hardware/software stack. - Cross-GPU reproducibility is measured, not assumed. - Single-GPU deterministic training is the primary supported baseline. diff --git a/RUNBOOK.md b/RUNBOOK.md index d721d44..6d30d56 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -74,11 +74,18 @@ The generated model card should say the repo includes: - Sigstore/model-transparency bundle - Merkle metadata +Rerun `prepare-publish` after model-card or manifest template changes so the +publish directory contains the current generated metadata. + Do not locally sign the published artifact. The project-scoped signing path is the **Publish Verified Model** GitHub Actions workflow, which uses GitHub OIDC so the Sigstore identity points at the workflow, not a personal local browser session. +Local signing is disabled by default to prevent developers from accidentally signing +with their personal accounts. If you need to test signing locally, set the environment +variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="true"` in PowerShell). + Run the workflow with: ```text @@ -100,10 +107,22 @@ It then verifies the signed directory without `--allow-unsigned`: ovllm verify dist/openverifiable-smoke --skip-replay ``` -If you already have a signed directory, manual Hugging Face upload is: +If you already have a signed directory, manual Hugging Face upload uses +`HF_TOKEN` directly and disables Hugging Face Xet transfers by default for these +small artifacts. This avoids local Hugging Face CLI token-cache and Xet-cache +permission issues: ```bash -huggingface-cli login +# bash/zsh +export HF_TOKEN= +export HF_HUB_DISABLE_XET=1 +ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +``` + +```powershell +# PowerShell +$env:HF_TOKEN = "" +$env:HF_HUB_DISABLE_XET = "1" ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke ``` @@ -113,6 +132,18 @@ Verify by model reference: ovllm verify /openverifiable-smoke --skip-replay ``` +Remote verification downloads into `.ovllm-cache/huggingface` by default. If a +machine has cache permission issues or you want a disposable cache, use: + +```bash +ovllm verify /openverifiable-smoke --skip-replay --cache-dir C:\tmp\ovllm-hf-cache +``` + +If hashes pass but `sigstore_bundle` fails with `manifest lacks +sigstore_identity/provider`, the uploaded repo was not the GitHub Actions-signed +artifact. Re-run **Publish Verified Model** with `publish_to_hf: true` and verify +the newly uploaded output. + Clean-machine smoke: ```bash diff --git a/src/publish.py b/src/publish.py index b27d7f2..57d171a 100644 --- a/src/publish.py +++ b/src/publish.py @@ -18,7 +18,15 @@ def _tensor_hash(weights: Path) -> Optional[str]: def build_model_card(name: str, manifest: Dict[str, Any]) -> str: - return f"""# {name} + return f"""--- +tags: +- openverifiablellm +- model-verification +- sigstore +- merkle-tree +--- + +# {name} This model is published with OpenVerifiableLLM verification metadata. @@ -142,6 +150,16 @@ def sign_model_dir( raise ValueError(f"signature path must be inside model directory: {signature_path}") signature_path.parent.mkdir(parents=True, exist_ok=True) + import os + if not dry_run and os.environ.get("GITHUB_ACTIONS") != "true" and os.environ.get("OVLLM_ALLOW_LOCAL_SIGNING") != "true": + print( + "Error: Local signing is disabled by default because it expects GitHub Actions.\n" + "The project-scoped signing path should use the GitHub Actions workflow.\n" + "To bypass this check for local testing, set the environment variable: OVLLM_ALLOW_LOCAL_SIGNING=true", + file=sys.stderr + ) + return 1 + cmd = [ sys.executable, "-m", @@ -185,6 +203,10 @@ def publish_huggingface(repo_id: str, model_dir: str, *, dry_run: bool = False) import importlib import os + # These verifier artifacts are small; disabling Xet avoids Windows cache + # permission failures in ~/.cache/huggingface/xet during manual uploads. + os.environ.setdefault("HF_HUB_DISABLE_XET", "1") + hf_hub = importlib.import_module("huggingface_hub") HfApi = getattr(hf_hub, "HfApi") except ImportError as e: diff --git a/src/verifier.py b/src/verifier.py index c000031..c575346 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -1,5 +1,6 @@ import importlib import json +import os import subprocess import sys from dataclasses import dataclass @@ -41,6 +42,13 @@ def resolve_model_reference(ref: str, cache_dir: Optional[str] = None) -> Path: "Install it with `pip install huggingface_hub` or pass a local directory." ) from exc + if cache_dir is None: + cache_dir = os.environ.get("OVLLM_HF_CACHE_DIR") or str( + Path.cwd() / ".ovllm-cache" / "huggingface" + ) + Path(cache_dir).mkdir(parents=True, exist_ok=True) + os.environ.setdefault("HF_HUB_DISABLE_XET", "1") + return Path( snapshot_download( repo_id=ref, diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 7ff7a7b..46a4a58 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -1,4 +1,5 @@ import json +import os import sys import tempfile import unittest @@ -16,9 +17,16 @@ except ImportError: HAS_TORCH = False -from publish import prepare_publish_dir, sign_model_dir # noqa: E402 +from publish import build_model_card, prepare_publish_dir, publish_huggingface, sign_model_dir # noqa: E402 from ovllm import main as ovllm_main # noqa: E402 -from verifier import FAIL, PASS, SKIP, check_sigstore_bundle, verify_model_reference # noqa: E402 +from verifier import ( # noqa: E402 + FAIL, + PASS, + SKIP, + check_sigstore_bundle, + resolve_model_reference, + verify_model_reference, +) @unittest.skipUnless(HAS_TORCH, "torch and safetensors required") @@ -106,6 +114,23 @@ def test_sign_rejects_signature_path_outside_model_dir(self): with self.assertRaises(ValueError): sign_model_dir(str(model_dir), signature=str(outside), dry_run=True) + def test_local_signing_fails_without_github_actions_or_bypass(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + with patch.dict("os.environ", {}, clear=True): + with patch("sys.stderr") as mock_stderr: + code = sign_model_dir(str(model_dir), dry_run=False) + self.assertEqual(code, 1) + + def test_local_signing_allows_bypass(self): + with tempfile.TemporaryDirectory() as tmp: + model_dir = self._prepared_model_dir(tmp) + with patch("publish.subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + with patch.dict("os.environ", {"OVLLM_ALLOW_LOCAL_SIGNING": "true"}): + code = sign_model_dir(str(model_dir), dry_run=False) + self.assertEqual(code, 0) + class CliErrorHandlingTests(unittest.TestCase): def test_verify_missing_manifest_returns_red_without_raising(self): @@ -115,5 +140,84 @@ def test_verify_missing_manifest_returns_red_without_raising(self): self.assertEqual(code, 1) +class PublishTests(unittest.TestCase): + def test_model_card_has_hf_yaml_metadata(self): + card = build_model_card( + "smoke", + { + "weights": "model.safetensors", + "sha256": "abc", + "merkle_root": "def", + "chunk_size_bytes": 1024, + "chunk_count": 1, + }, + ) + + self.assertTrue(card.startswith("---\n")) + self.assertIn("openverifiablellm", card) + + def test_publish_huggingface_disables_xet_by_default(self): + class FakeApi: + def __init__(self, token=None): + self.token = token + + def create_repo(self, **_kwargs): + return None + + def upload_folder(self, **_kwargs): + return None + + class FakeHub: + HfApi = FakeApi + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("os.environ", {}, clear=True): + with patch("importlib.import_module", return_value=FakeHub): + code = publish_huggingface("Ryoari/openverifiable-smoke", tmp) + self.assertEqual(code, 0) + self.assertEqual(os.environ["HF_HUB_DISABLE_XET"], "1") + + +class RemoteResolutionTests(unittest.TestCase): + def test_remote_refs_default_to_local_ovllm_cache(self): + class FakeHub: + @staticmethod + def snapshot_download(**kwargs): + self_cache = Path(kwargs["cache_dir"]) + return str(self_cache / "snapshot") + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("os.environ", {}, clear=True): + with patch.dict("sys.modules", {"huggingface_hub": FakeHub}): + with patch("verifier.Path.cwd", return_value=Path(tmp)): + resolved = resolve_model_reference("Ryoari/openverifiable-smoke") + + self.assertEqual(os.environ["HF_HUB_DISABLE_XET"], "1") + + self.assertEqual( + resolved, + Path(tmp) / ".ovllm-cache" / "huggingface" / "snapshot", + ) + + def test_remote_refs_honor_explicit_cache_dir(self): + class FakeHub: + @staticmethod + def snapshot_download(**kwargs): + return str(Path(kwargs["cache_dir"]) / "snapshot") + + with tempfile.TemporaryDirectory() as tmp: + with patch.dict("sys.modules", {"huggingface_hub": FakeHub}): + with patch("verifier.Path.cwd", return_value=Path(tmp)): + resolved = resolve_model_reference( + "Ryoari/openverifiable-smoke", + cache_dir=str(Path(tmp) / "custom-cache"), + ) + + self.assertEqual( + resolved, + Path(tmp) / "custom-cache" / "snapshot", + ) + + if __name__ == "__main__": unittest.main() From bd33f2ee43989bad98515a59fc4787d5b2a3e13d Mon Sep 17 00:00:00 2001 From: Rajat Date: Sun, 5 Jul 2026 02:47:39 +0530 Subject: [PATCH 08/10] fix: updated placeholder test of publishing to real model --- .github/workflows/publish-verified-model.yml | 39 ++++++++++---------- README.md | 32 ++++++++++------ RUNBOOK.md | 34 +++++++++-------- src/experiment.py | 10 ++++- src/publish.py | 15 ++++++++ 5 files changed, 81 insertions(+), 49 deletions(-) diff --git a/.github/workflows/publish-verified-model.yml b/.github/workflows/publish-verified-model.yml index 695f609..8030c6d 100644 --- a/.github/workflows/publish-verified-model.yml +++ b/.github/workflows/publish-verified-model.yml @@ -6,9 +6,9 @@ on: model_name: description: "Name to put in ovllm_manifest.json" required: true - default: "openverifiable-smoke" + default: "gpt10m-shakespeare" hf_repo_id: - description: "Optional Hugging Face repo id, e.g. owner/openverifiable-smoke" + description: "Optional Hugging Face repo id, e.g. owner/gpt10m-shakespeare" required: false default: "" publish_to_hf: @@ -40,24 +40,23 @@ jobs: python -m pip install -r requirements.txt python -m pip install -e . - - name: Build tiny safetensors artifact + - name: Train real model (smoke steps) run: | - python - <<'PY' - import torch - from safetensors.torch import save_file - - weights = { - "embed.weight": torch.arange(32, dtype=torch.float32).reshape(8, 4), - "head.bias": torch.linspace(-1, 1, steps=8, dtype=torch.float32), - } - save_file(weights, "openverifiable-smoke.safetensors") - PY + python run_experiment.py \ + --model gpt10m \ + --dataset shakespeare \ + --precision fp32 \ + --deterministic on \ + --steps 8 \ + --batch-size 8 \ + --block-size 64 \ + --keep-artifact - name: Prepare publish directory run: | ovllm prepare-publish \ - --weights openverifiable-smoke.safetensors \ - --out dist/openverifiable-smoke \ + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ --name "${{ inputs.model_name }}" - name: Sign with GitHub Actions OIDC @@ -65,23 +64,23 @@ jobs: SIGSTORE_IDENTITY: "https://github.com/${{ github.repository }}/.github/workflows/publish-verified-model.yml@${{ github.ref }}" SIGSTORE_IDENTITY_PROVIDER: "https://token.actions.githubusercontent.com" run: | - ovllm sign dist/openverifiable-smoke \ + ovllm sign dist/gpt10m-shakespeare \ --identity "$SIGSTORE_IDENTITY" \ --identity-provider "$SIGSTORE_IDENTITY_PROVIDER" \ --use-ambient-credentials - name: Verify signed artifact - run: ovllm verify dist/openverifiable-smoke --skip-replay + run: ovllm verify dist/gpt10m-shakespeare --skip-replay - name: Upload signed model directory uses: actions/upload-artifact@v4 with: - name: openverifiable-smoke-signed - path: dist/openverifiable-smoke + name: gpt10m-shakespeare-signed + path: dist/gpt10m-shakespeare - name: Publish to Hugging Face if: ${{ inputs.publish_to_hf && inputs.hf_repo_id != '' }} env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - ovllm publish-hf "${{ inputs.hf_repo_id }}" dist/openverifiable-smoke + ovllm publish-hf "${{ inputs.hf_repo_id }}" dist/gpt10m-shakespeare diff --git a/README.md b/README.md index 9deb793..6e32cc3 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,20 @@ Actions-signed artifact. Re-run the **Publish Verified Model** workflow and publish that signed output so the manifest includes the expected Sigstore identity metadata. -For local unsigned smoke tests: +For local unsigned smoke tests (skipping local retraining): ```bash ovllm verify path/to/model-dir --allow-unsigned --skip-replay ``` +For full end-to-end training verification (including local retraining and parameter hash matching): + +```bash +ovllm verify path/to/model-dir --allow-unsigned +``` + +This will automatically execute the local CPU retraining pipeline defined in the manifest's `segment_replay` block, then verify that your locally trained model matches the uploaded model's parameter hash bit-for-bit. + Run tests: ```bash @@ -109,13 +117,13 @@ For GPU demo commands and the full matrix, see [RUNBOOK.md](RUNBOOK.md). ## Publish Loop -Prepare a publishable model directory from a safetensors checkpoint: +Prepare a publishable model directory from the real trained safetensors checkpoint: ```bash ovllm prepare-publish \ - --weights mid_checkpoint.safetensors \ - --out dist/openverifiable-smoke \ - --name openverifiable-smoke + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ + --name gpt10m-shakespeare ``` This writes: @@ -140,8 +148,8 @@ variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="t Run the workflow with: ```text -model_name: openverifiable-smoke -hf_repo_id: /openverifiable-smoke +model_name: gpt10m-shakespeare +hf_repo_id: /gpt10m-shakespeare publish_to_hf: true ``` @@ -170,27 +178,27 @@ token-cache and Xet-cache permission issues: # bash/zsh export HF_TOKEN= export HF_HUB_DISABLE_XET=1 -ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare ``` ```powershell # PowerShell $env:HF_TOKEN = "" $env:HF_HUB_DISABLE_XET = "1" -ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare ``` Build an Ollama artifact from the generated `Modelfile`: ```bash -ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare ``` Dry-run wrappers are available for non-signing publish/build command-shape checks: ```bash -ovllm publish-hf dist/openverifiable-smoke --dry-run -ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run +ovllm publish-hf dist/gpt10m-shakespeare --dry-run +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare --dry-run ``` Signing is intentionally performed by the GitHub Actions workflow, because the diff --git a/RUNBOOK.md b/RUNBOOK.md index 6d30d56..f68b985 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -58,13 +58,13 @@ publish-hf ollama-build ``` -Prepare a publish directory from a safetensors artifact: +Prepare a publish directory from the real trained safetensors artifact: ```bash ovllm prepare-publish \ - --weights mid_checkpoint.safetensors \ - --out dist/openverifiable-smoke \ - --name openverifiable-smoke + --weights artifacts/gpt10m_shakespeare_fp32_deton_s99.safetensors \ + --out dist/gpt10m-shakespeare \ + --name gpt10m-shakespeare ``` The generated model card should say the repo includes: @@ -89,8 +89,8 @@ variable `OVLLM_ALLOW_LOCAL_SIGNING=true` (or `$env:OVLLM_ALLOW_LOCAL_SIGNING="t Run the workflow with: ```text -model_name: openverifiable-smoke -hf_repo_id: /openverifiable-smoke +model_name: gpt10m-shakespeare +hf_repo_id: /gpt10m-shakespeare publish_to_hf: true ``` @@ -104,7 +104,7 @@ provider: https://token.actions.githubusercontent.com It then verifies the signed directory without `--allow-unsigned`: ```bash -ovllm verify dist/openverifiable-smoke --skip-replay +ovllm verify dist/gpt10m-shakespeare --skip-replay ``` If you already have a signed directory, manual Hugging Face upload uses @@ -116,27 +116,31 @@ permission issues: # bash/zsh export HF_TOKEN= export HF_HUB_DISABLE_XET=1 -ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare ``` ```powershell # PowerShell $env:HF_TOKEN = "" $env:HF_HUB_DISABLE_XET = "1" -ovllm publish-hf /openverifiable-smoke dist/openverifiable-smoke +ovllm publish-hf /gpt10m-shakespeare dist/gpt10m-shakespeare ``` Verify by model reference: ```bash -ovllm verify /openverifiable-smoke --skip-replay +# Verify metadata and signatures only (fast) +ovllm verify /gpt10m-shakespeare --skip-replay + +# Verify with full local training/replay verification (bit-for-bit parameter match) +ovllm verify /gpt10m-shakespeare ``` Remote verification downloads into `.ovllm-cache/huggingface` by default. If a machine has cache permission issues or you want a disposable cache, use: ```bash -ovllm verify /openverifiable-smoke --skip-replay --cache-dir C:\tmp\ovllm-hf-cache +ovllm verify /gpt10m-shakespeare --skip-replay --cache-dir C:\tmp\ovllm-hf-cache ``` If hashes pass but `sigstore_bundle` fails with `manifest lacks @@ -152,20 +156,20 @@ python -m venv .venv . .venv/bin/activate # Windows: .venv\Scripts\activate pip install -r requirements.txt pip install -e . -ovllm verify /openverifiable-smoke --skip-replay +ovllm verify /gpt10m-shakespeare --skip-replay ``` Optional Ollama build path: ```bash -ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare ``` Dry-run wrappers are for non-signing publish/build command-shape checks: ```bash -ovllm publish-hf dist/openverifiable-smoke --dry-run -ovllm ollama-build openverifiable-smoke dist/openverifiable-smoke --dry-run +ovllm publish-hf dist/gpt10m-shakespeare --dry-run +ovllm ollama-build gpt10m-shakespeare dist/gpt10m-shakespeare --dry-run ``` Do not use local signing dry runs as part of the demo path. Published artifacts diff --git a/src/experiment.py b/src/experiment.py index b81b09b..fb922d8 100644 --- a/src/experiment.py +++ b/src/experiment.py @@ -41,7 +41,13 @@ from model import build_model, count_params, is_vision_model from dataset import get_dataset from config import model_config -from artifacts import build_merkle_manifest, model_parameters_sha256, save_model_safetensors +from artifacts import ( + _stable_cpu_state_dict, + build_merkle_manifest, + model_parameters_sha256, + save_model_safetensors, + tensor_mapping_sha256, +) REPO_ROOT = Path(__file__).resolve().parents[1] RESULTS_DIR = REPO_ROOT / "results" @@ -98,7 +104,7 @@ def _single_train(model_name, dataset_name, precision, deterministic, seed, dev, if track_full: step_hashes.append(model_parameters_sha256(model)) - return model, losses, model_parameters_sha256(model), step_hashes + return model, losses, tensor_mapping_sha256(_stable_cpu_state_dict(model)), step_hashes def _first_divergence(losses_a, losses_b, hashes_a=None, hashes_b=None): diff --git a/src/publish.py b/src/publish.py index 57d171a..717dd3e 100644 --- a/src/publish.py +++ b/src/publish.py @@ -104,6 +104,21 @@ def prepare_publish_dir( "param_sha256": _tensor_hash(dst), "signature": "model.sig", } + if name == "gpt10m-shakespeare": + manifest["segment_replay"] = { + "model": "gpt10m", + "dataset": "shakespeare", + "precision": "fp32", + "deterministic": True, + "seed": 99, + "device": "cpu", + "overrides": { + "total_steps": 8, + "batch_size": 8, + "block_size": 64 + }, + "expected_param_sha256": manifest["param_sha256"] + } if manifest_extra: manifest.update(manifest_extra) From 56e093dc5742e3efe975151b4df6f964af2ae6c8 Mon Sep 17 00:00:00 2001 From: Rajat Date: Sun, 5 Jul 2026 03:37:48 +0530 Subject: [PATCH 09/10] fix: active enviroment dependency check, distributed training roadmap --- src/artifacts.py | 13 +++++++++---- src/dataset.py | 6 ++++-- src/ddp_repro.py | 15 +++++++++++++++ src/verifier.py | 10 ++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/artifacts.py b/src/artifacts.py index 085751a..970b064 100644 --- a/src/artifacts.py +++ b/src/artifacts.py @@ -59,15 +59,19 @@ def tensor_mapping_sha256(tensors: Dict[str, "torch.Tensor"]) -> str: return h.hexdigest() +def _merkle_leaf(leaf_hash: bytes) -> bytes: + return compute_sha256_bytes(data=b"\x00" + leaf_hash) + + def _merkle_parent(left: bytes, right: bytes) -> bytes: - return compute_sha256_bytes(data=left + right) + return compute_sha256_bytes(data=b"\x01" + left + right) def merkle_root_from_leaf_hashes(leaf_hashes: List[str]) -> str: if not leaf_hashes: return compute_sha256(data=b"") - level = [bytes.fromhex(leaf) for leaf in leaf_hashes] + level = [_merkle_leaf(bytes.fromhex(leaf)) for leaf in leaf_hashes] while len(level) > 1: next_level = [] for i in range(0, len(level), 2): @@ -140,7 +144,7 @@ def generate_merkle_proof( if chunk_index < 0 or chunk_index >= manifest["chunk_count"]: raise IndexError("chunk_index out of range") - level = [bytes.fromhex(chunk["sha256"]) for chunk in manifest["chunks"]] + level = [_merkle_leaf(bytes.fromhex(chunk["sha256"])) for chunk in manifest["chunks"]] proof = [] index = chunk_index while len(level) > 1: @@ -169,7 +173,8 @@ def verify_merkle_proof( expected_root: str, ) -> bool: try: - current = compute_sha256_bytes(data=chunk_bytes) + raw_hash = compute_sha256_bytes(data=chunk_bytes) + current = _merkle_leaf(raw_hash) expected = bytes.fromhex(expected_root) except (TypeError, ValueError): return False diff --git a/src/dataset.py b/src/dataset.py index 9a0df8c..7e41580 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -100,7 +100,8 @@ def load_corpus(name): with zipfile.ZipFile(zpath) as zf: # Validate all members to prevent path traversal attacks for member in zf.namelist(): - if member.startswith("/") or ".." in member: + target_path = (DATA_DIR / member).resolve() + if not target_path.is_relative_to(DATA_DIR.resolve()): raise ValueError(f"Unsafe path in archive: {member}") raw = zf.read("enwik8")[:_ENWIK8_CHARS] local.write_bytes(raw) @@ -190,7 +191,8 @@ def _load(self): with tarfile.open(tgz) as tf: # Validate all members to prevent path traversal attacks for member in tf.getmembers(): - if member.name.startswith("/") or ".." in member.name: + target_path = (DATA_DIR / member.name).resolve() + if not target_path.is_relative_to(DATA_DIR.resolve()): raise ValueError(f"Unsafe path in archive: {member.name}") tf.extractall(DATA_DIR) import pickle diff --git a/src/ddp_repro.py b/src/ddp_repro.py index c09b436..02b6f01 100644 --- a/src/ddp_repro.py +++ b/src/ddp_repro.py @@ -12,6 +12,18 @@ A NEGATIVE result (cannot get bitwise-identical run-to-run under DDP) is the honest, interesting outcome -- it is the open problem, not a bug to hide. Keep this last; do not let it become the main thrust. + +Distributed Determinism Roadmap & Mitigation Strategies: +--------------------------------------------------------- +1. FP64 Gradient Accumulation: + Accumulating gradients in FP64 before applying optimizer steps reduces the + nondeterministic precision discrepancies introduced by varying floating-point + reduction orders in NCCL all-reduce operations. +2. Deterministic Communication Backends / Wrappers: + Forcing a deterministic reduction tree (such as sorting elements or forcing + a single reduction order) ensures that averaged gradients are mathematically + and bitwise identical across all parallel nodes/ranks, at the cost of some + inter-GPU communication overhead. """ import hashlib import os @@ -83,6 +95,9 @@ def main(): if not run_to_run_same: print("[DDP] NCCL all-reduce ordering breaks bitwise reproducibility even with " "deterministic per-rank kernels -- this is the open problem, present it as one.") + print("\n[DDP] Distributed Determinism Roadmap mitigations to consider:") + print(" 1. FP64 Gradient Accumulation: Accumulating gradients in FP64 reduces order-of-operation precision discrepancies.") + print(" 2. Deterministic Communication Wrappers: Force a deterministic reduction order across nodes/ranks.") dist.destroy_process_group() diff --git a/src/verifier.py b/src/verifier.py index c575346..7f8fb5d 100644 --- a/src/verifier.py +++ b/src/verifier.py @@ -214,6 +214,16 @@ def check_sigstore_bundle( "signature present, but manifest lacks sigstore_identity/provider", ) + # Verify model_signing is installed in the active python environment + import importlib.util + if importlib.util.find_spec("model_signing") is None: + return CheckResult( + "sigstore_bundle", + FAIL, + f"model_signing module is not installed in the active environment ({sys.executable}). " + "Please install it with `pip install model-signing` to enable Sigstore bundle verification.", + ) + cmd = [ sys.executable, "-m", From b308f1c3d11a7bb0ff6c19958105de6f3013bba1 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sat, 4 Jul 2026 22:22:07 +0000 Subject: [PATCH 10/10] fix: apply CodeRabbit auto-fixes Fixed 1 file(s) based on 3 unresolved review comments. Co-authored-by: CodeRabbit --- src/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publish.py b/src/publish.py index 717dd3e..3005be8 100644 --- a/src/publish.py +++ b/src/publish.py @@ -195,7 +195,7 @@ def sign_model_dir( manifest = _load_manifest(model_path) original_manifest = dict(manifest) - manifest["signature"] = signature_path.name if signature_path.parent == model_path else str(signature_path) + manifest["signature"] = str(signature_path.relative_to(model_path)) if identity: manifest["sigstore_identity"] = identity if identity_provider: