Gate releases on vendored template integrity

.github/workflows/ci.yml

@@ -26,6 +26,33 @@ jobs:
       - name: Run tests
         run: uv run --python ${{ matrix.python-version }} --with=".[dev]" pytest --cov --cov-report=''
 
+  templates:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        template: [coding, cua]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install template with the SDK from this checkout
+        working-directory: environments/${{ matrix.template }}
+        run: |
+          uv sync --all-extras
+          uv pip install -e ../..
+
+      - name: Run template checks
+        working-directory: environments/${{ matrix.template }}
+        run: |
+          uv run --no-sync ruff format . --check
+          uv run --no-sync ruff check .
+          uv run --no-sync pytest -q
+
   lint-ruff:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yml

@@ -6,8 +6,17 @@ on:
     types: [published]
 
 jobs:
+  # Release gate: every vendored template must pass its suite against the
+  # built wheel (and its image must build) before the wheel is published.
+  template-gate:
+    if: ${{ github.event_name == 'release' }}
+    uses: ./.github/workflows/templates-integrity.yml
+    with:
+      source: wheel
+
   deploy:
     if: ${{ github.event_name == 'release' }}
+    needs: template-gate
     runs-on: ubuntu-latest
     environment: pypi
     steps:

.github/workflows/templates-integrity.yml

@@ -0,0 +1,54 @@
+name: Template integrity
+
+# Runs each vendored environments/ template against the SDK: install the
+# template's pinned dependencies, override hud with this checkout (or with
+# the built wheel when called as a release gate), run the template's test
+# suite, and build its Docker image. The nightly run catches drift between
+# releases; the release gate blocks publishing a wheel that breaks them.
+on:
+  schedule:
+    - cron: "17 9 * * *"
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      source:
+        description: "Install hud from 'checkout' (editable) or the built 'wheel'"
+        type: string
+        default: checkout
+
+jobs:
+  integrity:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        template: [coding, cua]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build wheel
+        if: ${{ inputs.source == 'wheel' }}
+        run: uv build
+
+      - name: Install template
+        working-directory: environments/${{ matrix.template }}
+        run: |
+          uv sync --all-extras
+          if [ "${{ inputs.source || 'checkout' }}" = "wheel" ]; then
+            uv pip install ../../dist/hud-*.whl
+          else
+            uv pip install -e ../..
+          fi
+
+      - name: Run template tests
+        working-directory: environments/${{ matrix.template }}
+        run: uv run --no-sync pytest -q
+
+      - name: Build Docker image
+        working-directory: environments/${{ matrix.template }}
+        run: docker build -f Dockerfile.hud -t hud-template-${{ matrix.template }} .

.gitignore

@@ -61,7 +61,9 @@ hud/rl/checkpoints_test/
 
 docs/internal
 
-environments/
+environments/*
+!environments/coding/
+!environments/cua/
 
 experiments/
 .memories/

environments/coding/.dockerignore

@@ -0,0 +1,8 @@
+# The build context is this directory; keep it to what the images COPY, plus
+# the Dockerfile itself (remote builders read it from the context).
+*
+!Dockerfile.hud
+!env.py
+!coding
+!instances
+**/__pycache__

environments/coding/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.coverage
+.venv
+.env
+build/
+dist/
+*.egg-info/
+
+# OS / IDE
+.DS_Store
+.vscode/
+.idea/
+*.swp
+
+# HUD
+hud.lock.yaml
+.hud
+.hud_local/
+.hud_eval.toml
+.hud-images.json
+
+# SWE-bench Pro instance assets are fetched per-user (swe_tasks.py) and carry
+# the golden patches; only the empty .none dir (generic image builds) ships.
+instances/*
+!instances/.none/
+
+# The SDK repo ignores uv.lock at its root; this template pins its own so the
+# example resolves reproducibly.
+!uv.lock
+
+# Logs
+*.log

environments/coding/Dockerfile.hud

@@ -0,0 +1,72 @@
+# syntax=docker/dockerfile:1
+# One image definition for both flavors.
+#
+# Generic (default) — the repo-clone stage bakes your target repo into /app:
+#   docker build -f Dockerfile.hud --build-arg REPO_URL=https://github.com/you/repo .
+#   (private repos: --secret id=CODING_GITHUB_TOKEN,env=CODING_GITHUB_TOKEN)
+#
+# SWE-bench Pro (driven by swe_tasks.py) — BASE selects the instance's
+# prebuilt image (repo already at /app) and INSTANCE_ID bakes its assets:
+#   docker build -f Dockerfile.hud --build-arg BASE=jefzda/sweap-images:<tag> \
+#     --build-arg INSTANCE_ID=<id> .
+ARG BASE=repo-clone
+
+# ─── generic head: toolchain + target repo at /app ───
+FROM ubuntu:24.04 AS repo-clone
+
+RUN apt-get update -y \
+  && apt-get install -y --no-install-recommends \
+  bash build-essential ca-certificates curl git openssl \
+  python-is-python3 python3 python3-dev python3-pip python3-venv util-linux \
+  && rm -rf /var/lib/apt/lists/* \
+  && update-ca-certificates
+
+# The sample tasks' test command uses the image's python; adjust per repo.
+RUN pip3 install --break-system-packages pytest
+
+ARG REPO_URL="https://github.com/hud-evals/coding-template-sample"
+# chown in the same RUN as the clone (a separate layer would duplicate /app):
+# env.py serves agent shells as uid 1000, so the baked tree must be theirs.
+# Doing it at build makes it free at runtime, whatever the repo's size.
+RUN --mount=type=secret,id=CODING_GITHUB_TOKEN \
+    if [ -f /run/secrets/CODING_GITHUB_TOKEN ]; then \
+        git clone "https://$(cat /run/secrets/CODING_GITHUB_TOKEN)@${REPO_URL#https://}" /app; \
+    else \
+        git clone ${REPO_URL} /app; \
+    fi \
+    && chown -R 1000:1000 /app
+# Add a build step for the target repo here if it needs one (deps, compile).
+
+# ─── the served image: BASE + hud serving layer (+ instance assets) ───
+FROM ${BASE}
+
+# ".none" is a committed empty dir, so generic builds bake no instance and
+# env.py serves only the generic coding-task template.
+ARG INSTANCE_ID=".none"
+COPY env.py /hud/env.py
+COPY coding /hud/coding
+COPY instances/${INSTANCE_ID}/ /hud/instance/
+
+# Self-contained hud venv under /hud: bootstrap uv (with its own managed
+# Python), never touching the base image's Python or site-packages. chmod 700
+# keeps the vault and instance assets out of the uid-dropped agent's reach.
+RUN <<'INSTALL'
+set -eu
+export PATH="$HOME/.local/bin:$PATH" UV_INSTALL_DIR="$HOME/.local/bin"
+command -v uv >/dev/null 2>&1 || {
+  { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \
+    || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \
+    || apk add --no-cache curl ca-certificates
+  { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \
+    || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \
+    || pip install -q -U uv
+}
+uv python install 3.12
+uv venv /hud/venv --python 3.12
+uv pip install --python /hud/venv/bin/python 'hud>=0.6.11'
+chmod -R go-rwx /hud
+INSTALL
+ENV REPO_DIR=/app PYTHONPATH=/hud
+EXPOSE 8765
+ENTRYPOINT []
+CMD ["/hud/venv/bin/hud", "serve", "/hud/env.py", "--host", "0.0.0.0", "--port", "8765"]

environments/coding/README.md

@@ -0,0 +1,97 @@
+# Coding Environment (HUD v6)
+
+A coding environment: the agent gets a git repo over a sandboxed **`ssh` workspace** (the
+harness brings its own bash/file tools), and grading is diff-based — capture the agent's
+changes, reset to the pre-agent snapshot, re-apply the diff, bring in the hidden tests, run
+them. Generic tasks, SDLC workflow tasks, and SWE-bench Pro are flavors on this core.
+
+The agent never sees the answer key. Task setup moves the repo's real `.git` — which may hold
+solution branches or the fix commit — into a vault outside the workspace and leaves a fresh
+single-commit repo: git works normally, but there is no history, no refs, and no remotes.
+Grading discards the agent's `.git`, restores the vault, and checks hidden tests out *after*
+the agent's diff. In images, the vault and instance assets live under `/hud` (root, mode 700)
+and agent shells drop to a non-root uid via `setpriv`.
+
+This directory is a self-contained uv project — run every command below from it (`hud init`
+hands you a copy of it as your own environment package).
+
+## Layout
+
+- `env.py` — the environment: workspace wiring plus the three task templates.
+- `coding/repo.py` — the shared repo lifecycle: vault, snapshot, diff capture, reset, apply.
+- `coding/github.py` — mock GitHub for SDLC tasks: issue/PR store served as `github_*` tools.
+- `coding/swe_bench_pro.py` — the SWE-bench Pro grading pipeline.
+- `tasks.py` — sample generic and SDLC tasks.
+- `swe_tasks.py` — the SWE-bench Pro task source: fetches instances and builds their images
+  when run; task rows when imported.
+- `Dockerfile.hud` — the image definition for both flavors: `BASE` selects the generic
+  repo-clone head or a prebuilt instance image.
+
+## Generic tasks (`coding-task`)
+
+Point the env at a repo (`REPO_URL`; locally it clones per process, or bake it with
+`Dockerfile.hud`) and parameterize the template: a `base_ref` to start from, a `test_ref` whose
+`test_files` are the hidden tests (checked out from the vaulted history at grade time), and a
+`test_command` scored by exit code. Tasks follow the 3-branch convention — `{task}_baseline` /
+`{task}_test` / `{task}_golden`; `tasks.py` ships four sample bugs on
+[coding-template-sample](https://github.com/hud-evals/coding-template-sample):
+
+```bash
+uv sync
+hud set HUD_API_KEY=your-key-here
+hud eval tasks.py claude --task-ids sentry-fix -y --runtime local
+```
+
+## SDLC tasks (`sdlc-task`)
+
+The generic flavor plus workflow: the repo gets an `origin` remote (a bare mock-GitHub repo the
+agent pushes to) and `github_*` MCP tools seeded with the task's issues. The deliverable is a
+pushed branch with a pull request — grading checks the PR head out of the remote, brings in the
+hidden tests, runs the test command (weight 0.8), and scores the PR itself (0.2): a structural
+title/body check by default, or `pr_rubric` judged by `LLMJudgeGrader` when provided. See the
+`sentry-fix-pr` sample in `tasks.py`:
+
+```bash
+hud eval tasks.py claude --task-ids sentry-fix-pr -y --runtime local
+```
+
+## SWE-bench Pro tasks
+
+Each of the 731 public [SWE-bench Pro](https://github.com/scaleapi/SWE-bench_Pro-os) instances
+ships a prebuilt image (`jefzda/sweap-images:<tag>`, `linux/amd64`) with the repo and toolchain
+baked in. Running `swe_tasks.py` fetches the dataset row plus the official
+`run_script.sh`/`parser.py` into `instances/<id>/` and builds `Dockerfile.hud` with the
+instance's image as `BASE`, so the image serves this env from inside. Grading replays the
+official evaluator: resolved iff every `fail_to_pass` **and** `pass_to_pass` test passes.
+
+```bash
+uv run swe_tasks.py instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan
+hud eval swe_tasks.py claude --task-ids nodebb-04998908
+uv run swe_tasks.py <id>... --push registry.io/acme   # push for cloud runtimes
+```
+
+## Tests
+
+```bash
+uv run pytest tests/ -q --ignore=tests/test_integration.py   # offline + hermetic local e2e
+uv run pytest tests/test_integration.py -v                   # SWE-bench gold-patch check (Docker)
+```
+
+`test_local_rollout.py` runs the generic and SDLC flavors end to end against a fixture 3-branch
+repo (no Docker or network): the golden ref grades 1.0 and the untouched baseline 0.0. The
+integration suite is the same check for built SWE-bench Pro instances.
+
+## Caveats
+
+- Public benchmarks are public: a networked agent could fetch solutions from GitHub. Disable
+  network egress at the runtime layer if that matters for your run.
+- The uid wall needs `setpriv` (util-linux) in the image; the repo path comes from `REPO_DIR`
+  (`/app` in instance images).
+- The agent runs as uid 1000, so the baked repo must belong to it. The workspace only chowns
+  its own directory at start (O(1), keeps boot fast); the tree is owned where it's staged —
+  the generic build chowns `/app` in the clone step, and task setup re-chowns after root
+  mutates the worktree (checkout, vaulting).
+
+## Documentation
+
+See the [full docs](https://docs.hud.ai) for tasks, evaluation, and scaling.

environments/coding/coding/__init__.py

@@ -0,0 +1,5 @@
+"""The coding environment's core: repo-lifecycle primitives and task flavors."""
+
+from . import repo, swe_bench_pro
+
+__all__ = ["repo", "swe_bench_pro"]