From 6dc7bd7f40b317a20c326055b8525136072bf8a8 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Wed, 15 Jul 2026 09:40:48 -0400 Subject: [PATCH 1/8] Set up model validation infrastructure, script, and workflow from scratch using Claude's Fable. --- .github/workflows/model_validation.yml | 157 ++++++ .gitignore | 10 + model_validation/README.md | 127 +++++ model_validation/config.yml | 85 ++++ model_validation/requirements.txt | 3 + model_validation/test_data/.gitkeep | 0 model_validation/validate_models.py | 645 +++++++++++++++++++++++++ 7 files changed, 1027 insertions(+) create mode 100644 .github/workflows/model_validation.yml create mode 100644 model_validation/README.md create mode 100644 model_validation/config.yml create mode 100644 model_validation/requirements.txt create mode 100644 model_validation/test_data/.gitkeep create mode 100644 model_validation/validate_models.py diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml new file mode 100644 index 00000000..c04f6552 --- /dev/null +++ b/.github/workflows/model_validation.yml @@ -0,0 +1,157 @@ +# Daily validation of HARP model deployments. +# +# Two tiers: +# 1. baseline - local pyharp examples run inside the CI runner; no Hugging +# Face infrastructure involved. A failure here means pyharp +# or gradio itself is broken - fix that before debugging +# individual spaces. +# 2. spaces - every Hugging Face Space under teamup-tech, validated +# end-to-end (/controls + /process) via gradio_client. +# +# Models are validated independently; one failure never stops the rest. +# Results are published to the run summary and uploaded as artifacts. +# +# Model failures do NOT fail the workflow (deployments are not yet stable +# enough - a red run every day would just spam maintainers with emails). +# They appear as warning annotations and in the run summary instead. Only +# infrastructure errors (bad token, discovery failure, ...) turn the run +# red, since those mean validation itself has stopped working. To start +# getting notified on model failures later, remove the exit-code handling +# in the two Validate steps so exit code 1 propagates. +# +# Requires the repository secret HF_TOKEN: a Hugging Face token with read +# access to the teamup-tech spaces (write access if you want the workflow to +# auto-restart crashed spaces). NEVER commit the token; GitHub masks secrets +# in logs automatically. + +name: Model Validation + +on: + schedule: + - cron: '30 6 * * *' # daily, 06:30 UTC + workflow_dispatch: + inputs: + spaces: + description: 'Space ids to validate (space-separated); empty = all' + required: false + default: '' + skip_process: + description: 'Skip /process inference tests (availability only)' + type: boolean + default: false + restart_failed: + description: 'Attempt to restart crashed/stopped spaces (needs write token)' + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: model-validation + cancel-in-progress: false + +jobs: + + baseline: + name: Baseline (local pyharp examples) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install pyharp (local submodule) and example dependencies + run: | + python -m pip install --upgrade pip + # CPU torch/torchaudio must be installed BEFORE pyharp, otherwise + # descript-audiotools pulls mismatched CUDA builds from PyPI + pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install -e ./pyharp + pip install -r model_validation/requirements.txt + + - name: Validate local examples + run: | + set +e + python model_validation/validate_models.py --local-examples \ + --output-dir reports/baseline + CODE=$? + if [ "$CODE" -eq 1 ]; then + echo "::warning title=Baseline failures::Some local examples failed - see the run summary and report artifact." + exit 0 + fi + exit "$CODE" + + - name: Publish summary + if: always() + run: cat reports/baseline/report.md >> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: report-baseline + path: reports/baseline/report.* + if-no-files-found: ignore + + spaces: + name: Hugging Face Spaces + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r model_validation/requirements.txt + + - name: Validate spaces + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + INPUT_SPACES: ${{ inputs.spaces }} + INPUT_SKIP_PROCESS: ${{ inputs.skip_process }} + INPUT_RESTART_FAILED: ${{ inputs.restart_failed }} + run: | + ARGS=(--output-dir reports/spaces) + if [ -n "$INPUT_SPACES" ]; then + read -r -a SPACE_IDS <<< "$INPUT_SPACES" + ARGS+=(--spaces "${SPACE_IDS[@]}") + fi + if [ "$INPUT_SKIP_PROCESS" = "true" ]; then + ARGS+=(--skip-process) + fi + if [ "$INPUT_RESTART_FAILED" = "true" ]; then + ARGS+=(--restart-failed) + fi + set +e + python model_validation/validate_models.py "${ARGS[@]}" + CODE=$? + if [ "$CODE" -eq 1 ]; then + echo "::warning title=Space failures::Some spaces failed validation - see the run summary and report artifact." + exit 0 + fi + exit "$CODE" + + - name: Publish summary + if: always() + run: cat reports/spaces/report.md >> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: report-spaces + path: reports/spaces/report.* + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index b41816c2..5fbcbbc5 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,16 @@ dist testproj.RPP +# Python +__pycache__/ +*.pyc +*.egg-info/ +venv/ +.venv/ + +# model validation output +reports/ + # Ignore all the *.md files in website/HARP website/content/HARP/*.md website/content/pyHARP/*.md diff --git a/model_validation/README.md b/model_validation/README.md new file mode 100644 index 00000000..f15b65c0 --- /dev/null +++ b/model_validation/README.md @@ -0,0 +1,127 @@ +# HARP Model Validation + +Automated validation of HARP model deployments. Verifies that each deployment +is reachable, exposes the HARP gradio endpoints (`/controls`, `/process`), +and can actually process test inputs end-to-end. + +Two tiers share the same harness ([validate_models.py](validate_models.py)): + +| Tier | What it validates | What a failure means | +|---|---|---| +| **Baseline** (`--local-examples`) | The apps under `pyharp/examples/`, launched locally | pyharp or gradio itself is broken | +| **Spaces** (default) | Every Hugging Face Space under `teamup-tech` | That specific deployment is broken (build error, HF platform change, dependency drift, ...) | + +Models are validated **independently**: each model (and each test case within +it) runs inside its own error boundary, so a crash, hang, or timeout in one +model never stops validation of the others. All results are collected into a +single report and the process exits non-zero only after everything has run. + +A daily GitHub Action ([model_validation.yml](../.github/workflows/model_validation.yml)) +runs both tiers at 06:30 UTC and publishes the results to the run summary and +as artifacts. Model failures do not turn the run red (no notification emails); +check the run summary for results. + +## Token setup (IMPORTANT — read this) + +ZeroGPU spaces require an authenticated request to obtain GPU quota, and +private spaces require read access. Validation reads the token **only** from +the `HF_TOKEN` environment variable. + +- **Never** commit a token, paste it in an issue/PR, or pass it as a + command-line argument (argv is visible in process listings). +- **CI:** add the token as a repository secret named `HF_TOKEN` + (Settings → Secrets and variables → Actions). GitHub masks secrets in logs. +- **Locally:** `export HF_TOKEN=...` in your shell (consider `read -s` so it + stays out of shell history). +- Use a [fine-grained token](https://huggingface.co/settings/tokens) scoped to + read access on the org's spaces. Grant write access only if you want + `--restart-failed` / the workflow's restart option to work. +- **If a token is ever exposed, rotate it immediately** at + https://huggingface.co/settings/tokens. + +## Running locally + +```bash +pip install -r model_validation/requirements.txt + +# All spaces in the org +export HF_TOKEN=hf_... # see token setup above +python model_validation/validate_models.py + +# A single space, with verbose errors +python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter --verbose + +# Exclude specific models (also configurable via `exclude` in config.yml) +python model_validation/validate_models.py --exclude teamup-tech/broken-space + +# Availability + /controls only (fast; no inference, no GPU quota used) +python model_validation/validate_models.py --skip-process + +# Try to restart crashed/stopped spaces first (token needs write access) +python model_validation/validate_models.py --restart-failed + +# Baseline tier (needs pyharp + example deps; no token required) +pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu +pip install -e ./pyharp +python model_validation/validate_models.py --local-examples +``` + +Reports land in `reports/` as `report.json` (machine-readable) and +`report.md` (human-readable table). Exit code is `0` when everything passes, +`1` on any model failure, `2` on configuration/infrastructure errors. + +## Excluding models + +Two equivalent ways, merged together: + +- `exclude` in [config.yml](config.yml) — for permanent exclusions + (non-HARP spaces, archived deployments, known-broken examples). +- `--exclude ...` on the command line — for ad-hoc runs. + +Use the space id (`teamup-tech/some-space`) for remote models and +`local/` (e.g. `local/midi_synthesizer`) for baseline examples. + +## Per-model test cases + +By default every model gets one `default` test case with inputs synthesized +from its `/controls` spec (a sine-sweep WAV for audio tracks, a two-note MIDI +file for MIDI tracks, declared default values for sliders/toggles/etc.). + +To validate deployment-specific behavior, add named cases in +[config.yml](config.yml). Cases override control values by their **label** and +can substitute custom input files (checked into `model_validation/test_data/`): + +```yaml +overrides: + teamup-tech/pitch_shifter: + process_timeout: 900 # this model is slow; extend the timeout + test_cases: + - name: default # keep the synthesized case + - name: extreme-shift + controls: + "Pitch Shift (semitones)": 24 + - name: real-audio + files: + "Input Audio A": test_data/short_vocal.wav +``` + +Local examples use the key `local/`, e.g. `local/pitch_shifter`. +See the comment block at the top of config.yml for the full schema. + +## CI behavior + +- **Schedule:** daily at 06:30 UTC; also runnable manually from the Actions + tab (with options to validate specific spaces, skip inference, or + auto-restart crashed spaces). +- **Reports:** markdown summary on each run + JSON/markdown artifacts. +- **Failure signal:** model failures keep the run green (deployments are not + yet stable enough for daily failure emails to be useful) — they show up as + warning annotations and in the run summary/report instead. No issues are + opened and no emails are sent. Only infrastructure errors (bad token, + discovery failure) turn the run red, since those mean validation itself + has stopped working. +- **Opting back into notifications later:** remove the exit-code handling in + the two `Validate` steps of the workflow so the script's exit code 1 + propagates; failed runs then turn red and GitHub emails maintainers. +- Baseline failures indicate a pyharp/gradio-level breakage that likely + affects every deployment — fix those before debugging individual spaces. diff --git a/model_validation/config.yml b/model_validation/config.yml new file mode 100644 index 00000000..00ee2879 --- /dev/null +++ b/model_validation/config.yml @@ -0,0 +1,85 @@ +# Configuration for model_validation/validate_models.py +# +# All keys are optional. Spaces not listed here are still discovered and +# validated automatically with a single synthesized "default" test case. +# +# Schema: +# +# exclude: # models to exclude from validation: space ids, or +# # "local/" for local pyharp examples +# - teamup-tech/some-non-harp-space +# - local/some_example +# +# include_extra: # spaces outside the org that should also be validated +# - some-user/some-harp-space +# +# overrides: # per-model settings; keys are space ids, or +# # "local/" for local pyharp examples +# : +# connect_timeout: 600 # seconds to wait for build/wake (default 420) +# process_timeout: 900 # seconds to wait for /process (default 600) +# skip_process: true # only check availability + /controls +# test_cases: # named /process test cases; when omitted, a +# # single synthesized "default" case runs +# - name: default # empty case == synthesized defaults +# - name: my-case +# process_timeout: 300 # optional per-case override +# controls: # override control values by label +# "Some Slider Label": 12 +# "Some Toggle Label": true +# files: # override track/file inputs by label; +# # paths are relative to this file +# "Input Audio": test_data/my_clip.wav + +exclude: [] + +include_extra: [] + +overrides: + + # ---- Baseline: local pyharp examples ------------------------------------- + # These run with `--local-examples` and exercise pyharp itself with no + # Hugging Face infrastructure involved. A failure here indicates a + # pyharp/gradio-level breakage rather than a deployment-specific one. + + local/pitch_shifter: + test_cases: + - name: default + - name: shift-up-octave + controls: + "Pitch Shift (semitones)": 12 + - name: shift-down-octave + controls: + "Pitch Shift (semitones)": -12 + - name: no-shift + controls: + "Pitch Shift (semitones)": 0 + + local/midi_pitch_shifter: + test_cases: + - name: default + - name: shift-up-max + controls: + "Pitch Shift (semitones)": 24 + - name: shift-down-max + controls: + "Pitch Shift (semitones)": -24 + + local/midi_synthesizer: + test_cases: + - name: default + + local/ui_tester: + test_cases: + - name: default + + # ---- Remote space overrides --------------------------------------------- + # Add per-space timeouts and extra test cases here as needed, e.g.: + # + # teamup-tech/pitch_shifter: + # process_timeout: 900 + # test_cases: + # - name: default + # - name: extreme-shift + # controls: + # "Pitch Shift (semitones)": 24 diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt new file mode 100644 index 00000000..84582416 --- /dev/null +++ b/model_validation/requirements.txt @@ -0,0 +1,3 @@ +gradio_client>=1.0 +huggingface_hub>=0.23 +PyYAML>=6.0 diff --git a/model_validation/test_data/.gitkeep b/model_validation/test_data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/model_validation/validate_models.py b/model_validation/validate_models.py new file mode 100644 index 00000000..3c6f0f31 --- /dev/null +++ b/model_validation/validate_models.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +""" +HARP model validation. + +Validates HARP deployments in two modes, using the same black-box harness: + +1. Remote Hugging Face Spaces (default): discovers all Spaces under an + organization (default: teamup-tech), verifies each is running, exposes the + HARP gradio endpoints (/controls and /process), and processes test inputs + end-to-end. ZeroGPU spaces require an authenticated request for GPU quota, + so a token must be provided via the HF_TOKEN environment variable - never + on the command line or in the repository. + +2. Baseline (--local-examples): launches each app under pyharp/examples/ on + a local port and runs the identical endpoint tests. The baseline tier + exercises pyharp itself with no Hugging Face infrastructure in the loop, + so a failure indicates a pyharp/gradio-level breakage rather than a + deployment-specific one. + +Per-model test cases can be declared in config.yml (see README.md). When a +model has no configured cases, a single "default" case runs with inputs +synthesized automatically from the /controls spec. Models are validated +independently: a failure (or timeout) in one model never stops validation +of the others. + +Usage: + HF_TOKEN=... python model_validation/validate_models.py + HF_TOKEN=... python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter + HF_TOKEN=... python model_validation/validate_models.py --skip-process --workers 8 + HF_TOKEN=... python model_validation/validate_models.py --restart-failed + python model_validation/validate_models.py --local-examples + +Exit codes: + 0 - all validated models passed (or were explicitly skipped) + 1 - at least one model failed + 2 - infrastructure/configuration error (bad token, no spaces found, ...) +""" + +import argparse +import concurrent.futures +import dataclasses +import json +import math +import os +import struct +import subprocess +import sys +import time +import traceback +import urllib.request +import wave +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + +from gradio_client import Client, handle_file + + +DEFAULT_ORG = "teamup-tech" +SCRIPT_DIR = Path(__file__).parent +DEFAULT_CONFIG = SCRIPT_DIR / "config.yml" +DEFAULT_EXAMPLES_DIR = SCRIPT_DIR.parent / "pyharp" / "examples" + +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" + +# Space runtime stages that indicate a hard failure (no point connecting) +DEAD_STAGES = {"BUILD_ERROR", "RUNTIME_ERROR", "CONFIG_ERROR", "DELETING"} +# Stages that resolve on their own if we wait (or wake on first request) +TRANSIENT_STAGES = {"BUILDING", "RUNNING_BUILDING", "APP_STARTING", "SLEEPING"} + +LOCAL_PORT_BASE = 7861 + + +def get_token(required: bool) -> str: + token = os.environ.get("HF_TOKEN", "").strip() + if required and not token: + print("ERROR: HF_TOKEN environment variable is not set.", file=sys.stderr) + print("Set it locally (export HF_TOKEN=...) or as a GitHub Actions secret.", + file=sys.stderr) + sys.exit(2) + return token + + +def scrub(text: str, token: str) -> str: + """Remove the token from any string that might get printed or reported.""" + return text.replace(token, "***HF_TOKEN***") if token else text + + +# --------------------------------------------------------------------------- +# Synthesized test assets +# --------------------------------------------------------------------------- + +def make_test_wav(path: Path, duration: float = 2.0, sr: int = 44100) -> Path: + """Write a short mono 16-bit sine sweep - a valid input for any audio model.""" + n = int(duration * sr) + with wave.open(str(path), "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(sr) + frames = bytearray() + for i in range(n): + t = i / sr + freq = 220.0 + 440.0 * t / duration + sample = int(0.5 * 32767 * math.sin(2 * math.pi * freq * t)) + frames += struct.pack(" Path: + """Write a minimal standard MIDI file (format 0, two quarter notes).""" + track_events = bytes([ + 0x00, 0xC0, 0x00, # program change: acoustic grand + 0x00, 0x90, 0x3C, 0x64, # note on C4 + 0x83, 0x60, 0x80, 0x3C, 0x40, # note off C4 after 480 ticks + 0x00, 0x90, 0x40, 0x64, # note on E4 + 0x83, 0x60, 0x80, 0x40, 0x40, # note off E4 after 480 ticks + 0x00, 0xFF, 0x2F, 0x00, # end of track + ]) + header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, 480) + track = b"MTrk" + struct.pack(">I", len(track_events)) + track_events + path.write_bytes(header + track) + return path + + +AUDIO_EXTS = {".wav", ".mp3", ".flac", ".ogg", ".aif", ".aiff", ".m4a", "audio"} +MIDI_EXTS = {".mid", ".midi"} + + +class Assets: + """Synthesized test input files, shared across all model tests.""" + + def __init__(self, workdir: Path): + workdir.mkdir(parents=True, exist_ok=True) + self.wav = make_test_wav(workdir / "test_input.wav") + self.midi = make_test_midi(workdir / "test_input.mid") + self.text = workdir / "test_input.txt" + self.text.write_text("HARP model validation\n") + self.json = workdir / "test_input.json" + self.json.write_text("{}\n") + + def for_file_types(self, file_types: list) -> Path | None: + types = {str(t).lower() for t in (file_types or [])} + if not types or types & AUDIO_EXTS: + return self.wav + if types & MIDI_EXTS: + return self.midi + if ".json" in types: + return self.json + if ".txt" in types or ".text" in types: + return self.text + return None + + +# --------------------------------------------------------------------------- +# Input synthesis from the /controls spec + per-model test cases +# --------------------------------------------------------------------------- + +def synthesize_default_args(controls: dict, assets: Assets) -> tuple[list, dict]: + """ + Build the positional argument list for /process from the controls spec. + Returns (args, missing) where missing maps the label of every input we + could NOT synthesize to a reason. Such inputs get a None placeholder; a + test case can still run if its controls/files overrides cover them all. + """ + args, missing = [], {} + for spec in controls.get("inputs", []): + ctype = spec.get("type") + label = spec.get("label") + if ctype == "audio_track": + args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) + elif ctype == "midi_track": + args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) + elif ctype == "generic_file": + path = assets.for_file_types(spec.get("file_types")) + if path is None and spec.get("required", True): + missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" + args.append(handle_file(str(path)) if path is not None else None) + elif ctype in ("slider", "number_box"): + value = spec.get("value") + if value is None: + value = spec.get("minimum", 0) + args.append(value) + elif ctype == "text_box": + args.append(spec.get("value") or "test") + elif ctype == "toggle": + args.append(bool(spec.get("value", False))) + elif ctype == "dropdown": + value = spec.get("value") + if value is None: + choices = spec.get("choices") or [] + if not choices: + missing[label] = "dropdown has no choices" + else: + first = choices[0] + value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first + args.append(value) + else: + missing[label] = f"unsupported input control type '{ctype}'" + args.append(None) + return args, missing + + +def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: + """ + Overlay a configured test case onto the default argument list. + + A case may contain: + controls: {: } - override scalar control values + files: {: } - override track/file inputs + (paths relative to config.yml) + Raises ValueError if a label does not match any input. + """ + labels = [spec.get("label") for spec in controls.get("inputs", [])] + args = list(args) + + for label, value in (case.get("controls") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"control '{label}' (available: {labels})") + args[labels.index(label)] = value + + for label, rel_path in (case.get("files") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"input '{label}' (available: {labels})") + path = (config_dir / rel_path).resolve() + if not path.exists(): + raise ValueError(f"test case '{case.get('name')}': file not found: {path}") + args[labels.index(label)] = handle_file(str(path)) + + return args + + +def validate_outputs(result, controls: dict) -> str | None: + """Return an error string if the /process outputs look wrong, else None.""" + specs = controls.get("outputs", []) + outputs = result if isinstance(result, (list, tuple)) else [result] + if len(specs) > 1 and len(outputs) != len(specs): + return f"expected {len(specs)} outputs, got {len(outputs)}" + for spec, out in zip(specs, outputs): + if out is None: + return f"output '{spec.get('label')}' is None" + # gradio_client downloads file outputs and returns local paths + path = out.get("path") if isinstance(out, dict) and "path" in out else out + if isinstance(path, str) and os.path.sep in path and os.path.exists(path): + if os.path.getsize(path) == 0: + return f"output file for '{spec.get('label')}' is empty" + return None + + +# --------------------------------------------------------------------------- +# Results +# --------------------------------------------------------------------------- + +@dataclasses.dataclass +class CaseResult: + name: str + ok: bool | None = None # None => skipped + duration: float = 0.0 + error: str = "" + + +@dataclasses.dataclass +class ModelResult: + target: str # space id or "local/" + kind: str = "space" # "space" | "local" + status: str = FAIL + stage: str = "" + controls_ok: bool = False + cases: list = dataclasses.field(default_factory=list) + duration: float = 0.0 + error: str = "" + model_name: str = "" + + +def run_with_timeout(fn, timeout: float, what: str): + """Run fn() in a worker thread; raise TimeoutError if it exceeds timeout.""" + pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) + future = pool.submit(fn) + try: + return future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + raise TimeoutError(f"{what} timed out after {int(timeout)}s") + finally: + # wait=False: never block on a hung call; the worker thread is leaked, + # which is acceptable for a test script + pool.shutdown(wait=False) + + +# --------------------------------------------------------------------------- +# Shared endpoint tests (identical for remote spaces and local examples) +# --------------------------------------------------------------------------- + +def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, + overrides: dict, opts: argparse.Namespace) -> ModelResult: + """Verify /controls and run every configured /process test case.""" + process_timeout = overrides.get("process_timeout", opts.process_timeout) + skip_process = opts.skip_process or overrides.get("skip_process", False) + config_dir = opts.config.parent.resolve() + + endpoints = client.view_api(return_format="dict", print_info=False) or {} + named = endpoints.get("named_endpoints", {}) + if "/controls" not in named or "/process" not in named: + result.error = (f"missing HARP endpoints (found: {sorted(named)}); " + f"deployment may use an outdated pyharp") + return result + + controls = run_with_timeout( + lambda: client.predict(api_name="/controls"), 120, "/controls") + if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: + result.error = f"/controls returned malformed data: {str(controls)[:200]}" + return result + result.controls_ok = True + result.model_name = controls.get("card", {}).get("name", "") + + if skip_process: + result.status = PASS + return result + + default_args, missing = synthesize_default_args(controls, assets) + cases = overrides.get("test_cases") or [{"name": "default"}] + + for case in cases: + case_result = CaseResult(name=case.get("name", "unnamed")) + result.cases.append(case_result) + case_start = time.time() + try: + # Inputs we couldn't synthesize are fine if this case supplies them + supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) + unsatisfied = {k: v for k, v in missing.items() if k not in supplied} + if unsatisfied: + case_result.error = "skipped: " + "; ".join( + f"'{k}': {v}" for k, v in unsatisfied.items()) + continue + args = apply_case(default_args, controls, case, config_dir) + job = client.submit(*args, api_name="/process") + output = job.result(timeout=case.get("process_timeout", process_timeout)) + error = validate_outputs(output, controls) + if error: + case_result.ok = False + case_result.error = f"/process output invalid: {error}" + else: + case_result.ok = True + except Exception as exc: # noqa: BLE001 - any exception fails the case + case_result.ok = False + case_result.error = f"{type(exc).__name__}: {exc}" + finally: + case_result.duration = round(time.time() - case_start, 1) + + if any(c.ok is False for c in result.cases): + failed = [c.name for c in result.cases if c.ok is False] + result.error = "; ".join( + f"[{c.name}] {c.error}" for c in result.cases if c.ok is False) + result.status = FAIL + else: + result.status = PASS + return result + + +# --------------------------------------------------------------------------- +# Remote space tests +# --------------------------------------------------------------------------- + +def wait_for_stage(api, space_id: str, deadline: float) -> str: + """Poll runtime stage until RUNNING, a dead stage, or the deadline.""" + stage = api.get_space_runtime(space_id).stage + while stage in TRANSIENT_STAGES - {"SLEEPING"} and time.time() < deadline: + time.sleep(10) + stage = api.get_space_runtime(space_id).stage + return stage + + +def test_space(space_id: str, token: str, assets: Assets, + opts: argparse.Namespace, overrides: dict) -> ModelResult: + from huggingface_hub import HfApi + + result = ModelResult(target=space_id, kind="space") + start = time.time() + api = HfApi(token=token) + connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) + + try: + deadline = time.time() + connect_timeout + stage = wait_for_stage(api, space_id, deadline) + result.stage = stage + + if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): + if opts.restart_failed and stage != "DELETING": + print(f" [{space_id}] stage={stage}, requesting restart...") + api.restart_space(space_id) + stage = wait_for_stage(api, space_id, time.time() + connect_timeout) + result.stage = stage + if stage not in ("RUNNING", "SLEEPING"): + result.error = f"space is not running (stage={stage})" + return result + + # Connect (this wakes sleeping spaces; retry through the wake-up) + client, last_exc = None, None + deadline = time.time() + connect_timeout + while time.time() < deadline: + try: + client = run_with_timeout( + lambda: Client(space_id, hf_token=token, verbose=False), + max(10.0, deadline - time.time()), "connect") + break + except Exception as exc: # noqa: BLE001 - space may still be waking + last_exc = exc + time.sleep(15) + if client is None: + raise RuntimeError(f"could not connect: {last_exc}") + + return run_endpoint_tests(client, result, assets, overrides, opts) + + except Exception as exc: # noqa: BLE001 + result.error = scrub(f"{type(exc).__name__}: {exc}", token) + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + + +# --------------------------------------------------------------------------- +# Baseline: local pyharp example tests +# --------------------------------------------------------------------------- + +def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> None: + deadline = time.time() + timeout + url = f"http://127.0.0.1:{port}/config" + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"app exited early with code {proc.returncode}") + try: + with urllib.request.urlopen(url, timeout=5): + return + except Exception: # noqa: BLE001 - server not up yet + time.sleep(2) + raise TimeoutError(f"local app did not become ready within {int(timeout)}s") + + +def test_local_example(app_dir: Path, port: int, assets: Assets, + opts: argparse.Namespace, overrides: dict) -> ModelResult: + target = f"local/{app_dir.name}" + result = ModelResult(target=target, kind="local", stage="LOCAL") + start = time.time() + + env = os.environ.copy() + env["GRADIO_SERVER_NAME"] = "127.0.0.1" + env["GRADIO_SERVER_PORT"] = str(port) + env.pop("HF_TOKEN", None) # local examples must not need credentials + + log_path = opts.output_dir / f"{app_dir.name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + proc = None + try: + with open(log_path, "w") as log: + proc = subprocess.Popen( + [sys.executable, "app.py"], cwd=app_dir, env=env, + stdout=log, stderr=subprocess.STDOUT) + wait_for_local_server(port, proc, overrides.get( + "connect_timeout", opts.connect_timeout)) + client = Client(f"http://127.0.0.1:{port}", verbose=False) + return run_endpoint_tests(client, result, assets, overrides, opts) + except Exception as exc: # noqa: BLE001 + result.error = f"{type(exc).__name__}: {exc} (see {log_path.name})" + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + + +# --------------------------------------------------------------------------- +# Discovery, config, reporting +# --------------------------------------------------------------------------- + +def load_config(path: Path) -> dict: + if not path.exists(): + return {} + if yaml is None: + print(f"WARNING: PyYAML not installed; ignoring {path}", file=sys.stderr) + return {} + return yaml.safe_load(path.read_text()) or {} + + +def get_excluded(config: dict, opts: argparse.Namespace) -> set: + """Models excluded from validation: config `exclude` list + --exclude.""" + return set(config.get("exclude", [])) | set(opts.exclude or []) + + +def discover_spaces(api, org: str, config: dict, excluded: set) -> list[str]: + spaces = [s.id for s in api.list_spaces(author=org)] + spaces += [s for s in config.get("include_extra", []) if s not in spaces] + return sorted(s for s in spaces if s not in excluded) + + +def status_emoji(r: ModelResult) -> str: + return {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭️"}[r.status] + + +def write_reports(results: list[ModelResult], out_dir: Path, label: str) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + + payload = { + "label": label, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "total": len(results), + "passed": sum(r.status == PASS for r in results), + "failed": sum(r.status == FAIL for r in results), + "results": [dataclasses.asdict(r) for r in results], + } + (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) + + lines = [ + f"# HARP Model Validation Report - {label}", + "", + f"**{payload['passed']}/{payload['total']} models passed** ({payload['timestamp']})", + "", + "| Model | Status | Stage | Controls | Cases | Time (s) | Detail |", + "|---|---|---|---|---|---|---|", + ] + for r in sorted(results, key=lambda r: (r.status != FAIL, r.target)): + if r.cases: + cases = ", ".join( + f"{c.name} {'✅' if c.ok else '⏭️' if c.ok is None else '❌'}" + for c in r.cases) + else: + cases = "—" + link = (f"[{r.target}](https://huggingface.co/spaces/{r.target})" + if r.kind == "space" else f"`{r.target}`") + detail = r.error.replace("|", "\\|")[:300] if r.error else "" + lines.append(f"| {link} | {status_emoji(r)} {r.status} | {r.stage} " + f"| {'✅' if r.controls_ok else '❌'} | {cases} " + f"| {r.duration} | {detail} |") + (out_dir / "report.md").write_text("\n".join(lines) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate HARP model deployments.") + parser.add_argument("--org", default=DEFAULT_ORG, help="HF organization to scan") + parser.add_argument("--spaces", nargs="*", default=None, + help="Explicit space ids to validate (skips discovery)") + parser.add_argument("--exclude", nargs="*", default=None, metavar="MODEL", + help="Models to exclude from validation (space ids, or " + "local/); merged with the config " + "`exclude` list") + parser.add_argument("--local-examples", nargs="*", default=None, metavar="DIR", + help="Validate local pyharp example apps instead of remote " + "spaces (the baseline tier). With no DIRs given, tests " + "every app under pyharp/examples/.") + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG, + help="Optional YAML config (excludes, per-model overrides/cases)") + parser.add_argument("--skip-process", action="store_true", + help="Only verify availability and /controls, do not run inference") + parser.add_argument("--restart-failed", action="store_true", + help="Attempt to restart spaces found in an error/stopped state " + "(requires a token with write access)") + parser.add_argument("--workers", type=int, default=4, + help="Number of spaces to test concurrently (remote mode only)") + parser.add_argument("--connect-timeout", type=float, default=420, + help="Seconds to wait for a deployment to build/wake/start") + parser.add_argument("--process-timeout", type=float, default=600, + help="Seconds to wait for /process (includes ZeroGPU queue time)") + parser.add_argument("--output-dir", type=Path, default=Path("reports")) + parser.add_argument("--verbose", action="store_true") + opts = parser.parse_args() + + config = load_config(opts.config) + overrides = config.get("overrides", {}) + excluded = get_excluded(config, opts) + assets = Assets(opts.output_dir / "assets") + results = [] + + if opts.local_examples is not None: + # ---- Baseline mode: local pyharp examples (run sequentially) ---- + if opts.local_examples: + app_dirs = [Path(d) for d in opts.local_examples] + else: + app_dirs = sorted(d for d in DEFAULT_EXAMPLES_DIR.iterdir() + if (d / "app.py").exists()) + app_dirs = [d for d in app_dirs if f"local/{d.name}" not in excluded] + if not app_dirs: + print("ERROR: no local examples found", file=sys.stderr) + return 2 + print(f"Validating {len(app_dirs)} local pyharp examples (baseline)\n") + for i, app_dir in enumerate(app_dirs): + r = test_local_example(app_dir, LOCAL_PORT_BASE + i, assets, opts, + overrides.get(f"local/{app_dir.name}", {})) + results.append(r) + note = f" - {r.error}" if r.error else "" + print(f"{status_emoji(r)} {r.status:4s} {r.target} ({r.duration}s){note}") + label = "baseline (local pyharp examples)" + token = "" + else: + # ---- Remote mode: Hugging Face Spaces ---- + from huggingface_hub import HfApi + token = get_token(required=True) + api = HfApi(token=token) + if opts.spaces: + space_ids = [s for s in opts.spaces if s not in excluded] + else: + space_ids = discover_spaces(api, opts.org, config, excluded) + if not space_ids: + print(f"ERROR: no spaces found for org '{opts.org}'", file=sys.stderr) + return 2 + print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " + f"(process test: {'OFF' if opts.skip_process else 'ON'})\n") + with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: + futures = { + pool.submit(test_space, sid, token, assets, opts, + overrides.get(sid, {})): sid + for sid in space_ids + } + for future in concurrent.futures.as_completed(futures): + r = future.result() + results.append(r) + note = f" - {r.error}" if r.error else "" + print(f"{status_emoji(r)} {r.status:4s} {r.target} " + f"({r.duration}s){scrub(note, token)}") + label = f"{opts.org} spaces" + + write_reports(results, opts.output_dir, label) + + failed = [r for r in results if r.status == FAIL] + print(f"\n{len(results) - len(failed)}/{len(results)} models passed. " + f"Reports written to {opts.output_dir}/") + if failed: + print("\nFailed models:") + for r in sorted(failed, key=lambda r: r.target): + print(f" - {r.target}: {scrub(r.error, token)}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 516d72057a370f28d0a0fba790426342481829c9 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Fri, 17 Jul 2026 10:43:54 -0400 Subject: [PATCH 2/8] Refactored model validation into modules; added ZeroGPU-aware quota tracking, output inspection via expect rules and custom validators, and fixed the post-run hang. --- .github/workflows/model_validation.yml | 31 +- .gitignore | 11 +- model_validation/README.md | 227 ++++++-- model_validation/assets.py | 123 +++++ model_validation/cases.py | 265 +++++++++ model_validation/config.yml | 49 +- model_validation/harness.py | 346 ++++++++++++ model_validation/quota.py | 135 +++++ model_validation/results.py | 111 ++++ model_validation/utils.py | 163 ++++++ model_validation/validate_models.py | 730 ++++++------------------- model_validation/validators.py | 146 +++++ 12 files changed, 1691 insertions(+), 646 deletions(-) create mode 100644 model_validation/assets.py create mode 100644 model_validation/cases.py create mode 100644 model_validation/harness.py create mode 100644 model_validation/quota.py create mode 100644 model_validation/results.py create mode 100644 model_validation/utils.py create mode 100644 model_validation/validators.py diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml index c04f6552..d018ba65 100644 --- a/.github/workflows/model_validation.yml +++ b/.github/workflows/model_validation.yml @@ -1,7 +1,7 @@ # Daily validation of HARP model deployments. # # Two tiers: -# 1. baseline - local pyharp examples run inside the CI runner; no Hugging +# 1. examples - local pyharp examples run inside the CI runner; no Hugging # Face infrastructure involved. A failure here means pyharp # or gradio itself is broken - fix that before debugging # individual spaces. @@ -35,14 +35,14 @@ on: description: 'Space ids to validate (space-separated); empty = all' required: false default: '' - skip_process: + load_only: description: 'Skip /process inference tests (availability only)' type: boolean default: false restart_failed: description: 'Attempt to restart crashed/stopped spaces (needs write token)' type: boolean - default: false + default: true permissions: contents: read @@ -53,8 +53,8 @@ concurrency: jobs: - baseline: - name: Baseline (local pyharp examples) + examples: + name: Examples (local pyharp) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -80,24 +80,24 @@ jobs: run: | set +e python model_validation/validate_models.py --local-examples \ - --output-dir reports/baseline + --output-dir reports/examples CODE=$? if [ "$CODE" -eq 1 ]; then - echo "::warning title=Baseline failures::Some local examples failed - see the run summary and report artifact." + echo "::warning title=Example failures::Some local examples failed - see the run summary and report artifact." exit 0 fi exit "$CODE" - name: Publish summary if: always() - run: cat reports/baseline/report.md >> "$GITHUB_STEP_SUMMARY" || true + run: cat reports/examples/report.md >> "$GITHUB_STEP_SUMMARY" || true - name: Upload report if: always() uses: actions/upload-artifact@v4 with: - name: report-baseline - path: reports/baseline/report.* + name: report-examples + path: reports/examples/report.* if-no-files-found: ignore spaces: @@ -121,7 +121,7 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} INPUT_SPACES: ${{ inputs.spaces }} - INPUT_SKIP_PROCESS: ${{ inputs.skip_process }} + INPUT_LOAD_ONLY: ${{ inputs.load_only }} INPUT_RESTART_FAILED: ${{ inputs.restart_failed }} run: | ARGS=(--output-dir reports/spaces) @@ -129,11 +129,12 @@ jobs: read -r -a SPACE_IDS <<< "$INPUT_SPACES" ARGS+=(--spaces "${SPACE_IDS[@]}") fi - if [ "$INPUT_SKIP_PROCESS" = "true" ]; then - ARGS+=(--skip-process) + if [ "$INPUT_LOAD_ONLY" = "true" ]; then + ARGS+=(--load-only) fi - if [ "$INPUT_RESTART_FAILED" = "true" ]; then - ARGS+=(--restart-failed) + # restart is the script default; scheduled runs leave INPUT empty + if [ "$INPUT_RESTART_FAILED" = "false" ]; then + ARGS+=(--no-restart-failed) fi set +e python model_validation/validate_models.py "${ARGS[@]}" diff --git a/.gitignore b/.gitignore index 5fbcbbc5..d5b34a3c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ *.out *.app -# build +# Build build*/ .vscode/ libtorch/ @@ -53,12 +53,11 @@ py/client/dist/ py/client/*.spec dist*/ - dist/ Miniconda*/ Miniconda*.sh -# website +# Website node_modules temp cache @@ -73,9 +72,9 @@ __pycache__/ venv/ .venv/ -# model validation output -reports/ - # Ignore all the *.md files in website/HARP website/content/HARP/*.md website/content/pyHARP/*.md + +# Model validation output +reports/ diff --git a/model_validation/README.md b/model_validation/README.md index f15b65c0..76a6d9c3 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -2,24 +2,50 @@ Automated validation of HARP model deployments. Verifies that each deployment is reachable, exposes the HARP gradio endpoints (`/controls`, `/process`), -and can actually process test inputs end-to-end. +and can actually process test inputs end-to-end — the same interactions the +HARP client performs, driven headlessly. -Two tiers share the same harness ([validate_models.py](validate_models.py)): +## Overview -| Tier | What it validates | What a failure means | -|---|---|---| -| **Baseline** (`--local-examples`) | The apps under `pyharp/examples/`, launched locally | pyharp or gradio itself is broken | -| **Spaces** (default) | Every Hugging Face Space under `teamup-tech` | That specific deployment is broken (build error, HF platform change, dependency drift, ...) | +Two tiers share the same harness: -Models are validated **independently**: each model (and each test case within -it) runs inside its own error boundary, so a crash, hang, or timeout in one -model never stops validation of the others. All results are collected into a -single report and the process exits non-zero only after everything has run. +| Tier | Command | What it validates | What a failure means | +|---|---|---|---| +| **Examples** | `--local-examples` | The apps under `pyharp/examples/`, launched locally | pyharp or gradio itself is broken | +| **Spaces** | (default) | Every Hugging Face Space under `teamup-tech` | That specific deployment is broken (build error, HF platform change, dependency drift, ...) | -A daily GitHub Action ([model_validation.yml](../.github/workflows/model_validation.yml)) -runs both tiers at 06:30 UTC and publishes the results to the run summary and -as artifacts. Model failures do not turn the run red (no notification emails); -check the run summary for results. +Key behaviors: + +- **Models are validated independently.** Each model (and each test case + within it) runs inside its own error boundary, so a crash, hang, or + timeout in one model never stops validation of the others. All results are + collected into a single report and the process exits non-zero only after + everything has run. +- **Crashed/stopped spaces are restarted automatically** (sleeping spaces + are woken simply by connecting). Pass `--no-restart-failed` to disable; + restarting requires a token with write access. +- **ZeroGPU quota is reported** at the start of a spaces run and after every + model, so it is easy to tell if and when quota will be exceeded mid-run. +- A daily GitHub Action + ([model_validation.yml](../.github/workflows/model_validation.yml)) runs + both tiers at 06:30 UTC. Model failures do **not** turn the run red (no + notification emails while deployments stabilize) — results live in the + run summary and report artifacts. + +## Code layout + +| File | Purpose | +|---|---| +| [validate_models.py](validate_models.py) | Command-line entry point and per-tier orchestration | +| [harness.py](harness.py) | Core endpoint tests; space and local-example drivers | +| [cases.py](cases.py) | Input synthesis, test-case overlay, output validation/inspection | +| [validators.py](validators.py) | Registry of custom output validators (extend this) | +| [assets.py](assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | +| [quota.py](quota.py) | ZeroGPU usage tracking and account quota lookup | +| [results.py](results.py) | Result records and JSON/markdown report generation | +| [utils.py](utils.py) | Token handling, config loading, discovery, timeouts | +| [config.yml](config.yml) | Validation configuration (excludes, per-model test cases) | +| [test_data/](test_data/) | Real input files referenced by test cases | ## Token setup (IMPORTANT — read this) @@ -33,9 +59,9 @@ the `HF_TOKEN` environment variable. (Settings → Secrets and variables → Actions). GitHub masks secrets in logs. - **Locally:** `export HF_TOKEN=...` in your shell (consider `read -s` so it stays out of shell history). -- Use a [fine-grained token](https://huggingface.co/settings/tokens) scoped to - read access on the org's spaces. Grant write access only if you want - `--restart-failed` / the workflow's restart option to work. +- Use a [fine-grained token](https://huggingface.co/settings/tokens) with + write access to the org's spaces (needed for the default auto-restart of + crashed spaces; read access suffices with `--no-restart-failed`). - **If a token is ever exposed, rotate it immediately** at https://huggingface.co/settings/tokens. @@ -44,59 +70,90 @@ the `HF_TOKEN` environment variable. ```bash pip install -r model_validation/requirements.txt -# All spaces in the org +# All spaces in the org (crashed/stopped spaces are restarted by default) export HF_TOKEN=hf_... # see token setup above python model_validation/validate_models.py -# A single space, with verbose errors +# A single space, with verbose errors — the go-to while developing a test case python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter --verbose # Exclude specific models (also configurable via `exclude` in config.yml) python model_validation/validate_models.py --exclude teamup-tech/broken-space # Availability + /controls only (fast; no inference, no GPU quota used) -python model_validation/validate_models.py --skip-process +python model_validation/validate_models.py --load-only -# Try to restart crashed/stopped spaces first (token needs write access) -python model_validation/validate_models.py --restart-failed +# Never restart spaces (works with a read-only token) +python model_validation/validate_models.py --no-restart-failed -# Baseline tier (needs pyharp + example deps; no token required) +# Examples tier (needs pyharp + example deps; no token required) pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu pip install -e ./pyharp python model_validation/validate_models.py --local-examples ``` Reports land in `reports/` as `report.json` (machine-readable) and -`report.md` (human-readable table). Exit code is `0` when everything passes, -`1` on any model failure, `2` on configuration/infrastructure errors. +`report.md` (human-readable table); local example logs are saved alongside +them. Exit code is `0` when everything passes, `1` on any model failure, +`2` on configuration/infrastructure errors. -## Excluding models +## ZeroGPU quota reporting -Two equivalent ways, merged together: +Space validation prints the quota state at the start of the run and after +every model, on the same line as its pass/fail status: -- `exclude` in [config.yml](config.yml) — for permanent exclusions - (non-HARP spaces, archived deployments, known-broken examples). -- `--exclude ...` on the command line — for ad-hoc runs. +``` +✅ PASS teamup-tech/pitch_shifter (42.1s) [GPU time ~38s/1500s budget | ...] +``` -Use the space id (`teamup-tech/some-space`) for remote models and -`local/` (e.g. `local/midi_synthesizer`) for baseline examples. +Two signals are combined: + +- **GPU time this run** — cumulative `/process` wall time on ZeroGPU-hardware + spaces only (models on CPU or dedicated hardware do not draw on the + allowance and are never counted). It is an upper bound on GPU seconds + consumed, since it includes queue time. Set `zerogpu_budget_seconds` in + [config.yml](config.yml) (e.g. `1500` for the PRO 25 min/day allowance) + to show usage against a budget. +- **Account quota** — fetched from `huggingface.co/api/quota` when available. + Hugging Face has no documented public ZeroGPU quota API, so this part is + best-effort and silently omitted if the endpoint yields nothing usable. + +## Configuring test cases — a walkthrough + +Every model automatically gets one **`default`** test case, even with no +configuration at all: inputs are synthesized from the model's `/controls` +spec — a sine-sweep WAV for audio tracks, a two-note MIDI file for MIDI +tracks, and each control's declared default value for sliders, toggles, +dropdowns, number boxes, and text boxes. So configuration is only needed to +go beyond that: pinning specific control values, feeding real audio, or +inspecting outputs more deeply. -## Per-model test cases +### Step 1: find the model's control labels -By default every model gets one `default` test case with inputs synthesized -from its `/controls` spec (a sine-sweep WAV for audio tracks, a two-note MIDI -file for MIDI tracks, declared default values for sliders/toggles/etc.). +Test cases reference inputs and outputs by their **label** — the display +name each gradio component was given in the model's `app.py`. Three ways to +find them: -To validate deployment-specific behavior, add named cases in -[config.yml](config.yml). Cases override control values by their **label** and -can substitute custom input files (checked into `model_validation/test_data/`): +- Open the Space's gradio UI and read the component titles, or click its + "View Controls" button to see the full spec as JSON; +- Read the model's `app.py` (each component has a `label=...`); +- Reference a wrong label on purpose and run the validator — the error + message lists every available label. + +### Step 2: add the case to config.yml + +Cases live under `overrides:` keyed by space id (or `examples/` +for local examples). Each case has a `name` plus any of: `controls` (override +scalar values by label), `files` (substitute input files by label, paths +relative to config.yml — commit real inputs to `test_data/`), and a +per-case `process_timeout`. ```yaml overrides: teamup-tech/pitch_shifter: process_timeout: 900 # this model is slow; extend the timeout test_cases: - - name: default # keep the synthesized case + - name: default # keep the synthesized case too - name: extreme-shift controls: "Pitch Shift (semitones)": 24 @@ -105,14 +162,94 @@ overrides: "Input Audio A": test_data/short_vocal.wav ``` -Local examples use the key `local/`, e.g. `local/pitch_shifter`. -See the comment block at the top of config.yml for the full schema. +Note: once `test_cases` is present, **only** the listed cases run — include +`- name: default` to keep the synthesized one. + +### Step 3: check the outputs (optional, two levels) + +Every case already gets structural checks for free: `/process` must not +error, file outputs must exist and be non-empty, and JSON outputs (e.g. an +optional pyharp `LabelList`) must be well-formed when present (absent/None +is valid, since labels are optional). + +**Level 1 — declarative `expect` rules**, simple per-output assertions: + +```yaml + - name: extreme-shift + controls: + "Pitch Shift (semitones)": 24 + expect: + "Output Audio": + ext: .wav # downloaded file must have this extension + min_bytes: 10000 # ... and be at least this many bytes +``` + +**Level 2 — custom validators**, arbitrary Python registered in +[validators.py](validators.py) and referenced by name: + +```yaml + - name: extreme-shift + validator: wav_not_silent + min_rms_db: -40 # extra case keys parameterize the validator +``` + +A validator receives `(outputs, controls, case)` — `outputs` maps output +labels to local file paths (file outputs) or decoded objects (JSON +outputs) — and raises `AssertionError` with a helpful message on failure. +Three ship out of the box: + +- `wav_not_silent` — asserts WAV outputs carry signal, measured as RMS in + dBFS (0 dBFS = full scale). The default threshold of `-60` dBFS rejects + digital silence and near-silence; raise it (e.g. `min_rms_db: -40`) to + demand typical program levels. +- `wav_format` — checks `channels`, `sample_rate`, and/or `min_duration` on + WAV outputs — e.g. assert mono output at 44100 Hz and at least 1.5s long. +- `has_labels` — asserts a non-empty pyharp `LabelList` was returned + (optionally `min_labels`). + +To add one: + +```python +@validator("midi_not_empty") +def midi_not_empty(outputs, controls, case): + for label, value in outputs.items(): + if isinstance(value, str) and value.lower().endswith((".mid", ".midi")): + assert os.path.getsize(value) > 50, f"'{label}' looks empty" +``` + +### Step 4: run just that model to iterate + +```bash +python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter --verbose +``` + +The console shows each case's pass/fail; `reports/report.json` has per-case +timings and error details. Once green, the daily CI run picks the case up +automatically — no workflow changes needed. + +### Reference: full config.yml schema + +See the comment block at the top of [config.yml](config.yml) for the +complete schema in one place: `zerogpu_budget_seconds`, `exclude`, +`include_extra`, and per-model `overrides` (`connect_timeout`, +`process_timeout`, `load_only`, `test_cases`). + +## Excluding models + +Two equivalent ways, merged together: + +- `exclude` in [config.yml](config.yml) — for permanent exclusions + (non-HARP spaces, archived deployments, known-broken examples). +- `--exclude ...` on the command line — for ad-hoc runs. + +Use the space id (`teamup-tech/some-space`) for remote models and +`examples/` (e.g. `examples/midi_synthesizer`) for local examples. ## CI behavior - **Schedule:** daily at 06:30 UTC; also runnable manually from the Actions - tab (with options to validate specific spaces, skip inference, or - auto-restart crashed spaces). + tab (with options to validate specific spaces, skip inference, or disable + the default restart of crashed spaces). - **Reports:** markdown summary on each run + JSON/markdown artifacts. - **Failure signal:** model failures keep the run green (deployments are not yet stable enough for daily failure emails to be useful) — they show up as @@ -123,5 +260,5 @@ See the comment block at the top of config.yml for the full schema. - **Opting back into notifications later:** remove the exit-code handling in the two `Validate` steps of the workflow so the script's exit code 1 propagates; failed runs then turn red and GitHub emails maintainers. -- Baseline failures indicate a pyharp/gradio-level breakage that likely +- Example failures indicate a pyharp/gradio-level breakage that likely affects every deployment — fix those before debugging individual spaces. diff --git a/model_validation/assets.py b/model_validation/assets.py new file mode 100644 index 00000000..85911e3b --- /dev/null +++ b/model_validation/assets.py @@ -0,0 +1,123 @@ +""" +Synthesized test inputs for HARP model validation. + +Every input file is generated from scratch with the standard library, so +validation needs no binary fixtures checked into the repository. Real-world +inputs for specific models belong in test_data/ and are referenced from a +test case's `files` entry in config.yml. +""" + +import math +import struct +import wave +from pathlib import Path + + +__all__ = [ + 'Assets', + 'make_test_wav', + 'make_test_midi' +] + + +def make_test_wav(path: Path, duration: float = 2.0, sr: int = 44100) -> Path: + """ + Write a short mono 16-bit sine sweep - a valid input for any audio model. + + Args: + path (Path): Destination .wav path. + duration (float): Length of the sweep in seconds. + sr (int): Sample rate in Hz. + + Returns: + path (Path): The written file, for chaining. + """ + + n = int(duration * sr) + + with wave.open(str(path), "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(sr) + frames = bytearray() + for i in range(n): + t = i / sr + # Sweep from 220 Hz up one octave over the clip + freq = 220.0 + 440.0 * t / duration + sample = int(0.5 * 32767 * math.sin(2 * math.pi * freq * t)) + frames += struct.pack(" Path: + """ + Write a minimal standard MIDI file (format 0, two quarter notes). + + Args: + path (Path): Destination .mid path. + + Returns: + path (Path): The written file, for chaining. + """ + + track_events = bytes([ + 0x00, 0xC0, 0x00, # program change: acoustic grand + 0x00, 0x90, 0x3C, 0x64, # note on C4 + 0x83, 0x60, 0x80, 0x3C, 0x40, # note off C4 after 480 ticks + 0x00, 0x90, 0x40, 0x64, # note on E4 + 0x83, 0x60, 0x80, 0x40, 0x40, # note off E4 after 480 ticks + 0x00, 0xFF, 0x2F, 0x00, # end of track + ]) + header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, 480) + track = b"MTrk" + struct.pack(">I", len(track_events)) + track_events + path.write_bytes(header + track) + + return path + + +class Assets: + """ + Synthesized test input files, generated once and shared across all + model validations in a run. + """ + + def __init__(self, workdir: Path): + workdir.mkdir(parents=True, exist_ok=True) + self.wav = make_test_wav(workdir / "test_input.wav") + self.midi = make_test_midi(workdir / "test_input.mid") + self.text = workdir / "test_input.txt" + self.text.write_text("HARP model validation\n") + self.json = workdir / "test_input.json" + self.json.write_text("{}\n") + + def for_file_types(self, file_types: list) -> Path | None: + """ + Pick a synthesized file whose format actually matches the accepted + types. Only extensions we can genuinely produce are matched - e.g. + a component accepting only {".mp3", ".flac"} gets None (we cannot + synthesize those with the stdlib), NOT a mislabeled WAV. Supply a + real file via a test case's `files` entry in config.yml instead. + + Args: + file_types (list): Accepted extensions from the /controls spec; + empty or None means any file is accepted. + + Returns: + path (Path | None): A matching synthesized file, or None when + no synthesized format satisfies the component. + """ + + types = {str(t).lower() for t in (file_types or [])} + + if not types or ".wav" in types or "audio" in types: + return self.wav + if types & {".mid", ".midi"}: + return self.midi + if ".json" in types: + return self.json + if types & {".txt", ".text", "text"}: + return self.text + + return None diff --git a/model_validation/cases.py b/model_validation/cases.py new file mode 100644 index 00000000..82973a4f --- /dev/null +++ b/model_validation/cases.py @@ -0,0 +1,265 @@ +""" +Test-case handling for HARP model validation. + +Covers the full input/output cycle of one /process test case: +synthesizing default inputs from a model's /controls spec, overlaying a +configured case's control values and input files, and checking the outputs +(structural validation plus optional per-case `expect` rules and custom +validators). +""" + +import os +from pathlib import Path + +from gradio_client import handle_file + +from assets import Assets +from validators import VALIDATORS + + +__all__ = [ + 'synthesize_default_args', + 'apply_case', + 'validate_outputs', + 'inspect_outputs' +] + + +def synthesize_default_args(controls: dict, assets: Assets) -> tuple: + """ + Build the positional argument list for /process from the /controls spec. + + Args: + controls (dict): The /controls payload (card, inputs, outputs). + assets (Assets): Synthesized input files to draw from. + + Returns: + args (list): One argument per input component, in declaration order. + missing (dict): Label -> reason for every input that could NOT be + synthesized. Such inputs get a None placeholder; a test case can + still run if its controls/files overrides cover them all. + """ + + args, missing = [], {} + + for spec in controls.get("inputs", []): + ctype = spec.get("type") + label = spec.get("label") + + if ctype == "audio_track": + args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) + elif ctype == "midi_track": + args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) + elif ctype == "generic_file": + path = assets.for_file_types(spec.get("file_types")) + if path is None and spec.get("required", True): + missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" + args.append(handle_file(str(path)) if path is not None else None) + elif ctype in ("slider", "number_box"): + value = spec.get("value") + if value is None: + value = spec.get("minimum", 0) + args.append(value) + elif ctype == "text_box": + args.append(spec.get("value") or "test") + elif ctype == "toggle": + args.append(bool(spec.get("value", False))) + elif ctype == "dropdown": + value = spec.get("value") + if value is None: + choices = spec.get("choices") or [] + if not choices: + missing[label] = "dropdown has no choices" + else: + # Choices arrive as [label, value] pairs or plain values + first = choices[0] + value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first + args.append(value) + else: + missing[label] = f"unsupported input control type '{ctype}'" + args.append(None) + + return args, missing + + +def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: + """ + Overlay a configured test case onto the default argument list. + + Args: + args (list): Default arguments from synthesize_default_args(). + controls (dict): The /controls payload, used to match labels. + case (dict): Test case entry from config.yml. May contain: + controls: {: } - override scalar values. + files: {: } - override track/file inputs + (paths relative to config.yml). + config_dir (Path): Directory containing config.yml, the base for + relative file paths. + + Returns: + args (list): A new argument list with the overrides applied. + + Raises: + ValueError: If a label does not match any input, or a file is missing. + """ + + labels = [spec.get("label") for spec in controls.get("inputs", [])] + args = list(args) + + for label, value in (case.get("controls") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"control '{label}' (available: {labels})") + args[labels.index(label)] = value + + for label, rel_path in (case.get("files") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"input '{label}' (available: {labels})") + path = (config_dir / rel_path).resolve() + if not path.exists(): + raise ValueError(f"test case '{case.get('name')}': file not found: {path}") + args[labels.index(label)] = handle_file(str(path)) + + return args + + +def as_output_list(result, specs: list) -> list: + """ + Normalize a /process result to one value per output spec. + + With a single output the raw value is used as-is, so a JSON output + returning a list is not mistaken for multiple outputs. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (list): One value per output spec. + """ + + if len(specs) <= 1: + return [result] + + return list(result) if isinstance(result, (list, tuple)) else [result] + + +def validate_outputs(result, controls: dict) -> str | None: + """ + Structurally check the outputs every model must satisfy. + + File outputs must be present, on disk, and non-empty. JSON outputs carry + optional data such as a pyharp LabelList: None/absent is valid (labels + are optional), but a present value must be JSON-shaped and a LabelList + must be well-formed. + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + + Returns: + error (str | None): A description of the first problem found, or + None when the outputs look structurally sound. + """ + + specs = controls.get("outputs", []) + outputs = as_output_list(result, specs) + + if len(specs) > 1 and len(outputs) != len(specs): + return f"expected {len(specs)} outputs, got {len(outputs)}" + + for spec, out in zip(specs, outputs): + if spec.get("type") == "json": + if out is None: + continue + if not isinstance(out, (dict, list)): + return (f"JSON output '{spec.get('label')}' is not valid JSON " + f"data: {str(out)[:100]}") + if isinstance(out, dict) and "labels" in out and \ + not isinstance(out["labels"], list): + return f"label list output '{spec.get('label')}' is malformed" + continue + + if out is None: + return f"output '{spec.get('label')}' is None" + + # gradio_client downloads file outputs and returns local paths + path = out.get("path") if isinstance(out, dict) and "path" in out else out + if isinstance(path, str) and os.path.sep in path and os.path.exists(path): + if os.path.getsize(path) == 0: + return f"output file for '{spec.get('label')}' is empty" + + return None + + +def outputs_by_label(result, specs: list) -> dict: + """ + Map output labels to values for inspection by validators. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (dict): Label -> local file path (file outputs) or decoded + object (JSON outputs). + """ + + outputs = as_output_list(result, specs) + mapped = {} + + for spec, out in zip(specs, outputs): + value = out.get("path") if isinstance(out, dict) and "path" in out else out + mapped[spec.get("label")] = value + + return mapped + + +def inspect_outputs(result, controls: dict, case: dict) -> None: + """ + Apply a test case's deeper output checks (see README.md). + + Two mechanisms, both optional per case: + expect: declarative per-output rules ({ext, min_bytes}). + validator: name of a custom function registered in + validators.py; extra case keys parameterize it. + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + case (dict): Test case entry from config.yml. + + Raises: + AssertionError: If an expectation or validator check fails. + ValueError: If the case references an unknown output or validator. + """ + + out_map = outputs_by_label(result, controls.get("outputs", [])) + + for label, rules in (case.get("expect") or {}).items(): + if label not in out_map: + raise ValueError(f"expect references unknown output '{label}' " + f"(available: {list(out_map)})") + value = out_map[label] + + ext = rules.get("ext") + if ext and not (isinstance(value, str) and value.lower().endswith(ext.lower())): + raise AssertionError(f"output '{label}' is not a {ext} file: {value}") + + min_bytes = rules.get("min_bytes") + if min_bytes is not None: + if not (isinstance(value, str) and os.path.exists(value)): + raise AssertionError(f"output '{label}' is not a file on disk: {value}") + size = os.path.getsize(value) + if size < min_bytes: + raise AssertionError(f"output file '{label}' is {size} bytes, " + f"expected at least {min_bytes}") + + name = case.get("validator") + if name: + if name not in VALIDATORS: + raise ValueError(f"unknown validator '{name}' (available: " + f"{sorted(VALIDATORS)}); register it in " + f"validators.py") + VALIDATORS[name](out_map, controls, case) diff --git a/model_validation/config.yml b/model_validation/config.yml index 00ee2879..a2c195a7 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -1,24 +1,29 @@ # Configuration for model_validation/validate_models.py # -# All keys are optional. Spaces not listed here are still discovered and -# validated automatically with a single synthesized "default" test case. +# All keys are optional. Models not listed here are still discovered and +# validated automatically with a single synthesized "default" test case, so +# an entry is only needed to change a model's settings or add test cases. # # Schema: # +# zerogpu_budget_seconds: 1500 # optional ZeroGPU budget for quota +# # reporting (e.g. 1500 = PRO 25 min/day); +# # unset = report usage without a budget +# # exclude: # models to exclude from validation: space ids, or -# # "local/" for local pyharp examples +# # "examples/" for local pyharp examples # - teamup-tech/some-non-harp-space -# - local/some_example +# - examples/some_example # # include_extra: # spaces outside the org that should also be validated # - some-user/some-harp-space # # overrides: # per-model settings; keys are space ids, or -# # "local/" for local pyharp examples +# # "examples/" for local pyharp examples # : # connect_timeout: 600 # seconds to wait for build/wake (default 420) # process_timeout: 900 # seconds to wait for /process (default 600) -# skip_process: true # only check availability + /controls +# load_only: true # only check availability + /controls # test_cases: # named /process test cases; when omitted, a # # single synthesized "default" case runs # - name: default # empty case == synthesized defaults @@ -30,6 +35,13 @@ # files: # override track/file inputs by label; # # paths are relative to this file # "Input Audio": test_data/my_clip.wav +# expect: # declarative output checks by label +# "Output Audio": +# ext: .wav # downloaded file must have this ext +# min_bytes: 1000 # ... and be at least this many bytes +# validator: wav_not_silent # custom check registered in +# # validators.py; extra case keys +# # (e.g. min_rms_db) parameterize it exclude: [] @@ -37,17 +49,20 @@ include_extra: [] overrides: - # ---- Baseline: local pyharp examples ------------------------------------- + # ---- Local pyharp examples ---------------------------------------------- # These run with `--local-examples` and exercise pyharp itself with no # Hugging Face infrastructure involved. A failure here indicates a # pyharp/gradio-level breakage rather than a deployment-specific one. + # Examples are discovered automatically; entries are only needed to add + # test cases beyond the synthesized default. - local/pitch_shifter: + examples/pitch_shifter: test_cases: - name: default - name: shift-up-octave controls: "Pitch Shift (semitones)": 12 + validator: wav_not_silent - name: shift-down-octave controls: "Pitch Shift (semitones)": -12 @@ -55,7 +70,7 @@ overrides: controls: "Pitch Shift (semitones)": 0 - local/midi_pitch_shifter: + examples/midi_pitch_shifter: test_cases: - name: default - name: shift-up-max @@ -65,14 +80,6 @@ overrides: controls: "Pitch Shift (semitones)": -24 - local/midi_synthesizer: - test_cases: - - name: default - - local/ui_tester: - test_cases: - - name: default - # ---- Remote space overrides --------------------------------------------- # Add per-space timeouts and extra test cases here as needed, e.g.: # @@ -83,3 +90,11 @@ overrides: # - name: extreme-shift # controls: # "Pitch Shift (semitones)": 24 + # expect: + # "Output Audio": + # ext: .wav + # min_bytes: 10000 + # validator: wav_format + # channels: 1 + # sample_rate: 44100 + # min_duration: 1.5 diff --git a/model_validation/harness.py b/model_validation/harness.py new file mode 100644 index 00000000..8fac735d --- /dev/null +++ b/model_validation/harness.py @@ -0,0 +1,346 @@ +""" +The core validation harness for HARP model deployments. + +Both validation tiers funnel into run_endpoint_tests(), which performs the +identical black-box checks against any live HARP gradio app: + + 1. the /controls and /process endpoints exist; + 2. /controls returns a well-formed model card and component spec; + 3. every configured /process test case produces valid outputs. + +test_space() wraps this for a remote Hugging Face Space (runtime stage +checks, optional restart, connection with wake-up retries) and +test_local_example() for a pyharp example app launched locally. +""" + +import argparse +import os +import subprocess +import sys +import time +import traceback +import urllib.request +from pathlib import Path + +from gradio_client import Client + +from assets import Assets +from cases import synthesize_default_args, apply_case, validate_outputs, inspect_outputs +from results import ModelResult, CaseResult, PASS, FAIL +from utils import run_with_timeout, scrub + + +__all__ = [ + 'test_space', + 'test_local_example' +] + + +# Space runtime stages that indicate a hard failure (no point connecting) +DEAD_STAGES = {"BUILD_ERROR", "RUNTIME_ERROR", "CONFIG_ERROR", "DELETING"} +# Stages that resolve on their own if we wait (or wake on first request) +TRANSIENT_STAGES = {"BUILDING", "RUNNING_BUILDING", "APP_STARTING", "SLEEPING"} + + +def close_client(client) -> None: + """ + Shut down a gradio_client instance without letting cleanup errors + (or hangs in its heartbeat machinery) affect the result. + + Args: + client (Client | None): The client to close; None is a no-op. + """ + + if client is None: + return + + try: + run_with_timeout(client.close, 10, "client close") + except Exception: # noqa: BLE001 - cleanup is best-effort + pass + + +def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, + overrides: dict, opts: argparse.Namespace) -> ModelResult: + """ + Verify /controls and run every configured /process test case. + + Each case runs inside its own error boundary, so a failing case never + stops the remaining cases (or models) from running. + + Args: + client (Client): Connected gradio client for the deployment. + result (ModelResult): Result record to fill in. + assets (Assets): Synthesized input files. + overrides (dict): This model's entry from config.yml `overrides`. + opts (argparse.Namespace): Parsed command-line options. + + Returns: + result (ModelResult): The same record, completed. + """ + + process_timeout = overrides.get("process_timeout", opts.process_timeout) + load_only = opts.load_only or overrides.get("load_only", False) + config_dir = opts.config.parent.resolve() + + # --- Endpoint presence --------------------------------------------------- + endpoints = client.view_api(return_format="dict", print_info=False) or {} + named = endpoints.get("named_endpoints", {}) + if "/controls" not in named or "/process" not in named: + result.error = (f"missing HARP endpoints (found: {sorted(named)}); " + f"deployment may use an outdated pyharp") + return result + + # --- /controls ----------------------------------------------------------- + controls = run_with_timeout( + lambda: client.predict(api_name="/controls"), 120, "/controls") + if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: + result.error = f"/controls returned malformed data: {str(controls)[:200]}" + return result + result.controls_ok = True + result.model_name = controls.get("card", {}).get("name", "") + + if load_only: + result.status = PASS + return result + + # --- /process test cases ------------------------------------------------- + default_args, missing = synthesize_default_args(controls, assets) + cases = overrides.get("test_cases") or [{"name": "default"}] + + for case in cases: + case_result = CaseResult(name=case.get("name", "unnamed")) + result.cases.append(case_result) + case_start = time.time() + try: + # Inputs we couldn't synthesize are fine if this case supplies them + supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) + unsatisfied = {k: v for k, v in missing.items() if k not in supplied} + if unsatisfied: + case_result.error = "skipped: " + "; ".join( + f"'{k}': {v}" for k, v in unsatisfied.items()) + continue + + args = apply_case(default_args, controls, case, config_dir) + job = client.submit(*args, api_name="/process") + output = job.result(timeout=case.get("process_timeout", process_timeout)) + + error = validate_outputs(output, controls) + if error: + case_result.ok = False + case_result.error = f"/process output invalid: {error}" + else: + inspect_outputs(output, controls, case) + case_result.ok = True + except Exception as exc: # noqa: BLE001 - any exception fails the case + case_result.ok = False + case_result.error = f"{type(exc).__name__}: {exc}" + finally: + case_result.duration = round(time.time() - case_start, 1) + + if any(c.ok is False for c in result.cases): + result.error = "; ".join( + f"[{c.name}] {c.error}" for c in result.cases if c.ok is False) + result.status = FAIL + else: + result.status = PASS + + return result + + +# --------------------------------------------------------------------------- +# Remote Hugging Face Spaces +# --------------------------------------------------------------------------- + +def wait_for_runtime(api, space_id: str, deadline: float): + """ + Poll a space's runtime until its stage settles or the deadline passes. + + Args: + api (HfApi): Authenticated Hugging Face API client. + space_id (str): The space to poll. + deadline (float): time.time() value to stop polling at. + + Returns: + runtime (SpaceRuntime): The last observed runtime (stage, hardware). + """ + + runtime = api.get_space_runtime(space_id) + + # SLEEPING resolves on first request rather than by waiting + while runtime.stage in TRANSIENT_STAGES - {"SLEEPING"} and time.time() < deadline: + time.sleep(10) + runtime = api.get_space_runtime(space_id) + + return runtime + + +def test_space(space_id: str, token: str, assets: Assets, + opts: argparse.Namespace, overrides: dict) -> ModelResult: + """ + Validate one remote Hugging Face Space end-to-end. + + Args: + space_id (str): The space to validate (e.g. "teamup-tech/foo"). + token (str): Hugging Face access token. + assets (Assets): Synthesized input files. + opts (argparse.Namespace): Parsed command-line options. + overrides (dict): This space's entry from config.yml `overrides`. + + Returns: + result (ModelResult): The completed validation record. + """ + + # Imported lazily so the examples tier works without huggingface_hub + from huggingface_hub import HfApi + + result = ModelResult(target=space_id, kind="space") + start = time.time() + api = HfApi(token=token) + connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) + client = None + + try: + # --- Runtime stage (restart crashed/stopped spaces by default) ------- + deadline = time.time() + connect_timeout + runtime = wait_for_runtime(api, space_id, deadline) + stage = runtime.stage + result.stage = stage + # requested_hardware reflects the space's configuration even while + # it is sleeping or stopped (hardware itself is only set when live) + result.hardware = runtime.requested_hardware or runtime.hardware or "" + + if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): + if opts.restart_failed and stage != "DELETING": + print(f" [{space_id}] stage={stage}, requesting restart...") + try: + api.restart_space(space_id) + except Exception as exc: # noqa: BLE001 - e.g. read-only token + result.error = scrub( + f"space is not running (stage={stage}) and restart " + f"failed: {type(exc).__name__}: {exc}", token) + return result + runtime = wait_for_runtime(api, space_id, time.time() + connect_timeout) + stage = runtime.stage + result.stage = stage + if stage not in ("RUNNING", "SLEEPING"): + result.error = f"space is not running (stage={stage})" + return result + + # --- Connect (wakes sleeping spaces; retry through the wake-up) ------ + last_exc = None + deadline = time.time() + connect_timeout + while time.time() < deadline: + try: + client = run_with_timeout( + lambda: Client(space_id, hf_token=token, verbose=False), + max(10.0, deadline - time.time()), "connect") + break + except Exception as exc: # noqa: BLE001 - space may still be waking + last_exc = exc + time.sleep(15) + if client is None: + raise RuntimeError(f"could not connect: {last_exc}") + + return run_endpoint_tests(client, result, assets, overrides, opts) + + except Exception as exc: # noqa: BLE001 - any failure means invalid + result.error = scrub(f"{type(exc).__name__}: {exc}", token) + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + close_client(client) + + +# --------------------------------------------------------------------------- +# Local pyharp examples +# --------------------------------------------------------------------------- + +def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> None: + """ + Block until a locally launched gradio app starts serving. + + Args: + port (int): The port the app was told to bind (GRADIO_SERVER_PORT). + proc (subprocess.Popen): The app process, watched for early exit. + timeout (float): Seconds to wait before giving up. + + Raises: + RuntimeError: If the app process exits before serving. + TimeoutError: If the app is not serving within the timeout. + """ + + deadline = time.time() + timeout + url = f"http://127.0.0.1:{port}/config" + + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"app exited early with code {proc.returncode}") + try: + with urllib.request.urlopen(url, timeout=5): + return + except Exception: # noqa: BLE001 - server not up yet + time.sleep(2) + + raise TimeoutError(f"local app did not become ready within {int(timeout)}s") + + +def test_local_example(app_dir: Path, port: int, assets: Assets, + opts: argparse.Namespace, overrides: dict) -> ModelResult: + """ + Validate one pyharp example app by launching it locally. + + The app's stdout/stderr are captured to /.log for + debugging failures. + + Args: + app_dir (Path): Example directory containing app.py. + port (int): Local port to launch the app on. + assets (Assets): Synthesized input files. + opts (argparse.Namespace): Parsed command-line options. + overrides (dict): This example's entry from config.yml `overrides` + (keyed "examples/"). + + Returns: + result (ModelResult): The completed validation record. + """ + + target = f"examples/{app_dir.name}" + result = ModelResult(target=target, kind="local", stage="LOCAL") + start = time.time() + + env = os.environ.copy() + env["GRADIO_SERVER_NAME"] = "127.0.0.1" + env["GRADIO_SERVER_PORT"] = str(port) + env.pop("HF_TOKEN", None) # local examples must not need credentials + + log_path = opts.output_dir / f"{app_dir.name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + proc = None + client = None + + try: + with open(log_path, "w") as log: + proc = subprocess.Popen( + [sys.executable, "app.py"], cwd=app_dir, env=env, + stdout=log, stderr=subprocess.STDOUT) + wait_for_local_server(port, proc, overrides.get( + "connect_timeout", opts.connect_timeout)) + client = Client(f"http://127.0.0.1:{port}", verbose=False) + return run_endpoint_tests(client, result, assets, overrides, opts) + except Exception as exc: # noqa: BLE001 - any failure means invalid + result.error = f"{type(exc).__name__}: {exc} (see {log_path.name})" + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + close_client(client) + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() diff --git a/model_validation/quota.py b/model_validation/quota.py new file mode 100644 index 00000000..6d18bad4 --- /dev/null +++ b/model_validation/quota.py @@ -0,0 +1,135 @@ +""" +ZeroGPU quota tracking for HARP model validation. + +ZeroGPU allowances are consumed per account, so a long validation run can +exhaust the day's quota partway through. To make that visible, the quota +state is reported at the start of a run and after every model. +""" + +import json +import threading +import urllib.request + + +__all__ = [ + 'QuotaTracker', + 'fetch_account_quota', + 'is_zerogpu' +] + + +def is_zerogpu(hardware: str) -> bool: + """ + Whether a space's hardware consumes ZeroGPU quota. + + ZeroGPU hardware ids start with "zero" (e.g. "zero-a10g"); anything + else (cpu-basic, t4-small, ...) does not draw on the shared allowance. + + Args: + hardware (str): HF hardware id, possibly empty. + + Returns: + zerogpu (bool): True when the hardware is a ZeroGPU tier. + """ + + return bool(hardware) and hardware.lower().startswith("zero") + + +def fetch_account_quota(token: str) -> str | None: + """ + Best-effort fetch of the account's quota state from huggingface.co. + + Hugging Face has no *documented* public API for ZeroGPU quota; /api/quota + exists but its schema is unstable, so parse defensively: surface any + entries whose keys mention gpu/zero and return None when nothing useful + comes back. Never raises. + + Args: + token (str): Hugging Face access token. + + Returns: + summary (str | None): Compact "key=value" summary of GPU-related + quota entries, or None when unavailable. + """ + + if not token: + return None + + try: + req = urllib.request.Request( + "https://huggingface.co/api/quota", + headers={"Authorization": f"Bearer {token}"}) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode()) + except Exception: # noqa: BLE001 - quota reporting must never break a run + return None + + def gpu_entries(obj, prefix=""): + found = [] + if isinstance(obj, dict): + for key, value in obj.items(): + name = f"{prefix}{key}" + if any(s in key.lower() for s in ("zero", "gpu")) and \ + isinstance(value, (int, float, str)): + found.append(f"{name}={value}") + else: + found += gpu_entries(value, f"{name}.") + elif isinstance(obj, list): + for value in obj: + found += gpu_entries(value, prefix) + return found + + entries = gpu_entries(data) + + return ", ".join(entries[:4]) if entries else None + + +class QuotaTracker: + """ + Tracks ZeroGPU usage during a validation run. + + Two signals are combined into each status line: + - cumulative /process wall time this run on ZeroGPU spaces only + (CPU-hardware models do not draw on the allowance and are never + counted); an upper bound on GPU seconds consumed, since it + includes queue time, shown against an optional budget + (`zerogpu_budget_seconds` in config.yml); + - the account quota reported by huggingface.co, when available. + """ + + def __init__(self, token: str, budget: float | None): + self.token = token + self.budget = budget + self.used = 0.0 + self._lock = threading.Lock() + + def add(self, seconds: float) -> None: + """ + Record processing time consumed by a completed model validation. + + Args: + seconds (float): Wall time spent in /process calls. + """ + + with self._lock: + self.used += seconds + + def status(self) -> str: + """ + Format the current quota state for a console line. + + Returns: + status (str): e.g. "[GPU time ~38s/1500s budget | left=120s]". + """ + + with self._lock: + used = int(self.used) + + if self.budget: + usage = f"GPU time ~{used}s/{int(self.budget)}s budget" + else: + usage = f"GPU time ~{used}s this run" + + account = fetch_account_quota(self.token) + + return f"[{usage}]" if account is None else f"[{usage} | {account}]" diff --git a/model_validation/results.py b/model_validation/results.py new file mode 100644 index 00000000..5b58cdfe --- /dev/null +++ b/model_validation/results.py @@ -0,0 +1,111 @@ +""" +Result records and report generation for HARP model validation. +""" + +import dataclasses +import json +import time +from pathlib import Path + + +__all__ = [ + 'PASS', + 'FAIL', + 'SKIP', + 'CaseResult', + 'ModelResult', + 'status_emoji', + 'write_reports' +] + + +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" + + +@dataclasses.dataclass +class CaseResult: + """Outcome of a single /process test case.""" + + name: str + ok: bool | None = None # None => skipped + duration: float = 0.0 + error: str = "" + + +@dataclasses.dataclass +class ModelResult: + """Outcome of validating one model deployment.""" + + target: str # space id or "examples/" + kind: str = "space" # "space" | "local" + status: str = FAIL + stage: str = "" # HF runtime stage, or "LOCAL" + hardware: str = "" # HF hardware (e.g. "zero-a10g", "cpu-basic") + controls_ok: bool = False + cases: list = dataclasses.field(default_factory=list) + duration: float = 0.0 + error: str = "" + model_name: str = "" # from the model card + + +def status_emoji(result: ModelResult) -> str: + """ + Symbol used for a result in console output and reports. + + Args: + result (ModelResult): The result to represent. + + Returns: + emoji (str): One of the pass/fail/skip symbols. + """ + + return {PASS: "✅", FAIL: "❌", SKIP: "⏭️"}[result.status] + + +def write_reports(results: list, out_dir: Path, label: str) -> None: + """ + Write the machine-readable and human-readable reports. + + Args: + results (list): ModelResult objects for every validated model. + out_dir (Path): Directory receiving report.json and report.md. + label (str): Report heading (e.g. "teamup-tech spaces"). + """ + + out_dir.mkdir(parents=True, exist_ok=True) + + payload = { + "label": label, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "total": len(results), + "passed": sum(r.status == PASS for r in results), + "failed": sum(r.status == FAIL for r in results), + "results": [dataclasses.asdict(r) for r in results], + } + (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) + + lines = [ + f"# HARP Model Validation Report - {label}", + "", + f"**{payload['passed']}/{payload['total']} models passed** ({payload['timestamp']})", + "", + "| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", + "|---|---|---|---|---|---|---|---|", + ] + + # Failures first, then alphabetical, so problems are visible at a glance + for r in sorted(results, key=lambda r: (r.status != FAIL, r.target)): + if r.cases: + cases = ", ".join( + f"{c.name} {'✅' if c.ok else '⏭️' if c.ok is None else '❌'}" + for c in r.cases) + else: + cases = "—" + link = (f"[{r.target}](https://huggingface.co/spaces/{r.target})" + if r.kind == "space" else f"`{r.target}`") + detail = r.error.replace("|", "\\|")[:300] if r.error else "" + lines.append(f"| {link} | {status_emoji(r)} {r.status} | {r.stage} " + f"| {r.hardware or '—'} | {'✅' if r.controls_ok else '❌'} " + f"| {cases} | {r.duration} | {detail} |") + + (out_dir / "report.md").write_text("\n".join(lines) + "\n") diff --git a/model_validation/utils.py b/model_validation/utils.py new file mode 100644 index 00000000..78439b22 --- /dev/null +++ b/model_validation/utils.py @@ -0,0 +1,163 @@ +""" +Shared utilities for HARP model validation: credentials, configuration, +model discovery, and timeout handling. +""" + +import os +import sys +import threading +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None + + +__all__ = [ + 'get_token', + 'scrub', + 'run_with_timeout', + 'load_config', + 'get_excluded', + 'discover_spaces' +] + + +def get_token(required: bool) -> str: + """ + Read the Hugging Face access token from the HF_TOKEN environment variable. + + The token is intentionally never accepted as a command-line argument + (argv is visible in process listings) and must never be committed. + + Args: + required (bool): Exit with code 2 if the token is missing. + + Returns: + token (str): The token, or an empty string when absent and optional. + """ + + token = os.environ.get("HF_TOKEN", "").strip() + + if required and not token: + print("ERROR: HF_TOKEN environment variable is not set.", file=sys.stderr) + print("Set it locally (export HF_TOKEN=...) or as a GitHub Actions secret.", + file=sys.stderr) + sys.exit(2) + + return token + + +def scrub(text: str, token: str) -> str: + """ + Remove the token from any string that might get printed or reported. + + Args: + text (str): Text that may contain the token (e.g. an error message). + token (str): The token to redact; empty string disables scrubbing. + + Returns: + text (str): The text with any token occurrence replaced. + """ + + return text.replace(token, "***HF_TOKEN***") if token else text + + +def run_with_timeout(fn, timeout: float, what: str): + """ + Run fn() in a daemon thread; raise TimeoutError if it exceeds timeout. + + A daemon thread (not a ThreadPoolExecutor) is essential here: executor + workers are non-daemon and are joined at interpreter shutdown, so a hung + call leaked by a timeout would make the whole process hang after the + final summary prints. + + Args: + fn (callable): Zero-argument function to execute. + timeout (float): Seconds to wait before giving up. + what (str): Short description used in the timeout message. + + Returns: + result: Whatever fn() returns. + + Raises: + TimeoutError: If fn() does not finish within timeout seconds. + Exception: Whatever fn() raised, re-raised in the calling thread. + """ + + outcome = {} + + def target(): + try: + outcome["result"] = fn() + except BaseException as exc: # noqa: BLE001 - re-raised in caller + outcome["error"] = exc + + thread = threading.Thread(target=target, daemon=True, name=f"timeout-{what}") + thread.start() + thread.join(timeout) + + if thread.is_alive(): + raise TimeoutError(f"{what} timed out after {int(timeout)}s") + if "error" in outcome: + raise outcome["error"] + + return outcome.get("result") + + +def load_config(path: Path) -> dict: + """ + Load the validation configuration (see config.yml for the schema). + + Args: + path (Path): Path to the YAML configuration file. + + Returns: + config (dict): Parsed configuration; empty when the file is absent. + """ + + if not path.exists(): + return {} + + if yaml is None: + print(f"WARNING: PyYAML not installed; ignoring {path}", file=sys.stderr) + return {} + + return yaml.safe_load(path.read_text()) or {} + + +def get_excluded(config: dict, cli_exclude: list | None) -> set: + """ + Combine the models excluded from validation. + + Args: + config (dict): Parsed configuration (its `exclude` list is used). + cli_exclude (list | None): Models passed via --exclude, if any. + + Returns: + excluded (set): Model keys (space ids or "examples/"). + """ + + return set(config.get("exclude", [])) | set(cli_exclude or []) + + +def discover_spaces(api, org: str, config: dict, excluded: set) -> list: + """ + Enumerate the Hugging Face Spaces to validate. + + Args: + api (HfApi): Authenticated Hugging Face API client. + org (str): Organization whose spaces are discovered. + config (dict): Parsed configuration (`include_extra` adds spaces + outside the organization). + excluded (set): Model keys to leave out. + + Returns: + space_ids (list): Sorted space ids to validate. + """ + + spaces = [s.id for s in api.list_spaces(author=org)] + spaces += [s for s in config.get("include_extra", []) if s not in spaces] + + return sorted(s for s in spaces if s not in excluded) diff --git a/model_validation/validate_models.py b/model_validation/validate_models.py index 3c6f0f31..e689bd69 100644 --- a/model_validation/validate_models.py +++ b/model_validation/validate_models.py @@ -1,33 +1,36 @@ #!/usr/bin/env python3 """ -HARP model validation. - -Validates HARP deployments in two modes, using the same black-box harness: - -1. Remote Hugging Face Spaces (default): discovers all Spaces under an - organization (default: teamup-tech), verifies each is running, exposes the - HARP gradio endpoints (/controls and /process), and processes test inputs - end-to-end. ZeroGPU spaces require an authenticated request for GPU quota, - so a token must be provided via the HF_TOKEN environment variable - never - on the command line or in the repository. - -2. Baseline (--local-examples): launches each app under pyharp/examples/ on - a local port and runs the identical endpoint tests. The baseline tier +HARP model validation - command-line entry point. + +Validates HARP deployments in two tiers, using the same black-box harness +(see harness.py): + +1. Spaces (default): discovers all Spaces under an organization (default: + teamup-tech), verifies each is running (restarting crashed/stopped spaces + by default), exposes the HARP gradio endpoints (/controls and /process), + and processes test inputs end-to-end. ZeroGPU spaces require an + authenticated request for GPU quota, so a token must be provided via the + HF_TOKEN environment variable - never on the command line or in the + repository. ZeroGPU quota usage is reported at the start of the run and + after every model. + +2. Examples (--local-examples): launches each app under pyharp/examples/ on + a local port and runs the identical endpoint tests. The examples tier exercises pyharp itself with no Hugging Face infrastructure in the loop, so a failure indicates a pyharp/gradio-level breakage rather than a deployment-specific one. -Per-model test cases can be declared in config.yml (see README.md). When a -model has no configured cases, a single "default" case runs with inputs -synthesized automatically from the /controls spec. Models are validated -independently: a failure (or timeout) in one model never stops validation -of the others. +Per-model test cases are declared in config.yml (see README.md for a full +walkthrough). When a model has no configured cases, a single "default" case +runs with inputs synthesized automatically from the /controls spec. Models +are validated independently: a failure (or timeout) in one model never stops +validation of the others. Usage: HF_TOKEN=... python model_validation/validate_models.py HF_TOKEN=... python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter - HF_TOKEN=... python model_validation/validate_models.py --skip-process --workers 8 - HF_TOKEN=... python model_validation/validate_models.py --restart-failed + HF_TOKEN=... python model_validation/validate_models.py --load-only --workers 8 + HF_TOKEN=... python model_validation/validate_models.py --no-restart-failed python model_validation/validate_models.py --local-examples Exit codes: @@ -38,25 +41,15 @@ import argparse import concurrent.futures -import dataclasses -import json -import math import os -import struct -import subprocess import sys -import time -import traceback -import urllib.request -import wave from pathlib import Path -try: - import yaml -except ImportError: - yaml = None - -from gradio_client import Client, handle_file +from assets import Assets +from harness import test_space, test_local_example +from quota import QuotaTracker, is_zerogpu +from results import FAIL, status_emoji, write_reports +from utils import get_token, scrub, load_config, get_excluded, discover_spaces DEFAULT_ORG = "teamup-tech" @@ -64,570 +57,173 @@ DEFAULT_CONFIG = SCRIPT_DIR / "config.yml" DEFAULT_EXAMPLES_DIR = SCRIPT_DIR.parent / "pyharp" / "examples" -PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" - -# Space runtime stages that indicate a hard failure (no point connecting) -DEAD_STAGES = {"BUILD_ERROR", "RUNTIME_ERROR", "CONFIG_ERROR", "DELETING"} -# Stages that resolve on their own if we wait (or wake on first request) -TRANSIENT_STAGES = {"BUILDING", "RUNNING_BUILDING", "APP_STARTING", "SLEEPING"} - LOCAL_PORT_BASE = 7861 -def get_token(required: bool) -> str: - token = os.environ.get("HF_TOKEN", "").strip() - if required and not token: - print("ERROR: HF_TOKEN environment variable is not set.", file=sys.stderr) - print("Set it locally (export HF_TOKEN=...) or as a GitHub Actions secret.", - file=sys.stderr) - sys.exit(2) - return token - - -def scrub(text: str, token: str) -> str: - """Remove the token from any string that might get printed or reported.""" - return text.replace(token, "***HF_TOKEN***") if token else text - - -# --------------------------------------------------------------------------- -# Synthesized test assets -# --------------------------------------------------------------------------- - -def make_test_wav(path: Path, duration: float = 2.0, sr: int = 44100) -> Path: - """Write a short mono 16-bit sine sweep - a valid input for any audio model.""" - n = int(duration * sr) - with wave.open(str(path), "wb") as f: - f.setnchannels(1) - f.setsampwidth(2) - f.setframerate(sr) - frames = bytearray() - for i in range(n): - t = i / sr - freq = 220.0 + 440.0 * t / duration - sample = int(0.5 * 32767 * math.sin(2 * math.pi * freq * t)) - frames += struct.pack(" Path: - """Write a minimal standard MIDI file (format 0, two quarter notes).""" - track_events = bytes([ - 0x00, 0xC0, 0x00, # program change: acoustic grand - 0x00, 0x90, 0x3C, 0x64, # note on C4 - 0x83, 0x60, 0x80, 0x3C, 0x40, # note off C4 after 480 ticks - 0x00, 0x90, 0x40, 0x64, # note on E4 - 0x83, 0x60, 0x80, 0x40, 0x40, # note off E4 after 480 ticks - 0x00, 0xFF, 0x2F, 0x00, # end of track - ]) - header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, 480) - track = b"MTrk" + struct.pack(">I", len(track_events)) + track_events - path.write_bytes(header + track) - return path - - -AUDIO_EXTS = {".wav", ".mp3", ".flac", ".ogg", ".aif", ".aiff", ".m4a", "audio"} -MIDI_EXTS = {".mid", ".midi"} - - -class Assets: - """Synthesized test input files, shared across all model tests.""" - - def __init__(self, workdir: Path): - workdir.mkdir(parents=True, exist_ok=True) - self.wav = make_test_wav(workdir / "test_input.wav") - self.midi = make_test_midi(workdir / "test_input.mid") - self.text = workdir / "test_input.txt" - self.text.write_text("HARP model validation\n") - self.json = workdir / "test_input.json" - self.json.write_text("{}\n") - - def for_file_types(self, file_types: list) -> Path | None: - types = {str(t).lower() for t in (file_types or [])} - if not types or types & AUDIO_EXTS: - return self.wav - if types & MIDI_EXTS: - return self.midi - if ".json" in types: - return self.json - if ".txt" in types or ".text" in types: - return self.text - return None - - -# --------------------------------------------------------------------------- -# Input synthesis from the /controls spec + per-model test cases -# --------------------------------------------------------------------------- - -def synthesize_default_args(controls: dict, assets: Assets) -> tuple[list, dict]: - """ - Build the positional argument list for /process from the controls spec. - Returns (args, missing) where missing maps the label of every input we - could NOT synthesize to a reason. Such inputs get a None placeholder; a - test case can still run if its controls/files overrides cover them all. - """ - args, missing = [], {} - for spec in controls.get("inputs", []): - ctype = spec.get("type") - label = spec.get("label") - if ctype == "audio_track": - args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) - elif ctype == "midi_track": - args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) - elif ctype == "generic_file": - path = assets.for_file_types(spec.get("file_types")) - if path is None and spec.get("required", True): - missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" - args.append(handle_file(str(path)) if path is not None else None) - elif ctype in ("slider", "number_box"): - value = spec.get("value") - if value is None: - value = spec.get("minimum", 0) - args.append(value) - elif ctype == "text_box": - args.append(spec.get("value") or "test") - elif ctype == "toggle": - args.append(bool(spec.get("value", False))) - elif ctype == "dropdown": - value = spec.get("value") - if value is None: - choices = spec.get("choices") or [] - if not choices: - missing[label] = "dropdown has no choices" - else: - first = choices[0] - value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first - args.append(value) - else: - missing[label] = f"unsupported input control type '{ctype}'" - args.append(None) - return args, missing - - -def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: +def parse_args() -> argparse.Namespace: """ - Overlay a configured test case onto the default argument list. + Define and parse the command-line interface. - A case may contain: - controls: {: } - override scalar control values - files: {: } - override track/file inputs - (paths relative to config.yml) - Raises ValueError if a label does not match any input. + Returns: + opts (argparse.Namespace): Parsed options. """ - labels = [spec.get("label") for spec in controls.get("inputs", [])] - args = list(args) - - for label, value in (case.get("controls") or {}).items(): - if label not in labels: - raise ValueError(f"test case '{case.get('name')}' references unknown " - f"control '{label}' (available: {labels})") - args[labels.index(label)] = value - - for label, rel_path in (case.get("files") or {}).items(): - if label not in labels: - raise ValueError(f"test case '{case.get('name')}' references unknown " - f"input '{label}' (available: {labels})") - path = (config_dir / rel_path).resolve() - if not path.exists(): - raise ValueError(f"test case '{case.get('name')}': file not found: {path}") - args[labels.index(label)] = handle_file(str(path)) - - return args - - -def validate_outputs(result, controls: dict) -> str | None: - """Return an error string if the /process outputs look wrong, else None.""" - specs = controls.get("outputs", []) - outputs = result if isinstance(result, (list, tuple)) else [result] - if len(specs) > 1 and len(outputs) != len(specs): - return f"expected {len(specs)} outputs, got {len(outputs)}" - for spec, out in zip(specs, outputs): - if out is None: - return f"output '{spec.get('label')}' is None" - # gradio_client downloads file outputs and returns local paths - path = out.get("path") if isinstance(out, dict) and "path" in out else out - if isinstance(path, str) and os.path.sep in path and os.path.exists(path): - if os.path.getsize(path) == 0: - return f"output file for '{spec.get('label')}' is empty" - return None - - -# --------------------------------------------------------------------------- -# Results -# --------------------------------------------------------------------------- - -@dataclasses.dataclass -class CaseResult: - name: str - ok: bool | None = None # None => skipped - duration: float = 0.0 - error: str = "" - - -@dataclasses.dataclass -class ModelResult: - target: str # space id or "local/" - kind: str = "space" # "space" | "local" - status: str = FAIL - stage: str = "" - controls_ok: bool = False - cases: list = dataclasses.field(default_factory=list) - duration: float = 0.0 - error: str = "" - model_name: str = "" - - -def run_with_timeout(fn, timeout: float, what: str): - """Run fn() in a worker thread; raise TimeoutError if it exceeds timeout.""" - pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) - future = pool.submit(fn) - try: - return future.result(timeout=timeout) - except concurrent.futures.TimeoutError: - raise TimeoutError(f"{what} timed out after {int(timeout)}s") - finally: - # wait=False: never block on a hung call; the worker thread is leaked, - # which is acceptable for a test script - pool.shutdown(wait=False) - - -# --------------------------------------------------------------------------- -# Shared endpoint tests (identical for remote spaces and local examples) -# --------------------------------------------------------------------------- - -def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, - overrides: dict, opts: argparse.Namespace) -> ModelResult: - """Verify /controls and run every configured /process test case.""" - process_timeout = overrides.get("process_timeout", opts.process_timeout) - skip_process = opts.skip_process or overrides.get("skip_process", False) - config_dir = opts.config.parent.resolve() - - endpoints = client.view_api(return_format="dict", print_info=False) or {} - named = endpoints.get("named_endpoints", {}) - if "/controls" not in named or "/process" not in named: - result.error = (f"missing HARP endpoints (found: {sorted(named)}); " - f"deployment may use an outdated pyharp") - return result - - controls = run_with_timeout( - lambda: client.predict(api_name="/controls"), 120, "/controls") - if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: - result.error = f"/controls returned malformed data: {str(controls)[:200]}" - return result - result.controls_ok = True - result.model_name = controls.get("card", {}).get("name", "") - - if skip_process: - result.status = PASS - return result - - default_args, missing = synthesize_default_args(controls, assets) - cases = overrides.get("test_cases") or [{"name": "default"}] - - for case in cases: - case_result = CaseResult(name=case.get("name", "unnamed")) - result.cases.append(case_result) - case_start = time.time() - try: - # Inputs we couldn't synthesize are fine if this case supplies them - supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) - unsatisfied = {k: v for k, v in missing.items() if k not in supplied} - if unsatisfied: - case_result.error = "skipped: " + "; ".join( - f"'{k}': {v}" for k, v in unsatisfied.items()) - continue - args = apply_case(default_args, controls, case, config_dir) - job = client.submit(*args, api_name="/process") - output = job.result(timeout=case.get("process_timeout", process_timeout)) - error = validate_outputs(output, controls) - if error: - case_result.ok = False - case_result.error = f"/process output invalid: {error}" - else: - case_result.ok = True - except Exception as exc: # noqa: BLE001 - any exception fails the case - case_result.ok = False - case_result.error = f"{type(exc).__name__}: {exc}" - finally: - case_result.duration = round(time.time() - case_start, 1) - - if any(c.ok is False for c in result.cases): - failed = [c.name for c in result.cases if c.ok is False] - result.error = "; ".join( - f"[{c.name}] {c.error}" for c in result.cases if c.ok is False) - result.status = FAIL - else: - result.status = PASS - return result - - -# --------------------------------------------------------------------------- -# Remote space tests -# --------------------------------------------------------------------------- - -def wait_for_stage(api, space_id: str, deadline: float) -> str: - """Poll runtime stage until RUNNING, a dead stage, or the deadline.""" - stage = api.get_space_runtime(space_id).stage - while stage in TRANSIENT_STAGES - {"SLEEPING"} and time.time() < deadline: - time.sleep(10) - stage = api.get_space_runtime(space_id).stage - return stage - - -def test_space(space_id: str, token: str, assets: Assets, - opts: argparse.Namespace, overrides: dict) -> ModelResult: - from huggingface_hub import HfApi - - result = ModelResult(target=space_id, kind="space") - start = time.time() - api = HfApi(token=token) - connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) - - try: - deadline = time.time() + connect_timeout - stage = wait_for_stage(api, space_id, deadline) - result.stage = stage - - if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): - if opts.restart_failed and stage != "DELETING": - print(f" [{space_id}] stage={stage}, requesting restart...") - api.restart_space(space_id) - stage = wait_for_stage(api, space_id, time.time() + connect_timeout) - result.stage = stage - if stage not in ("RUNNING", "SLEEPING"): - result.error = f"space is not running (stage={stage})" - return result - - # Connect (this wakes sleeping spaces; retry through the wake-up) - client, last_exc = None, None - deadline = time.time() + connect_timeout - while time.time() < deadline: - try: - client = run_with_timeout( - lambda: Client(space_id, hf_token=token, verbose=False), - max(10.0, deadline - time.time()), "connect") - break - except Exception as exc: # noqa: BLE001 - space may still be waking - last_exc = exc - time.sleep(15) - if client is None: - raise RuntimeError(f"could not connect: {last_exc}") - - return run_endpoint_tests(client, result, assets, overrides, opts) - - except Exception as exc: # noqa: BLE001 - result.error = scrub(f"{type(exc).__name__}: {exc}", token) - if opts.verbose: - traceback.print_exc() - return result - finally: - result.duration = round(time.time() - start, 1) - - -# --------------------------------------------------------------------------- -# Baseline: local pyharp example tests -# --------------------------------------------------------------------------- - -def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> None: - deadline = time.time() + timeout - url = f"http://127.0.0.1:{port}/config" - while time.time() < deadline: - if proc.poll() is not None: - raise RuntimeError(f"app exited early with code {proc.returncode}") - try: - with urllib.request.urlopen(url, timeout=5): - return - except Exception: # noqa: BLE001 - server not up yet - time.sleep(2) - raise TimeoutError(f"local app did not become ready within {int(timeout)}s") - - -def test_local_example(app_dir: Path, port: int, assets: Assets, - opts: argparse.Namespace, overrides: dict) -> ModelResult: - target = f"local/{app_dir.name}" - result = ModelResult(target=target, kind="local", stage="LOCAL") - start = time.time() - - env = os.environ.copy() - env["GRADIO_SERVER_NAME"] = "127.0.0.1" - env["GRADIO_SERVER_PORT"] = str(port) - env.pop("HF_TOKEN", None) # local examples must not need credentials - - log_path = opts.output_dir / f"{app_dir.name}.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - proc = None - try: - with open(log_path, "w") as log: - proc = subprocess.Popen( - [sys.executable, "app.py"], cwd=app_dir, env=env, - stdout=log, stderr=subprocess.STDOUT) - wait_for_local_server(port, proc, overrides.get( - "connect_timeout", opts.connect_timeout)) - client = Client(f"http://127.0.0.1:{port}", verbose=False) - return run_endpoint_tests(client, result, assets, overrides, opts) - except Exception as exc: # noqa: BLE001 - result.error = f"{type(exc).__name__}: {exc} (see {log_path.name})" - if opts.verbose: - traceback.print_exc() - return result - finally: - result.duration = round(time.time() - start, 1) - if proc is not None and proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=15) - except subprocess.TimeoutExpired: - proc.kill() - - -# --------------------------------------------------------------------------- -# Discovery, config, reporting -# --------------------------------------------------------------------------- - -def load_config(path: Path) -> dict: - if not path.exists(): - return {} - if yaml is None: - print(f"WARNING: PyYAML not installed; ignoring {path}", file=sys.stderr) - return {} - return yaml.safe_load(path.read_text()) or {} - - -def get_excluded(config: dict, opts: argparse.Namespace) -> set: - """Models excluded from validation: config `exclude` list + --exclude.""" - return set(config.get("exclude", [])) | set(opts.exclude or []) - - -def discover_spaces(api, org: str, config: dict, excluded: set) -> list[str]: - spaces = [s.id for s in api.list_spaces(author=org)] - spaces += [s for s in config.get("include_extra", []) if s not in spaces] - return sorted(s for s in spaces if s not in excluded) - - -def status_emoji(r: ModelResult) -> str: - return {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭️"}[r.status] - - -def write_reports(results: list[ModelResult], out_dir: Path, label: str) -> None: - out_dir.mkdir(parents=True, exist_ok=True) - - payload = { - "label": label, - "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - "total": len(results), - "passed": sum(r.status == PASS for r in results), - "failed": sum(r.status == FAIL for r in results), - "results": [dataclasses.asdict(r) for r in results], - } - (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) - - lines = [ - f"# HARP Model Validation Report - {label}", - "", - f"**{payload['passed']}/{payload['total']} models passed** ({payload['timestamp']})", - "", - "| Model | Status | Stage | Controls | Cases | Time (s) | Detail |", - "|---|---|---|---|---|---|---|", - ] - for r in sorted(results, key=lambda r: (r.status != FAIL, r.target)): - if r.cases: - cases = ", ".join( - f"{c.name} {'✅' if c.ok else '⏭️' if c.ok is None else '❌'}" - for c in r.cases) - else: - cases = "—" - link = (f"[{r.target}](https://huggingface.co/spaces/{r.target})" - if r.kind == "space" else f"`{r.target}`") - detail = r.error.replace("|", "\\|")[:300] if r.error else "" - lines.append(f"| {link} | {status_emoji(r)} {r.status} | {r.stage} " - f"| {'✅' if r.controls_ok else '❌'} | {cases} " - f"| {r.duration} | {detail} |") - (out_dir / "report.md").write_text("\n".join(lines) + "\n") - -def main() -> int: parser = argparse.ArgumentParser(description="Validate HARP model deployments.") parser.add_argument("--org", default=DEFAULT_ORG, help="HF organization to scan") parser.add_argument("--spaces", nargs="*", default=None, help="Explicit space ids to validate (skips discovery)") parser.add_argument("--exclude", nargs="*", default=None, metavar="MODEL", help="Models to exclude from validation (space ids, or " - "local/); merged with the config " + "examples/); merged with the config " "`exclude` list") parser.add_argument("--local-examples", nargs="*", default=None, metavar="DIR", help="Validate local pyharp example apps instead of remote " - "spaces (the baseline tier). With no DIRs given, tests " + "spaces (the examples tier). With no DIRs given, tests " "every app under pyharp/examples/.") parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG, help="Optional YAML config (excludes, per-model overrides/cases)") - parser.add_argument("--skip-process", action="store_true", + parser.add_argument("--load-only", action="store_true", help="Only verify availability and /controls, do not run inference") - parser.add_argument("--restart-failed", action="store_true", - help="Attempt to restart spaces found in an error/stopped state " - "(requires a token with write access)") + parser.add_argument("--restart-failed", action=argparse.BooleanOptionalAction, + default=True, + help="Attempt to restart spaces found in an error/stopped state; " + "enabled by default (requires a token with write access), " + "disable with --no-restart-failed") parser.add_argument("--workers", type=int, default=4, - help="Number of spaces to test concurrently (remote mode only)") + help="Number of spaces to validate concurrently (spaces tier only)") parser.add_argument("--connect-timeout", type=float, default=420, help="Seconds to wait for a deployment to build/wake/start") parser.add_argument("--process-timeout", type=float, default=600, help="Seconds to wait for /process (includes ZeroGPU queue time)") parser.add_argument("--output-dir", type=Path, default=Path("reports")) parser.add_argument("--verbose", action="store_true") - opts = parser.parse_args() - config = load_config(opts.config) + return parser.parse_args() + + +def validate_examples(opts: argparse.Namespace, config: dict, excluded: set, + assets: Assets) -> list: + """ + Run the examples tier: launch and validate each local pyharp example. + + Examples run sequentially so concurrent model loads cannot exhaust the + machine's memory. + + Args: + opts (argparse.Namespace): Parsed command-line options. + config (dict): Parsed configuration. + excluded (set): Model keys to leave out. + assets (Assets): Synthesized input files. + + Returns: + results (list): ModelResult objects, or None when no examples exist. + """ + overrides = config.get("overrides", {}) - excluded = get_excluded(config, opts) - assets = Assets(opts.output_dir / "assets") + + if opts.local_examples: + app_dirs = [Path(d) for d in opts.local_examples] + else: + app_dirs = sorted(d for d in DEFAULT_EXAMPLES_DIR.iterdir() + if (d / "app.py").exists()) + app_dirs = [d for d in app_dirs if f"examples/{d.name}" not in excluded] + + if not app_dirs: + print("ERROR: no local examples found", file=sys.stderr) + return None + + print(f"Validating {len(app_dirs)} pyharp examples\n") + results = [] + for i, app_dir in enumerate(app_dirs): + r = test_local_example(app_dir, LOCAL_PORT_BASE + i, assets, opts, + overrides.get(f"examples/{app_dir.name}", {})) + results.append(r) + note = f" - {r.error}" if r.error else "" + print(f"{status_emoji(r)} {r.status:4s} {r.target} ({r.duration}s){note}") - if opts.local_examples is not None: - # ---- Baseline mode: local pyharp examples (run sequentially) ---- - if opts.local_examples: - app_dirs = [Path(d) for d in opts.local_examples] - else: - app_dirs = sorted(d for d in DEFAULT_EXAMPLES_DIR.iterdir() - if (d / "app.py").exists()) - app_dirs = [d for d in app_dirs if f"local/{d.name}" not in excluded] - if not app_dirs: - print("ERROR: no local examples found", file=sys.stderr) - return 2 - print(f"Validating {len(app_dirs)} local pyharp examples (baseline)\n") - for i, app_dir in enumerate(app_dirs): - r = test_local_example(app_dir, LOCAL_PORT_BASE + i, assets, opts, - overrides.get(f"local/{app_dir.name}", {})) + return results + + +def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, + assets: Assets, token: str) -> list: + """ + Run the spaces tier: validate remote Hugging Face Spaces concurrently. + + Args: + opts (argparse.Namespace): Parsed command-line options. + config (dict): Parsed configuration. + excluded (set): Model keys to leave out. + assets (Assets): Synthesized input files. + token (str): Hugging Face access token. + + Returns: + results (list): ModelResult objects, or None when no spaces exist. + """ + + from huggingface_hub import HfApi + + overrides = config.get("overrides", {}) + api = HfApi(token=token) + + if opts.spaces: + space_ids = [s for s in opts.spaces if s not in excluded] + else: + space_ids = discover_spaces(api, opts.org, config, excluded) + + if not space_ids: + print(f"ERROR: no spaces found for org '{opts.org}'", file=sys.stderr) + return None + + tracker = QuotaTracker(token, config.get("zerogpu_budget_seconds")) + print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " + f"(process test: {'OFF' if opts.load_only else 'ON'})") + print(f"ZeroGPU quota at start: {tracker.status()}\n") + + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: + futures = { + pool.submit(test_space, sid, token, assets, opts, + overrides.get(sid, {})): sid + for sid in space_ids + } + for future in concurrent.futures.as_completed(futures): + r = future.result() results.append(r) + # CPU-hardware models do not draw on the ZeroGPU allowance + if is_zerogpu(r.hardware): + tracker.add(sum(c.duration for c in r.cases)) note = f" - {r.error}" if r.error else "" - print(f"{status_emoji(r)} {r.status:4s} {r.target} ({r.duration}s){note}") - label = "baseline (local pyharp examples)" + print(f"{status_emoji(r)} {r.status:4s} {r.target} " + f"({r.duration}s) {tracker.status()}{scrub(note, token)}") + + return results + + +def main() -> int: + """ + Entry point: run the selected tier and write reports. + + Returns: + code (int): Process exit code (see module docstring). + """ + + opts = parse_args() + config = load_config(opts.config) + excluded = get_excluded(config, opts.exclude) + assets = Assets(opts.output_dir / "assets") + + if opts.local_examples is not None: token = "" + label = "pyharp examples" + results = validate_examples(opts, config, excluded, assets) else: - # ---- Remote mode: Hugging Face Spaces ---- - from huggingface_hub import HfApi token = get_token(required=True) - api = HfApi(token=token) - if opts.spaces: - space_ids = [s for s in opts.spaces if s not in excluded] - else: - space_ids = discover_spaces(api, opts.org, config, excluded) - if not space_ids: - print(f"ERROR: no spaces found for org '{opts.org}'", file=sys.stderr) - return 2 - print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " - f"(process test: {'OFF' if opts.skip_process else 'ON'})\n") - with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: - futures = { - pool.submit(test_space, sid, token, assets, opts, - overrides.get(sid, {})): sid - for sid in space_ids - } - for future in concurrent.futures.as_completed(futures): - r = future.result() - results.append(r) - note = f" - {r.error}" if r.error else "" - print(f"{status_emoji(r)} {r.status:4s} {r.target} " - f"({r.duration}s){scrub(note, token)}") label = f"{opts.org} spaces" + results = validate_spaces(opts, config, excluded, assets, token) + + if results is None: + return 2 write_reports(results, opts.output_dir, label) @@ -638,8 +234,16 @@ def main() -> int: print("\nFailed models:") for r in sorted(failed, key=lambda r: r.target): print(f" - {r.target}: {scrub(r.error, token)}") + return 1 if failed else 0 if __name__ == "__main__": - sys.exit(main()) + code = main() + # Hard-exit rather than sys.exit: gradio_client and timed-out calls can + # leave non-daemon threads behind, and a normal interpreter shutdown + # would join them - hanging the process after all results are printed. + # Reports are already flushed to disk at this point. + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) diff --git a/model_validation/validators.py b/model_validation/validators.py new file mode 100644 index 00000000..7b79d6b0 --- /dev/null +++ b/model_validation/validators.py @@ -0,0 +1,146 @@ +""" +Custom output validators for HARP model validation. + +A validator is a function that inspects the outputs of one /process test case +and raises AssertionError (with a helpful message) when something is wrong. +Register one with the @validator decorator and reference it from a test case +in config.yml: + + overrides: + teamup-tech/pitch_shifter: + test_cases: + - name: shift-up-octave + controls: + "Pitch Shift (semitones)": 12 + validator: wav_not_silent + +Each validator receives: + outputs (dict): output label -> value. File outputs (audio, MIDI, ...) + are local filesystem paths downloaded by gradio_client; JSON outputs + (e.g. pyharp LabelList) are the decoded objects (dict/list) or None. + controls (dict): the full /controls payload (card, inputs, outputs). + case (dict): the test case entry from config.yml, so a validator can + read its own parameters (e.g. thresholds) from extra keys. +""" + +import math +import struct +import wave + + +__all__ = [ + 'VALIDATORS', + 'validator' +] + + +VALIDATORS = {} + + +def validator(name): + """ + Register a validator function under a config-referenceable name. + + Args: + name (str): The name test cases use in their `validator` entry. + + Returns: + register (callable): Decorator that records the function. + """ + + def register(fn): + VALIDATORS[name] = fn + return fn + return register + + +@validator("wav_not_silent") +def wav_not_silent(outputs, controls, case): + """ + Assert every 16-bit WAV output contains an audible (non-silent) signal. + + The signal level is measured as RMS in dBFS (0 dBFS = full scale), a + standard, bit-depth-independent unit that is easy to set thresholds in: + digital silence is -inf, the noise floor of quiet recordings sits around + -60 dBFS, and typical program material is above -40 dBFS. RMS is also + robust where a peak measurement is not - a single stray click cannot + make an otherwise-silent file pass. + + Optional case key `min_rms_db` (default -60.0) sets the quietest + acceptable RMS level in dBFS. + + Raises: + AssertionError: If a WAV output is empty/silent, or none exist. + """ + min_rms_db = case.get("min_rms_db", -60.0) + checked = 0 + for label, value in outputs.items(): + if not (isinstance(value, str) and value.lower().endswith(".wav")): + continue + with wave.open(value, "rb") as f: + assert f.getsampwidth() == 2, \ + f"'{label}': expected 16-bit audio, got {8 * f.getsampwidth()}-bit" + frames = f.readframes(f.getnframes()) + assert frames, f"'{label}' contains no audio frames" + samples = struct.unpack(f"<{len(frames) // 2}h", frames) + rms = math.sqrt(sum(s * s for s in samples) / len(samples)) / 32768.0 + rms_db = 20 * math.log10(rms) if rms > 0 else float("-inf") + assert rms_db >= min_rms_db, \ + f"'{label}' appears silent (RMS {rms_db:.1f} dBFS < {min_rms_db} dBFS)" + checked += 1 + assert checked > 0, "no WAV outputs found to check" + + +@validator("wav_format") +def wav_format(outputs, controls, case): + """ + Assert every WAV output matches the supplied format constraints. + + Optional case keys (all optional; only the supplied ones are checked): + channels (int): expected channel count (1 = mono, 2 = stereo). + sample_rate (int): expected sample rate in Hz. + min_duration (float): minimum length in seconds. + + Raises: + AssertionError: If a WAV output fails a supplied check, or none exist. + """ + channels = case.get("channels") + sample_rate = case.get("sample_rate") + min_duration = case.get("min_duration") + checked = 0 + for label, value in outputs.items(): + if not (isinstance(value, str) and value.lower().endswith(".wav")): + continue + with wave.open(value, "rb") as f: + if channels is not None: + assert f.getnchannels() == channels, \ + f"'{label}': expected {channels} channel(s), got {f.getnchannels()}" + if sample_rate is not None: + assert f.getframerate() == sample_rate, \ + f"'{label}': expected {sample_rate} Hz, got {f.getframerate()} Hz" + if min_duration is not None: + duration = f.getnframes() / f.getframerate() + assert duration >= min_duration, \ + f"'{label}' is {duration:.2f}s, expected at least {min_duration}s" + checked += 1 + assert checked > 0, "no WAV outputs found to check" + + +@validator("has_labels") +def has_labels(outputs, controls, case): + """ + Assert at least one JSON output contains a non-empty pyharp LabelList. + + Optional case key `min_labels` (default 1) sets the minimum count. + + Raises: + AssertionError: If no LabelList is present or it has too few labels. + """ + min_labels = case.get("min_labels", 1) + for label, value in outputs.items(): + if isinstance(value, dict) and isinstance(value.get("labels"), list): + count = len(value["labels"]) + assert count >= min_labels, \ + f"'{label}' has {count} labels, expected at least {min_labels}" + return + raise AssertionError("no output contains a pyharp LabelList") From a26900980fb1177068bf074ee1dbded634e553b0 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Tue, 21 Jul 2026 06:51:01 -0400 Subject: [PATCH 3/8] Updates to structure, documentation, and expect/validator checks. --- .github/workflows/model_validation.yml | 4 +- .gitignore | 2 +- model_validation/README.md | 168 +++++-- model_validation/cases.py | 265 ---------- model_validation/config.yml | 78 ++- model_validation/requirements.txt | 1 + model_validation/{ => src}/assets.py | 0 model_validation/src/audio.py | 104 ++++ model_validation/src/cases.py | 460 ++++++++++++++++++ model_validation/{ => src}/harness.py | 4 +- model_validation/{ => src}/quota.py | 0 model_validation/{ => src}/results.py | 0 model_validation/{ => src}/utils.py | 0 model_validation/{ => src}/validate_models.py | 18 +- model_validation/src/validators.py | 110 +++++ model_validation/validators.py | 146 ------ 16 files changed, 873 insertions(+), 487 deletions(-) delete mode 100644 model_validation/cases.py rename model_validation/{ => src}/assets.py (100%) create mode 100644 model_validation/src/audio.py create mode 100644 model_validation/src/cases.py rename model_validation/{ => src}/harness.py (98%) rename model_validation/{ => src}/quota.py (100%) rename model_validation/{ => src}/results.py (100%) rename model_validation/{ => src}/utils.py (100%) rename model_validation/{ => src}/validate_models.py (93%) create mode 100644 model_validation/src/validators.py delete mode 100644 model_validation/validators.py diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml index d018ba65..b4ea8a4c 100644 --- a/.github/workflows/model_validation.yml +++ b/.github/workflows/model_validation.yml @@ -79,7 +79,7 @@ jobs: - name: Validate local examples run: | set +e - python model_validation/validate_models.py --local-examples \ + python model_validation/src/validate_models.py --local-examples \ --output-dir reports/examples CODE=$? if [ "$CODE" -eq 1 ]; then @@ -137,7 +137,7 @@ jobs: ARGS+=(--no-restart-failed) fi set +e - python model_validation/validate_models.py "${ARGS[@]}" + python model_validation/src/validate_models.py "${ARGS[@]}" CODE=$? if [ "$CODE" -eq 1 ]; then echo "::warning title=Space failures::Some spaces failed validation - see the run summary and report artifact." diff --git a/.gitignore b/.gitignore index d5b34a3c..36170ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,4 @@ website/content/HARP/*.md website/content/pyHARP/*.md # Model validation output -reports/ +model_validation/reports/ diff --git a/model_validation/README.md b/model_validation/README.md index 76a6d9c3..1af8fad1 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -34,18 +34,23 @@ Key behaviors: ## Code layout +Implementation lives under `src/`, separate from configuration, real test +inputs, and generated output — mirroring how pyharp itself keeps its source +(`pyharp/pyharp/`) apart from its top-level docs, examples, and packaging. + | File | Purpose | |---|---| -| [validate_models.py](validate_models.py) | Command-line entry point and per-tier orchestration | -| [harness.py](harness.py) | Core endpoint tests; space and local-example drivers | -| [cases.py](cases.py) | Input synthesis, test-case overlay, output validation/inspection | -| [validators.py](validators.py) | Registry of custom output validators (extend this) | -| [assets.py](assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | -| [quota.py](quota.py) | ZeroGPU usage tracking and account quota lookup | -| [results.py](results.py) | Result records and JSON/markdown report generation | -| [utils.py](utils.py) | Token handling, config loading, discovery, timeouts | -| [config.yml](config.yml) | Validation configuration (excludes, per-model test cases) | +| [src/validate_models.py](src/validate_models.py) | Command-line entry point and per-tier orchestration | +| [src/harness.py](src/harness.py) | The test harness: drives /controls + /process against a live model | +| [src/cases.py](src/cases.py) | Synthesis of inputs, test-case overlay, output validation/inspection | +| [src/validators.py](src/validators.py) | Registry of custom output validators (to be extended) | +| [src/assets.py](src/assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | +| [src/quota.py](src/quota.py) | ZeroGPU usage tracking and account quota lookup | +| [src/results.py](src/results.py) | Result records and JSON/markdown report generation | +| [src/utils.py](src/utils.py) | Token handling, config loading, discovery, timeouts | +| [config.yml](config.yml) | Validation configuration (excludes, per-model test cases, etc.) | | [test_data/](test_data/) | Real input files referenced by test cases | +| `reports/` | Generated report output (gitignored) | ## Token setup (IMPORTANT — read this) @@ -72,24 +77,24 @@ pip install -r model_validation/requirements.txt # All spaces in the org (crashed/stopped spaces are restarted by default) export HF_TOKEN=hf_... # see token setup above -python model_validation/validate_models.py +python model_validation/src/validate_models.py -# A single space, with verbose errors — the go-to while developing a test case -python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter --verbose +# A single space, with verbose errors (useful while developing a test case) +python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose # Exclude specific models (also configurable via `exclude` in config.yml) -python model_validation/validate_models.py --exclude teamup-tech/broken-space +python model_validation/src/validate_models.py --exclude teamup-tech/broken-space # Availability + /controls only (fast; no inference, no GPU quota used) -python model_validation/validate_models.py --load-only +python model_validation/src/validate_models.py --load-only # Never restart spaces (works with a read-only token) -python model_validation/validate_models.py --no-restart-failed +python model_validation/src/validate_models.py --no-restart-failed # Examples tier (needs pyharp + example deps; no token required) pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu pip install -e ./pyharp -python model_validation/validate_models.py --local-examples +python model_validation/src/validate_models.py --local-examples ``` Reports land in `reports/` as `report.json` (machine-readable) and @@ -165,14 +170,25 @@ overrides: Note: once `test_cases` is present, **only** the listed cases run — include `- name: default` to keep the synthesized one. -### Step 3: check the outputs (optional, two levels) +### Step 3: check the outputs (optional) Every case already gets structural checks for free: `/process` must not error, file outputs must exist and be non-empty, and JSON outputs (e.g. an optional pyharp `LabelList`) must be well-formed when present (absent/None is valid, since labels are optional). -**Level 1 — declarative `expect` rules**, simple per-output assertions: +Beyond that, there are two mechanisms, divided by what they can express: + +- **`expect`** asserts properties of a *single* output, and covers almost + every check worth writing. +- **`validators`** run custom Python for checks that cannot be expressed as + a property of one output — typically relationships spanning several + outputs. + +#### `expect` — declarative per-output rules + +Keyed by output label, then by rule. Every rule is optional, and an output +with no rules is still subject to the structural checks above: ```yaml - name: extreme-shift @@ -180,47 +196,117 @@ is valid, since labels are optional). "Pitch Shift (semitones)": 24 expect: "Output Audio": - ext: .wav # downloaded file must have this extension - min_bytes: 10000 # ... and be at least this many bytes + ext: .wav # extension: a string, or a list of accepted ones + min_bytes: 10000 # minimum file size + channels: 1 # exact channel count (1 = mono, 2 = stereo) + sample_rate: 44100 # exact sample rate, in Hz + min_duration: 1.5 # minimum length, in seconds + max_duration: 10.0 # maximum length, in seconds + bit_depth: 16 # exact PCM bit depth + min_rms_db: -60 # minimum RMS level, in dBFS + "Output Labels": + min_labels: 1 # minimum labels in a pyharp LabelList ``` -**Level 2 — custom validators**, arbitrary Python registered in -[validators.py](validators.py) and referenced by name: +The full vocabulary, and which output types each rule covers: + +| Rule | Applies to | Asserts | +|---|---|---| +| `ext` | any file output | Extension matches (string, or list of accepted extensions) | +| `min_bytes` | any file output | File is at least this many bytes | +| `channels` | audio output | Exact channel count | +| `sample_rate` | audio output | Exact sample rate in Hz | +| `min_duration` / `max_duration` | audio output | Length in seconds is within bounds | +| `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...); errors on compressed formats | +| `min_rms_db` | audio output | RMS level is at least this many dBFS | +| `min_labels` | JSON (`LabelList`) output | At least this many labels returned | + +**Targeting outputs.** A model with several outputs gets one block per +output label, each checked independently. Use `"*"` instead of a label to +apply rules to every output a rule covers — with mixed outputs, `"*"` sends +`min_bytes` to all file outputs and `min_rms_db` only to the audio ones: ```yaml - - name: extreme-shift - validator: wav_not_silent - min_rms_db: -40 # extra case keys parameterize the validator + expect: + "*": + min_bytes: 1000 # every file output must be non-trivial ``` -A validator receives `(outputs, controls, case)` — `outputs` maps output -labels to local file paths (file outputs) or decoded objects (JSON -outputs) — and raises `AssertionError` with a helpful message on failure. -Three ship out of the box: +Labels are preferred over positional indices: they survive a model reordering +its outputs, and they make the config readable without cross-referencing the +model's `app.py`. + +**Mistakes are reported as configuration errors, not model failures.** An +unrecognized rule name, an unknown output label, or a rule aimed at an output +type it does not cover (`min_labels` on an audio output, say) raises an error +naming the problem and listing what is valid. A `"*"` rule matching no output +at all is also an error, since it would otherwise silently check nothing. + +**On `min_rms_db`:** level is measured as RMS in dBFS (0 dBFS = full scale), +a bit-depth-independent unit that is easy to set thresholds in. Digital +silence is `-inf`, quiet noise floors sit near `-60`, and typical program +material is above `-40`. RMS is also more robust than a peak measurement — a +single stray click cannot make an otherwise-silent file pass. `min_rms_db: +-60` is the usual "this output is not silent" check. + +**On `min_labels`:** `min_labels: 0` is meaningful and is the right rule when +a model may legitimately return no labels — it asserts a well-formed +`LabelList` came back while permitting it to be empty. Omitting the rule +entirely is weaker: a missing or `None` label output also passes, because +labels are optional in pyharp. + +**On audio formats:** audio outputs are decoded with +[soundfile](https://python-soundfile.readthedocs.io/) (a listed dependency), +so audio rules work on any libsndfile format — WAV, FLAC, OGG, MP3, AIFF, and +more — not just pyharp's `save_audio()` WAV default. Decoding to float also +makes every audio rule bit-depth-independent, so a level threshold means the +same thing whether the source is 16-bit, 24-bit, or float. `bit_depth` is the +exception, by definition: it reads the file's PCM encoding and errors on a +compressed format, which has no fixed bit depth. + +#### `validators` — custom Python + +Registered in [validators.py](src/validators.py) and referenced by name, with +each validator's parameters nested beneath it, so they stay separate from the +test case's own fields. A case may run several: + +```yaml + - name: labels-line-up + validators: + labels_within_audio: + tolerance: 0.1 +``` -- `wav_not_silent` — asserts WAV outputs carry signal, measured as RMS in - dBFS (0 dBFS = full scale). The default threshold of `-60` dBFS rejects - digital silence and near-silence; raise it (e.g. `min_rms_db: -40`) to - demand typical program levels. -- `wav_format` — checks `channels`, `sample_rate`, and/or `min_duration` on - WAV outputs — e.g. assert mono output at 44100 Hz and at least 1.5s long. -- `has_labels` — asserts a non-empty pyharp `LabelList` was returned - (optionally `min_labels`). +A validator receives `(outputs, controls, params)` — `outputs` maps output +labels to local file paths (file outputs) or decoded objects (JSON outputs) — +and raises `AssertionError` with a message naming the output and what went +wrong. One ships as an example: `labels_within_audio`, which asserts every +returned label falls inside the audio output's timespan. It belongs here +because it relates two different outputs to each other; had it concerned only +one output, it would belong in `expect` instead. To add one: ```python -@validator("midi_not_empty") -def midi_not_empty(outputs, controls, case): +@validator("midi_note_count") +def midi_note_count(outputs, controls, params): + """Assert the MIDI output has at least params['min_notes'] note-ons.""" for label, value in outputs.items(): if isinstance(value, str) and value.lower().endswith((".mid", ".midi")): - assert os.path.getsize(value) > 50, f"'{label}' looks empty" + notes = count_note_ons(value) + assert notes >= params.get("min_notes", 1), \ + f"'{label}' has {notes} notes, expected at least {params['min_notes']}" ``` +Before writing one, check whether an `expect` rule would do — and if the +check is a generally useful property of a single output, consider adding it +to the `expect` vocabulary (`EXPECT_RULES` in [cases.py](src/cases.py)) +instead of putting it in a one-off validator. + ### Step 4: run just that model to iterate ```bash -python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter --verbose +python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose ``` The console shows each case's pass/fail; `reports/report.json` has per-case diff --git a/model_validation/cases.py b/model_validation/cases.py deleted file mode 100644 index 82973a4f..00000000 --- a/model_validation/cases.py +++ /dev/null @@ -1,265 +0,0 @@ -""" -Test-case handling for HARP model validation. - -Covers the full input/output cycle of one /process test case: -synthesizing default inputs from a model's /controls spec, overlaying a -configured case's control values and input files, and checking the outputs -(structural validation plus optional per-case `expect` rules and custom -validators). -""" - -import os -from pathlib import Path - -from gradio_client import handle_file - -from assets import Assets -from validators import VALIDATORS - - -__all__ = [ - 'synthesize_default_args', - 'apply_case', - 'validate_outputs', - 'inspect_outputs' -] - - -def synthesize_default_args(controls: dict, assets: Assets) -> tuple: - """ - Build the positional argument list for /process from the /controls spec. - - Args: - controls (dict): The /controls payload (card, inputs, outputs). - assets (Assets): Synthesized input files to draw from. - - Returns: - args (list): One argument per input component, in declaration order. - missing (dict): Label -> reason for every input that could NOT be - synthesized. Such inputs get a None placeholder; a test case can - still run if its controls/files overrides cover them all. - """ - - args, missing = [], {} - - for spec in controls.get("inputs", []): - ctype = spec.get("type") - label = spec.get("label") - - if ctype == "audio_track": - args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) - elif ctype == "midi_track": - args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) - elif ctype == "generic_file": - path = assets.for_file_types(spec.get("file_types")) - if path is None and spec.get("required", True): - missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" - args.append(handle_file(str(path)) if path is not None else None) - elif ctype in ("slider", "number_box"): - value = spec.get("value") - if value is None: - value = spec.get("minimum", 0) - args.append(value) - elif ctype == "text_box": - args.append(spec.get("value") or "test") - elif ctype == "toggle": - args.append(bool(spec.get("value", False))) - elif ctype == "dropdown": - value = spec.get("value") - if value is None: - choices = spec.get("choices") or [] - if not choices: - missing[label] = "dropdown has no choices" - else: - # Choices arrive as [label, value] pairs or plain values - first = choices[0] - value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first - args.append(value) - else: - missing[label] = f"unsupported input control type '{ctype}'" - args.append(None) - - return args, missing - - -def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: - """ - Overlay a configured test case onto the default argument list. - - Args: - args (list): Default arguments from synthesize_default_args(). - controls (dict): The /controls payload, used to match labels. - case (dict): Test case entry from config.yml. May contain: - controls: {: } - override scalar values. - files: {: } - override track/file inputs - (paths relative to config.yml). - config_dir (Path): Directory containing config.yml, the base for - relative file paths. - - Returns: - args (list): A new argument list with the overrides applied. - - Raises: - ValueError: If a label does not match any input, or a file is missing. - """ - - labels = [spec.get("label") for spec in controls.get("inputs", [])] - args = list(args) - - for label, value in (case.get("controls") or {}).items(): - if label not in labels: - raise ValueError(f"test case '{case.get('name')}' references unknown " - f"control '{label}' (available: {labels})") - args[labels.index(label)] = value - - for label, rel_path in (case.get("files") or {}).items(): - if label not in labels: - raise ValueError(f"test case '{case.get('name')}' references unknown " - f"input '{label}' (available: {labels})") - path = (config_dir / rel_path).resolve() - if not path.exists(): - raise ValueError(f"test case '{case.get('name')}': file not found: {path}") - args[labels.index(label)] = handle_file(str(path)) - - return args - - -def as_output_list(result, specs: list) -> list: - """ - Normalize a /process result to one value per output spec. - - With a single output the raw value is used as-is, so a JSON output - returning a list is not mistaken for multiple outputs. - - Args: - result: The raw value returned by gradio_client for /process. - specs (list): Output component specs from /controls. - - Returns: - outputs (list): One value per output spec. - """ - - if len(specs) <= 1: - return [result] - - return list(result) if isinstance(result, (list, tuple)) else [result] - - -def validate_outputs(result, controls: dict) -> str | None: - """ - Structurally check the outputs every model must satisfy. - - File outputs must be present, on disk, and non-empty. JSON outputs carry - optional data such as a pyharp LabelList: None/absent is valid (labels - are optional), but a present value must be JSON-shaped and a LabelList - must be well-formed. - - Args: - result: The raw value returned by gradio_client for /process. - controls (dict): The /controls payload. - - Returns: - error (str | None): A description of the first problem found, or - None when the outputs look structurally sound. - """ - - specs = controls.get("outputs", []) - outputs = as_output_list(result, specs) - - if len(specs) > 1 and len(outputs) != len(specs): - return f"expected {len(specs)} outputs, got {len(outputs)}" - - for spec, out in zip(specs, outputs): - if spec.get("type") == "json": - if out is None: - continue - if not isinstance(out, (dict, list)): - return (f"JSON output '{spec.get('label')}' is not valid JSON " - f"data: {str(out)[:100]}") - if isinstance(out, dict) and "labels" in out and \ - not isinstance(out["labels"], list): - return f"label list output '{spec.get('label')}' is malformed" - continue - - if out is None: - return f"output '{spec.get('label')}' is None" - - # gradio_client downloads file outputs and returns local paths - path = out.get("path") if isinstance(out, dict) and "path" in out else out - if isinstance(path, str) and os.path.sep in path and os.path.exists(path): - if os.path.getsize(path) == 0: - return f"output file for '{spec.get('label')}' is empty" - - return None - - -def outputs_by_label(result, specs: list) -> dict: - """ - Map output labels to values for inspection by validators. - - Args: - result: The raw value returned by gradio_client for /process. - specs (list): Output component specs from /controls. - - Returns: - outputs (dict): Label -> local file path (file outputs) or decoded - object (JSON outputs). - """ - - outputs = as_output_list(result, specs) - mapped = {} - - for spec, out in zip(specs, outputs): - value = out.get("path") if isinstance(out, dict) and "path" in out else out - mapped[spec.get("label")] = value - - return mapped - - -def inspect_outputs(result, controls: dict, case: dict) -> None: - """ - Apply a test case's deeper output checks (see README.md). - - Two mechanisms, both optional per case: - expect: declarative per-output rules ({ext, min_bytes}). - validator: name of a custom function registered in - validators.py; extra case keys parameterize it. - - Args: - result: The raw value returned by gradio_client for /process. - controls (dict): The /controls payload. - case (dict): Test case entry from config.yml. - - Raises: - AssertionError: If an expectation or validator check fails. - ValueError: If the case references an unknown output or validator. - """ - - out_map = outputs_by_label(result, controls.get("outputs", [])) - - for label, rules in (case.get("expect") or {}).items(): - if label not in out_map: - raise ValueError(f"expect references unknown output '{label}' " - f"(available: {list(out_map)})") - value = out_map[label] - - ext = rules.get("ext") - if ext and not (isinstance(value, str) and value.lower().endswith(ext.lower())): - raise AssertionError(f"output '{label}' is not a {ext} file: {value}") - - min_bytes = rules.get("min_bytes") - if min_bytes is not None: - if not (isinstance(value, str) and os.path.exists(value)): - raise AssertionError(f"output '{label}' is not a file on disk: {value}") - size = os.path.getsize(value) - if size < min_bytes: - raise AssertionError(f"output file '{label}' is {size} bytes, " - f"expected at least {min_bytes}") - - name = case.get("validator") - if name: - if name not in VALIDATORS: - raise ValueError(f"unknown validator '{name}' (available: " - f"{sorted(VALIDATORS)}); register it in " - f"validators.py") - VALIDATORS[name](out_map, controls, case) diff --git a/model_validation/config.yml b/model_validation/config.yml index a2c195a7..27038926 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -1,4 +1,4 @@ -# Configuration for model_validation/validate_models.py +# Configuration for model_validation/src/validate_models.py # # All keys are optional. Models not listed here are still discovered and # validated automatically with a single synthesized "default" test case, so @@ -21,27 +21,57 @@ # overrides: # per-model settings; keys are space ids, or # # "examples/" for local pyharp examples # : -# connect_timeout: 600 # seconds to wait for build/wake (default 420) -# process_timeout: 900 # seconds to wait for /process (default 600) -# load_only: true # only check availability + /controls -# test_cases: # named /process test cases; when omitted, a -# # single synthesized "default" case runs -# - name: default # empty case == synthesized defaults +# connect_timeout: 600 # seconds to wait for build/wake (default 420) +# process_timeout: 900 # seconds to wait for /process (default 600) +# load_only: true # only check availability + /controls +# +# test_cases: # named /process test cases; when omitted, a +# # single synthesized "default" case runs +# - name: default # empty case == synthesized defaults +# # - name: my-case -# process_timeout: 300 # optional per-case override -# controls: # override control values by label +# process_timeout: 300 # optional per-case override +# +# controls: # override control values by label # "Some Slider Label": 12 # "Some Toggle Label": true -# files: # override track/file inputs by label; -# # paths are relative to this file +# +# files: # override track/file inputs by label; +# # paths are relative to this file # "Input Audio": test_data/my_clip.wav -# expect: # declarative output checks by label -# "Output Audio": -# ext: .wav # downloaded file must have this ext -# min_bytes: 1000 # ... and be at least this many bytes -# validator: wav_not_silent # custom check registered in -# # validators.py; extra case keys -# # (e.g. min_rms_db) parameterize it +# +# expect: # per-output checks, keyed by output label +# "Output Audio": # ("*" applies rules to every compatible +# # output; see the rule table below) +# ext: .wav # extension: string or list of accepted +# min_bytes: 1000 # minimum file size, in bytes +# channels: 1 # audio: exact channel count +# sample_rate: 44100 # audio: exact sample rate, in Hz +# min_duration: 1.5 # audio: minimum length, in seconds +# max_duration: 10.0 # audio: maximum length, in seconds +# bit_depth: 16 # audio: exact PCM bit depth (PCM formats) +# min_rms_db: -60 # audio: minimum RMS level, in dBFS +# "Output Labels": +# min_labels: 1 # LabelList: minimum count (0 permits an +# # empty list, but requires a valid one) +# +# validators: # only for checks `expect` cannot express, +# # e.g. spanning several outputs; each +# # validator's parameters nest beneath it +# labels_within_audio: +# tolerance: 0.1 +# +# Which expect rules apply to which output types: +# +# ext, min_bytes any file output +# channels, sample_rate, min_duration, audio outputs +# max_duration, bit_depth, min_rms_db +# min_labels JSON (LabelList) outputs +# +# Applying a rule to an output type it does not cover is reported as a +# configuration error, not a model failure. Audio outputs are decoded with +# soundfile, so any libsndfile format works (WAV, FLAC, OGG, MP3, AIFF, ...); +# bit_depth applies only to PCM formats, not compressed ones. exclude: [] @@ -62,7 +92,10 @@ overrides: - name: shift-up-octave controls: "Pitch Shift (semitones)": 12 - validator: wav_not_silent + expect: + "Output Audio": + ext: .wav + min_rms_db: -60 # shifted audio must still carry signal - name: shift-down-octave controls: "Pitch Shift (semitones)": -12 @@ -94,7 +127,6 @@ overrides: # "Output Audio": # ext: .wav # min_bytes: 10000 - # validator: wav_format - # channels: 1 - # sample_rate: 44100 - # min_duration: 1.5 + # channels: 1 + # sample_rate: 44100 + # min_duration: 1.5 diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt index 84582416..f080f281 100644 --- a/model_validation/requirements.txt +++ b/model_validation/requirements.txt @@ -1,3 +1,4 @@ gradio_client>=1.0 huggingface_hub>=0.23 PyYAML>=6.0 +soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules diff --git a/model_validation/assets.py b/model_validation/src/assets.py similarity index 100% rename from model_validation/assets.py rename to model_validation/src/assets.py diff --git a/model_validation/src/audio.py b/model_validation/src/audio.py new file mode 100644 index 00000000..e71176f8 --- /dev/null +++ b/model_validation/src/audio.py @@ -0,0 +1,104 @@ +""" +Audio decoding for HARP model validation. + +Models return audio in whatever format their process function writes - +pyharp's save_audio() defaults to WAV, but a model passing its own output +path can produce FLAC, OGG, MP3, AIFF, and so on. All formats are decoded +through soundfile (libsndfile), which also frees the checks from caring +about bit depth: samples always arrive as floats in [-1, 1], so a level +threshold means the same thing for 16-bit, 24-bit, and float sources. +""" + +import math +import os + +import soundfile + + +__all__ = [ + 'read_audio_props', + 'supported_formats' +] + + +def supported_formats() -> list: + """ + List the audio formats this installation can decode. + + Returns: + formats (list): Sorted format names (e.g. WAV, FLAC, MP3, OGG). + """ + + try: + return sorted(soundfile.available_formats()) + except Exception: # noqa: BLE001 - diagnostics must never raise + return [] + + +def bit_depth_from_subtype(subtype: str): + """ + Map a libsndfile subtype to a PCM bit depth, when it has one. + + Args: + subtype (str): soundfile subtype, e.g. "PCM_16", "FLOAT", "VORBIS". + + Returns: + depth (int | None): Bits per sample for PCM/float encodings; None + for compressed encodings that have no fixed bit depth. + """ + + if subtype.startswith("PCM_"): + tail = subtype[len("PCM_"):] + if tail in ("S8", "U8"): + return 8 + if tail.isdigit(): + return int(tail) + return {"FLOAT": 32, "DOUBLE": 64}.get(subtype) + + +def read_audio_props(label: str, path: str) -> dict: + """ + Decode an audio file and measure the properties `expect` rules check. + + Args: + label (str): Output label, for error messages. + path (str): Path to the audio file. + + Returns: + props (dict): channels, sample_rate, duration (seconds), rms_db (RMS + level in dBFS; -inf for digital silence), subtype (libsndfile + encoding name), and bit_depth (int, or None for compressed + encodings). + + Raises: + AssertionError: If the file cannot be decoded or contains no audio. + """ + + try: + # A single header-and-data open: always_2d gives a consistent + # (frames, channels) shape, and float64 output normalizes any bit + # depth to [-1, 1] for the level measurement + with soundfile.SoundFile(path) as f: + sample_rate = f.samplerate + channels = f.channels + subtype = f.subtype + data = f.read(always_2d=True, dtype="float64") + except Exception as exc: # noqa: BLE001 - any decode failure + ext = os.path.splitext(path)[1] or "(no extension)" + raise AssertionError( + f"output '{label}' could not be decoded as audio (format {ext}): " + f"{exc}. Decodable formats: {', '.join(supported_formats())}") + + frames = data.shape[0] + assert frames, f"output '{label}' contains no audio frames" + + rms = math.sqrt(float((data * data).sum()) / data.size) + + return { + "channels": channels, + "sample_rate": sample_rate, + "duration": frames / sample_rate, + "rms_db": 20 * math.log10(rms) if rms > 0 else float("-inf"), + "subtype": subtype, + "bit_depth": bit_depth_from_subtype(subtype), + } diff --git a/model_validation/src/cases.py b/model_validation/src/cases.py new file mode 100644 index 00000000..8f61f2c6 --- /dev/null +++ b/model_validation/src/cases.py @@ -0,0 +1,460 @@ +""" +Test-case handling for HARP model validation. + +Covers the full input/output cycle of one /process test case: +synthesizing default inputs from a model's /controls spec, overlaying a +configured case's control values and input files, and checking the outputs +(structural validation plus optional per-case `expect` rules and custom +validators). +""" + +import os +from pathlib import Path + +from gradio_client import handle_file + +from assets import Assets +from audio import read_audio_props +from validators import VALIDATORS + + +__all__ = [ + 'synthesize_default_args', + 'apply_case', + 'validate_outputs', + 'inspect_outputs', + 'EXPECT_RULES' +] + + +# Output component types that produce a file on disk +FILE_TYPES = {"audio_track", "midi_track", "generic_file"} + +# The declarative `expect` vocabulary: rule name -> (applicable output types, +# what it asserts). Anything expressible as "read a property of one output and +# compare it" belongs here rather than in a custom validator (validators.py). +# Applying a rule to an output type it does not cover is a configuration +# error, not a model failure. +EXPECT_RULES = { + "ext": (FILE_TYPES, + "file extension, as a string or list of accepted extensions"), + "min_bytes": (FILE_TYPES, + "minimum file size in bytes"), + "channels": ({"audio_track"}, + "exact channel count (1 = mono, 2 = stereo)"), + "sample_rate": ({"audio_track"}, + "exact sample rate, in Hz"), + "min_duration": ({"audio_track"}, + "minimum length, in seconds"), + "max_duration": ({"audio_track"}, + "maximum length, in seconds"), + "bit_depth": ({"audio_track"}, + "exact PCM bit depth, e.g. 16 or 24 (not valid for " + "compressed formats such as MP3 or OGG)"), + "min_rms_db": ({"audio_track"}, + "minimum RMS level, in dBFS (0 = full scale); -60 is the " + "usual threshold for 'this output is not silent'"), + "min_labels": ({"json"}, + "minimum number of labels in a pyharp LabelList; 0 asserts " + "a well-formed LabelList that may legitimately be empty"), +} + +# Rules requiring the audio properties of the output to be decoded +AUDIO_RULES = {rule for rule, (types, _) in EXPECT_RULES.items() + if types == {"audio_track"}} + +# Key selecting every compatible output rather than one named output +ALL_OUTPUTS = "*" + + +def synthesize_default_args(controls: dict, assets: Assets) -> tuple: + """ + Build the positional argument list for /process from the /controls spec. + + Args: + controls (dict): The /controls payload (card, inputs, outputs). + assets (Assets): Synthesized input files to draw from. + + Returns: + args (list): One argument per input component, in declaration order. + missing (dict): Label -> reason for every input that could NOT be + synthesized. Such inputs get a None placeholder; a test case can + still run if its controls/files overrides cover them all. + """ + + args, missing = [], {} + + for spec in controls.get("inputs", []): + ctype = spec.get("type") + label = spec.get("label") + + if ctype == "audio_track": + args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) + elif ctype == "midi_track": + args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) + elif ctype == "generic_file": + path = assets.for_file_types(spec.get("file_types")) + if path is None and spec.get("required", True): + missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" + args.append(handle_file(str(path)) if path is not None else None) + elif ctype in ("slider", "number_box"): + value = spec.get("value") + if value is None: + value = spec.get("minimum", 0) + args.append(value) + elif ctype == "text_box": + args.append(spec.get("value") or "test") + elif ctype == "toggle": + args.append(bool(spec.get("value", False))) + elif ctype == "dropdown": + value = spec.get("value") + if value is None: + choices = spec.get("choices") or [] + if not choices: + missing[label] = "dropdown has no choices" + else: + # Choices arrive as [label, value] pairs or plain values + first = choices[0] + value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first + args.append(value) + else: + missing[label] = f"unsupported input control type '{ctype}'" + args.append(None) + + return args, missing + + +def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: + """ + Overlay a configured test case onto the default argument list. + + Args: + args (list): Default arguments from synthesize_default_args(). + controls (dict): The /controls payload, used to match labels. + case (dict): Test case entry from config.yml. May contain: + controls: {: } - override scalar values. + files: {: } - override track/file inputs + (paths relative to config.yml). + config_dir (Path): Directory containing config.yml, the base for + relative file paths. + + Returns: + args (list): A new argument list with the overrides applied. + + Raises: + ValueError: If a label does not match any input, or a file is missing. + """ + + labels = [spec.get("label") for spec in controls.get("inputs", [])] + args = list(args) + + for label, value in (case.get("controls") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"control '{label}' (available: {labels})") + args[labels.index(label)] = value + + for label, rel_path in (case.get("files") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"input '{label}' (available: {labels})") + path = (config_dir / rel_path).resolve() + if not path.exists(): + raise ValueError(f"test case '{case.get('name')}': file not found: {path}") + args[labels.index(label)] = handle_file(str(path)) + + return args + + +def as_output_list(result, specs: list) -> list: + """ + Normalize a /process result to one value per output spec. + + With a single output the raw value is used as-is, so a JSON output + returning a list is not mistaken for multiple outputs. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (list): One value per output spec. + """ + + if len(specs) <= 1: + return [result] + + return list(result) if isinstance(result, (list, tuple)) else [result] + + +def validate_outputs(result, controls: dict) -> str | None: + """ + Structurally check the outputs every model must satisfy. + + File outputs must be present, on disk, and non-empty. JSON outputs carry + optional data such as a pyharp LabelList: None/absent is valid (labels + are optional), but a present value must be JSON-shaped and a LabelList + must be well-formed. + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + + Returns: + error (str | None): A description of the first problem found, or + None when the outputs look structurally sound. + """ + + specs = controls.get("outputs", []) + outputs = as_output_list(result, specs) + + if len(specs) > 1 and len(outputs) != len(specs): + return f"expected {len(specs)} outputs, got {len(outputs)}" + + for spec, out in zip(specs, outputs): + if spec.get("type") == "json": + if out is None: + continue + if not isinstance(out, (dict, list)): + return (f"JSON output '{spec.get('label')}' is not valid JSON " + f"data: {str(out)[:100]}") + if isinstance(out, dict) and "labels" in out and \ + not isinstance(out["labels"], list): + return f"label list output '{spec.get('label')}' is malformed" + continue + + if out is None: + return f"output '{spec.get('label')}' is None" + + # gradio_client downloads file outputs and returns local paths + path = out.get("path") if isinstance(out, dict) and "path" in out else out + if isinstance(path, str) and os.path.sep in path and os.path.exists(path): + if os.path.getsize(path) == 0: + return f"output file for '{spec.get('label')}' is empty" + + return None + + +def outputs_by_label(result, specs: list) -> dict: + """ + Map output labels to values for inspection by validators. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (dict): Label -> local file path (file outputs) or decoded + object (JSON outputs). + """ + + outputs = as_output_list(result, specs) + mapped = {} + + for spec, out in zip(specs, outputs): + value = out.get("path") if isinstance(out, dict) and "path" in out else out + mapped[spec.get("label")] = value + + return mapped + + +def require_file(label: str, value) -> str: + """ + Assert an output is a file on disk and return its path. + + Args: + label (str): Output label, for the error message. + value: The mapped output value. + + Returns: + path (str): The verified filesystem path. + + Raises: + AssertionError: If the output is not an existing file. + """ + + if not (isinstance(value, str) and os.path.exists(value)): + raise AssertionError(f"output '{label}' is not a file on disk: {value}") + + return value + + +def resolve_expect_targets(expect: dict, out_types: dict) -> list: + """ + Resolve an `expect` block into concrete (label, rules) pairs. + + Keys are output labels, or "*" to apply rules to every output the rule + is compatible with (e.g. `min_rms_db` under "*" reaches only the audio + outputs). Rules are checked against the output's type here, so a rule + aimed at the wrong kind of output is reported as a configuration error + rather than a model failure. + + Args: + expect (dict): The case's `expect` block. + out_types (dict): Output label -> component type from /controls. + + Returns: + targets (list): (label, rules) pairs to check. + + Raises: + ValueError: If a label, rule name, or rule/output pairing is invalid. + """ + + targets = [] + + for key, rules in (expect or {}).items(): + rules = rules or {} + + unknown = set(rules) - set(EXPECT_RULES) + if unknown: + raise ValueError(f"unknown expect rule(s) {sorted(unknown)} for " + f"'{key}'; supported rules: {sorted(EXPECT_RULES)}") + + if key == ALL_OUTPUTS: + # Fan each rule out to the outputs whose type it covers + per_label = {} + for rule, value in rules.items(): + applicable, _ = EXPECT_RULES[rule] + matched = [label for label, otype in out_types.items() + if otype in applicable] + if not matched: + raise ValueError( + f"expect rule '{rule}' under '{ALL_OUTPUTS}' matches no " + f"output; it applies to {sorted(applicable)} outputs, " + f"but this model has {sorted(set(out_types.values()))}") + for label in matched: + per_label.setdefault(label, {})[rule] = value + targets.extend(per_label.items()) + continue + + if key not in out_types: + raise ValueError(f"expect references unknown output '{key}' " + f"(available: {list(out_types)})") + + for rule in rules: + applicable, _ = EXPECT_RULES[rule] + if out_types[key] not in applicable: + raise ValueError( + f"expect rule '{rule}' does not apply to output '{key}' " + f"of type '{out_types[key]}'; it applies to " + f"{sorted(applicable)} outputs") + + targets.append((key, rules)) + + return targets + + +def check_expectations(label: str, value, rules: dict) -> None: + """ + Apply one output's declarative `expect` rules. + + Rule names and their applicability to this output are validated upstream + by resolve_expect_targets(). Audio properties are decoded in a single + pass, and only when an audio rule is actually requested. + + Args: + label (str): Output label the rules apply to. + value: The mapped output value (file path or decoded JSON). + rules (dict): Rule name -> expected value. + + Raises: + AssertionError: If any rule is not satisfied. + """ + + # --- Any file output ----------------------------------------------------- + if "ext" in rules: + allowed = rules["ext"] + allowed = [allowed] if isinstance(allowed, str) else list(allowed) + assert isinstance(value, str) and \ + any(value.lower().endswith(e.lower()) for e in allowed), \ + f"output '{label}' is not a {' or '.join(allowed)} file: {value}" + + if "min_bytes" in rules: + size = os.path.getsize(require_file(label, value)) + assert size >= rules["min_bytes"], \ + f"output file '{label}' is {size} bytes, expected at least {rules['min_bytes']}" + + # --- Audio outputs ------------------------------------------------------- + if set(rules) & AUDIO_RULES: + props = read_audio_props(label, require_file(label, value)) + + if "channels" in rules: + assert props["channels"] == rules["channels"], \ + (f"output '{label}': expected {rules['channels']} channel(s), " + f"got {props['channels']}") + + if "sample_rate" in rules: + assert props["sample_rate"] == rules["sample_rate"], \ + (f"output '{label}': expected {rules['sample_rate']} Hz, " + f"got {props['sample_rate']} Hz") + + if "min_duration" in rules: + assert props["duration"] >= rules["min_duration"], \ + (f"output '{label}' is {props['duration']:.2f}s, expected at " + f"least {rules['min_duration']}s") + + if "max_duration" in rules: + assert props["duration"] <= rules["max_duration"], \ + (f"output '{label}' is {props['duration']:.2f}s, expected at " + f"most {rules['max_duration']}s") + + if "bit_depth" in rules: + assert props["bit_depth"] is not None, \ + (f"output '{label}' is a compressed format ({props['subtype']}) " + f"with no PCM bit depth to check") + assert props["bit_depth"] == rules["bit_depth"], \ + (f"output '{label}': expected {rules['bit_depth']}-bit audio, " + f"got {props['bit_depth']}-bit ({props['subtype']})") + + if "min_rms_db" in rules: + assert props["rms_db"] >= rules["min_rms_db"], \ + (f"output '{label}' appears silent (RMS {props['rms_db']:.1f} dBFS " + f"< {rules['min_rms_db']} dBFS)") + + # --- JSON / LabelList outputs -------------------------------------------- + if "min_labels" in rules: + labels = value.get("labels") if isinstance(value, dict) else None + assert isinstance(labels, list), \ + f"output '{label}' does not contain a pyharp LabelList: {value}" + assert len(labels) >= rules["min_labels"], \ + (f"output '{label}' has {len(labels)} label(s), expected at least " + f"{rules['min_labels']}") + + +def inspect_outputs(result, controls: dict, case: dict) -> None: + """ + Apply a test case's deeper output checks (see README.md). + + Both mechanisms are optional per case; without either, an output is still + subject to the structural checks in validate_outputs(). + + expect: declarative per-output rules, the common path - see + EXPECT_RULES for the vocabulary. + validators: mapping of validator name -> parameters, for checks that + cannot be expressed declaratively (e.g. spanning several + outputs); registered in validators.py. + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + case (dict): Test case entry from config.yml. + + Raises: + AssertionError: If an expectation or validator check fails. + ValueError: If the case references an unknown output, rule, or + validator, or applies a rule to an incompatible output. + """ + + specs = controls.get("outputs", []) + out_map = outputs_by_label(result, specs) + out_types = {spec.get("label"): spec.get("type") for spec in specs} + + for label, rules in resolve_expect_targets(case.get("expect"), out_types): + check_expectations(label, out_map[label], rules) + + for name, params in (case.get("validators") or {}).items(): + if name not in VALIDATORS: + raise ValueError(f"unknown validator '{name}' (available: " + f"{sorted(VALIDATORS)}); register it in " + f"validators.py") + VALIDATORS[name](out_map, controls, params or {}) diff --git a/model_validation/harness.py b/model_validation/src/harness.py similarity index 98% rename from model_validation/harness.py rename to model_validation/src/harness.py index 8fac735d..60f8f275 100644 --- a/model_validation/harness.py +++ b/model_validation/src/harness.py @@ -1,5 +1,7 @@ """ -The core validation harness for HARP model deployments. +The core validation harness for HARP model deployments: the code that sets +up a connection to a live model, drives it through /controls and /process, +and reports what happened, independent of where the model is running. Both validation tiers funnel into run_endpoint_tests(), which performs the identical black-box checks against any live HARP gradio app: diff --git a/model_validation/quota.py b/model_validation/src/quota.py similarity index 100% rename from model_validation/quota.py rename to model_validation/src/quota.py diff --git a/model_validation/results.py b/model_validation/src/results.py similarity index 100% rename from model_validation/results.py rename to model_validation/src/results.py diff --git a/model_validation/utils.py b/model_validation/src/utils.py similarity index 100% rename from model_validation/utils.py rename to model_validation/src/utils.py diff --git a/model_validation/validate_models.py b/model_validation/src/validate_models.py similarity index 93% rename from model_validation/validate_models.py rename to model_validation/src/validate_models.py index e689bd69..9500a11c 100644 --- a/model_validation/validate_models.py +++ b/model_validation/src/validate_models.py @@ -27,11 +27,11 @@ validation of the others. Usage: - HF_TOKEN=... python model_validation/validate_models.py - HF_TOKEN=... python model_validation/validate_models.py --spaces teamup-tech/pitch_shifter - HF_TOKEN=... python model_validation/validate_models.py --load-only --workers 8 - HF_TOKEN=... python model_validation/validate_models.py --no-restart-failed - python model_validation/validate_models.py --local-examples + HF_TOKEN=... python model_validation/src/validate_models.py + HF_TOKEN=... python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter + HF_TOKEN=... python model_validation/src/validate_models.py --load-only --workers 8 + HF_TOKEN=... python model_validation/src/validate_models.py --no-restart-failed + python model_validation/src/validate_models.py --local-examples Exit codes: 0 - all validated models passed (or were explicitly skipped) @@ -53,9 +53,11 @@ DEFAULT_ORG = "teamup-tech" -SCRIPT_DIR = Path(__file__).parent -DEFAULT_CONFIG = SCRIPT_DIR / "config.yml" -DEFAULT_EXAMPLES_DIR = SCRIPT_DIR.parent / "pyharp" / "examples" +SCRIPT_DIR = Path(__file__).parent # model_validation/src +MODEL_VALIDATION_DIR = SCRIPT_DIR.parent # model_validation +REPO_ROOT = MODEL_VALIDATION_DIR.parent +DEFAULT_CONFIG = MODEL_VALIDATION_DIR / "config.yml" +DEFAULT_EXAMPLES_DIR = REPO_ROOT / "pyharp" / "examples" LOCAL_PORT_BASE = 7861 diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py new file mode 100644 index 00000000..32cfb615 --- /dev/null +++ b/model_validation/src/validators.py @@ -0,0 +1,110 @@ +""" +Custom output validators for HARP model validation. + +Most output checks do not belong here: anything expressible as "read a +property of one output and compare it" (extension, size, channel count, +sample rate, duration, signal level, label count) is a declarative `expect` +rule in config.yml - see EXPECT_RULES in cases.py. + +A validator covers the checks that cannot be written that way: relationships +spanning several outputs, or logic needing real computation. Register one +with the @validator decorator and reference it from a test case, with its +parameters nested beneath the name: + + overrides: + teamup-tech/some_transcriber: + test_cases: + - name: labels-line-up + validators: + labels_within_audio: + tolerance: 0.1 + +Each validator receives: + outputs (dict): output label -> value. File outputs (audio, MIDI, ...) + are local filesystem paths downloaded by gradio_client; JSON outputs + (e.g. pyharp LabelList) are the decoded objects (dict/list) or None. + controls (dict): the full /controls payload (card, inputs, outputs). + params (dict): the parameters nested under this validator's name, keeping + its settings separate from the test case's own fields. + +Validators signal failure by raising AssertionError with a message naming the +output and what was wrong. +""" + +from audio import read_audio_props + + +__all__ = [ + 'VALIDATORS', + 'validator' +] + + +VALIDATORS = {} + + +def validator(name): + """ + Register a validator function under a config-referenceable name. + + Args: + name (str): The name test cases use in their `validator` entry. + + Returns: + register (callable): Decorator that records the function. + """ + + def register(fn): + VALIDATORS[name] = fn + return fn + return register + + +@validator("labels_within_audio") +def labels_within_audio(outputs, controls, params): + """ + Assert every returned label falls inside the audio output's timespan. + + This check warrants a validator rather than an `expect` rule because it + relates two different outputs to each other, which no per-output property + comparison can express. + + Params: + tolerance (float): seconds a label may exceed the audio duration + before it is treated as out of bounds (default 0.05). + + Raises: + AssertionError: If a label lies outside the audio, or the outputs + needed for the comparison are missing. + """ + + tolerance = params.get("tolerance", 0.05) + + # Identify the audio output from the component spec rather than by file + # extension, since models may return any format soundfile can decode + audio_labels = [spec.get("label") for spec in controls.get("outputs", []) + if spec.get("type") == "audio_track"] + + duration = None + for label in audio_labels: + value = outputs.get(label) + if isinstance(value, str): + duration = read_audio_props(label, value)["duration"] + break + assert duration is not None, "no audio output found to compare labels against" + + for label, value in outputs.items(): + if not (isinstance(value, dict) and isinstance(value.get("labels"), list)): + continue + for entry in value["labels"]: + start = entry.get("t", 0.0) + end = start + entry.get("duration", 0.0) + assert -tolerance <= start <= duration + tolerance, \ + (f"'{label}': label \"{entry.get('label')}\" starts at {start:.2f}s, " + f"outside the {duration:.2f}s audio output") + assert end <= duration + tolerance, \ + (f"'{label}': label \"{entry.get('label')}\" ends at {end:.2f}s, " + f"past the {duration:.2f}s audio output") + return + + raise AssertionError("no pyharp LabelList output found to check") diff --git a/model_validation/validators.py b/model_validation/validators.py deleted file mode 100644 index 7b79d6b0..00000000 --- a/model_validation/validators.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Custom output validators for HARP model validation. - -A validator is a function that inspects the outputs of one /process test case -and raises AssertionError (with a helpful message) when something is wrong. -Register one with the @validator decorator and reference it from a test case -in config.yml: - - overrides: - teamup-tech/pitch_shifter: - test_cases: - - name: shift-up-octave - controls: - "Pitch Shift (semitones)": 12 - validator: wav_not_silent - -Each validator receives: - outputs (dict): output label -> value. File outputs (audio, MIDI, ...) - are local filesystem paths downloaded by gradio_client; JSON outputs - (e.g. pyharp LabelList) are the decoded objects (dict/list) or None. - controls (dict): the full /controls payload (card, inputs, outputs). - case (dict): the test case entry from config.yml, so a validator can - read its own parameters (e.g. thresholds) from extra keys. -""" - -import math -import struct -import wave - - -__all__ = [ - 'VALIDATORS', - 'validator' -] - - -VALIDATORS = {} - - -def validator(name): - """ - Register a validator function under a config-referenceable name. - - Args: - name (str): The name test cases use in their `validator` entry. - - Returns: - register (callable): Decorator that records the function. - """ - - def register(fn): - VALIDATORS[name] = fn - return fn - return register - - -@validator("wav_not_silent") -def wav_not_silent(outputs, controls, case): - """ - Assert every 16-bit WAV output contains an audible (non-silent) signal. - - The signal level is measured as RMS in dBFS (0 dBFS = full scale), a - standard, bit-depth-independent unit that is easy to set thresholds in: - digital silence is -inf, the noise floor of quiet recordings sits around - -60 dBFS, and typical program material is above -40 dBFS. RMS is also - robust where a peak measurement is not - a single stray click cannot - make an otherwise-silent file pass. - - Optional case key `min_rms_db` (default -60.0) sets the quietest - acceptable RMS level in dBFS. - - Raises: - AssertionError: If a WAV output is empty/silent, or none exist. - """ - min_rms_db = case.get("min_rms_db", -60.0) - checked = 0 - for label, value in outputs.items(): - if not (isinstance(value, str) and value.lower().endswith(".wav")): - continue - with wave.open(value, "rb") as f: - assert f.getsampwidth() == 2, \ - f"'{label}': expected 16-bit audio, got {8 * f.getsampwidth()}-bit" - frames = f.readframes(f.getnframes()) - assert frames, f"'{label}' contains no audio frames" - samples = struct.unpack(f"<{len(frames) // 2}h", frames) - rms = math.sqrt(sum(s * s for s in samples) / len(samples)) / 32768.0 - rms_db = 20 * math.log10(rms) if rms > 0 else float("-inf") - assert rms_db >= min_rms_db, \ - f"'{label}' appears silent (RMS {rms_db:.1f} dBFS < {min_rms_db} dBFS)" - checked += 1 - assert checked > 0, "no WAV outputs found to check" - - -@validator("wav_format") -def wav_format(outputs, controls, case): - """ - Assert every WAV output matches the supplied format constraints. - - Optional case keys (all optional; only the supplied ones are checked): - channels (int): expected channel count (1 = mono, 2 = stereo). - sample_rate (int): expected sample rate in Hz. - min_duration (float): minimum length in seconds. - - Raises: - AssertionError: If a WAV output fails a supplied check, or none exist. - """ - channels = case.get("channels") - sample_rate = case.get("sample_rate") - min_duration = case.get("min_duration") - checked = 0 - for label, value in outputs.items(): - if not (isinstance(value, str) and value.lower().endswith(".wav")): - continue - with wave.open(value, "rb") as f: - if channels is not None: - assert f.getnchannels() == channels, \ - f"'{label}': expected {channels} channel(s), got {f.getnchannels()}" - if sample_rate is not None: - assert f.getframerate() == sample_rate, \ - f"'{label}': expected {sample_rate} Hz, got {f.getframerate()} Hz" - if min_duration is not None: - duration = f.getnframes() / f.getframerate() - assert duration >= min_duration, \ - f"'{label}' is {duration:.2f}s, expected at least {min_duration}s" - checked += 1 - assert checked > 0, "no WAV outputs found to check" - - -@validator("has_labels") -def has_labels(outputs, controls, case): - """ - Assert at least one JSON output contains a non-empty pyharp LabelList. - - Optional case key `min_labels` (default 1) sets the minimum count. - - Raises: - AssertionError: If no LabelList is present or it has too few labels. - """ - min_labels = case.get("min_labels", 1) - for label, value in outputs.items(): - if isinstance(value, dict) and isinstance(value.get("labels"), list): - count = len(value["labels"]) - assert count >= min_labels, \ - f"'{label}' has {count} labels, expected at least {min_labels}" - return - raise AssertionError("no output contains a pyharp LabelList") From 3f3ea004976d6d8cd1936292b375d561f44db278 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Tue, 21 Jul 2026 07:29:13 -0400 Subject: [PATCH 4/8] Minor README cleanup. --- model_validation/README.md | 48 +++++++++++++------------------ model_validation/requirements.txt | 2 +- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/model_validation/README.md b/model_validation/README.md index 1af8fad1..6589ae14 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -1,13 +1,13 @@ # HARP Model Validation Automated validation of HARP model deployments. Verifies that each deployment -is reachable, exposes the HARP gradio endpoints (`/controls`, `/process`), +is reachable, exposes the HARP gradio endpoints (_i.e._, `/controls`, `/process`), and can actually process test inputs end-to-end — the same interactions the HARP client performs, driven headlessly. ## Overview -Two tiers share the same harness: +Two tiers share the same test harness: | Tier | Command | What it validates | What a failure means | |---|---|---|---| @@ -34,23 +34,20 @@ Key behaviors: ## Code layout -Implementation lives under `src/`, separate from configuration, real test -inputs, and generated output — mirroring how pyharp itself keeps its source -(`pyharp/pyharp/`) apart from its top-level docs, examples, and packaging. - -| File | Purpose | + File | Purpose | |---|---| | [src/validate_models.py](src/validate_models.py) | Command-line entry point and per-tier orchestration | | [src/harness.py](src/harness.py) | The test harness: drives /controls + /process against a live model | | [src/cases.py](src/cases.py) | Synthesis of inputs, test-case overlay, output validation/inspection | | [src/validators.py](src/validators.py) | Registry of custom output validators (to be extended) | +| [src/audio.py](src/audio.py) | Audio decoding (any libsndfile format) for output checks | | [src/assets.py](src/assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | | [src/quota.py](src/quota.py) | ZeroGPU usage tracking and account quota lookup | | [src/results.py](src/results.py) | Result records and JSON/markdown report generation | | [src/utils.py](src/utils.py) | Token handling, config loading, discovery, timeouts | -| [config.yml](config.yml) | Validation configuration (excludes, per-model test cases, etc.) | +| [config.yml](config.yml) | Validation configuration (_e.g._, excludes, per-model test cases) | | [test_data/](test_data/) | Real input files referenced by test cases | -| `reports/` | Generated report output (gitignored) | +| `reports/` | Generated report output | ## Token setup (IMPORTANT — read this) @@ -117,7 +114,7 @@ Two signals are combined: spaces only (models on CPU or dedicated hardware do not draw on the allowance and are never counted). It is an upper bound on GPU seconds consumed, since it includes queue time. Set `zerogpu_budget_seconds` in - [config.yml](config.yml) (e.g. `1500` for the PRO 25 min/day allowance) + [config.yml](config.yml) (_e.g._, `1500` for the PRO 25 min/day allowance) to show usage against a budget. - **Account quota** — fetched from `huggingface.co/api/quota` when available. Hugging Face has no documented public ZeroGPU quota API, so this part is @@ -173,7 +170,7 @@ Note: once `test_cases` is present, **only** the listed cases run — include ### Step 3: check the outputs (optional) Every case already gets structural checks for free: `/process` must not -error, file outputs must exist and be non-empty, and JSON outputs (e.g. an +error, file outputs must exist and be non-empty, and JSON outputs (_i.e._, an optional pyharp `LabelList`) must be well-formed when present (absent/None is valid, since labels are optional). @@ -217,7 +214,7 @@ The full vocabulary, and which output types each rule covers: | `channels` | audio output | Exact channel count | | `sample_rate` | audio output | Exact sample rate in Hz | | `min_duration` / `max_duration` | audio output | Length in seconds is within bounds | -| `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...); errors on compressed formats | +| `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...); errors on compressed formats (_e.g._, MP3 or OGG) | | `min_rms_db` | audio output | RMS level is at least this many dBFS | | `min_labels` | JSON (`LabelList`) output | At least this many labels returned | @@ -232,13 +229,9 @@ apply rules to every output a rule covers — with mixed outputs, `"*"` sends min_bytes: 1000 # every file output must be non-trivial ``` -Labels are preferred over positional indices: they survive a model reordering -its outputs, and they make the config readable without cross-referencing the -model's `app.py`. - **Mistakes are reported as configuration errors, not model failures.** An unrecognized rule name, an unknown output label, or a rule aimed at an output -type it does not cover (`min_labels` on an audio output, say) raises an error +type it does not cover (_e.g._, `min_labels` on an audio output) raises an error naming the problem and listing what is valid. A `"*"` rule matching no output at all is also an error, since it would otherwise silently check nothing. @@ -261,8 +254,8 @@ so audio rules work on any libsndfile format — WAV, FLAC, OGG, MP3, AIFF, and more — not just pyharp's `save_audio()` WAV default. Decoding to float also makes every audio rule bit-depth-independent, so a level threshold means the same thing whether the source is 16-bit, 24-bit, or float. `bit_depth` is the -exception, by definition: it reads the file's PCM encoding and errors on a -compressed format, which has no fixed bit depth. +exception, by definition: it reads the file's PCM encoding and errors on any +compressed formats, which have no fixed bit depth. #### `validators` — custom Python @@ -280,12 +273,11 @@ test case's own fields. A case may run several: A validator receives `(outputs, controls, params)` — `outputs` maps output labels to local file paths (file outputs) or decoded objects (JSON outputs) — and raises `AssertionError` with a message naming the output and what went -wrong. One ships as an example: `labels_within_audio`, which asserts every -returned label falls inside the audio output's timespan. It belongs here -because it relates two different outputs to each other; had it concerned only -one output, it would belong in `expect` instead. +wrong. As an example: `labels_within_audio` asserts every +returned label falls inside the audio output's timespan, +relating two different outputs to each other. -To add one: +To add a validator: ```python @validator("midi_note_count") @@ -328,8 +320,8 @@ Two equivalent ways, merged together: (non-HARP spaces, archived deployments, known-broken examples). - `--exclude ...` on the command line — for ad-hoc runs. -Use the space id (`teamup-tech/some-space`) for remote models and -`examples/` (e.g. `examples/midi_synthesizer`) for local examples. +Use the space id (`teamup-tech/`) for remote models and +`examples/` for local examples. ## CI behavior @@ -346,5 +338,5 @@ Use the space id (`teamup-tech/some-space`) for remote models and - **Opting back into notifications later:** remove the exit-code handling in the two `Validate` steps of the workflow so the script's exit code 1 propagates; failed runs then turn red and GitHub emails maintainers. -- Example failures indicate a pyharp/gradio-level breakage that likely - affects every deployment — fix those before debugging individual spaces. +- Failures with example models likely indicate a pyharp/gradio-level breakage that may + affect every deployment — fix those before debugging individual spaces. diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt index f080f281..50724c5b 100644 --- a/model_validation/requirements.txt +++ b/model_validation/requirements.txt @@ -1,4 +1,4 @@ gradio_client>=1.0 huggingface_hub>=0.23 PyYAML>=6.0 -soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules +soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules From 8943f84ea83e2036a39cc4f6144dfd9f550125a9 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Tue, 21 Jul 2026 08:53:34 -0400 Subject: [PATCH 5/8] Fixed bug with reports/ path, transcoding synthesized WAV when necessary, and abstracted harness timeouts. --- model_validation/README.md | 17 +++--- model_validation/src/assets.py | 69 ++++++++++++++++++++----- model_validation/src/cases.py | 4 +- model_validation/src/harness.py | 19 +++++-- model_validation/src/validate_models.py | 5 +- 5 files changed, 87 insertions(+), 27 deletions(-) diff --git a/model_validation/README.md b/model_validation/README.md index 6589ae14..3955b06b 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -47,7 +47,7 @@ Key behaviors: | [src/utils.py](src/utils.py) | Token handling, config loading, discovery, timeouts | | [config.yml](config.yml) | Validation configuration (_e.g._, excludes, per-model test cases) | | [test_data/](test_data/) | Real input files referenced by test cases | -| `reports/` | Generated report output | +| `reports/` | Generated reports, synthesized assets, and example logs | ## Token setup (IMPORTANT — read this) @@ -94,10 +94,11 @@ pip install -e ./pyharp python model_validation/src/validate_models.py --local-examples ``` -Reports land in `reports/` as `report.json` (machine-readable) and -`report.md` (human-readable table); local example logs are saved alongside -them. Exit code is `0` when everything passes, `1` on any model failure, -`2` on configuration/infrastructure errors. +Reports land in `model_validation/reports/` (override with `--output-dir`) as +`report.json` (machine-readable) and `report.md` (human-readable table); +synthesized test inputs and local example logs are saved alongside them. Exit +code is `0` when everything passes, `1` on any model failure, `2` on +configuration/infrastructure errors. ## ZeroGPU quota reporting @@ -301,9 +302,9 @@ instead of putting it in a one-off validator. python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose ``` -The console shows each case's pass/fail; `reports/report.json` has per-case -timings and error details. Once green, the daily CI run picks the case up -automatically — no workflow changes needed. +The console shows each case's pass/fail; `report.json` has per-case timings +and error details. Once the case passes reliably, the daily CI run picks it +up automatically — no workflow changes needed. ### Reference: full config.yml schema diff --git a/model_validation/src/assets.py b/model_validation/src/assets.py index 85911e3b..78c73b5b 100644 --- a/model_validation/src/assets.py +++ b/model_validation/src/assets.py @@ -1,10 +1,11 @@ """ Synthesized test inputs for HARP model validation. -Every input file is generated from scratch with the standard library, so -validation needs no binary fixtures checked into the repository. Real-world -inputs for specific models belong in test_data/ and are referenced from a -test case's `files` entry in config.yml. +Every input file is generated from scratch - the base WAV and MIDI with the +standard library, other audio formats transcoded from the WAV with soundfile +- so validation needs no binary fixtures checked into the repository. +Real-world inputs for specific models belong in test_data/ and are referenced +from a test case's `files` entry in config.yml. """ import math @@ -12,6 +13,8 @@ import wave from pathlib import Path +import soundfile + __all__ = [ 'Assets', @@ -20,6 +23,13 @@ ] +# Extensions understood as audio; a component accepting one of these gets a +# synthesized clip in that format (transcoded from the base WAV via soundfile, +# so only formats this libsndfile build can write actually succeed). +AUDIO_EXTS = {".wav", ".flac", ".ogg", ".oga", ".opus", ".aiff", ".aif", + ".aifc", ".au", ".snd", ".w64", ".caf", ".mp3"} + + def make_test_wav(path: Path, duration: float = 2.0, sr: int = 44100) -> Path: """ Write a short mono 16-bit sine sweep - a valid input for any audio model. @@ -85,34 +95,69 @@ class Assets: def __init__(self, workdir: Path): workdir.mkdir(parents=True, exist_ok=True) + self.workdir = workdir self.wav = make_test_wav(workdir / "test_input.wav") self.midi = make_test_midi(workdir / "test_input.mid") self.text = workdir / "test_input.txt" self.text.write_text("HARP model validation\n") self.json = workdir / "test_input.json" self.json.write_text("{}\n") + self._audio_cache = {".wav": self.wav} + + def audio_in(self, ext: str) -> Path | None: + """ + Get the synthesized test clip in a given audio format, transcoding + it from the base WAV on first request and caching the result. + + Args: + ext (str): Target extension, e.g. ".flac" or ".ogg". + + Returns: + path (Path | None): The clip in that format, or None when this + libsndfile build cannot write it. + """ + + if ext not in self._audio_cache: + path = self.workdir / f"test_input{ext}" + try: + data, sr = soundfile.read(str(self.wav)) + soundfile.write(str(path), data, sr) + self._audio_cache[ext] = path + except Exception: # noqa: BLE001 - format not writable here + self._audio_cache[ext] = None + return self._audio_cache[ext] def for_file_types(self, file_types: list) -> Path | None: """ - Pick a synthesized file whose format actually matches the accepted - types. Only extensions we can genuinely produce are matched - e.g. - a component accepting only {".mp3", ".flac"} gets None (we cannot - synthesize those with the stdlib), NOT a mislabeled WAV. Supply a - real file via a test case's `files` entry in config.yml instead. + Pick a synthesized file whose format matches the accepted types. + + Audio components get a real clip in an accepted format (WAV, FLAC, + OGG, AIFF, ...), transcoded on demand; MIDI, JSON, and text inputs + are served from their fixed synthesized files. A component whose + accepted types are all unsupported (e.g. a bespoke binary format) + gets None - supply a real file via a test case's `files` entry. Args: file_types (list): Accepted extensions from the /controls spec; empty or None means any file is accepted. Returns: - path (Path | None): A matching synthesized file, or None when - no synthesized format satisfies the component. + path (Path | None): A matching synthesized file, or None when no + synthesized format satisfies the component. """ types = {str(t).lower() for t in (file_types or [])} - if not types or ".wav" in types or "audio" in types: + if not types or "audio" in types: return self.wav + + # Prefer WAV, then any other accepted audio format we can write + for ext in [".wav"] + sorted(types & AUDIO_EXTS - {".wav"}): + if ext in types: + clip = self.audio_in(ext) + if clip is not None: + return clip + if types & {".mid", ".midi"}: return self.midi if ".json" in types: diff --git a/model_validation/src/cases.py b/model_validation/src/cases.py index 8f61f2c6..07f0a94c 100644 --- a/model_validation/src/cases.py +++ b/model_validation/src/cases.py @@ -95,7 +95,9 @@ def synthesize_default_args(controls: dict, assets: Assets) -> tuple: elif ctype == "generic_file": path = assets.for_file_types(spec.get("file_types")) if path is None and spec.get("required", True): - missing[label] = f"cannot synthesize input for file_types={spec.get('file_types')}" + missing[label] = (f"cannot synthesize input for " + f"file_types={spec.get('file_types')}; supply " + f"one via a test case's `files` entry") args.append(handle_file(str(path)) if path is not None else None) elif ctype in ("slider", "number_box"): value = spec.get("value") diff --git a/model_validation/src/harness.py b/model_validation/src/harness.py index 60f8f275..74596df9 100644 --- a/model_validation/src/harness.py +++ b/model_validation/src/harness.py @@ -43,6 +43,15 @@ # Stages that resolve on their own if we wait (or wake on first request) TRANSIENT_STAGES = {"BUILDING", "RUNNING_BUILDING", "APP_STARTING", "SLEEPING"} +# Fixed sub-timeouts, in seconds. Unlike connect_timeout / process_timeout, +# these bound quick metadata calls whose duration does not vary by model, so +# they are module constants rather than per-model config. +CONTROLS_TIMEOUT = 120 # fetch the /controls spec (model already loaded) +CLIENT_CLOSE_TIMEOUT = 10 # best-effort gradio_client shutdown +SERVER_PROBE_INTERVAL = 2 # poll cadence while a local example boots +SERVER_PROBE_TIMEOUT = 5 # per-probe HTTP timeout against a local example +PROCESS_TERMINATE_TIMEOUT = 15 # grace period before killing a local example + def close_client(client) -> None: """ @@ -57,7 +66,7 @@ def close_client(client) -> None: return try: - run_with_timeout(client.close, 10, "client close") + run_with_timeout(client.close, CLIENT_CLOSE_TIMEOUT, "client close") except Exception: # noqa: BLE001 - cleanup is best-effort pass @@ -95,7 +104,7 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, # --- /controls ----------------------------------------------------------- controls = run_with_timeout( - lambda: client.predict(api_name="/controls"), 120, "/controls") + lambda: client.predict(api_name="/controls"), CONTROLS_TIMEOUT, "/controls") if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: result.error = f"/controls returned malformed data: {str(controls)[:200]}" return result @@ -281,10 +290,10 @@ def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> if proc.poll() is not None: raise RuntimeError(f"app exited early with code {proc.returncode}") try: - with urllib.request.urlopen(url, timeout=5): + with urllib.request.urlopen(url, timeout=SERVER_PROBE_TIMEOUT): return except Exception: # noqa: BLE001 - server not up yet - time.sleep(2) + time.sleep(SERVER_PROBE_INTERVAL) raise TimeoutError(f"local app did not become ready within {int(timeout)}s") @@ -343,6 +352,6 @@ def test_local_example(app_dir: Path, port: int, assets: Assets, if proc is not None and proc.poll() is None: proc.terminate() try: - proc.wait(timeout=15) + proc.wait(timeout=PROCESS_TERMINATE_TIMEOUT) except subprocess.TimeoutExpired: proc.kill() diff --git a/model_validation/src/validate_models.py b/model_validation/src/validate_models.py index 9500a11c..ad15e298 100644 --- a/model_validation/src/validate_models.py +++ b/model_validation/src/validate_models.py @@ -58,6 +58,7 @@ REPO_ROOT = MODEL_VALIDATION_DIR.parent DEFAULT_CONFIG = MODEL_VALIDATION_DIR / "config.yml" DEFAULT_EXAMPLES_DIR = REPO_ROOT / "pyharp" / "examples" +DEFAULT_OUTPUT_DIR = MODEL_VALIDATION_DIR / "reports" LOCAL_PORT_BASE = 7861 @@ -97,7 +98,9 @@ def parse_args() -> argparse.Namespace: help="Seconds to wait for a deployment to build/wake/start") parser.add_argument("--process-timeout", type=float, default=600, help="Seconds to wait for /process (includes ZeroGPU queue time)") - parser.add_argument("--output-dir", type=Path, default=Path("reports")) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, + help=f"Directory for reports, synthesized assets, and " + f"example logs (default: {DEFAULT_OUTPUT_DIR})") parser.add_argument("--verbose", action="store_true") return parser.parse_args() From 786bca24e86962c7ab126c7bcbbb03448306a376 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Thu, 23 Jul 2026 08:09:54 -0400 Subject: [PATCH 6/8] No longer attempting to track ZeroGPU quota explicitly, skip-zerogpu option, retrying loading ZeroGPU models, and updates to printouts and README. --- model_validation/README.md | 49 +++++++----- model_validation/src/harness.py | 85 ++++++++++++++++++-- model_validation/src/quota.py | 102 +++++------------------- model_validation/src/results.py | 6 +- model_validation/src/validate_models.py | 31 ++++--- 5 files changed, 150 insertions(+), 123 deletions(-) diff --git a/model_validation/README.md b/model_validation/README.md index 3955b06b..f251673a 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -16,16 +16,14 @@ Two tiers share the same test harness: Key behaviors: -- **Models are validated independently.** Each model (and each test case - within it) runs inside its own error boundary, so a crash, hang, or - timeout in one model never stops validation of the others. All results are - collected into a single report and the process exits non-zero only after - everything has run. +- **Models are validated independently** — a crash, hang, or timeout in one + never stops the rest; all results are collected into a single report and + the process exits non-zero only after everything has run. - **Crashed/stopped spaces are restarted automatically** (sleeping spaces are woken simply by connecting). Pass `--no-restart-failed` to disable; restarting requires a token with write access. -- **ZeroGPU quota is reported** at the start of a spaces run and after every - model, so it is easy to tell if and when quota will be exceeded mid-run. +- **ZeroGPU time is tracked** across the run and reported after each ZeroGPU + model; `--skip-zerogpu` skips those models to spend no allowance at all. - A daily GitHub Action ([model_validation.yml](../.github/workflows/model_validation.yml)) runs both tiers at 06:30 UTC. Model failures do **not** turn the run red (no @@ -85,6 +83,9 @@ python model_validation/src/validate_models.py --exclude teamup-tech/broken-spac # Availability + /controls only (fast; no inference, no GPU quota used) python model_validation/src/validate_models.py --load-only +# Skip ZeroGPU models, to spend none of the shared ZeroGPU allowance +python model_validation/src/validate_models.py --skip-zerogpu + # Never restart spaces (works with a read-only token) python model_validation/src/validate_models.py --no-restart-failed @@ -100,26 +101,30 @@ synthesized test inputs and local example logs are saved alongside them. Exit code is `0` when everything passes, `1` on any model failure, `2` on configuration/infrastructure errors. -## ZeroGPU quota reporting +## ZeroGPU time tracking -Space validation prints the quota state at the start of the run and after -every model, on the same line as its pass/fail status: +Each model's line shows its hardware, and ZeroGPU models additionally show +the running ZeroGPU time consumed so far this run: ``` -✅ PASS teamup-tech/pitch_shifter (42.1s) [GPU time ~38s/1500s budget | ...] +✅ PASS teamup-tech/pitch_shifter (42.1s) [zero-a10g | ZeroGPU time ~38s this run] +✅ PASS teamup-tech/cpu_model (18.3s) [cpu-basic] ``` -Two signals are combined: - -- **GPU time this run** — cumulative `/process` wall time on ZeroGPU-hardware - spaces only (models on CPU or dedicated hardware do not draw on the - allowance and are never counted). It is an upper bound on GPU seconds - consumed, since it includes queue time. Set `zerogpu_budget_seconds` in - [config.yml](config.yml) (_e.g._, `1500` for the PRO 25 min/day allowance) - to show usage against a budget. -- **Account quota** — fetched from `huggingface.co/api/quota` when available. - Hugging Face has no documented public ZeroGPU quota API, so this part is - best-effort and silently omitted if the endpoint yields nothing usable. +The tracked total is cumulative `/process` wall time across ZeroGPU models +only — CPU and dedicated-hardware models never contribute. It is an upper +bound on the GPU seconds charged, since wall time includes queue time. Set +`zerogpu_budget_seconds` in [config.yml](config.yml) (_e.g._, `1500` for the +PRO 25 min/day allowance) to show it against a budget. + +This is the ZeroGPU time *this run* spends, not your account's remaining +quota — Hugging Face exposes no reliable public API for the latter, so it is +not reported. To avoid spending any allowance, pass `--skip-zerogpu`, which +skips ZeroGPU models entirely (they appear as `SKIP` in the report). + +Transient "model is still loading" responses from a ZeroGPU space waking its +GPU worker are retried automatically (up to `connect_timeout`), rather than +counted as failures. ## Configuring test cases — a walkthrough diff --git a/model_validation/src/harness.py b/model_validation/src/harness.py index 74596df9..e5bb40b8 100644 --- a/model_validation/src/harness.py +++ b/model_validation/src/harness.py @@ -28,7 +28,8 @@ from assets import Assets from cases import synthesize_default_args, apply_case, validate_outputs, inspect_outputs -from results import ModelResult, CaseResult, PASS, FAIL +from quota import is_zerogpu +from results import ModelResult, CaseResult, PASS, FAIL, SKIP from utils import run_with_timeout, scrub @@ -51,6 +52,59 @@ SERVER_PROBE_INTERVAL = 2 # poll cadence while a local example boots SERVER_PROBE_TIMEOUT = 5 # per-probe HTTP timeout against a local example PROCESS_TERMINATE_TIMEOUT = 15 # grace period before killing a local example +LOADING_RETRY_INTERVAL = 15 # wait between retries while a model warms up + +# Substrings marking a "model is still loading" response - a ZeroGPU space +# waking its GPU worker returns this immediately rather than blocking, so it +# is retried (within connect_timeout) instead of treated as a failure. +LOADING_MARKERS = ("still loading", "loading, please wait", "is loading", + "currently loading", "warming up") + + +def is_loading_error(exc: Exception) -> bool: + """ + Whether an exception is a transient "model still loading" response. + + Args: + exc (Exception): The exception raised by a gradio call. + + Returns: + loading (bool): True if the model reported it was still loading. + """ + + message = str(exc).lower() + return any(marker in message for marker in LOADING_MARKERS) + + +def call_through_loading(fn, deadline: float, what: str): + """ + Call fn(), retrying while the model reports it is still loading. + + A ZeroGPU space that has to spin up its GPU worker answers the first + request with a "still loading" error immediately; retrying until the + deadline lets validation ride through the warm-up instead of failing. + + Args: + fn (callable): Zero-argument call to make. + deadline (float): time.time() value to stop retrying at. + what (str): Short description for the timeout message. + + Returns: + result: Whatever fn() returns once the model is ready. + + Raises: + Exception: fn()'s error, once it is not a loading error or the + deadline has passed. + """ + + while True: + try: + return fn() + except Exception as exc: # noqa: BLE001 - inspected below, else re-raised + if is_loading_error(exc) and time.time() + LOADING_RETRY_INTERVAL < deadline: + time.sleep(LOADING_RETRY_INTERVAL) + continue + raise def close_client(client) -> None: @@ -76,8 +130,8 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, """ Verify /controls and run every configured /process test case. - Each case runs inside its own error boundary, so a failing case never - stops the remaining cases (or models) from running. + Each case runs inside its own error boundary, so a failing case does not + stop the remaining cases. Args: client (Client): Connected gradio client for the deployment. @@ -91,6 +145,7 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, """ process_timeout = overrides.get("process_timeout", opts.process_timeout) + connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) load_only = opts.load_only or overrides.get("load_only", False) config_dir = opts.config.parent.resolve() @@ -102,9 +157,11 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, f"deployment may use an outdated pyharp") return result - # --- /controls ----------------------------------------------------------- - controls = run_with_timeout( - lambda: client.predict(api_name="/controls"), CONTROLS_TIMEOUT, "/controls") + # --- /controls (retry through a warming-up model) ------------------------ + controls = call_through_loading( + lambda: run_with_timeout( + lambda: client.predict(api_name="/controls"), CONTROLS_TIMEOUT, "/controls"), + time.time() + connect_timeout, "/controls") if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: result.error = f"/controls returned malformed data: {str(controls)[:200]}" return result @@ -133,8 +190,14 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, continue args = apply_case(default_args, controls, case, config_dir) - job = client.submit(*args, api_name="/process") - output = job.result(timeout=case.get("process_timeout", process_timeout)) + case_timeout = case.get("process_timeout", process_timeout) + + def run_process(): + job = client.submit(*args, api_name="/process") + return job.result(timeout=case_timeout) + + output = call_through_loading( + run_process, time.time() + connect_timeout, "/process") error = validate_outputs(output, controls) if error: @@ -221,6 +284,12 @@ def test_space(space_id: str, token: str, assets: Assets, # it is sleeping or stopped (hardware itself is only set when live) result.hardware = runtime.requested_hardware or runtime.hardware or "" + # Skip ZeroGPU models before doing any work that would spend quota + if opts.skip_zerogpu and is_zerogpu(result.hardware): + result.status = SKIP + result.error = "skipped: ZeroGPU hardware (--skip-zerogpu)" + return result + if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): if opts.restart_failed and stage != "DELETING": print(f" [{space_id}] stage={stage}, requesting restart...") diff --git a/model_validation/src/quota.py b/model_validation/src/quota.py index 6d18bad4..7ae6fe84 100644 --- a/model_validation/src/quota.py +++ b/model_validation/src/quota.py @@ -1,19 +1,18 @@ """ -ZeroGPU quota tracking for HARP model validation. +ZeroGPU usage tracking for HARP model validation. -ZeroGPU allowances are consumed per account, so a long validation run can -exhaust the day's quota partway through. To make that visible, the quota -state is reported at the start of a run and after every model. +ZeroGPU allowances are consumed per account, so a long validation run can eat +into the day's quota. There is no documented public API for the remaining +account quota, so rather than guess at it this module tracks the ZeroGPU time +*this run* consumes - the part attributable to validation - and reports it +after every ZeroGPU model. """ -import json import threading -import urllib.request __all__ = [ - 'QuotaTracker', - 'fetch_account_quota', + 'ZeroGPUTracker', 'is_zerogpu' ] @@ -35,101 +34,44 @@ def is_zerogpu(hardware: str) -> bool: return bool(hardware) and hardware.lower().startswith("zero") -def fetch_account_quota(token: str) -> str | None: +class ZeroGPUTracker: """ - Best-effort fetch of the account's quota state from huggingface.co. + Accumulates the ZeroGPU processing time consumed during a run. - Hugging Face has no *documented* public API for ZeroGPU quota; /api/quota - exists but its schema is unstable, so parse defensively: surface any - entries whose keys mention gpu/zero and return None when nothing useful - comes back. Never raises. - - Args: - token (str): Hugging Face access token. - - Returns: - summary (str | None): Compact "key=value" summary of GPU-related - quota entries, or None when unavailable. + The total is the cumulative /process wall time across ZeroGPU models + (CPU and dedicated-hardware models never contribute). It is an upper + bound on the GPU seconds charged, since wall time includes queue time, + and is shown against an optional budget (`zerogpu_budget_seconds` in + config.yml). """ - if not token: - return None - - try: - req = urllib.request.Request( - "https://huggingface.co/api/quota", - headers={"Authorization": f"Bearer {token}"}) - with urllib.request.urlopen(req, timeout=10) as resp: - data = json.loads(resp.read().decode()) - except Exception: # noqa: BLE001 - quota reporting must never break a run - return None - - def gpu_entries(obj, prefix=""): - found = [] - if isinstance(obj, dict): - for key, value in obj.items(): - name = f"{prefix}{key}" - if any(s in key.lower() for s in ("zero", "gpu")) and \ - isinstance(value, (int, float, str)): - found.append(f"{name}={value}") - else: - found += gpu_entries(value, f"{name}.") - elif isinstance(obj, list): - for value in obj: - found += gpu_entries(value, prefix) - return found - - entries = gpu_entries(data) - - return ", ".join(entries[:4]) if entries else None - - -class QuotaTracker: - """ - Tracks ZeroGPU usage during a validation run. - - Two signals are combined into each status line: - - cumulative /process wall time this run on ZeroGPU spaces only - (CPU-hardware models do not draw on the allowance and are never - counted); an upper bound on GPU seconds consumed, since it - includes queue time, shown against an optional budget - (`zerogpu_budget_seconds` in config.yml); - - the account quota reported by huggingface.co, when available. - """ - - def __init__(self, token: str, budget: float | None): - self.token = token + def __init__(self, budget: float | None): self.budget = budget self.used = 0.0 self._lock = threading.Lock() def add(self, seconds: float) -> None: """ - Record processing time consumed by a completed model validation. + Record ZeroGPU processing time from a completed model validation. Args: - seconds (float): Wall time spent in /process calls. + seconds (float): Wall time spent in this model's /process calls. """ with self._lock: self.used += seconds - def status(self) -> str: + def summary(self) -> str: """ - Format the current quota state for a console line. + Format the ZeroGPU time consumed so far for a console line. Returns: - status (str): e.g. "[GPU time ~38s/1500s budget | left=120s]". + summary (str): e.g. "ZeroGPU time ~38s/1500s budget this run". """ with self._lock: used = int(self.used) if self.budget: - usage = f"GPU time ~{used}s/{int(self.budget)}s budget" - else: - usage = f"GPU time ~{used}s this run" - - account = fetch_account_quota(self.token) - - return f"[{usage}]" if account is None else f"[{usage} | {account}]" + return f"ZeroGPU time ~{used}s/{int(self.budget)}s budget this run" + return f"ZeroGPU time ~{used}s this run" diff --git a/model_validation/src/results.py b/model_validation/src/results.py index 5b58cdfe..b6d7fd25 100644 --- a/model_validation/src/results.py +++ b/model_validation/src/results.py @@ -80,14 +80,18 @@ def write_reports(results: list, out_dir: Path, label: str) -> None: "total": len(results), "passed": sum(r.status == PASS for r in results), "failed": sum(r.status == FAIL for r in results), + "skipped": sum(r.status == SKIP for r in results), "results": [dataclasses.asdict(r) for r in results], } (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) + headline = f"**{payload['passed']}/{payload['total']} models passed**" + if payload["skipped"]: + headline += f", {payload['skipped']} skipped" lines = [ f"# HARP Model Validation Report - {label}", "", - f"**{payload['passed']}/{payload['total']} models passed** ({payload['timestamp']})", + f"{headline} ({payload['timestamp']})", "", "| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", "|---|---|---|---|---|---|---|---|", diff --git a/model_validation/src/validate_models.py b/model_validation/src/validate_models.py index ad15e298..96ae3da9 100644 --- a/model_validation/src/validate_models.py +++ b/model_validation/src/validate_models.py @@ -22,9 +22,7 @@ Per-model test cases are declared in config.yml (see README.md for a full walkthrough). When a model has no configured cases, a single "default" case -runs with inputs synthesized automatically from the /controls spec. Models -are validated independently: a failure (or timeout) in one model never stops -validation of the others. +runs with inputs synthesized automatically from the /controls spec. Usage: HF_TOKEN=... python model_validation/src/validate_models.py @@ -47,8 +45,8 @@ from assets import Assets from harness import test_space, test_local_example -from quota import QuotaTracker, is_zerogpu -from results import FAIL, status_emoji, write_reports +from quota import ZeroGPUTracker, is_zerogpu +from results import PASS, FAIL, SKIP, status_emoji, write_reports from utils import get_token, scrub, load_config, get_excluded, discover_spaces @@ -87,6 +85,9 @@ def parse_args() -> argparse.Namespace: help="Optional YAML config (excludes, per-model overrides/cases)") parser.add_argument("--load-only", action="store_true", help="Only verify availability and /controls, do not run inference") + parser.add_argument("--skip-zerogpu", action="store_true", + help="Skip models on ZeroGPU hardware, to avoid spending " + "the shared ZeroGPU allowance") parser.add_argument("--restart-failed", action=argparse.BooleanOptionalAction, default=True, help="Attempt to restart spaces found in an error/stopped state; " @@ -180,10 +181,9 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, print(f"ERROR: no spaces found for org '{opts.org}'", file=sys.stderr) return None - tracker = QuotaTracker(token, config.get("zerogpu_budget_seconds")) + tracker = ZeroGPUTracker(config.get("zerogpu_budget_seconds")) print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " - f"(process test: {'OFF' if opts.load_only else 'ON'})") - print(f"ZeroGPU quota at start: {tracker.status()}\n") + f"(process test: {'OFF' if opts.load_only else 'ON'})\n") results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: @@ -195,12 +195,15 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, for future in concurrent.futures.as_completed(futures): r = future.result() results.append(r) - # CPU-hardware models do not draw on the ZeroGPU allowance + # Show the hardware for every model; the running ZeroGPU total is + # only meaningful (and only accrues) for ZeroGPU models + info = r.hardware or "?" if is_zerogpu(r.hardware): tracker.add(sum(c.duration for c in r.cases)) + info = f"{info} | {tracker.summary()}" note = f" - {r.error}" if r.error else "" print(f"{status_emoji(r)} {r.status:4s} {r.target} " - f"({r.duration}s) {tracker.status()}{scrub(note, token)}") + f"({r.duration}s) [{info}]{scrub(note, token)}") return results @@ -232,9 +235,13 @@ def main() -> int: write_reports(results, opts.output_dir, label) + passed = [r for r in results if r.status == PASS] failed = [r for r in results if r.status == FAIL] - print(f"\n{len(results) - len(failed)}/{len(results)} models passed. " - f"Reports written to {opts.output_dir}/") + skipped = [r for r in results if r.status == SKIP] + summary = f"\n{len(passed)}/{len(results)} models passed" + if skipped: + summary += f", {len(skipped)} skipped" + print(f"{summary}. Reports written to {opts.output_dir}/") if failed: print("\nFailed models:") for r in sorted(failed, key=lambda r: r.target): From bdbaf1b637e6a8154f958ba50967175057c8f6fb Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Fri, 24 Jul 2026 09:43:54 -0400 Subject: [PATCH 7/8] Added MIDI output rules and reusable test cases, lowered timeout for ZeroGPU models, graceful handling of remaining ZeroGPU models after quota exceeded, and improved reporting. --- .github/workflows/model_validation.yml | 8 +- model_validation/README.md | 151 +++++++++++++++++------- model_validation/config.yml | 41 +++++-- model_validation/requirements.txt | 3 +- model_validation/src/cases.py | 115 ++++++++++++------ model_validation/src/harness.py | 132 +++++++++++++++++++-- model_validation/src/midi.py | 53 +++++++++ model_validation/src/quota.py | 79 ++++++++++--- model_validation/src/results.py | 33 ++++-- model_validation/src/validate_models.py | 39 ++++-- model_validation/src/validators.py | 24 +++- 11 files changed, 537 insertions(+), 141 deletions(-) create mode 100644 model_validation/src/midi.py diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml index b4ea8a4c..2dd95b46 100644 --- a/.github/workflows/model_validation.yml +++ b/.github/workflows/model_validation.yml @@ -90,14 +90,14 @@ jobs: - name: Publish summary if: always() - run: cat reports/examples/report.md >> "$GITHUB_STEP_SUMMARY" || true + run: cat reports/examples/*/report.md >> "$GITHUB_STEP_SUMMARY" || true - name: Upload report if: always() uses: actions/upload-artifact@v4 with: name: report-examples - path: reports/examples/report.* + path: reports/examples/*/report.* if-no-files-found: ignore spaces: @@ -147,12 +147,12 @@ jobs: - name: Publish summary if: always() - run: cat reports/spaces/report.md >> "$GITHUB_STEP_SUMMARY" || true + run: cat reports/spaces/*/report.md >> "$GITHUB_STEP_SUMMARY" || true - name: Upload report if: always() uses: actions/upload-artifact@v4 with: name: report-spaces - path: reports/spaces/report.* + path: reports/spaces/*/report.* if-no-files-found: ignore diff --git a/model_validation/README.md b/model_validation/README.md index f251673a..0dd60fae 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -39,6 +39,7 @@ Key behaviors: | [src/cases.py](src/cases.py) | Synthesis of inputs, test-case overlay, output validation/inspection | | [src/validators.py](src/validators.py) | Registry of custom output validators (to be extended) | | [src/audio.py](src/audio.py) | Audio decoding (any libsndfile format) for output checks | +| [src/midi.py](src/midi.py) | MIDI parsing (via mido) for output checks | | [src/assets.py](src/assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | | [src/quota.py](src/quota.py) | ZeroGPU usage tracking and account quota lookup | | [src/results.py](src/results.py) | Result records and JSON/markdown report generation | @@ -95,32 +96,58 @@ pip install -e ./pyharp python model_validation/src/validate_models.py --local-examples ``` -Reports land in `model_validation/reports/` (override with `--output-dir`) as -`report.json` (machine-readable) and `report.md` (human-readable table); -synthesized test inputs and local example logs are saved alongside them. Exit -code is `0` when everything passes, `1` on any model failure, `2` on +Each run writes to its own timestamped directory under +`model_validation/reports/` (override the base with `--output-dir`), so runs +never overwrite each other — e.g. `model_validation/reports/2026-07-18T14-30-00Z/`. +It contains `report.json` (machine-readable) and `report.md` (human-readable +table), with synthesized test inputs and local example logs alongside. Both +reports record the command line the run was invoked with. Exit code is `0` +when everything passes, `1` on any model failure, `2` on configuration/infrastructure errors. -## ZeroGPU time tracking +## ZeroGPU usage tracking Each model's line shows its hardware, and ZeroGPU models additionally show -the running ZeroGPU time consumed so far this run: +the ZeroGPU work done so far this run: ``` -✅ PASS teamup-tech/pitch_shifter (42.1s) [zero-a10g | ZeroGPU time ~38s this run] +✅ PASS teamup-tech/pitch_shifter (42.1s) [zero-a10g | ZeroGPU: 2 calls, ~84s wall (approx)] ✅ PASS teamup-tech/cpu_model (18.3s) [cpu-basic] ``` -The tracked total is cumulative `/process` wall time across ZeroGPU models -only — CPU and dedicated-hardware models never contribute. It is an upper -bound on the GPU seconds charged, since wall time includes queue time. Set -`zerogpu_budget_seconds` in [config.yml](config.yml) (_e.g._, `1500` for the -PRO 25 min/day allowance) to show it against a budget. - -This is the ZeroGPU time *this run* spends, not your account's remaining -quota — Hugging Face exposes no reliable public API for the latter, so it is -not reported. To avoid spending any allowance, pass `--skip-zerogpu`, which -skips ZeroGPU models entirely (they appear as `SKIP` in the report). +Two figures are tracked, because the exact billed amount is not observable +from the client: + +- **Call count** — the number of `/process` calls that reached the GPU. This + is exact and is the most reliable signal of how much of the allowance a run + will use. Only calls that ran count; a queued or input-skipped case does + not. CPU and dedicated-hardware models never contribute. +- **Wall time** — the total `/process` wall time (queue plus execution) of + those calls, marked `(approx)`. Hugging Face bills ZeroGPU **dynamically**: + each call reserves its declared `@spaces.GPU(duration=)` time up front and + refunds the unused part when the function returns, so an account's usage + rises during a run and settles lower afterwards. This wall figure is an + over-estimate of the settled bill — closer to that mid-run reservation peak + — and is not returned to the client, so treat it as indicative. Set + `zerogpu_budget_seconds` in [config.yml](config.yml) (_e.g._, `1500` for the + PRO 25 min/day allowance) to show it against a budget. + +Each case's wall time is also recorded in `report.json` (`duration`). These +figures reflect the work *this run* does, not your account's remaining quota — +Hugging Face exposes no reliable public API for the latter. To spend none of +the allowance, pass `--skip-zerogpu`, which skips ZeroGPU models entirely +(they appear as `SKIP` in the report). + +Two mechanisms limit the ZeroGPU allowance a run can consume: + +- **Quota exhaustion stops the remaining ZeroGPU models.** If a ZeroGPU model + fails with a "quota exceeded" error, every remaining ZeroGPU model is + skipped (as `SKIP`) rather than run against an exhausted allowance. +- **ZeroGPU models use a lower execution timeout** — `--zerogpu-process-timeout` + (default 120s) instead of `--process-timeout` (default 600s). This bounds + execution only; the queue wait before a job runs is bounded separately by + `--connect-timeout`, so a long queue does not trip the execution timeout. A + per-model `process_timeout` override takes precedence. Transient "model is still loading" responses from a ZeroGPU space waking its GPU worker are retried automatically (up to `connect_timeout`), rather than @@ -182,8 +209,8 @@ is valid, since labels are optional). Beyond that, there are two mechanisms, divided by what they can express: -- **`expect`** asserts properties of a *single* output, and covers almost - every check worth writing. +- **`expect`** asserts properties of a *single* output, and covers most + checks. - **`validators`** run custom Python for checks that cannot be expressed as a property of one output — typically relationships spanning several outputs. @@ -219,15 +246,17 @@ The full vocabulary, and which output types each rule covers: | `min_bytes` | any file output | File is at least this many bytes | | `channels` | audio output | Exact channel count | | `sample_rate` | audio output | Exact sample rate in Hz | -| `min_duration` / `max_duration` | audio output | Length in seconds is within bounds | | `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...); errors on compressed formats (_e.g._, MP3 or OGG) | | `min_rms_db` | audio output | RMS level is at least this many dBFS | +| `min_duration` / `max_duration` | audio or MIDI output | Length in seconds is within bounds | +| `min_notes` | MIDI output | At least this many note-on events | | `min_labels` | JSON (`LabelList`) output | At least this many labels returned | **Targeting outputs.** A model with several outputs gets one block per output label, each checked independently. Use `"*"` instead of a label to apply rules to every output a rule covers — with mixed outputs, `"*"` sends -`min_bytes` to all file outputs and `min_rms_db` only to the audio ones: +`min_bytes` to all file outputs, `min_rms_db` only to the audio ones, and +`min_notes` only to the MIDI ones: ```yaml expect: @@ -235,11 +264,14 @@ apply rules to every output a rule covers — with mixed outputs, `"*"` sends min_bytes: 1000 # every file output must be non-trivial ``` -**Mistakes are reported as configuration errors, not model failures.** An -unrecognized rule name, an unknown output label, or a rule aimed at an output -type it does not cover (_e.g._, `min_labels` on an audio output) raises an error -naming the problem and listing what is valid. A `"*"` rule matching no output -at all is also an error, since it would otherwise silently check nothing. +**Mistakes on a named output are configuration errors, not model failures.** +An unrecognized rule name, an unknown output label, or a rule aimed at a named +output whose type it does not cover (_e.g._, `min_labels` on an audio output) +raises an error naming the problem and listing what is valid. A rule under +`"*"` is more forgiving: when it matches no compatible output, it is skipped +instead of raising an error. This lets a generic case (see +[common test cases](#step-4-reuse-a-case-across-models)) target output types +that a given model does not have. **On `min_rms_db`:** level is measured as RMS in dBFS (0 dBFS = full scale), a bit-depth-independent unit that is easy to set thresholds in. Digital @@ -260,8 +292,13 @@ so audio rules work on any libsndfile format — WAV, FLAC, OGG, MP3, AIFF, and more — not just pyharp's `save_audio()` WAV default. Decoding to float also makes every audio rule bit-depth-independent, so a level threshold means the same thing whether the source is 16-bit, 24-bit, or float. `bit_depth` is the -exception, by definition: it reads the file's PCM encoding and errors on any -compressed formats, which have no fixed bit depth. +exception: it reads the file's PCM encoding and errors on compressed formats, +which have no fixed bit depth. + +**On MIDI:** MIDI outputs are parsed with +[mido](https://mido.readthedocs.io/) (a listed dependency). `min_duration` / +`max_duration` are shared with audio and read the MIDI's own length in +seconds; `min_notes` counts note-on events. #### `validators` — custom Python @@ -279,21 +316,24 @@ test case's own fields. A case may run several: A validator receives `(outputs, controls, params)` — `outputs` maps output labels to local file paths (file outputs) or decoded objects (JSON outputs) — and raises `AssertionError` with a message naming the output and what went -wrong. As an example: `labels_within_audio` asserts every -returned label falls inside the audio output's timespan, -relating two different outputs to each other. +wrong. It may raise `ValidatorNotApplicable` to opt out on a model that lacks +the outputs it needs; that is treated as a skip, not a failure, so a validator +can be used in a common test case. As an example, `labels_within_audio` +asserts every returned label falls inside the audio output's timespan (a +relationship between two outputs), and raises `ValidatorNotApplicable` when +the model has no audio or no label output. To add a validator: ```python -@validator("midi_note_count") -def midi_note_count(outputs, controls, params): - """Assert the MIDI output has at least params['min_notes'] note-ons.""" +@validator("labels_sorted") +def labels_sorted(outputs, controls, params): + """Assert every LabelList output is in chronological order - an ordering + property no single-value `expect` rule can express.""" for label, value in outputs.items(): - if isinstance(value, str) and value.lower().endswith((".mid", ".midi")): - notes = count_note_ons(value) - assert notes >= params.get("min_notes", 1), \ - f"'{label}' has {notes} notes, expected at least {params['min_notes']}" + if isinstance(value, dict) and isinstance(value.get("labels"), list): + times = [entry.get("t", 0.0) for entry in value["labels"]] + assert times == sorted(times), f"'{label}' labels are out of order" ``` Before writing one, check whether an `expect` rule would do — and if the @@ -301,7 +341,35 @@ check is a generally useful property of a single output, consider adding it to the `expect` vocabulary (`EXPECT_RULES` in [cases.py](src/cases.py)) instead of putting it in a one-off validator. -### Step 4: run just that model to iterate +### Step 4: reuse a case across models + +A case under a model's `test_cases` only applies to that model. For a check +that should hold for *every* model — "no file output is empty", "audio is +never silent" — add a `common_test_cases` entry at the top level of +config.yml instead. Common cases run on every model in addition to its own, +and their names are shown in the report prefixed with `common:`. + +```yaml +common_test_cases: + - name: outputs-nontrivial + expect: + "*": + min_bytes: 100 # every file output, on every model + - name: audio-not-silent + expect: + "*": + min_rms_db: -60 # applies only to models that have audio outputs +``` + +Keep common cases model-agnostic: target outputs with `"*"` (never a specific +label, which will not exist on most models). Because a `"*"` rule is skipped +when it matches no compatible output, `audio-not-silent` above checks audio +models and is skipped on MIDI-only ones. A validator can do the same by +raising `ValidatorNotApplicable` when the model lacks the outputs it needs +(the shipped `labels_within_audio` does this). A model can opt out of common +cases entirely with `skip_common_cases: true` in its `overrides` entry. + +### Step 5: run just that model to iterate ```bash python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose @@ -315,8 +383,9 @@ up automatically — no workflow changes needed. See the comment block at the top of [config.yml](config.yml) for the complete schema in one place: `zerogpu_budget_seconds`, `exclude`, -`include_extra`, and per-model `overrides` (`connect_timeout`, -`process_timeout`, `load_only`, `test_cases`). +`include_extra`, top-level `common_test_cases`, and per-model `overrides` +(`connect_timeout`, `process_timeout`, `load_only`, `test_cases`, +`skip_common_cases`). ## Excluding models diff --git a/model_validation/config.yml b/model_validation/config.yml index 27038926..5b2b8d45 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -6,9 +6,11 @@ # # Schema: # -# zerogpu_budget_seconds: 1500 # optional ZeroGPU budget for quota -# # reporting (e.g. 1500 = PRO 25 min/day); -# # unset = report usage without a budget +# zerogpu_budget_seconds: 1500 # optional budget (seconds) the measured +# # ZeroGPU execution time is shown against +# # (e.g. 1500 = PRO 25 min/day). Note the +# # measured time is approximate, not the +# # amount Hugging Face bills (see README) # # exclude: # models to exclude from validation: space ids, or # # "examples/" for local pyharp examples @@ -47,10 +49,12 @@ # min_bytes: 1000 # minimum file size, in bytes # channels: 1 # audio: exact channel count # sample_rate: 44100 # audio: exact sample rate, in Hz -# min_duration: 1.5 # audio: minimum length, in seconds -# max_duration: 10.0 # audio: maximum length, in seconds +# min_duration: 1.5 # audio/MIDI: minimum length, in seconds +# max_duration: 10.0 # audio/MIDI: maximum length, in seconds # bit_depth: 16 # audio: exact PCM bit depth (PCM formats) # min_rms_db: -60 # audio: minimum RMS level, in dBFS +# "Output MIDI": +# min_notes: 1 # MIDI: minimum note-on count # "Output Labels": # min_labels: 1 # LabelList: minimum count (0 permits an # # empty list, but requires a valid one) @@ -61,17 +65,29 @@ # labels_within_audio: # tolerance: 0.1 # +# skip_common_cases: true # opt this model out of common_test_cases +# +# common_test_cases: # generic cases run on EVERY model, on top of its +# # own; keep them model-agnostic - target outputs with +# # "*" (rules that match nothing are simply skipped) +# - name: outputs-nontrivial +# expect: +# "*": +# min_bytes: 100 # every file output is at least minimally sized +# # Which expect rules apply to which output types: # # ext, min_bytes any file output -# channels, sample_rate, min_duration, audio outputs -# max_duration, bit_depth, min_rms_db +# channels, sample_rate, bit_depth, min_rms_db audio outputs +# min_duration, max_duration audio and MIDI outputs +# min_notes MIDI outputs # min_labels JSON (LabelList) outputs # -# Applying a rule to an output type it does not cover is reported as a -# configuration error, not a model failure. Audio outputs are decoded with -# soundfile, so any libsndfile format works (WAV, FLAC, OGG, MP3, AIFF, ...); -# bit_depth applies only to PCM formats, not compressed ones. +# Applying a rule to a NAMED output whose type it does not cover is reported +# as a configuration error, not a model failure; under "*" such a rule is +# simply skipped. Audio outputs are decoded with soundfile, so any libsndfile +# format works (WAV, FLAC, OGG, MP3, AIFF, ...); bit_depth applies only to +# PCM formats, not compressed ones. MIDI outputs are parsed with mido. exclude: [] @@ -109,6 +125,9 @@ overrides: - name: shift-up-max controls: "Pitch Shift (semitones)": 24 + expect: + "Output Midi": + min_notes: 1 # shifted MIDI must still contain notes - name: shift-down-max controls: "Pitch Shift (semitones)": -24 diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt index 50724c5b..f8483b7e 100644 --- a/model_validation/requirements.txt +++ b/model_validation/requirements.txt @@ -1,4 +1,5 @@ gradio_client>=1.0 huggingface_hub>=0.23 PyYAML>=6.0 -soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules +soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules +mido>=1.3 # parse MIDI outputs for expect rules diff --git a/model_validation/src/cases.py b/model_validation/src/cases.py index 07f0a94c..6bfc958f 100644 --- a/model_validation/src/cases.py +++ b/model_validation/src/cases.py @@ -15,7 +15,8 @@ from assets import Assets from audio import read_audio_props -from validators import VALIDATORS +from midi import read_midi_props +from validators import VALIDATORS, ValidatorNotApplicable __all__ = [ @@ -40,28 +41,32 @@ "file extension, as a string or list of accepted extensions"), "min_bytes": (FILE_TYPES, "minimum file size in bytes"), + "min_duration": ({"audio_track", "midi_track"}, + "minimum length, in seconds"), + "max_duration": ({"audio_track", "midi_track"}, + "maximum length, in seconds"), "channels": ({"audio_track"}, "exact channel count (1 = mono, 2 = stereo)"), "sample_rate": ({"audio_track"}, "exact sample rate, in Hz"), - "min_duration": ({"audio_track"}, - "minimum length, in seconds"), - "max_duration": ({"audio_track"}, - "maximum length, in seconds"), "bit_depth": ({"audio_track"}, "exact PCM bit depth, e.g. 16 or 24 (not valid for " "compressed formats such as MP3 or OGG)"), "min_rms_db": ({"audio_track"}, "minimum RMS level, in dBFS (0 = full scale); -60 is the " "usual threshold for 'this output is not silent'"), + "min_notes": ({"midi_track"}, + "minimum number of note-on events"), "min_labels": ({"json"}, "minimum number of labels in a pyharp LabelList; 0 asserts " "a well-formed LabelList that may legitimately be empty"), } -# Rules requiring the audio properties of the output to be decoded -AUDIO_RULES = {rule for rule, (types, _) in EXPECT_RULES.items() - if types == {"audio_track"}} +# Rule groups, by what they need decoded. Duration rules are shared: they read +# from whichever of audio/MIDI props matches the output's type. +DURATION_RULES = {"min_duration", "max_duration"} +AUDIO_PROP_RULES = {"channels", "sample_rate", "bit_depth", "min_rms_db"} +MIDI_PROP_RULES = {"min_notes"} # Key selecting every compatible output rather than one named output ALL_OUTPUTS = "*" @@ -287,9 +292,10 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: Keys are output labels, or "*" to apply rules to every output the rule is compatible with (e.g. `min_rms_db` under "*" reaches only the audio - outputs). Rules are checked against the output's type here, so a rule - aimed at the wrong kind of output is reported as a configuration error - rather than a model failure. + outputs). A "*" rule that matches no output is simply dropped, so a + generic case can safely target output types a given model lacks. An + explicit label, by contrast, must exist and must accept the rule - + otherwise it is a configuration error, not a model failure. Args: expect (dict): The case's `expect` block. @@ -313,19 +319,14 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: f"'{key}'; supported rules: {sorted(EXPECT_RULES)}") if key == ALL_OUTPUTS: - # Fan each rule out to the outputs whose type it covers + # Fan each rule out to the outputs whose type it covers; a rule + # matching nothing is dropped (lenient, so generic cases apply) per_label = {} for rule, value in rules.items(): applicable, _ = EXPECT_RULES[rule] - matched = [label for label, otype in out_types.items() - if otype in applicable] - if not matched: - raise ValueError( - f"expect rule '{rule}' under '{ALL_OUTPUTS}' matches no " - f"output; it applies to {sorted(applicable)} outputs, " - f"but this model has {sorted(set(out_types.values()))}") - for label in matched: - per_label.setdefault(label, {})[rule] = value + for label, otype in out_types.items(): + if otype in applicable: + per_label.setdefault(label, {})[rule] = value targets.extend(per_label.items()) continue @@ -346,16 +347,48 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: return targets -def check_expectations(label: str, value, rules: dict) -> None: +def check_duration(label: str, duration, rules: dict) -> None: + """ + Apply the shared min_duration / max_duration rules to a decoded length. + + Args: + label (str): Output label, for error messages. + duration (float | None): Length in seconds, or None if undetermined. + rules (dict): The output's rules (only duration keys are read). + + Raises: + AssertionError: If a duration bound is not met, or duration is + required but could not be determined. + """ + + if not (set(rules) & DURATION_RULES): + return + + assert duration is not None, \ + f"output '{label}': could not determine its duration" + + if "min_duration" in rules: + assert duration >= rules["min_duration"], \ + (f"output '{label}' is {duration:.2f}s, expected at least " + f"{rules['min_duration']}s") + + if "max_duration" in rules: + assert duration <= rules["max_duration"], \ + (f"output '{label}' is {duration:.2f}s, expected at most " + f"{rules['max_duration']}s") + + +def check_expectations(label: str, otype: str, value, rules: dict) -> None: """ Apply one output's declarative `expect` rules. Rule names and their applicability to this output are validated upstream - by resolve_expect_targets(). Audio properties are decoded in a single - pass, and only when an audio rule is actually requested. + by resolve_expect_targets(). Audio/MIDI files are decoded in a single + pass, and only when a rule that needs the decoded properties is present. Args: label (str): Output label the rules apply to. + otype (str): The output's component type from /controls. value: The mapped output value (file path or decoded JSON). rules (dict): Rule name -> expected value. @@ -377,7 +410,7 @@ def check_expectations(label: str, value, rules: dict) -> None: f"output file '{label}' is {size} bytes, expected at least {rules['min_bytes']}" # --- Audio outputs ------------------------------------------------------- - if set(rules) & AUDIO_RULES: + if otype == "audio_track" and set(rules) & (AUDIO_PROP_RULES | DURATION_RULES): props = read_audio_props(label, require_file(label, value)) if "channels" in rules: @@ -390,16 +423,6 @@ def check_expectations(label: str, value, rules: dict) -> None: (f"output '{label}': expected {rules['sample_rate']} Hz, " f"got {props['sample_rate']} Hz") - if "min_duration" in rules: - assert props["duration"] >= rules["min_duration"], \ - (f"output '{label}' is {props['duration']:.2f}s, expected at " - f"least {rules['min_duration']}s") - - if "max_duration" in rules: - assert props["duration"] <= rules["max_duration"], \ - (f"output '{label}' is {props['duration']:.2f}s, expected at " - f"most {rules['max_duration']}s") - if "bit_depth" in rules: assert props["bit_depth"] is not None, \ (f"output '{label}' is a compressed format ({props['subtype']}) " @@ -413,6 +436,19 @@ def check_expectations(label: str, value, rules: dict) -> None: (f"output '{label}' appears silent (RMS {props['rms_db']:.1f} dBFS " f"< {rules['min_rms_db']} dBFS)") + check_duration(label, props["duration"], rules) + + # --- MIDI outputs -------------------------------------------------------- + if otype == "midi_track" and set(rules) & (MIDI_PROP_RULES | DURATION_RULES): + props = read_midi_props(label, require_file(label, value)) + + if "min_notes" in rules: + assert props["num_notes"] >= rules["min_notes"], \ + (f"output '{label}' has {props['num_notes']} note(s), expected " + f"at least {rules['min_notes']}") + + check_duration(label, props["duration"], rules) + # --- JSON / LabelList outputs -------------------------------------------- if "min_labels" in rules: labels = value.get("labels") if isinstance(value, dict) else None @@ -452,11 +488,16 @@ def inspect_outputs(result, controls: dict, case: dict) -> None: out_types = {spec.get("label"): spec.get("type") for spec in specs} for label, rules in resolve_expect_targets(case.get("expect"), out_types): - check_expectations(label, out_map[label], rules) + check_expectations(label, out_types[label], out_map[label], rules) for name, params in (case.get("validators") or {}).items(): if name not in VALIDATORS: raise ValueError(f"unknown validator '{name}' (available: " f"{sorted(VALIDATORS)}); register it in " f"validators.py") - VALIDATORS[name](out_map, controls, params or {}) + try: + VALIDATORS[name](out_map, controls, params or {}) + except ValidatorNotApplicable: + # The model lacks the outputs this validator needs; skip it so a + # common case's validator does not fail on models it does not fit + continue diff --git a/model_validation/src/harness.py b/model_validation/src/harness.py index e5bb40b8..6054e8ba 100644 --- a/model_validation/src/harness.py +++ b/model_validation/src/harness.py @@ -53,6 +53,11 @@ SERVER_PROBE_TIMEOUT = 5 # per-probe HTTP timeout against a local example PROCESS_TERMINATE_TIMEOUT = 15 # grace period before killing a local example LOADING_RETRY_INTERVAL = 15 # wait between retries while a model warms up +JOB_POLL_INTERVAL = 1 # cadence for polling a /process job's status + +# gradio job status codes that mean the job is still waiting in the ZeroGPU +# queue (not yet executing on the GPU) +QUEUE_STATUS_CODES = {"IN_QUEUE", "JOINING_QUEUE", "QUEUE_FULL"} # Substrings marking a "model is still loading" response - a ZeroGPU space # waking its GPU worker returns this immediately rather than blocking, so it @@ -60,6 +65,12 @@ LOADING_MARKERS = ("still loading", "loading, please wait", "is loading", "currently loading", "warming up") +# Substrings marking a ZeroGPU quota-exhausted response. Hitting this on one +# model means every other ZeroGPU model would hit it too, so the rest are +# skipped rather than retried. +QUOTA_EXHAUSTED_MARKERS = ("exceeded your gpu quota", "gpu quota exceeded", + "zerogpu quota", "quota exceeded", "gpu quota") + def is_loading_error(exc: Exception) -> bool: """ @@ -76,13 +87,28 @@ def is_loading_error(exc: Exception) -> bool: return any(marker in message for marker in LOADING_MARKERS) +def is_quota_exhausted(text: str) -> bool: + """ + Whether an error message indicates the ZeroGPU allowance is exhausted. + + Args: + text (str): An error message to inspect. + + Returns: + exhausted (bool): True if it looks like a ZeroGPU quota error. + """ + + lowered = text.lower() + return any(marker in lowered for marker in QUOTA_EXHAUSTED_MARKERS) + + def call_through_loading(fn, deadline: float, what: str): """ Call fn(), retrying while the model reports it is still loading. - A ZeroGPU space that has to spin up its GPU worker answers the first - request with a "still loading" error immediately; retrying until the - deadline lets validation ride through the warm-up instead of failing. + A ZeroGPU space that must start its GPU worker answers the first request + with a "still loading" error immediately; retrying until the deadline + tolerates that start-up interval instead of failing on it. Args: fn (callable): Zero-argument call to make. @@ -107,6 +133,52 @@ def call_through_loading(fn, deadline: float, what: str): raise +def job_status_code(job) -> str | None: + """ + Best-effort read of a gradio job's current status code name. + + Args: + job: A gradio_client Job handle. + + Returns: + code (str | None): The status code name (e.g. "IN_QUEUE", + "PROCESSING"), or None if the status cannot be read. + """ + + try: + status = job.status() + return getattr(getattr(status, "code", None), "name", None) + except Exception: # noqa: BLE001 - status polling is best-effort + return None + + +def wait_out_queue(job, queue_deadline: float, api_name: str) -> None: + """ + Block until a /process job leaves the ZeroGPU queue and begins executing. + + A ZeroGPU job waits in a shared queue before running; that wait is not + part of the model's execution and is not charged against quota, so it is + bounded by queue_deadline (the connect timeout) rather than by the shorter + execution timeout the caller applies afterwards. + + Args: + job: A gradio_client Job handle. + queue_deadline (float): time.time() by which the job must leave the + queue, or it is cancelled. + api_name (str): Endpoint name, for the timeout message. + + Raises: + TimeoutError: If the job never leaves the queue by queue_deadline. + """ + + while not job.done() and job_status_code(job) in QUEUE_STATUS_CODES: + if time.time() >= queue_deadline: + job.cancel() + raise TimeoutError(f"{api_name} still queued after waiting for the " + f"connect timeout") + time.sleep(JOB_POLL_INTERVAL) + + def close_client(client) -> None: """ Shut down a gradio_client instance without letting cleanup errors @@ -144,7 +216,12 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, result (ModelResult): The same record, completed. """ - process_timeout = overrides.get("process_timeout", opts.process_timeout) + # ZeroGPU models default to a lower process timeout (their jobs are short + # once running); a per-model process_timeout override still wins + default_process_timeout = (opts.zerogpu_process_timeout + if is_zerogpu(result.hardware) + else opts.process_timeout) + process_timeout = overrides.get("process_timeout", default_process_timeout) connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) load_only = opts.load_only or overrides.get("load_only", False) config_dir = opts.config.parent.resolve() @@ -174,12 +251,24 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, # --- /process test cases ------------------------------------------------- default_args, missing = synthesize_default_args(controls, assets) - cases = overrides.get("test_cases") or [{"name": "default"}] + # Common cases (generic, applied to every model) run alongside this + # model's own cases; their names are namespaced so both are legible in + # the report. A model can opt out with `skip_common_cases`. + own_cases = overrides.get("test_cases") or [{"name": "default"}] + if overrides.get("skip_common_cases"): + common_cases = [] + else: + common_cases = [dict(c, name=f"common:{c.get('name', 'unnamed')}") + for c in (opts.common_test_cases or [])] + cases = common_cases + own_cases for case in cases: case_result = CaseResult(name=case.get("name", "unnamed")) result.cases.append(case_result) case_start = time.time() + # Shared with run_process so `executed` is recorded even when the job + # leaves the queue (and thus reserves GPU) but then fails or times out + run_state = {"executed": False} try: # Inputs we couldn't synthesize are fine if this case supplies them supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) @@ -194,6 +283,11 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, def run_process(): job = client.submit(*args, api_name="/process") + # The queue wait is bounded by connect_timeout; only once the + # job is dequeued does the (shorter) execution timeout apply + wait_out_queue(job, time.time() + connect_timeout, "/process") + # Left the queue: GPU is now reserved, so this call is billed + run_state["executed"] = True return job.result(timeout=case_timeout) output = call_through_loading( @@ -211,6 +305,7 @@ def run_process(): case_result.error = f"{type(exc).__name__}: {exc}" finally: case_result.duration = round(time.time() - case_start, 1) + case_result.executed = run_state["executed"] if any(c.ok is False for c in result.cases): result.error = "; ".join( @@ -250,7 +345,8 @@ def wait_for_runtime(api, space_id: str, deadline: float): def test_space(space_id: str, token: str, assets: Assets, - opts: argparse.Namespace, overrides: dict) -> ModelResult: + opts: argparse.Namespace, overrides: dict, + quota_guard=None) -> ModelResult: """ Validate one remote Hugging Face Space end-to-end. @@ -260,6 +356,9 @@ def test_space(space_id: str, token: str, assets: Assets, assets (Assets): Synthesized input files. opts (argparse.Namespace): Parsed command-line options. overrides (dict): This space's entry from config.yml `overrides`. + quota_guard (ZeroGPUQuotaGuard | None): Shared flag; when tripped, + ZeroGPU models are skipped, and this model trips it if its own + run reveals the allowance is exhausted. Returns: result (ModelResult): The completed validation record. @@ -283,12 +382,17 @@ def test_space(space_id: str, token: str, assets: Assets, # requested_hardware reflects the space's configuration even while # it is sleeping or stopped (hardware itself is only set when live) result.hardware = runtime.requested_hardware or runtime.hardware or "" + zerogpu = is_zerogpu(result.hardware) # Skip ZeroGPU models before doing any work that would spend quota - if opts.skip_zerogpu and is_zerogpu(result.hardware): + if zerogpu and opts.skip_zerogpu: result.status = SKIP result.error = "skipped: ZeroGPU hardware (--skip-zerogpu)" return result + if zerogpu and quota_guard is not None and quota_guard.exhausted: + result.status = SKIP + result.error = "skipped: ZeroGPU allowance already exhausted this run" + return result if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): if opts.restart_failed and stage != "DELETING": @@ -321,8 +425,20 @@ def test_space(space_id: str, token: str, assets: Assets, time.sleep(15) if client is None: raise RuntimeError(f"could not connect: {last_exc}") + # Connecting has woken the space; reflect that rather than the stale + # SLEEPING/STARTING stage seen before the wake-up + if result.stage != "RUNNING": + result.stage = "RUNNING" - return run_endpoint_tests(client, result, assets, overrides, opts) + run_endpoint_tests(client, result, assets, overrides, opts) + + # If a ZeroGPU model failed because the allowance is gone, trip the + # guard so the remaining ZeroGPU models are skipped + if zerogpu and quota_guard is not None and result.status == FAIL \ + and is_quota_exhausted(result.error): + quota_guard.mark_exhausted() + + return result except Exception as exc: # noqa: BLE001 - any failure means invalid result.error = scrub(f"{type(exc).__name__}: {exc}", token) diff --git a/model_validation/src/midi.py b/model_validation/src/midi.py new file mode 100644 index 00000000..d72af2d5 --- /dev/null +++ b/model_validation/src/midi.py @@ -0,0 +1,53 @@ +""" +MIDI decoding for HARP model validation. + +Parallels audio.py: reads a MIDI output and measures the handful of +properties `expect` rules check. Uses mido, a small pure-Python parser, so +it handles running status and tempo maps correctly without a native +dependency. +""" + +import mido + + +__all__ = [ + 'read_midi_props' +] + + +def read_midi_props(label: str, path: str) -> dict: + """ + Parse a MIDI file and measure the properties `expect` rules check. + + Args: + label (str): Output label, for error messages. + path (str): Path to the MIDI file. + + Returns: + props (dict): num_tracks, num_notes (note-on events with non-zero + velocity), and duration (seconds, or None when it cannot be + determined, e.g. an asynchronous format-2 file). + + Raises: + AssertionError: If the file cannot be parsed as MIDI. + """ + + try: + midi = mido.MidiFile(path) + except Exception as exc: # noqa: BLE001 - any parse failure + raise AssertionError(f"output '{label}' could not be parsed as MIDI: {exc}") + + num_notes = sum(1 for track in midi.tracks for msg in track + if msg.type == "note_on" and msg.velocity > 0) + + try: + duration = midi.length + except (ValueError, KeyError): + # length is undefined for asynchronous (format 2) files + duration = None + + return { + "num_tracks": len(midi.tracks), + "num_notes": num_notes, + "duration": duration, + } diff --git a/model_validation/src/quota.py b/model_validation/src/quota.py index 7ae6fe84..c80ae8e5 100644 --- a/model_validation/src/quota.py +++ b/model_validation/src/quota.py @@ -13,6 +13,7 @@ __all__ = [ 'ZeroGPUTracker', + 'ZeroGPUQuotaGuard', 'is_zerogpu' ] @@ -36,42 +37,86 @@ def is_zerogpu(hardware: str) -> bool: class ZeroGPUTracker: """ - Accumulates the ZeroGPU processing time consumed during a run. - - The total is the cumulative /process wall time across ZeroGPU models - (CPU and dedicated-hardware models never contribute). It is an upper - bound on the GPU seconds charged, since wall time includes queue time, - and is shown against an optional budget (`zerogpu_budget_seconds` in - config.yml). + Tracks ZeroGPU work done during a run. + + ZeroGPU bills dynamically: each call reserves its declared + `@spaces.GPU(duration=...)` time up front, then refunds the unused portion + once the function returns - so the account's usage rises during a run and + settles lower afterwards, and the exact billed amount is not readable from + the gradio client. Two figures are tracked instead: + + - the number of /process calls that reached the GPU - exact, and the + most reliable signal of how much of the allowance a run will use; + - the total /process wall time of those calls (queue plus execution) - + an over-estimate of the settled bill, in the range of the mid-run + reservation peak, not the amount that remains after refunds. + + Only calls that reached the GPU count; queued or input-skipped cases do + not. CPU and dedicated-hardware models never contribute. """ def __init__(self, budget: float | None): self.budget = budget - self.used = 0.0 + self.calls = 0 + self.wall_seconds = 0.0 self._lock = threading.Lock() - def add(self, seconds: float) -> None: + def add(self, calls: int, wall_seconds: float) -> None: """ - Record ZeroGPU processing time from a completed model validation. + Record ZeroGPU work from a completed model validation. Args: - seconds (float): Wall time spent in this model's /process calls. + calls (int): Number of /process calls that ran on the GPU. + wall_seconds (float): Total /process wall time of those calls. """ with self._lock: - self.used += seconds + self.calls += calls + self.wall_seconds += wall_seconds def summary(self) -> str: """ - Format the ZeroGPU time consumed so far for a console line. + Format the ZeroGPU work done so far for a console line. Returns: - summary (str): e.g. "ZeroGPU time ~38s/1500s budget this run". + summary (str): e.g. + "ZeroGPU: 25 calls, ~340s wall (approx)". """ with self._lock: - used = int(self.used) + calls, wall = self.calls, int(self.wall_seconds) + plural = "s" if calls != 1 else "" if self.budget: - return f"ZeroGPU time ~{used}s/{int(self.budget)}s budget this run" - return f"ZeroGPU time ~{used}s this run" + detail = f"{calls} call{plural}, ~{wall}s/{int(self.budget)}s wall" + else: + detail = f"{calls} call{plural}, ~{wall}s wall" + + return f"ZeroGPU: {detail} (approx)" + + +class ZeroGPUQuotaGuard: + """ + A shared, thread-safe flag tripped when the ZeroGPU allowance runs out. + + Spaces are validated concurrently. Once any ZeroGPU model reports its + quota is exhausted, this guard causes the remaining ZeroGPU models to be + skipped rather than run against an exhausted allowance. + """ + + def __init__(self): + self._exhausted = False + self._lock = threading.Lock() + + def mark_exhausted(self) -> None: + """Record that the ZeroGPU allowance has been exhausted.""" + + with self._lock: + self._exhausted = True + + @property + def exhausted(self) -> bool: + """Whether the ZeroGPU allowance has been reported exhausted.""" + + with self._lock: + return self._exhausted diff --git a/model_validation/src/results.py b/model_validation/src/results.py index b6d7fd25..f77f5fcc 100644 --- a/model_validation/src/results.py +++ b/model_validation/src/results.py @@ -27,8 +27,9 @@ class CaseResult: """Outcome of a single /process test case.""" name: str - ok: bool | None = None # None => skipped - duration: float = 0.0 + ok: bool | None = None # None => skipped + duration: float = 0.0 # total /process wall time (queue + execution) + executed: bool = False # the /process job left the queue and ran on GPU error: str = "" @@ -62,7 +63,8 @@ def status_emoji(result: ModelResult) -> str: return {PASS: "✅", FAIL: "❌", SKIP: "⏭️"}[result.status] -def write_reports(results: list, out_dir: Path, label: str) -> None: +def write_reports(results: list, out_dir: Path, label: str, + command: str = "", options: dict | None = None) -> None: """ Write the machine-readable and human-readable reports. @@ -70,29 +72,42 @@ def write_reports(results: list, out_dir: Path, label: str) -> None: results (list): ModelResult objects for every validated model. out_dir (Path): Directory receiving report.json and report.md. label (str): Report heading (e.g. "teamup-tech spaces"). + command (str): The command line the run was invoked with. + options (dict | None): The resolved run options, recorded in the + report so a run's parameters are reproducible from it. """ out_dir.mkdir(parents=True, exist_ok=True) + passed = sum(r.status == PASS for r in results) + failed = sum(r.status == FAIL for r in results) + skipped = sum(r.status == SKIP for r in results) + validated = len(results) - skipped # denominator excludes skipped models + payload = { "label": label, "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "command": command, + "options": options or {}, "total": len(results), - "passed": sum(r.status == PASS for r in results), - "failed": sum(r.status == FAIL for r in results), - "skipped": sum(r.status == SKIP for r in results), + "validated": validated, + "passed": passed, + "failed": failed, + "skipped": skipped, "results": [dataclasses.asdict(r) for r in results], } (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) - headline = f"**{payload['passed']}/{payload['total']} models passed**" - if payload["skipped"]: - headline += f", {payload['skipped']} skipped" + headline = f"**{passed}/{validated} models passed**" + if skipped: + headline += f", {skipped} skipped" lines = [ f"# HARP Model Validation Report - {label}", "", f"{headline} ({payload['timestamp']})", "", + f"Command: `{command}`" if command else "", + "", "| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", "|---|---|---|---|---|---|---|---|", ] diff --git a/model_validation/src/validate_models.py b/model_validation/src/validate_models.py index 96ae3da9..3370d302 100644 --- a/model_validation/src/validate_models.py +++ b/model_validation/src/validate_models.py @@ -41,11 +41,12 @@ import concurrent.futures import os import sys +import time from pathlib import Path from assets import Assets from harness import test_space, test_local_example -from quota import ZeroGPUTracker, is_zerogpu +from quota import ZeroGPUTracker, ZeroGPUQuotaGuard, is_zerogpu from results import PASS, FAIL, SKIP, status_emoji, write_reports from utils import get_token, scrub, load_config, get_excluded, discover_spaces @@ -98,7 +99,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--connect-timeout", type=float, default=420, help="Seconds to wait for a deployment to build/wake/start") parser.add_argument("--process-timeout", type=float, default=600, - help="Seconds to wait for /process (includes ZeroGPU queue time)") + help="Seconds to wait for /process on non-ZeroGPU models") + parser.add_argument("--zerogpu-process-timeout", type=float, default=120, + help="Seconds allowed for /process EXECUTION on ZeroGPU " + "models once they leave the queue (queue wait is " + "bounded separately by --connect-timeout; default 120)") parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, help=f"Directory for reports, synthesized assets, and " f"example logs (default: {DEFAULT_OUTPUT_DIR})") @@ -182,6 +187,7 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, return None tracker = ZeroGPUTracker(config.get("zerogpu_budget_seconds")) + guard = ZeroGPUQuotaGuard() print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " f"(process test: {'OFF' if opts.load_only else 'ON'})\n") @@ -189,17 +195,20 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: futures = { pool.submit(test_space, sid, token, assets, opts, - overrides.get(sid, {})): sid + overrides.get(sid, {}), guard): sid for sid in space_ids } for future in concurrent.futures.as_completed(futures): r = future.result() results.append(r) - # Show the hardware for every model; the running ZeroGPU total is - # only meaningful (and only accrues) for ZeroGPU models + # Show the hardware for every model; ZeroGPU work only accrues for + # ZeroGPU models. Count each /process call that reached the GPU and + # its total wall time - a queued or input-skipped case never ran + # and is excluded info = r.hardware or "?" if is_zerogpu(r.hardware): - tracker.add(sum(c.duration for c in r.cases)) + ran = [c for c in r.cases if c.executed] + tracker.add(len(ran), sum(c.duration for c in ran)) info = f"{info} | {tracker.summary()}" note = f" - {r.error}" if r.error else "" print(f"{status_emoji(r)} {r.status:4s} {r.target} " @@ -219,6 +228,14 @@ def main() -> int: opts = parse_args() config = load_config(opts.config) excluded = get_excluded(config, opts.exclude) + # Generic cases applied to every model, on top of its own (see config.yml) + opts.common_test_cases = config.get("common_test_cases", []) + + # Each run writes to its own timestamped directory so runs never overwrite + # each other; assets, logs, and reports all live under it. Local time is + # used for the directory name (on a UTC CI runner this is naturally UTC). + run_stamp = time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime()) + opts.output_dir = opts.output_dir / run_stamp assets = Assets(opts.output_dir / "assets") if opts.local_examples is not None: @@ -233,12 +250,18 @@ def main() -> int: if results is None: return 2 - write_reports(results, opts.output_dir, label) + # Record how the run was invoked (argv holds no secrets - the token comes + # from HF_TOKEN) plus the resolved options, so the report is reproducible + command = " ".join(sys.argv) + options = {k: str(v) if isinstance(v, Path) else v + for k, v in vars(opts).items()} + write_reports(results, opts.output_dir, label, command, options) passed = [r for r in results if r.status == PASS] failed = [r for r in results if r.status == FAIL] skipped = [r for r in results if r.status == SKIP] - summary = f"\n{len(passed)}/{len(results)} models passed" + validated = len(results) - len(skipped) # exclude skipped from the total + summary = f"\n{len(passed)}/{validated} models passed" if skipped: summary += f", {len(skipped)} skipped" print(f"{summary}. Reports written to {opts.output_dir}/") diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py index 32cfb615..90e0e59b 100644 --- a/model_validation/src/validators.py +++ b/model_validation/src/validators.py @@ -36,10 +36,22 @@ __all__ = [ 'VALIDATORS', - 'validator' + 'validator', + 'ValidatorNotApplicable' ] +class ValidatorNotApplicable(Exception): + """ + Raised by a validator when the current model lacks the outputs it needs. + + Treated as a skip, not a failure. Raising this (rather than asserting) + for absent outputs lets a validator run in a common test case: it applies + to models that have the relevant outputs and is skipped on those that do + not, mirroring the leniency of a "*" expect rule. + """ + + VALIDATORS = {} @@ -74,8 +86,9 @@ def labels_within_audio(outputs, controls, params): before it is treated as out of bounds (default 0.05). Raises: - AssertionError: If a label lies outside the audio, or the outputs - needed for the comparison are missing. + AssertionError: If a label lies outside the audio output. + ValidatorNotApplicable: If the model has no audio output or no + LabelList output for the comparison. """ tolerance = params.get("tolerance", 0.05) @@ -91,7 +104,8 @@ def labels_within_audio(outputs, controls, params): if isinstance(value, str): duration = read_audio_props(label, value)["duration"] break - assert duration is not None, "no audio output found to compare labels against" + if duration is None: + raise ValidatorNotApplicable("no audio output to compare labels against") for label, value in outputs.items(): if not (isinstance(value, dict) and isinstance(value.get("labels"), list)): @@ -107,4 +121,4 @@ def labels_within_audio(outputs, controls, params): f"past the {duration:.2f}s audio output") return - raise AssertionError("no pyharp LabelList output found to check") + raise ValidatorNotApplicable("no pyharp LabelList output to check") From 359ba5552a5c835c087c7625dac7572f09c6cb78 Mon Sep 17 00:00:00 2001 From: Frank Cwitkowitz Date: Fri, 31 Jul 2026 10:24:01 -0400 Subject: [PATCH 8/8] Added ability to set number of ZeroGPU workers, added ability to specify properties of synthesized inputs, only None for text_box value goes to default string, and some overall polishing. --- .github/workflows/model_validation.yml | 14 +- model_validation/README.md | 39 +++- model_validation/config.yml | 24 ++ model_validation/requirements.txt | 3 +- model_validation/src/assets.py | 295 +++++++++++++++++------- model_validation/src/cases.py | 124 +++++----- model_validation/src/harness.py | 216 ++++++++++++----- model_validation/src/quota.py | 45 +--- model_validation/src/results.py | 86 +++++-- model_validation/src/utils.py | 48 +++- model_validation/src/validate_models.py | 54 +++-- 11 files changed, 635 insertions(+), 313 deletions(-) diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml index 2dd95b46..c9242a00 100644 --- a/.github/workflows/model_validation.yml +++ b/.github/workflows/model_validation.yml @@ -8,16 +8,14 @@ # 2. spaces - every Hugging Face Space under teamup-tech, validated # end-to-end (/controls + /process) via gradio_client. # -# Models are validated independently; one failure never stops the rest. # Results are published to the run summary and uploaded as artifacts. # -# Model failures do NOT fail the workflow (deployments are not yet stable -# enough - a red run every day would just spam maintainers with emails). -# They appear as warning annotations and in the run summary instead. Only -# infrastructure errors (bad token, discovery failure, ...) turn the run -# red, since those mean validation itself has stopped working. To start -# getting notified on model failures later, remove the exit-code handling -# in the two Validate steps so exit code 1 propagates. +# Model failures do NOT fail the workflow, since deployments are not yet +# stable enough for a daily red run to be actionable; they appear as warning +# annotations instead. Only infrastructure errors (bad token, discovery +# failure, ...) turn the run red, since those mean validation itself has +# stopped working. To get notified on model failures later, remove the +# exit-code handling in the two Validate steps so exit code 1 propagates. # # Requires the repository secret HF_TOKEN: a Hugging Face token with read # access to the teamup-tech spaces (write access if you want the workflow to diff --git a/model_validation/README.md b/model_validation/README.md index 0dd60fae..bfeafb6f 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -148,6 +148,11 @@ Two mechanisms limit the ZeroGPU allowance a run can consume: execution only; the queue wait before a job runs is bounded separately by `--connect-timeout`, so a long queue does not trip the execution timeout. A per-model `process_timeout` override takes precedence. +- **ZeroGPU models run one at a time** — `--zerogpu-workers` (default 1). Each + concurrent GPU call reserves its declared duration, so overlapping them ties + up allowance that is not being used. Non-ZeroGPU models still run at full + `--workers` concurrency alongside, and the limit is held only around the + endpoint tests, so slow restarts and connections still overlap. Transient "model is still loading" responses from a ZeroGPU space waking its GPU worker are retried automatically (up to `connect_timeout`), rather than @@ -157,11 +162,35 @@ counted as failures. Every model automatically gets one **`default`** test case, even with no configuration at all: inputs are synthesized from the model's `/controls` -spec — a sine-sweep WAV for audio tracks, a two-note MIDI file for MIDI -tracks, and each control's declared default value for sliders, toggles, -dropdowns, number boxes, and text boxes. So configuration is only needed to -go beyond that: pinning specific control values, feeding real audio, or -inspecting outputs more deeply. +spec — a sine-sweep clip for audio tracks, a short MIDI file for MIDI tracks, +and each control's declared default value for sliders, toggles, dropdowns, +number boxes, and text boxes. So configuration is only needed to go beyond +that: pinning specific control values, feeding real audio, adjusting the +properties of the synthesized inputs, or inspecting outputs more deeply. + +### Synthesized input properties + +The generated inputs default to a 2-second mono 44.1 kHz WAV and a two-note +MIDI file. Override any of those properties with a `synthesized_inputs` block +— globally in [config.yml](config.yml), per model under its `overrides` entry, +or per test case; each level overrides the last: + +```yaml +synthesized_inputs: # global default for every model + audio: + sample_rate: 48000 + channels: 2 + duration: 5.0 + ext: .flac + midi: + num_notes: 8 + note_duration: 0.25 +``` + +Audio is written with soundfile, so `ext` accepts any libsndfile format. It is +a preference rather than a guarantee: a component that only accepts other +extensions still gets one it accepts. Each distinct set of properties is +generated once and reused for the rest of the run. ### Step 1: find the model's control labels diff --git a/model_validation/config.yml b/model_validation/config.yml index 5b2b8d45..94892dda 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -12,6 +12,21 @@ # # measured time is approximate, not the # # amount Hugging Face bills (see README) # +# synthesized_inputs: # properties of the generated test inputs. Set here +# # as the global default, per model under `overrides`, +# # or per test case; each level overrides the last. +# audio: +# sample_rate: 44100 # Hz +# channels: 1 # 1 = mono, 2 = stereo +# duration: 2.0 # seconds +# ext: .wav # format to write (any libsndfile format); a +# # component that accepts only other extensions +# # still gets one it accepts +# midi: +# num_notes: 2 # ascending notes from middle C +# note_duration: 0.5 # seconds per note +# ext: .mid +# # exclude: # models to exclude from validation: space ids, or # # "examples/" for local pyharp examples # - teamup-tech/some-non-harp-space @@ -27,6 +42,11 @@ # process_timeout: 900 # seconds to wait for /process (default 600) # load_only: true # only check availability + /controls # +# synthesized_inputs: # input properties for this model's cases +# audio: +# sample_rate: 48000 +# channels: 2 +# # test_cases: # named /process test cases; when omitted, a # # single synthesized "default" case runs # - name: default # empty case == synthesized defaults @@ -42,6 +62,10 @@ # # paths are relative to this file # "Input Audio": test_data/my_clip.wav # +# synthesized_inputs: # input properties for this case only +# audio: +# duration: 10.0 +# # expect: # per-output checks, keyed by output label # "Output Audio": # ("*" applies rules to every compatible # # output; see the rule table below) diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt index f8483b7e..59dcb4e9 100644 --- a/model_validation/requirements.txt +++ b/model_validation/requirements.txt @@ -1,5 +1,6 @@ gradio_client>=1.0 huggingface_hub>=0.23 PyYAML>=6.0 -soundfile>=0.12 # decode audio outputs (any libsndfile format) for expect rules +numpy # synthesize audio inputs +soundfile>=0.12 # read/write audio (any libsndfile format) for inputs and expect rules mido>=1.3 # parse MIDI outputs for expect rules diff --git a/model_validation/src/assets.py b/model_validation/src/assets.py index 78c73b5b..4f0c7159 100644 --- a/model_validation/src/assets.py +++ b/model_validation/src/assets.py @@ -1,145 +1,190 @@ """ Synthesized test inputs for HARP model validation. -Every input file is generated from scratch - the base WAV and MIDI with the -standard library, other audio formats transcoded from the WAV with soundfile -- so validation needs no binary fixtures checked into the repository. -Real-world inputs for specific models belong in test_data/ and are referenced -from a test case's `files` entry in config.yml. +Every input file is generated from scratch - audio with soundfile, MIDI with +the standard library - so validation needs no binary fixtures checked into the +repository. Real-world inputs for specific models belong in test_data/ and are +referenced from a test case's `files` entry in config.yml. + +Input properties (sample rate, channels, length, format, ...) are configurable +via `synthesized_inputs` - see config.yml for the schema. Each distinct set of +properties is generated once and cached for the run. """ -import math import struct -import wave from pathlib import Path +import numpy import soundfile __all__ = [ 'Assets', - 'make_test_wav', - 'make_test_midi' + 'AUDIO_DEFAULTS', + 'MIDI_DEFAULTS', + 'merge_specs' ] +# Configurable properties of synthesized inputs, with their default values +AUDIO_DEFAULTS = { + "sample_rate": 44100, # Hz + "channels": 1, # 1 = mono, 2 = stereo, ... + "duration": 2.0, # seconds + "ext": ".wav", # container/format to write +} +MIDI_DEFAULTS = { + "num_notes": 2, # ascending notes from middle C + "note_duration": 0.5, # seconds per note (at the default 120 BPM) + "ext": ".mid", +} + # Extensions understood as audio; a component accepting one of these gets a -# synthesized clip in that format (transcoded from the base WAV via soundfile, -# so only formats this libsndfile build can write actually succeed). +# synthesized clip in that format (written via soundfile, so only formats this +# libsndfile build supports actually succeed). AUDIO_EXTS = {".wav", ".flac", ".ogg", ".oga", ".opus", ".aiff", ".aif", ".aifc", ".au", ".snd", ".w64", ".caf", ".mp3"} +MIDI_EXTS = {".mid", ".midi"} -def make_test_wav(path: Path, duration: float = 2.0, sr: int = 44100) -> Path: - """ - Write a short mono 16-bit sine sweep - a valid input for any audio model. - - Args: - path (Path): Destination .wav path. - duration (float): Length of the sweep in seconds. - sr (int): Sample rate in Hz. - - Returns: - path (Path): The written file, for chaining. - """ - - n = int(duration * sr) - - with wave.open(str(path), "wb") as f: - f.setnchannels(1) - f.setsampwidth(2) - f.setframerate(sr) - frames = bytearray() - for i in range(n): - t = i / sr - # Sweep from 220 Hz up one octave over the clip - freq = 220.0 + 440.0 * t / duration - sample = int(0.5 * 32767 * math.sin(2 * math.pi * freq * t)) - frames += struct.pack(" Path: +def merge_specs(*specs) -> dict: """ - Write a minimal standard MIDI file (format 0, two quarter notes). + Merge `synthesized_inputs` blocks, later entries winning per property. Args: - path (Path): Destination .mid path. + *specs (dict | None): Blocks keyed by media kind ("audio", "midi"), + in increasing order of precedence. Returns: - path (Path): The written file, for chaining. + merged (dict): One block with per-kind properties merged. """ - track_events = bytes([ - 0x00, 0xC0, 0x00, # program change: acoustic grand - 0x00, 0x90, 0x3C, 0x64, # note on C4 - 0x83, 0x60, 0x80, 0x3C, 0x40, # note off C4 after 480 ticks - 0x00, 0x90, 0x40, 0x64, # note on E4 - 0x83, 0x60, 0x80, 0x40, 0x40, # note off E4 after 480 ticks - 0x00, 0xFF, 0x2F, 0x00, # end of track - ]) - header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, 480) - track = b"MTrk" + struct.pack(">I", len(track_events)) + track_events - path.write_bytes(header + track) + merged = {} - return path + for spec in specs: + for kind, props in (spec or {}).items(): + merged.setdefault(kind, {}).update(props or {}) + + return merged class Assets: """ - Synthesized test input files, generated once and shared across all - model validations in a run. + Synthesized test input files, generated on demand and cached for the run. + + Args: + workdir (Path): Directory the generated files are written to. + defaults (dict | None): Global `synthesized_inputs` block from + config.yml, overriding the built-in property defaults. """ - def __init__(self, workdir: Path): + def __init__(self, workdir: Path, defaults: dict | None = None): workdir.mkdir(parents=True, exist_ok=True) self.workdir = workdir - self.wav = make_test_wav(workdir / "test_input.wav") - self.midi = make_test_midi(workdir / "test_input.mid") + self.defaults = defaults or {} + self._cache = {} + self.text = workdir / "test_input.txt" self.text.write_text("HARP model validation\n") self.json = workdir / "test_input.json" self.json.write_text("{}\n") - self._audio_cache = {".wav": self.wav} - def audio_in(self, ext: str) -> Path | None: + def resolve(self, kind: str, overrides: dict | None = None) -> dict: + """ + Resolve the properties for one media kind. + + Precedence is built-in defaults, then the global `synthesized_inputs` + block, then the per-model/per-case overrides. + + Args: + kind (str): "audio" or "midi". + overrides (dict | None): A merged `synthesized_inputs` block. + + Returns: + props (dict): The resolved properties for that kind. + """ + + props = dict(AUDIO_DEFAULTS if kind == "audio" else MIDI_DEFAULTS) + props.update(self.defaults.get(kind) or {}) + props.update((overrides or {}).get(kind) or {}) + + return props + + def audio(self, overrides: dict | None = None, ext: str | None = None) -> Path | None: """ - Get the synthesized test clip in a given audio format, transcoding - it from the base WAV on first request and caching the result. + Get a synthesized audio clip with the resolved properties. Args: - ext (str): Target extension, e.g. ".flac" or ".ogg". + overrides (dict | None): A merged `synthesized_inputs` block. + ext (str | None): Format to write, overriding the resolved `ext` + (used when a component only accepts certain extensions). Returns: - path (Path | None): The clip in that format, or None when this - libsndfile build cannot write it. + path (Path | None): The clip, or None when this libsndfile build + cannot write the requested format. """ - if ext not in self._audio_cache: - path = self.workdir / f"test_input{ext}" + props = self.resolve("audio", overrides) + sample_rate = int(props["sample_rate"]) + channels = int(props["channels"]) + duration = float(props["duration"]) + ext = (ext or props["ext"]).lower() + + key = ("audio", sample_rate, channels, duration, ext) + if key not in self._cache: + path = (self.workdir / + f"test_input_{sample_rate}hz_{channels}ch_{duration:g}s{ext}") try: - data, sr = soundfile.read(str(self.wav)) - soundfile.write(str(path), data, sr) - self._audio_cache[ext] = path + self._cache[key] = write_audio(path, duration, sample_rate, channels) except Exception: # noqa: BLE001 - format not writable here - self._audio_cache[ext] = None - return self._audio_cache[ext] + self._cache[key] = None + + return self._cache[key] - def for_file_types(self, file_types: list) -> Path | None: + def midi(self, overrides: dict | None = None, ext: str | None = None) -> Path: + """ + Get a synthesized MIDI file with the resolved properties. + + Args: + overrides (dict | None): A merged `synthesized_inputs` block. + ext (str | None): Extension to write, overriding the resolved one. + + Returns: + path (Path): The MIDI file. + """ + + props = self.resolve("midi", overrides) + num_notes = int(props["num_notes"]) + note_duration = float(props["note_duration"]) + ext = (ext or props["ext"]).lower() + + key = ("midi", num_notes, note_duration, ext) + if key not in self._cache: + path = self.workdir / f"test_input_{num_notes}n_{note_duration:g}s{ext}" + self._cache[key] = write_midi(path, num_notes, note_duration) + + return self._cache[key] + + def for_file_types(self, file_types: list, + overrides: dict | None = None) -> Path | None: """ Pick a synthesized file whose format matches the accepted types. - Audio components get a real clip in an accepted format (WAV, FLAC, - OGG, AIFF, ...), transcoded on demand; MIDI, JSON, and text inputs - are served from their fixed synthesized files. A component whose - accepted types are all unsupported (e.g. a bespoke binary format) - gets None - supply a real file via a test case's `files` entry. + Audio components get a clip in an accepted format, preferring the + configured `ext` when the component allows it; MIDI, JSON, and text + inputs are served from their synthesized files. A component whose + accepted types are all unsupported (e.g. a bespoke binary format) gets + None - supply a real file via a test case's `files` entry. Args: file_types (list): Accepted extensions from the /controls spec; empty or None means any file is accepted. + overrides (dict | None): A merged `synthesized_inputs` block. Returns: path (Path | None): A matching synthesized file, or None when no @@ -149,20 +194,92 @@ def for_file_types(self, file_types: list) -> Path | None: types = {str(t).lower() for t in (file_types or [])} if not types or "audio" in types: - return self.wav - - # Prefer WAV, then any other accepted audio format we can write - for ext in [".wav"] + sorted(types & AUDIO_EXTS - {".wav"}): - if ext in types: - clip = self.audio_in(ext) + return self.audio(overrides) + + # Try the configured format first, then any other accepted audio format + accepted_audio = types & AUDIO_EXTS + if accepted_audio: + preferred = self.resolve("audio", overrides)["ext"].lower() + order = ([preferred] if preferred in accepted_audio else []) + \ + sorted(accepted_audio - {preferred}) + for candidate in order: + clip = self.audio(overrides, ext=candidate) if clip is not None: return clip - if types & {".mid", ".midi"}: - return self.midi + accepted_midi = types & MIDI_EXTS + if accepted_midi: + preferred = self.resolve("midi", overrides)["ext"].lower() + return self.midi(overrides, + ext=preferred if preferred in accepted_midi + else sorted(accepted_midi)[0]) + if ".json" in types: return self.json if types & {".txt", ".text", "text"}: return self.text return None + + +def write_audio(path: Path, duration: float, sample_rate: int, + channels: int) -> Path: + """ + Write a sine sweep - a valid input for any audio model. + + Args: + path (Path): Destination path; its extension selects the format. + duration (float): Length of the sweep in seconds. + sample_rate (int): Sample rate in Hz. + channels (int): Number of (identical) channels to write. + + Returns: + path (Path): The written file, for chaining. + """ + + n = max(1, int(duration * sample_rate)) + t = numpy.arange(n) / sample_rate + # Sweep from 220 Hz up one octave over the clip + freq = 220.0 + 220.0 * numpy.arange(n) / n + mono = 0.5 * numpy.sin(2 * numpy.pi * freq * t) + soundfile.write(str(path), numpy.tile(mono[:, None], (1, channels)), sample_rate) + + return path + + +def write_midi(path: Path, num_notes: int, note_duration: float) -> Path: + """ + Write a standard MIDI file (format 0) of ascending notes from middle C. + + Args: + path (Path): Destination .mid path. + num_notes (int): Number of notes to write. + note_duration (float): Seconds each note sounds, at 120 BPM. + + Returns: + path (Path): The written file, for chaining. + """ + + ticks = max(1, int(round(note_duration / MIDI_SECONDS_PER_BEAT * MIDI_TICKS_PER_BEAT))) + + def delta(value): + """Encode a delta time as a MIDI variable-length quantity.""" + out = [value & 0x7F] + value >>= 7 + while value: + out.insert(0, (value & 0x7F) | 0x80) + value >>= 7 + return bytes(out) + + events = bytearray([0x00, 0xC0, 0x00]) # program change: grand piano + for i in range(max(0, num_notes)): + pitch = 60 + (i * 2) % 24 # ascending from middle C + events += bytes([0x00, 0x90, pitch, 0x64]) # note on + events += delta(ticks) + bytes([0x80, pitch, 0x40]) + events += bytes([0x00, 0xFF, 0x2F, 0x00]) # end of track + + header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, MIDI_TICKS_PER_BEAT) + track = b"MTrk" + struct.pack(">I", len(events)) + bytes(events) + path.write_bytes(header + track) + + return path diff --git a/model_validation/src/cases.py b/model_validation/src/cases.py index 6bfc958f..90c7d0f1 100644 --- a/model_validation/src/cases.py +++ b/model_validation/src/cases.py @@ -30,55 +30,48 @@ # Output component types that produce a file on disk FILE_TYPES = {"audio_track", "midi_track", "generic_file"} +AUDIO, MIDI, JSON = {"audio_track"}, {"midi_track"}, {"json"} -# The declarative `expect` vocabulary: rule name -> (applicable output types, -# what it asserts). Anything expressible as "read a property of one output and -# compare it" belongs here rather than in a custom validator (validators.py). -# Applying a rule to an output type it does not cover is a configuration -# error, not a model failure. +# The declarative `expect` vocabulary: rule name -> output types it applies to. +# Anything expressible as "read a property of one output and compare it" +# belongs here rather than in a custom validator (validators.py). Applying a +# rule to an output type it does not cover is a configuration error, not a +# model failure. config.yml documents what each rule asserts. EXPECT_RULES = { - "ext": (FILE_TYPES, - "file extension, as a string or list of accepted extensions"), - "min_bytes": (FILE_TYPES, - "minimum file size in bytes"), - "min_duration": ({"audio_track", "midi_track"}, - "minimum length, in seconds"), - "max_duration": ({"audio_track", "midi_track"}, - "maximum length, in seconds"), - "channels": ({"audio_track"}, - "exact channel count (1 = mono, 2 = stereo)"), - "sample_rate": ({"audio_track"}, - "exact sample rate, in Hz"), - "bit_depth": ({"audio_track"}, - "exact PCM bit depth, e.g. 16 or 24 (not valid for " - "compressed formats such as MP3 or OGG)"), - "min_rms_db": ({"audio_track"}, - "minimum RMS level, in dBFS (0 = full scale); -60 is the " - "usual threshold for 'this output is not silent'"), - "min_notes": ({"midi_track"}, - "minimum number of note-on events"), - "min_labels": ({"json"}, - "minimum number of labels in a pyharp LabelList; 0 asserts " - "a well-formed LabelList that may legitimately be empty"), + "ext": FILE_TYPES, + "min_bytes": FILE_TYPES, + "min_duration": AUDIO | MIDI, + "max_duration": AUDIO | MIDI, + "channels": AUDIO, + "sample_rate": AUDIO, + "bit_depth": AUDIO, + "min_rms_db": AUDIO, + "min_notes": MIDI, + "min_labels": JSON, } # Rule groups, by what they need decoded. Duration rules are shared: they read # from whichever of audio/MIDI props matches the output's type. DURATION_RULES = {"min_duration", "max_duration"} -AUDIO_PROP_RULES = {"channels", "sample_rate", "bit_depth", "min_rms_db"} -MIDI_PROP_RULES = {"min_notes"} +DECODED_RULES = { + "audio_track": {"channels", "sample_rate", "bit_depth", "min_rms_db"} | DURATION_RULES, + "midi_track": {"min_notes"} | DURATION_RULES, +} # Key selecting every compatible output rather than one named output ALL_OUTPUTS = "*" -def synthesize_default_args(controls: dict, assets: Assets) -> tuple: +def synthesize_default_args(controls: dict, assets: Assets, + synth: dict | None = None) -> tuple: """ Build the positional argument list for /process from the /controls spec. Args: controls (dict): The /controls payload (card, inputs, outputs). assets (Assets): Synthesized input files to draw from. + synth (dict | None): Merged `synthesized_inputs` block controlling the + properties of the generated inputs (see config.yml). Returns: args (list): One argument per input component, in declaration order. @@ -92,15 +85,22 @@ def synthesize_default_args(controls: dict, assets: Assets) -> tuple: for spec in controls.get("inputs", []): ctype = spec.get("type") label = spec.get("label") + required = spec.get("required", True) - if ctype == "audio_track": - args.append(handle_file(str(assets.wav)) if spec.get("required", True) else None) - elif ctype == "midi_track": - args.append(handle_file(str(assets.midi)) if spec.get("required", True) else None) - elif ctype == "generic_file": - path = assets.for_file_types(spec.get("file_types")) - if path is None and spec.get("required", True): - missing[label] = (f"cannot synthesize input for " + if ctype in FILE_TYPES: + # Optional file inputs are left unsupplied, exercising the model's + # own handling of their absence + if not required: + args.append(None) + continue + if ctype == "audio_track": + path = assets.audio(synth) + elif ctype == "midi_track": + path = assets.midi(synth) + else: + path = assets.for_file_types(spec.get("file_types"), synth) + if path is None: + missing[label] = (f"cannot synthesize a {ctype} input for " f"file_types={spec.get('file_types')}; supply " f"one via a test case's `files` entry") args.append(handle_file(str(path)) if path is not None else None) @@ -110,7 +110,10 @@ def synthesize_default_args(controls: dict, assets: Assets) -> tuple: value = spec.get("minimum", 0) args.append(value) elif ctype == "text_box": - args.append(spec.get("value") or "test") + # An empty string is a legitimate declared default, so only fall + # back when no default was declared at all + value = spec.get("value") + args.append(value if value is not None else "test") elif ctype == "toggle": args.append(bool(spec.get("value", False))) elif ctype == "dropdown": @@ -235,9 +238,8 @@ def validate_outputs(result, controls: dict) -> str | None: # gradio_client downloads file outputs and returns local paths path = out.get("path") if isinstance(out, dict) and "path" in out else out - if isinstance(path, str) and os.path.sep in path and os.path.exists(path): - if os.path.getsize(path) == 0: - return f"output file for '{spec.get('label')}' is empty" + if isinstance(path, str) and os.path.isfile(path) and os.path.getsize(path) == 0: + return f"output file for '{spec.get('label')}' is empty" return None @@ -323,7 +325,7 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: # matching nothing is dropped (lenient, so generic cases apply) per_label = {} for rule, value in rules.items(): - applicable, _ = EXPECT_RULES[rule] + applicable = EXPECT_RULES[rule] for label, otype in out_types.items(): if otype in applicable: per_label.setdefault(label, {})[rule] = value @@ -335,7 +337,7 @@ def resolve_expect_targets(expect: dict, out_types: dict) -> list: f"(available: {list(out_types)})") for rule in rules: - applicable, _ = EXPECT_RULES[rule] + applicable = EXPECT_RULES[rule] if out_types[key] not in applicable: raise ValueError( f"expect rule '{rule}' does not apply to output '{key}' " @@ -409,10 +411,14 @@ def check_expectations(label: str, otype: str, value, rules: dict) -> None: assert size >= rules["min_bytes"], \ f"output file '{label}' is {size} bytes, expected at least {rules['min_bytes']}" - # --- Audio outputs ------------------------------------------------------- - if otype == "audio_track" and set(rules) & (AUDIO_PROP_RULES | DURATION_RULES): - props = read_audio_props(label, require_file(label, value)) + # --- Decoded audio/MIDI properties --------------------------------------- + # Decode once, and only when a rule actually needs the decoded properties + props = None + if set(rules) & DECODED_RULES.get(otype, set()): + reader = read_audio_props if otype == "audio_track" else read_midi_props + props = reader(label, require_file(label, value)) + if props is not None and otype == "audio_track": if "channels" in rules: assert props["channels"] == rules["channels"], \ (f"output '{label}': expected {rules['channels']} channel(s), " @@ -436,17 +442,13 @@ def check_expectations(label: str, otype: str, value, rules: dict) -> None: (f"output '{label}' appears silent (RMS {props['rms_db']:.1f} dBFS " f"< {rules['min_rms_db']} dBFS)") - check_duration(label, props["duration"], rules) - - # --- MIDI outputs -------------------------------------------------------- - if otype == "midi_track" and set(rules) & (MIDI_PROP_RULES | DURATION_RULES): - props = read_midi_props(label, require_file(label, value)) - + if props is not None and otype == "midi_track": if "min_notes" in rules: assert props["num_notes"] >= rules["min_notes"], \ (f"output '{label}' has {props['num_notes']} note(s), expected " f"at least {rules['min_notes']}") + if props is not None: check_duration(label, props["duration"], rules) # --- JSON / LabelList outputs -------------------------------------------- @@ -461,16 +463,10 @@ def check_expectations(label: str, otype: str, value, rules: dict) -> None: def inspect_outputs(result, controls: dict, case: dict) -> None: """ - Apply a test case's deeper output checks (see README.md). - - Both mechanisms are optional per case; without either, an output is still - subject to the structural checks in validate_outputs(). - - expect: declarative per-output rules, the common path - see - EXPECT_RULES for the vocabulary. - validators: mapping of validator name -> parameters, for checks that - cannot be expressed declaratively (e.g. spanning several - outputs); registered in validators.py. + Apply a test case's deeper output checks: its `expect` rules (declarative, + per output - see EXPECT_RULES) and its `validators` (custom Python for + checks spanning several outputs - see validators.py). Both are optional; + without either, outputs are still subject to validate_outputs(). Args: result: The raw value returned by gradio_client for /process. diff --git a/model_validation/src/harness.py b/model_validation/src/harness.py index 6054e8ba..0e1ea43d 100644 --- a/model_validation/src/harness.py +++ b/model_validation/src/harness.py @@ -16,6 +16,7 @@ """ import argparse +import contextlib import os import subprocess import sys @@ -26,11 +27,11 @@ from gradio_client import Client -from assets import Assets +from assets import Assets, merge_specs from cases import synthesize_default_args, apply_case, validate_outputs, inspect_outputs from quota import is_zerogpu from results import ModelResult, CaseResult, PASS, FAIL, SKIP -from utils import run_with_timeout, scrub +from utils import run_with_timeout, scrub, describe_exception __all__ = [ @@ -197,6 +198,79 @@ def close_client(client) -> None: pass +def run_case(client: Client, case: dict, controls: dict, assets: Assets, + overrides: dict, config_dir: Path, process_timeout: float, + connect_timeout: float) -> CaseResult: + """ + Run one /process test case and check its outputs. + + The whole case sits inside an error boundary: any failure is recorded on + the returned record rather than raised, so remaining cases still run. + + Args: + client (Client): Connected gradio client for the deployment. + case (dict): Test case entry from config.yml. + controls (dict): The /controls payload. + assets (Assets): Synthesized input files. + overrides (dict): This model's entry from config.yml `overrides`. + config_dir (Path): Directory containing config.yml. + process_timeout (float): Seconds allowed for /process execution. + connect_timeout (float): Seconds allowed for the queue wait and for + retrying a model that is still loading. + + Returns: + case_result (CaseResult): The completed case record. + """ + + case_result = CaseResult(name=case.get("name", "unnamed")) + start = time.time() + # Set inside run_process so `executed` is recorded even when the job leaves + # the queue (and so reserves GPU) but then fails or times out + state = {"executed": False} + + try: + synth = merge_specs(overrides.get("synthesized_inputs"), + case.get("synthesized_inputs")) + default_args, missing = synthesize_default_args(controls, assets, synth) + + # Inputs we could not synthesize are fine if this case supplies them + supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) + unsatisfied = {k: v for k, v in missing.items() if k not in supplied} + if unsatisfied: + case_result.error = "skipped: " + "; ".join( + f"'{k}': {v}" for k, v in unsatisfied.items()) + return case_result + + args = apply_case(default_args, controls, case, config_dir) + + def run_process(): + job = client.submit(*args, api_name="/process") + # The queue wait is bounded by connect_timeout; only once the job + # is dequeued does the (shorter) execution timeout apply + wait_out_queue(job, time.time() + connect_timeout, "/process") + state["executed"] = True + return job.result(timeout=process_timeout) + + output = call_through_loading( + run_process, time.time() + connect_timeout, "/process") + + error = validate_outputs(output, controls) + if error: + case_result.ok = False + case_result.error = f"/process output invalid: {error}" + else: + inspect_outputs(output, controls, case) + case_result.ok = True + except Exception as exc: # noqa: BLE001 - any exception fails the case + case_result.ok = False + case_result.error = describe_exception(exc) + finally: + case_result.duration = round(time.time() - start, 1) + case_result.executed = state["executed"] + + return case_result + + def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, overrides: dict, opts: argparse.Namespace) -> ModelResult: """ @@ -250,7 +324,6 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, return result # --- /process test cases ------------------------------------------------- - default_args, missing = synthesize_default_args(controls, assets) # Common cases (generic, applied to every model) run alongside this # model's own cases; their names are namespaced so both are legible in # the report. A model can opt out with `skip_common_cases`. @@ -263,49 +336,9 @@ def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, cases = common_cases + own_cases for case in cases: - case_result = CaseResult(name=case.get("name", "unnamed")) - result.cases.append(case_result) - case_start = time.time() - # Shared with run_process so `executed` is recorded even when the job - # leaves the queue (and thus reserves GPU) but then fails or times out - run_state = {"executed": False} - try: - # Inputs we couldn't synthesize are fine if this case supplies them - supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) - unsatisfied = {k: v for k, v in missing.items() if k not in supplied} - if unsatisfied: - case_result.error = "skipped: " + "; ".join( - f"'{k}': {v}" for k, v in unsatisfied.items()) - continue - - args = apply_case(default_args, controls, case, config_dir) - case_timeout = case.get("process_timeout", process_timeout) - - def run_process(): - job = client.submit(*args, api_name="/process") - # The queue wait is bounded by connect_timeout; only once the - # job is dequeued does the (shorter) execution timeout apply - wait_out_queue(job, time.time() + connect_timeout, "/process") - # Left the queue: GPU is now reserved, so this call is billed - run_state["executed"] = True - return job.result(timeout=case_timeout) - - output = call_through_loading( - run_process, time.time() + connect_timeout, "/process") - - error = validate_outputs(output, controls) - if error: - case_result.ok = False - case_result.error = f"/process output invalid: {error}" - else: - inspect_outputs(output, controls, case) - case_result.ok = True - except Exception as exc: # noqa: BLE001 - any exception fails the case - case_result.ok = False - case_result.error = f"{type(exc).__name__}: {exc}" - finally: - case_result.duration = round(time.time() - case_start, 1) - case_result.executed = run_state["executed"] + result.cases.append(run_case( + client, case, controls, assets, overrides, config_dir, + case.get("process_timeout", process_timeout), connect_timeout)) if any(c.ok is False for c in result.cases): result.error = "; ".join( @@ -346,7 +379,7 @@ def wait_for_runtime(api, space_id: str, deadline: float): def test_space(space_id: str, token: str, assets: Assets, opts: argparse.Namespace, overrides: dict, - quota_guard=None) -> ModelResult: + quota_exhausted=None, zerogpu_limiter=None) -> ModelResult: """ Validate one remote Hugging Face Space end-to-end. @@ -356,9 +389,12 @@ def test_space(space_id: str, token: str, assets: Assets, assets (Assets): Synthesized input files. opts (argparse.Namespace): Parsed command-line options. overrides (dict): This space's entry from config.yml `overrides`. - quota_guard (ZeroGPUQuotaGuard | None): Shared flag; when tripped, - ZeroGPU models are skipped, and this model trips it if its own - run reveals the allowance is exhausted. + quota_exhausted (threading.Event | None): Shared flag; when set, + ZeroGPU models are skipped, and this model sets it if its own run + reveals the allowance is exhausted. + zerogpu_limiter (threading.Semaphore | None): Caps how many ZeroGPU + models make GPU calls at once. Held only around the endpoint + tests, so slow restarts and connections still overlap. Returns: result (ModelResult): The completed validation record. @@ -383,13 +419,19 @@ def test_space(space_id: str, token: str, assets: Assets, # it is sleeping or stopped (hardware itself is only set when live) result.hardware = runtime.requested_hardware or runtime.hardware or "" zerogpu = is_zerogpu(result.hardware) + # Hardware is occasionally unreported for a sleeping or starting space. + # Throttle those as if they were ZeroGPU: being wrong only costs run + # time, whereas leaving a real ZeroGPU model unthrottled overlaps GPU + # reservations. Skipping, by contrast, stays strict - wrongly skipping + # a CPU model would silently drop it from validation. + throttle = zerogpu or not result.hardware # Skip ZeroGPU models before doing any work that would spend quota if zerogpu and opts.skip_zerogpu: result.status = SKIP result.error = "skipped: ZeroGPU hardware (--skip-zerogpu)" return result - if zerogpu and quota_guard is not None and quota_guard.exhausted: + if zerogpu and quota_exhausted is not None and quota_exhausted.is_set(): result.status = SKIP result.error = "skipped: ZeroGPU allowance already exhausted this run" return result @@ -402,7 +444,7 @@ def test_space(space_id: str, token: str, assets: Assets, except Exception as exc: # noqa: BLE001 - e.g. read-only token result.error = scrub( f"space is not running (stage={stage}) and restart " - f"failed: {type(exc).__name__}: {exc}", token) + f"failed: {describe_exception(exc)}", token) return result runtime = wait_for_runtime(api, space_id, time.time() + connect_timeout) stage = runtime.stage @@ -426,22 +468,42 @@ def test_space(space_id: str, token: str, assets: Assets, if client is None: raise RuntimeError(f"could not connect: {last_exc}") # Connecting has woken the space; reflect that rather than the stale - # SLEEPING/STARTING stage seen before the wake-up + # SLEEPING/STARTING stage seen before the wake-up, and fill in the + # hardware if it was not reported while the space was asleep if result.stage != "RUNNING": result.stage = "RUNNING" + if not result.hardware: + try: + awake = api.get_space_runtime(space_id) + result.hardware = awake.hardware or awake.requested_hardware or "" + zerogpu = is_zerogpu(result.hardware) + except Exception: # noqa: BLE001 - hardware stays unknown + pass + + # Serialize ZeroGPU models: concurrent GPU calls each reserve their + # declared duration, so overlapping them ties up allowance that is not + # actually being used + limiter = zerogpu_limiter if (throttle and zerogpu_limiter is not None) \ + else contextlib.nullcontext() + with limiter: + # The allowance may have run out while waiting for a slot + if zerogpu and quota_exhausted is not None and quota_exhausted.is_set(): + result.status = SKIP + result.error = "skipped: ZeroGPU allowance already exhausted this run" + return result - run_endpoint_tests(client, result, assets, overrides, opts) + run_endpoint_tests(client, result, assets, overrides, opts) # If a ZeroGPU model failed because the allowance is gone, trip the # guard so the remaining ZeroGPU models are skipped - if zerogpu and quota_guard is not None and result.status == FAIL \ + if zerogpu and quota_exhausted is not None and result.status == FAIL \ and is_quota_exhausted(result.error): - quota_guard.mark_exhausted() + quota_exhausted.set() return result except Exception as exc: # noqa: BLE001 - any failure means invalid - result.error = scrub(f"{type(exc).__name__}: {exc}", token) + result.error = scrub(describe_exception(exc), token) if opts.verbose: traceback.print_exc() return result @@ -483,6 +545,34 @@ def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> raise TimeoutError(f"local app did not become ready within {int(timeout)}s") +def read_app_traceback(log_path: Path, max_chars: int = 400) -> str: + """ + Extract the final exception line from a local app's captured log. + + Args: + log_path (Path): The app's stdout/stderr log. + max_chars (int): Cap on the returned text. + + Returns: + detail (str): The last traceback's exception line, or "" if the log + holds no traceback (or cannot be read). + """ + + try: + lines = log_path.read_text(errors="replace").splitlines() + except OSError: + return "" + + # The exception line is the last non-indented line after the last "Traceback" + starts = [i for i, line in enumerate(lines) if line.startswith("Traceback")] + if not starts: + return "" + tail = [line for line in lines[starts[-1] + 1:] + if line.strip() and not line.startswith((" ", "\t"))] + + return tail[-1].strip()[:max_chars] if tail else "" + + def test_local_example(app_dir: Path, port: int, assets: Assets, opts: argparse.Namespace, overrides: dict) -> ModelResult: """ @@ -525,9 +615,10 @@ def test_local_example(app_dir: Path, port: int, assets: Assets, wait_for_local_server(port, proc, overrides.get( "connect_timeout", opts.connect_timeout)) client = Client(f"http://127.0.0.1:{port}", verbose=False) - return run_endpoint_tests(client, result, assets, overrides, opts) + run_endpoint_tests(client, result, assets, overrides, opts) + return result except Exception as exc: # noqa: BLE001 - any failure means invalid - result.error = f"{type(exc).__name__}: {exc} (see {log_path.name})" + result.error = f"{describe_exception(exc)} (see {log_path.name})" if opts.verbose: traceback.print_exc() return result @@ -540,3 +631,10 @@ def test_local_example(app_dir: Path, port: int, assets: Assets, proc.wait(timeout=PROCESS_TERMINATE_TIMEOUT) except subprocess.TimeoutExpired: proc.kill() + # A local app logs the real traceback even when the client is only + # told "an error occurred", so fold it into the report rather than + # leaving it in a file the reader has to go find + if result.error: + detail = read_app_traceback(log_path) + if detail and detail not in result.error: + result.error = f"{result.error} | app log: {detail}" diff --git a/model_validation/src/quota.py b/model_validation/src/quota.py index c80ae8e5..02426d9f 100644 --- a/model_validation/src/quota.py +++ b/model_validation/src/quota.py @@ -1,11 +1,9 @@ """ ZeroGPU usage tracking for HARP model validation. -ZeroGPU allowances are consumed per account, so a long validation run can eat -into the day's quota. There is no documented public API for the remaining -account quota, so rather than guess at it this module tracks the ZeroGPU time -*this run* consumes - the part attributable to validation - and reports it -after every ZeroGPU model. +ZeroGPU allowances are consumed per account, so a long validation run draws on +the day's quota. Hugging Face publishes no reliable API for the remaining +allowance, so this module reports the work attributable to the run itself. """ import threading @@ -13,7 +11,6 @@ __all__ = [ 'ZeroGPUTracker', - 'ZeroGPUQuotaGuard', 'is_zerogpu' ] @@ -86,37 +83,7 @@ def summary(self) -> str: with self._lock: calls, wall = self.calls, int(self.wall_seconds) - plural = "s" if calls != 1 else "" - if self.budget: - detail = f"{calls} call{plural}, ~{wall}s/{int(self.budget)}s wall" - else: - detail = f"{calls} call{plural}, ~{wall}s wall" + budget = f"/{int(self.budget)}s" if self.budget else "" - return f"ZeroGPU: {detail} (approx)" - - -class ZeroGPUQuotaGuard: - """ - A shared, thread-safe flag tripped when the ZeroGPU allowance runs out. - - Spaces are validated concurrently. Once any ZeroGPU model reports its - quota is exhausted, this guard causes the remaining ZeroGPU models to be - skipped rather than run against an exhausted allowance. - """ - - def __init__(self): - self._exhausted = False - self._lock = threading.Lock() - - def mark_exhausted(self) -> None: - """Record that the ZeroGPU allowance has been exhausted.""" - - with self._lock: - self._exhausted = True - - @property - def exhausted(self) -> bool: - """Whether the ZeroGPU allowance has been reported exhausted.""" - - with self._lock: - return self._exhausted + return (f"ZeroGPU: {calls} call{'s' if calls != 1 else ''}, " + f"~{wall}s{budget} wall (approx)") diff --git a/model_validation/src/results.py b/model_validation/src/results.py index f77f5fcc..77639ec0 100644 --- a/model_validation/src/results.py +++ b/model_validation/src/results.py @@ -15,6 +15,7 @@ 'CaseResult', 'ModelResult', 'status_emoji', + 'case_emoji', 'write_reports' ] @@ -63,6 +64,20 @@ def status_emoji(result: ModelResult) -> str: return {PASS: "✅", FAIL: "❌", SKIP: "⏭️"}[result.status] +def case_emoji(ok: bool | None) -> str: + """ + Symbol for one test case's outcome. + + Args: + ok (bool | None): True passed, False failed, None skipped. + + Returns: + emoji (str): The corresponding symbol. + """ + + return {True: "✅", False: "❌", None: "⏭️"}[ok] + + def write_reports(results: list, out_dir: Path, label: str, command: str = "", options: dict | None = None) -> None: """ @@ -98,33 +113,60 @@ def write_reports(results: list, out_dir: Path, label: str, } (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) - headline = f"**{passed}/{validated} models passed**" - if skipped: - headline += f", {skipped} skipped" - lines = [ - f"# HARP Model Validation Report - {label}", - "", - f"{headline} ({payload['timestamp']})", - "", - f"Command: `{command}`" if command else "", - "", - "| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", - "|---|---|---|---|---|---|---|---|", - ] + (out_dir / "report.md").write_text(render_markdown(results, payload)) + + +def render_markdown(results: list, payload: dict) -> str: + """ + Render the human-readable report: a summary table plus full error text. + + Args: + results (list): ModelResult objects for every validated model. + payload (dict): The report.json payload (headline counts, command). + + Returns: + markdown (str): The rendered report. + """ # Failures first, then alphabetical, so problems are visible at a glance - for r in sorted(results, key=lambda r: (r.status != FAIL, r.target)): - if r.cases: - cases = ", ".join( - f"{c.name} {'✅' if c.ok else '⏭️' if c.ok is None else '❌'}" - for c in r.cases) - else: - cases = "—" + ranked = sorted(results, key=lambda r: (r.status != FAIL, r.target)) + + headline = f"**{payload['passed']}/{payload['validated']} models passed**" + if payload["skipped"]: + headline += f", {payload['skipped']} skipped" + + lines = [f"# HARP Model Validation Report - {payload['label']}", ""] + lines += [f"{headline} ({payload['timestamp']})", ""] + if payload["command"]: + lines += [f"Command: `{payload['command']}`", ""] + lines += ["| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", + "|---|---|---|---|---|---|---|---|"] + + for r in ranked: + cases = ", ".join(f"{c.name} {case_emoji(c.ok)}" for c in r.cases) or "—" link = (f"[{r.target}](https://huggingface.co/spaces/{r.target})" if r.kind == "space" else f"`{r.target}`") - detail = r.error.replace("|", "\\|")[:300] if r.error else "" + # Keep the table scannable; the untruncated text follows below + detail = r.error.replace("|", "\\|") if r.error else "" + if len(detail) > 200: + detail = detail[:200] + " […]" lines.append(f"| {link} | {status_emoji(r)} {r.status} | {r.stage} " f"| {r.hardware or '—'} | {'✅' if r.controls_ok else '❌'} " f"| {cases} | {r.duration} | {detail} |") - (out_dir / "report.md").write_text("\n".join(lines) + "\n") + # Full, untruncated text wherever something was reported - including a + # skipped case on an otherwise passing model + problems = [r for r in ranked + if r.error or r.status == FAIL or any(c.error for c in r.cases)] + if problems: + lines += ["", "## Details", ""] + for r in problems: + lines += [f"### {r.target} — {r.status}", ""] + if r.error and not r.cases: + lines += ["```", r.error, "```", ""] + for c in r.cases: + if c.error: + lines += [f"- **{c.name}** {case_emoji(c.ok)}", "", + "```", c.error, "```", ""] + + return "\n".join(lines) + "\n" diff --git a/model_validation/src/utils.py b/model_validation/src/utils.py index 78439b22..78eac5b4 100644 --- a/model_validation/src/utils.py +++ b/model_validation/src/utils.py @@ -1,6 +1,6 @@ """ -Shared utilities for HARP model validation: credentials, configuration, -model discovery, and timeout handling. +Shared utilities for HARP model validation: credentials, error rendering, +configuration, model discovery, and timeout handling. """ import os @@ -8,15 +8,13 @@ import threading from pathlib import Path -try: - import yaml -except ImportError: - yaml = None +import yaml __all__ = [ 'get_token', 'scrub', + 'describe_exception', 'run_with_timeout', 'load_config', 'get_excluded', @@ -24,6 +22,40 @@ ] +def describe_exception(exc: BaseException) -> str: + """ + Render an exception with as much detail as it carries. + + Beyond the usual "Type: message", this surfaces a gradio AppError's + `title` when the upstream app set an informative one, and follows the + exception chain so the underlying cause is not lost. Note that a Space + launched with `show_error=False` deliberately returns only the exception + class name, so for those there is genuinely nothing further to report. + + Args: + exc (BaseException): The exception to describe. + + Returns: + description (str): A single-line description of the exception. + """ + + parts = [f"{type(exc).__name__}: {exc}".strip()] + + # gradio's AppError carries a title alongside the message + title = getattr(exc, "title", None) + if title and title not in ("Error", str(exc)): + parts.append(f"[{title}]") + + # Follow the chain so the root cause survives (bounded to stay one line) + seen, cause = {id(exc)}, exc.__cause__ or exc.__context__ + while cause is not None and id(cause) not in seen and len(parts) < 4: + seen.add(id(cause)) + parts.append(f"caused by {type(cause).__name__}: {cause}".strip()) + cause = cause.__cause__ or cause.__context__ + + return " | ".join(parts) + + def get_token(required: bool) -> str: """ Read the Hugging Face access token from the HF_TOKEN environment variable. @@ -120,10 +152,6 @@ def load_config(path: Path) -> dict: if not path.exists(): return {} - if yaml is None: - print(f"WARNING: PyYAML not installed; ignoring {path}", file=sys.stderr) - return {} - return yaml.safe_load(path.read_text()) or {} diff --git a/model_validation/src/validate_models.py b/model_validation/src/validate_models.py index 3370d302..8eebf255 100644 --- a/model_validation/src/validate_models.py +++ b/model_validation/src/validate_models.py @@ -11,8 +11,7 @@ and processes test inputs end-to-end. ZeroGPU spaces require an authenticated request for GPU quota, so a token must be provided via the HF_TOKEN environment variable - never on the command line or in the - repository. ZeroGPU quota usage is reported at the start of the run and - after every model. + repository. 2. Examples (--local-examples): launches each app under pyharp/examples/ on a local port and runs the identical endpoint tests. The examples tier @@ -41,12 +40,13 @@ import concurrent.futures import os import sys +import threading import time from pathlib import Path from assets import Assets from harness import test_space, test_local_example -from quota import ZeroGPUTracker, ZeroGPUQuotaGuard, is_zerogpu +from quota import ZeroGPUTracker, is_zerogpu from results import PASS, FAIL, SKIP, status_emoji, write_reports from utils import get_token, scrub, load_config, get_excluded, discover_spaces @@ -96,6 +96,11 @@ def parse_args() -> argparse.Namespace: "disable with --no-restart-failed") parser.add_argument("--workers", type=int, default=4, help="Number of spaces to validate concurrently (spaces tier only)") + parser.add_argument("--zerogpu-workers", type=int, default=1, + help="How many ZeroGPU models may make GPU calls at once. " + "Each concurrent call reserves its declared duration, " + "so 1 (the default) keeps overlapping reservations " + "from tying up the allowance") parser.add_argument("--connect-timeout", type=float, default=420, help="Seconds to wait for a deployment to build/wake/start") parser.add_argument("--process-timeout", type=float, default=600, @@ -112,6 +117,25 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def result_line(result, extra: str = "", token: str = "") -> str: + """ + Format one model's console line. + + Args: + result (ModelResult): The completed validation record. + extra (str): Text inserted before the error note (e.g. hardware). + token (str): Token to scrub from the error text. + + Returns: + line (str): The formatted line. + """ + + note = f" - {result.error}" if result.error else "" + + return (f"{status_emoji(result)} {result.status:4s} {result.target} " + f"({result.duration}s){extra}{scrub(note, token)}") + + def validate_examples(opts: argparse.Namespace, config: dict, excluded: set, assets: Assets) -> list: """ @@ -150,8 +174,7 @@ def validate_examples(opts: argparse.Namespace, config: dict, excluded: set, r = test_local_example(app_dir, LOCAL_PORT_BASE + i, assets, opts, overrides.get(f"examples/{app_dir.name}", {})) results.append(r) - note = f" - {r.error}" if r.error else "" - print(f"{status_emoji(r)} {r.status:4s} {r.target} ({r.duration}s){note}") + print(result_line(r)) return results @@ -187,17 +210,17 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, return None tracker = ZeroGPUTracker(config.get("zerogpu_budget_seconds")) - guard = ZeroGPUQuotaGuard() + quota_exhausted = threading.Event() + zerogpu_limiter = threading.Semaphore(max(1, opts.zerogpu_workers)) print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " - f"(process test: {'OFF' if opts.load_only else 'ON'})\n") + f"({opts.zerogpu_workers} concurrent on ZeroGPU; " + f"process test: {'OFF' if opts.load_only else 'ON'})\n") results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: - futures = { - pool.submit(test_space, sid, token, assets, opts, - overrides.get(sid, {}), guard): sid - for sid in space_ids - } + futures = [pool.submit(test_space, sid, token, assets, opts, + overrides.get(sid, {}), quota_exhausted, zerogpu_limiter) + for sid in space_ids] for future in concurrent.futures.as_completed(futures): r = future.result() results.append(r) @@ -210,9 +233,7 @@ def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, ran = [c for c in r.cases if c.executed] tracker.add(len(ran), sum(c.duration for c in ran)) info = f"{info} | {tracker.summary()}" - note = f" - {r.error}" if r.error else "" - print(f"{status_emoji(r)} {r.status:4s} {r.target} " - f"({r.duration}s) [{info}]{scrub(note, token)}") + print(result_line(r, f" [{info}]", token)) return results @@ -236,7 +257,8 @@ def main() -> int: # used for the directory name (on a UTC CI runner this is naturally UTC). run_stamp = time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime()) opts.output_dir = opts.output_dir / run_stamp - assets = Assets(opts.output_dir / "assets") + assets = Assets(opts.output_dir / "assets", + config.get("synthesized_inputs")) if opts.local_examples is not None: token = ""