diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml new file mode 100644 index 00000000..c9242a00 --- /dev/null +++ b/.github/workflows/model_validation.yml @@ -0,0 +1,156 @@ +# Daily validation of HARP model deployments. +# +# Two tiers: +# 1. examples - local pyharp examples run inside the CI runner; no Hugging +# Face infrastructure involved. A failure here means pyharp +# or gradio itself is broken - fix that before debugging +# individual spaces. +# 2. spaces - every Hugging Face Space under teamup-tech, validated +# end-to-end (/controls + /process) via gradio_client. +# +# Results are published to the run summary and uploaded as artifacts. +# +# Model failures do NOT fail the workflow, since deployments are not yet +# stable enough for a daily red run to be actionable; they appear as warning +# annotations instead. Only infrastructure errors (bad token, discovery +# failure, ...) turn the run red, since those mean validation itself has +# stopped working. To get notified on model failures later, remove the +# exit-code handling in the two Validate steps so exit code 1 propagates. +# +# Requires the repository secret HF_TOKEN: a Hugging Face token with read +# access to the teamup-tech spaces (write access if you want the workflow to +# auto-restart crashed spaces). NEVER commit the token; GitHub masks secrets +# in logs automatically. + +name: Model Validation + +on: + schedule: + - cron: '30 6 * * *' # daily, 06:30 UTC + workflow_dispatch: + inputs: + spaces: + description: 'Space ids to validate (space-separated); empty = all' + required: false + default: '' + load_only: + description: 'Skip /process inference tests (availability only)' + type: boolean + default: false + restart_failed: + description: 'Attempt to restart crashed/stopped spaces (needs write token)' + type: boolean + default: true + +permissions: + contents: read + +concurrency: + group: model-validation + cancel-in-progress: false + +jobs: + + examples: + name: Examples (local pyharp) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install pyharp (local submodule) and example dependencies + run: | + python -m pip install --upgrade pip + # CPU torch/torchaudio must be installed BEFORE pyharp, otherwise + # descript-audiotools pulls mismatched CUDA builds from PyPI + pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install -e ./pyharp + pip install -r model_validation/requirements.txt + + - name: Validate local examples + run: | + set +e + python model_validation/src/validate_models.py --local-examples \ + --output-dir reports/examples + CODE=$? + if [ "$CODE" -eq 1 ]; then + echo "::warning title=Example failures::Some local examples failed - see the run summary and report artifact." + exit 0 + fi + exit "$CODE" + + - name: Publish summary + if: always() + run: cat reports/examples/*/report.md >> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: report-examples + path: reports/examples/*/report.* + if-no-files-found: ignore + + spaces: + name: Hugging Face Spaces + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r model_validation/requirements.txt + + - name: Validate spaces + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + INPUT_SPACES: ${{ inputs.spaces }} + INPUT_LOAD_ONLY: ${{ inputs.load_only }} + INPUT_RESTART_FAILED: ${{ inputs.restart_failed }} + run: | + ARGS=(--output-dir reports/spaces) + if [ -n "$INPUT_SPACES" ]; then + read -r -a SPACE_IDS <<< "$INPUT_SPACES" + ARGS+=(--spaces "${SPACE_IDS[@]}") + fi + if [ "$INPUT_LOAD_ONLY" = "true" ]; then + ARGS+=(--load-only) + fi + # restart is the script default; scheduled runs leave INPUT empty + if [ "$INPUT_RESTART_FAILED" = "false" ]; then + ARGS+=(--no-restart-failed) + fi + set +e + python model_validation/src/validate_models.py "${ARGS[@]}" + CODE=$? + if [ "$CODE" -eq 1 ]; then + echo "::warning title=Space failures::Some spaces failed validation - see the run summary and report artifact." + exit 0 + fi + exit "$CODE" + + - name: Publish summary + if: always() + run: cat reports/spaces/*/report.md >> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: report-spaces + path: reports/spaces/*/report.* + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index b41816c2..36170ec0 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ *.out *.app -# build +# Build build*/ .vscode/ libtorch/ @@ -53,12 +53,11 @@ py/client/dist/ py/client/*.spec dist*/ - dist/ Miniconda*/ Miniconda*.sh -# website +# Website node_modules temp cache @@ -66,6 +65,16 @@ dist testproj.RPP +# Python +__pycache__/ +*.pyc +*.egg-info/ +venv/ +.venv/ + # Ignore all the *.md files in website/HARP website/content/HARP/*.md website/content/pyHARP/*.md + +# Model validation output +model_validation/reports/ diff --git a/model_validation/README.md b/model_validation/README.md new file mode 100644 index 00000000..bfeafb6f --- /dev/null +++ b/model_validation/README.md @@ -0,0 +1,446 @@ +# HARP Model Validation + +Automated validation of HARP model deployments. Verifies that each deployment +is reachable, exposes the HARP gradio endpoints (_i.e._, `/controls`, `/process`), +and can actually process test inputs end-to-end — the same interactions the +HARP client performs, driven headlessly. + +## Overview + +Two tiers share the same test harness: + +| Tier | Command | What it validates | What a failure means | +|---|---|---|---| +| **Examples** | `--local-examples` | The apps under `pyharp/examples/`, launched locally | pyharp or gradio itself is broken | +| **Spaces** | (default) | Every Hugging Face Space under `teamup-tech` | That specific deployment is broken (build error, HF platform change, dependency drift, ...) | + +Key behaviors: + +- **Models are validated independently** — a crash, hang, or timeout in one + never stops the rest; all results are collected into a single report and + the process exits non-zero only after everything has run. +- **Crashed/stopped spaces are restarted automatically** (sleeping spaces + are woken simply by connecting). Pass `--no-restart-failed` to disable; + restarting requires a token with write access. +- **ZeroGPU time is tracked** across the run and reported after each ZeroGPU + model; `--skip-zerogpu` skips those models to spend no allowance at all. +- A daily GitHub Action + ([model_validation.yml](../.github/workflows/model_validation.yml)) runs + both tiers at 06:30 UTC. Model failures do **not** turn the run red (no + notification emails while deployments stabilize) — results live in the + run summary and report artifacts. + +## Code layout + + File | Purpose | +|---|---| +| [src/validate_models.py](src/validate_models.py) | Command-line entry point and per-tier orchestration | +| [src/harness.py](src/harness.py) | The test harness: drives /controls + /process against a live model | +| [src/cases.py](src/cases.py) | Synthesis of inputs, test-case overlay, output validation/inspection | +| [src/validators.py](src/validators.py) | Registry of custom output validators (to be extended) | +| [src/audio.py](src/audio.py) | Audio decoding (any libsndfile format) for output checks | +| [src/midi.py](src/midi.py) | MIDI parsing (via mido) for output checks | +| [src/assets.py](src/assets.py) | Synthesized WAV/MIDI/text/JSON test inputs | +| [src/quota.py](src/quota.py) | ZeroGPU usage tracking and account quota lookup | +| [src/results.py](src/results.py) | Result records and JSON/markdown report generation | +| [src/utils.py](src/utils.py) | Token handling, config loading, discovery, timeouts | +| [config.yml](config.yml) | Validation configuration (_e.g._, excludes, per-model test cases) | +| [test_data/](test_data/) | Real input files referenced by test cases | +| `reports/` | Generated reports, synthesized assets, and example logs | + +## Token setup (IMPORTANT — read this) + +ZeroGPU spaces require an authenticated request to obtain GPU quota, and +private spaces require read access. Validation reads the token **only** from +the `HF_TOKEN` environment variable. + +- **Never** commit a token, paste it in an issue/PR, or pass it as a + command-line argument (argv is visible in process listings). +- **CI:** add the token as a repository secret named `HF_TOKEN` + (Settings → Secrets and variables → Actions). GitHub masks secrets in logs. +- **Locally:** `export HF_TOKEN=...` in your shell (consider `read -s` so it + stays out of shell history). +- Use a [fine-grained token](https://huggingface.co/settings/tokens) with + write access to the org's spaces (needed for the default auto-restart of + crashed spaces; read access suffices with `--no-restart-failed`). +- **If a token is ever exposed, rotate it immediately** at + https://huggingface.co/settings/tokens. + +## Running locally + +```bash +pip install -r model_validation/requirements.txt + +# All spaces in the org (crashed/stopped spaces are restarted by default) +export HF_TOKEN=hf_... # see token setup above +python model_validation/src/validate_models.py + +# A single space, with verbose errors (useful while developing a test case) +python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose + +# Exclude specific models (also configurable via `exclude` in config.yml) +python model_validation/src/validate_models.py --exclude teamup-tech/broken-space + +# Availability + /controls only (fast; no inference, no GPU quota used) +python model_validation/src/validate_models.py --load-only + +# Skip ZeroGPU models, to spend none of the shared ZeroGPU allowance +python model_validation/src/validate_models.py --skip-zerogpu + +# Never restart spaces (works with a read-only token) +python model_validation/src/validate_models.py --no-restart-failed + +# Examples tier (needs pyharp + example deps; no token required) +pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu +pip install -e ./pyharp +python model_validation/src/validate_models.py --local-examples +``` + +Each run writes to its own timestamped directory under +`model_validation/reports/` (override the base with `--output-dir`), so runs +never overwrite each other — e.g. `model_validation/reports/2026-07-18T14-30-00Z/`. +It contains `report.json` (machine-readable) and `report.md` (human-readable +table), with synthesized test inputs and local example logs alongside. Both +reports record the command line the run was invoked with. Exit code is `0` +when everything passes, `1` on any model failure, `2` on +configuration/infrastructure errors. + +## ZeroGPU usage tracking + +Each model's line shows its hardware, and ZeroGPU models additionally show +the ZeroGPU work done so far this run: + +``` +✅ PASS teamup-tech/pitch_shifter (42.1s) [zero-a10g | ZeroGPU: 2 calls, ~84s wall (approx)] +✅ PASS teamup-tech/cpu_model (18.3s) [cpu-basic] +``` + +Two figures are tracked, because the exact billed amount is not observable +from the client: + +- **Call count** — the number of `/process` calls that reached the GPU. This + is exact and is the most reliable signal of how much of the allowance a run + will use. Only calls that ran count; a queued or input-skipped case does + not. CPU and dedicated-hardware models never contribute. +- **Wall time** — the total `/process` wall time (queue plus execution) of + those calls, marked `(approx)`. Hugging Face bills ZeroGPU **dynamically**: + each call reserves its declared `@spaces.GPU(duration=)` time up front and + refunds the unused part when the function returns, so an account's usage + rises during a run and settles lower afterwards. This wall figure is an + over-estimate of the settled bill — closer to that mid-run reservation peak + — and is not returned to the client, so treat it as indicative. Set + `zerogpu_budget_seconds` in [config.yml](config.yml) (_e.g._, `1500` for the + PRO 25 min/day allowance) to show it against a budget. + +Each case's wall time is also recorded in `report.json` (`duration`). These +figures reflect the work *this run* does, not your account's remaining quota — +Hugging Face exposes no reliable public API for the latter. To spend none of +the allowance, pass `--skip-zerogpu`, which skips ZeroGPU models entirely +(they appear as `SKIP` in the report). + +Two mechanisms limit the ZeroGPU allowance a run can consume: + +- **Quota exhaustion stops the remaining ZeroGPU models.** If a ZeroGPU model + fails with a "quota exceeded" error, every remaining ZeroGPU model is + skipped (as `SKIP`) rather than run against an exhausted allowance. +- **ZeroGPU models use a lower execution timeout** — `--zerogpu-process-timeout` + (default 120s) instead of `--process-timeout` (default 600s). This bounds + execution only; the queue wait before a job runs is bounded separately by + `--connect-timeout`, so a long queue does not trip the execution timeout. A + per-model `process_timeout` override takes precedence. +- **ZeroGPU models run one at a time** — `--zerogpu-workers` (default 1). Each + concurrent GPU call reserves its declared duration, so overlapping them ties + up allowance that is not being used. Non-ZeroGPU models still run at full + `--workers` concurrency alongside, and the limit is held only around the + endpoint tests, so slow restarts and connections still overlap. + +Transient "model is still loading" responses from a ZeroGPU space waking its +GPU worker are retried automatically (up to `connect_timeout`), rather than +counted as failures. + +## Configuring test cases — a walkthrough + +Every model automatically gets one **`default`** test case, even with no +configuration at all: inputs are synthesized from the model's `/controls` +spec — a sine-sweep clip for audio tracks, a short MIDI file for MIDI tracks, +and each control's declared default value for sliders, toggles, dropdowns, +number boxes, and text boxes. So configuration is only needed to go beyond +that: pinning specific control values, feeding real audio, adjusting the +properties of the synthesized inputs, or inspecting outputs more deeply. + +### Synthesized input properties + +The generated inputs default to a 2-second mono 44.1 kHz WAV and a two-note +MIDI file. Override any of those properties with a `synthesized_inputs` block +— globally in [config.yml](config.yml), per model under its `overrides` entry, +or per test case; each level overrides the last: + +```yaml +synthesized_inputs: # global default for every model + audio: + sample_rate: 48000 + channels: 2 + duration: 5.0 + ext: .flac + midi: + num_notes: 8 + note_duration: 0.25 +``` + +Audio is written with soundfile, so `ext` accepts any libsndfile format. It is +a preference rather than a guarantee: a component that only accepts other +extensions still gets one it accepts. Each distinct set of properties is +generated once and reused for the rest of the run. + +### Step 1: find the model's control labels + +Test cases reference inputs and outputs by their **label** — the display +name each gradio component was given in the model's `app.py`. Three ways to +find them: + +- Open the Space's gradio UI and read the component titles, or click its + "View Controls" button to see the full spec as JSON; +- Read the model's `app.py` (each component has a `label=...`); +- Reference a wrong label on purpose and run the validator — the error + message lists every available label. + +### Step 2: add the case to config.yml + +Cases live under `overrides:` keyed by space id (or `examples/` +for local examples). Each case has a `name` plus any of: `controls` (override +scalar values by label), `files` (substitute input files by label, paths +relative to config.yml — commit real inputs to `test_data/`), and a +per-case `process_timeout`. + +```yaml +overrides: + teamup-tech/pitch_shifter: + process_timeout: 900 # this model is slow; extend the timeout + test_cases: + - name: default # keep the synthesized case too + - name: extreme-shift + controls: + "Pitch Shift (semitones)": 24 + - name: real-audio + files: + "Input Audio A": test_data/short_vocal.wav +``` + +Note: once `test_cases` is present, **only** the listed cases run — include +`- name: default` to keep the synthesized one. + +### Step 3: check the outputs (optional) + +Every case already gets structural checks for free: `/process` must not +error, file outputs must exist and be non-empty, and JSON outputs (_i.e._, an +optional pyharp `LabelList`) must be well-formed when present (absent/None +is valid, since labels are optional). + +Beyond that, there are two mechanisms, divided by what they can express: + +- **`expect`** asserts properties of a *single* output, and covers most + checks. +- **`validators`** run custom Python for checks that cannot be expressed as + a property of one output — typically relationships spanning several + outputs. + +#### `expect` — declarative per-output rules + +Keyed by output label, then by rule. Every rule is optional, and an output +with no rules is still subject to the structural checks above: + +```yaml + - name: extreme-shift + controls: + "Pitch Shift (semitones)": 24 + expect: + "Output Audio": + ext: .wav # extension: a string, or a list of accepted ones + min_bytes: 10000 # minimum file size + channels: 1 # exact channel count (1 = mono, 2 = stereo) + sample_rate: 44100 # exact sample rate, in Hz + min_duration: 1.5 # minimum length, in seconds + max_duration: 10.0 # maximum length, in seconds + bit_depth: 16 # exact PCM bit depth + min_rms_db: -60 # minimum RMS level, in dBFS + "Output Labels": + min_labels: 1 # minimum labels in a pyharp LabelList +``` + +The full vocabulary, and which output types each rule covers: + +| Rule | Applies to | Asserts | +|---|---|---| +| `ext` | any file output | Extension matches (string, or list of accepted extensions) | +| `min_bytes` | any file output | File is at least this many bytes | +| `channels` | audio output | Exact channel count | +| `sample_rate` | audio output | Exact sample rate in Hz | +| `bit_depth` | audio output | Exact PCM bit depth (16, 24, ...); errors on compressed formats (_e.g._, MP3 or OGG) | +| `min_rms_db` | audio output | RMS level is at least this many dBFS | +| `min_duration` / `max_duration` | audio or MIDI output | Length in seconds is within bounds | +| `min_notes` | MIDI output | At least this many note-on events | +| `min_labels` | JSON (`LabelList`) output | At least this many labels returned | + +**Targeting outputs.** A model with several outputs gets one block per +output label, each checked independently. Use `"*"` instead of a label to +apply rules to every output a rule covers — with mixed outputs, `"*"` sends +`min_bytes` to all file outputs, `min_rms_db` only to the audio ones, and +`min_notes` only to the MIDI ones: + +```yaml + expect: + "*": + min_bytes: 1000 # every file output must be non-trivial +``` + +**Mistakes on a named output are configuration errors, not model failures.** +An unrecognized rule name, an unknown output label, or a rule aimed at a named +output whose type it does not cover (_e.g._, `min_labels` on an audio output) +raises an error naming the problem and listing what is valid. A rule under +`"*"` is more forgiving: when it matches no compatible output, it is skipped +instead of raising an error. This lets a generic case (see +[common test cases](#step-4-reuse-a-case-across-models)) target output types +that a given model does not have. + +**On `min_rms_db`:** level is measured as RMS in dBFS (0 dBFS = full scale), +a bit-depth-independent unit that is easy to set thresholds in. Digital +silence is `-inf`, quiet noise floors sit near `-60`, and typical program +material is above `-40`. RMS is also more robust than a peak measurement — a +single stray click cannot make an otherwise-silent file pass. `min_rms_db: +-60` is the usual "this output is not silent" check. + +**On `min_labels`:** `min_labels: 0` is meaningful and is the right rule when +a model may legitimately return no labels — it asserts a well-formed +`LabelList` came back while permitting it to be empty. Omitting the rule +entirely is weaker: a missing or `None` label output also passes, because +labels are optional in pyharp. + +**On audio formats:** audio outputs are decoded with +[soundfile](https://python-soundfile.readthedocs.io/) (a listed dependency), +so audio rules work on any libsndfile format — WAV, FLAC, OGG, MP3, AIFF, and +more — not just pyharp's `save_audio()` WAV default. Decoding to float also +makes every audio rule bit-depth-independent, so a level threshold means the +same thing whether the source is 16-bit, 24-bit, or float. `bit_depth` is the +exception: it reads the file's PCM encoding and errors on compressed formats, +which have no fixed bit depth. + +**On MIDI:** MIDI outputs are parsed with +[mido](https://mido.readthedocs.io/) (a listed dependency). `min_duration` / +`max_duration` are shared with audio and read the MIDI's own length in +seconds; `min_notes` counts note-on events. + +#### `validators` — custom Python + +Registered in [validators.py](src/validators.py) and referenced by name, with +each validator's parameters nested beneath it, so they stay separate from the +test case's own fields. A case may run several: + +```yaml + - name: labels-line-up + validators: + labels_within_audio: + tolerance: 0.1 +``` + +A validator receives `(outputs, controls, params)` — `outputs` maps output +labels to local file paths (file outputs) or decoded objects (JSON outputs) — +and raises `AssertionError` with a message naming the output and what went +wrong. It may raise `ValidatorNotApplicable` to opt out on a model that lacks +the outputs it needs; that is treated as a skip, not a failure, so a validator +can be used in a common test case. As an example, `labels_within_audio` +asserts every returned label falls inside the audio output's timespan (a +relationship between two outputs), and raises `ValidatorNotApplicable` when +the model has no audio or no label output. + +To add a validator: + +```python +@validator("labels_sorted") +def labels_sorted(outputs, controls, params): + """Assert every LabelList output is in chronological order - an ordering + property no single-value `expect` rule can express.""" + for label, value in outputs.items(): + if isinstance(value, dict) and isinstance(value.get("labels"), list): + times = [entry.get("t", 0.0) for entry in value["labels"]] + assert times == sorted(times), f"'{label}' labels are out of order" +``` + +Before writing one, check whether an `expect` rule would do — and if the +check is a generally useful property of a single output, consider adding it +to the `expect` vocabulary (`EXPECT_RULES` in [cases.py](src/cases.py)) +instead of putting it in a one-off validator. + +### Step 4: reuse a case across models + +A case under a model's `test_cases` only applies to that model. For a check +that should hold for *every* model — "no file output is empty", "audio is +never silent" — add a `common_test_cases` entry at the top level of +config.yml instead. Common cases run on every model in addition to its own, +and their names are shown in the report prefixed with `common:`. + +```yaml +common_test_cases: + - name: outputs-nontrivial + expect: + "*": + min_bytes: 100 # every file output, on every model + - name: audio-not-silent + expect: + "*": + min_rms_db: -60 # applies only to models that have audio outputs +``` + +Keep common cases model-agnostic: target outputs with `"*"` (never a specific +label, which will not exist on most models). Because a `"*"` rule is skipped +when it matches no compatible output, `audio-not-silent` above checks audio +models and is skipped on MIDI-only ones. A validator can do the same by +raising `ValidatorNotApplicable` when the model lacks the outputs it needs +(the shipped `labels_within_audio` does this). A model can opt out of common +cases entirely with `skip_common_cases: true` in its `overrides` entry. + +### Step 5: run just that model to iterate + +```bash +python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter --verbose +``` + +The console shows each case's pass/fail; `report.json` has per-case timings +and error details. Once the case passes reliably, the daily CI run picks it +up automatically — no workflow changes needed. + +### Reference: full config.yml schema + +See the comment block at the top of [config.yml](config.yml) for the +complete schema in one place: `zerogpu_budget_seconds`, `exclude`, +`include_extra`, top-level `common_test_cases`, and per-model `overrides` +(`connect_timeout`, `process_timeout`, `load_only`, `test_cases`, +`skip_common_cases`). + +## Excluding models + +Two equivalent ways, merged together: + +- `exclude` in [config.yml](config.yml) — for permanent exclusions + (non-HARP spaces, archived deployments, known-broken examples). +- `--exclude ...` on the command line — for ad-hoc runs. + +Use the space id (`teamup-tech/`) for remote models and +`examples/` for local examples. + +## CI behavior + +- **Schedule:** daily at 06:30 UTC; also runnable manually from the Actions + tab (with options to validate specific spaces, skip inference, or disable + the default restart of crashed spaces). +- **Reports:** markdown summary on each run + JSON/markdown artifacts. +- **Failure signal:** model failures keep the run green (deployments are not + yet stable enough for daily failure emails to be useful) — they show up as + warning annotations and in the run summary/report instead. No issues are + opened and no emails are sent. Only infrastructure errors (bad token, + discovery failure) turn the run red, since those mean validation itself + has stopped working. +- **Opting back into notifications later:** remove the exit-code handling in + the two `Validate` steps of the workflow so the script's exit code 1 + propagates; failed runs then turn red and GitHub emails maintainers. +- Failures with example models likely indicate a pyharp/gradio-level breakage that may + affect every deployment — fix those before debugging individual spaces. diff --git a/model_validation/config.yml b/model_validation/config.yml new file mode 100644 index 00000000..94892dda --- /dev/null +++ b/model_validation/config.yml @@ -0,0 +1,175 @@ +# Configuration for model_validation/src/validate_models.py +# +# All keys are optional. Models not listed here are still discovered and +# validated automatically with a single synthesized "default" test case, so +# an entry is only needed to change a model's settings or add test cases. +# +# Schema: +# +# zerogpu_budget_seconds: 1500 # optional budget (seconds) the measured +# # ZeroGPU execution time is shown against +# # (e.g. 1500 = PRO 25 min/day). Note the +# # measured time is approximate, not the +# # amount Hugging Face bills (see README) +# +# synthesized_inputs: # properties of the generated test inputs. Set here +# # as the global default, per model under `overrides`, +# # or per test case; each level overrides the last. +# audio: +# sample_rate: 44100 # Hz +# channels: 1 # 1 = mono, 2 = stereo +# duration: 2.0 # seconds +# ext: .wav # format to write (any libsndfile format); a +# # component that accepts only other extensions +# # still gets one it accepts +# midi: +# num_notes: 2 # ascending notes from middle C +# note_duration: 0.5 # seconds per note +# ext: .mid +# +# exclude: # models to exclude from validation: space ids, or +# # "examples/" for local pyharp examples +# - teamup-tech/some-non-harp-space +# - examples/some_example +# +# include_extra: # spaces outside the org that should also be validated +# - some-user/some-harp-space +# +# overrides: # per-model settings; keys are space ids, or +# # "examples/" for local pyharp examples +# : +# connect_timeout: 600 # seconds to wait for build/wake (default 420) +# process_timeout: 900 # seconds to wait for /process (default 600) +# load_only: true # only check availability + /controls +# +# synthesized_inputs: # input properties for this model's cases +# audio: +# sample_rate: 48000 +# channels: 2 +# +# test_cases: # named /process test cases; when omitted, a +# # single synthesized "default" case runs +# - name: default # empty case == synthesized defaults +# +# - name: my-case +# process_timeout: 300 # optional per-case override +# +# controls: # override control values by label +# "Some Slider Label": 12 +# "Some Toggle Label": true +# +# files: # override track/file inputs by label; +# # paths are relative to this file +# "Input Audio": test_data/my_clip.wav +# +# synthesized_inputs: # input properties for this case only +# audio: +# duration: 10.0 +# +# expect: # per-output checks, keyed by output label +# "Output Audio": # ("*" applies rules to every compatible +# # output; see the rule table below) +# ext: .wav # extension: string or list of accepted +# min_bytes: 1000 # minimum file size, in bytes +# channels: 1 # audio: exact channel count +# sample_rate: 44100 # audio: exact sample rate, in Hz +# min_duration: 1.5 # audio/MIDI: minimum length, in seconds +# max_duration: 10.0 # audio/MIDI: maximum length, in seconds +# bit_depth: 16 # audio: exact PCM bit depth (PCM formats) +# min_rms_db: -60 # audio: minimum RMS level, in dBFS +# "Output MIDI": +# min_notes: 1 # MIDI: minimum note-on count +# "Output Labels": +# min_labels: 1 # LabelList: minimum count (0 permits an +# # empty list, but requires a valid one) +# +# validators: # only for checks `expect` cannot express, +# # e.g. spanning several outputs; each +# # validator's parameters nest beneath it +# labels_within_audio: +# tolerance: 0.1 +# +# skip_common_cases: true # opt this model out of common_test_cases +# +# common_test_cases: # generic cases run on EVERY model, on top of its +# # own; keep them model-agnostic - target outputs with +# # "*" (rules that match nothing are simply skipped) +# - name: outputs-nontrivial +# expect: +# "*": +# min_bytes: 100 # every file output is at least minimally sized +# +# Which expect rules apply to which output types: +# +# ext, min_bytes any file output +# channels, sample_rate, bit_depth, min_rms_db audio outputs +# min_duration, max_duration audio and MIDI outputs +# min_notes MIDI outputs +# min_labels JSON (LabelList) outputs +# +# Applying a rule to a NAMED output whose type it does not cover is reported +# as a configuration error, not a model failure; under "*" such a rule is +# simply skipped. Audio outputs are decoded with soundfile, so any libsndfile +# format works (WAV, FLAC, OGG, MP3, AIFF, ...); bit_depth applies only to +# PCM formats, not compressed ones. MIDI outputs are parsed with mido. + +exclude: [] + +include_extra: [] + +overrides: + + # ---- Local pyharp examples ---------------------------------------------- + # These run with `--local-examples` and exercise pyharp itself with no + # Hugging Face infrastructure involved. A failure here indicates a + # pyharp/gradio-level breakage rather than a deployment-specific one. + # Examples are discovered automatically; entries are only needed to add + # test cases beyond the synthesized default. + + examples/pitch_shifter: + test_cases: + - name: default + - name: shift-up-octave + controls: + "Pitch Shift (semitones)": 12 + expect: + "Output Audio": + ext: .wav + min_rms_db: -60 # shifted audio must still carry signal + - name: shift-down-octave + controls: + "Pitch Shift (semitones)": -12 + - name: no-shift + controls: + "Pitch Shift (semitones)": 0 + + examples/midi_pitch_shifter: + test_cases: + - name: default + - name: shift-up-max + controls: + "Pitch Shift (semitones)": 24 + expect: + "Output Midi": + min_notes: 1 # shifted MIDI must still contain notes + - name: shift-down-max + controls: + "Pitch Shift (semitones)": -24 + + # ---- Remote space overrides --------------------------------------------- + # Add per-space timeouts and extra test cases here as needed, e.g.: + # + # teamup-tech/pitch_shifter: + # process_timeout: 900 + # test_cases: + # - name: default + # - name: extreme-shift + # controls: + # "Pitch Shift (semitones)": 24 + # expect: + # "Output Audio": + # ext: .wav + # min_bytes: 10000 + # channels: 1 + # sample_rate: 44100 + # min_duration: 1.5 diff --git a/model_validation/requirements.txt b/model_validation/requirements.txt new file mode 100644 index 00000000..59dcb4e9 --- /dev/null +++ b/model_validation/requirements.txt @@ -0,0 +1,6 @@ +gradio_client>=1.0 +huggingface_hub>=0.23 +PyYAML>=6.0 +numpy # synthesize audio inputs +soundfile>=0.12 # read/write audio (any libsndfile format) for inputs and expect rules +mido>=1.3 # parse MIDI outputs for expect rules diff --git a/model_validation/src/assets.py b/model_validation/src/assets.py new file mode 100644 index 00000000..4f0c7159 --- /dev/null +++ b/model_validation/src/assets.py @@ -0,0 +1,285 @@ +""" +Synthesized test inputs for HARP model validation. + +Every input file is generated from scratch - audio with soundfile, MIDI with +the standard library - so validation needs no binary fixtures checked into the +repository. Real-world inputs for specific models belong in test_data/ and are +referenced from a test case's `files` entry in config.yml. + +Input properties (sample rate, channels, length, format, ...) are configurable +via `synthesized_inputs` - see config.yml for the schema. Each distinct set of +properties is generated once and cached for the run. +""" + +import struct +from pathlib import Path + +import numpy +import soundfile + + +__all__ = [ + 'Assets', + 'AUDIO_DEFAULTS', + 'MIDI_DEFAULTS', + 'merge_specs' +] + + +# Configurable properties of synthesized inputs, with their default values +AUDIO_DEFAULTS = { + "sample_rate": 44100, # Hz + "channels": 1, # 1 = mono, 2 = stereo, ... + "duration": 2.0, # seconds + "ext": ".wav", # container/format to write +} +MIDI_DEFAULTS = { + "num_notes": 2, # ascending notes from middle C + "note_duration": 0.5, # seconds per note (at the default 120 BPM) + "ext": ".mid", +} + +# Extensions understood as audio; a component accepting one of these gets a +# synthesized clip in that format (written via soundfile, so only formats this +# libsndfile build supports actually succeed). +AUDIO_EXTS = {".wav", ".flac", ".ogg", ".oga", ".opus", ".aiff", ".aif", + ".aifc", ".au", ".snd", ".w64", ".caf", ".mp3"} + +MIDI_EXTS = {".mid", ".midi"} + +# MIDI ticks per quarter note, and the default tempo those ticks imply +MIDI_TICKS_PER_BEAT = 480 +MIDI_SECONDS_PER_BEAT = 0.5 + + +def merge_specs(*specs) -> dict: + """ + Merge `synthesized_inputs` blocks, later entries winning per property. + + Args: + *specs (dict | None): Blocks keyed by media kind ("audio", "midi"), + in increasing order of precedence. + + Returns: + merged (dict): One block with per-kind properties merged. + """ + + merged = {} + + for spec in specs: + for kind, props in (spec or {}).items(): + merged.setdefault(kind, {}).update(props or {}) + + return merged + + +class Assets: + """ + Synthesized test input files, generated on demand and cached for the run. + + Args: + workdir (Path): Directory the generated files are written to. + defaults (dict | None): Global `synthesized_inputs` block from + config.yml, overriding the built-in property defaults. + """ + + def __init__(self, workdir: Path, defaults: dict | None = None): + workdir.mkdir(parents=True, exist_ok=True) + self.workdir = workdir + self.defaults = defaults or {} + self._cache = {} + + self.text = workdir / "test_input.txt" + self.text.write_text("HARP model validation\n") + self.json = workdir / "test_input.json" + self.json.write_text("{}\n") + + def resolve(self, kind: str, overrides: dict | None = None) -> dict: + """ + Resolve the properties for one media kind. + + Precedence is built-in defaults, then the global `synthesized_inputs` + block, then the per-model/per-case overrides. + + Args: + kind (str): "audio" or "midi". + overrides (dict | None): A merged `synthesized_inputs` block. + + Returns: + props (dict): The resolved properties for that kind. + """ + + props = dict(AUDIO_DEFAULTS if kind == "audio" else MIDI_DEFAULTS) + props.update(self.defaults.get(kind) or {}) + props.update((overrides or {}).get(kind) or {}) + + return props + + def audio(self, overrides: dict | None = None, ext: str | None = None) -> Path | None: + """ + Get a synthesized audio clip with the resolved properties. + + Args: + overrides (dict | None): A merged `synthesized_inputs` block. + ext (str | None): Format to write, overriding the resolved `ext` + (used when a component only accepts certain extensions). + + Returns: + path (Path | None): The clip, or None when this libsndfile build + cannot write the requested format. + """ + + props = self.resolve("audio", overrides) + sample_rate = int(props["sample_rate"]) + channels = int(props["channels"]) + duration = float(props["duration"]) + ext = (ext or props["ext"]).lower() + + key = ("audio", sample_rate, channels, duration, ext) + if key not in self._cache: + path = (self.workdir / + f"test_input_{sample_rate}hz_{channels}ch_{duration:g}s{ext}") + try: + self._cache[key] = write_audio(path, duration, sample_rate, channels) + except Exception: # noqa: BLE001 - format not writable here + self._cache[key] = None + + return self._cache[key] + + def midi(self, overrides: dict | None = None, ext: str | None = None) -> Path: + """ + Get a synthesized MIDI file with the resolved properties. + + Args: + overrides (dict | None): A merged `synthesized_inputs` block. + ext (str | None): Extension to write, overriding the resolved one. + + Returns: + path (Path): The MIDI file. + """ + + props = self.resolve("midi", overrides) + num_notes = int(props["num_notes"]) + note_duration = float(props["note_duration"]) + ext = (ext or props["ext"]).lower() + + key = ("midi", num_notes, note_duration, ext) + if key not in self._cache: + path = self.workdir / f"test_input_{num_notes}n_{note_duration:g}s{ext}" + self._cache[key] = write_midi(path, num_notes, note_duration) + + return self._cache[key] + + def for_file_types(self, file_types: list, + overrides: dict | None = None) -> Path | None: + """ + Pick a synthesized file whose format matches the accepted types. + + Audio components get a clip in an accepted format, preferring the + configured `ext` when the component allows it; MIDI, JSON, and text + inputs are served from their synthesized files. A component whose + accepted types are all unsupported (e.g. a bespoke binary format) gets + None - supply a real file via a test case's `files` entry. + + Args: + file_types (list): Accepted extensions from the /controls spec; + empty or None means any file is accepted. + overrides (dict | None): A merged `synthesized_inputs` block. + + Returns: + path (Path | None): A matching synthesized file, or None when no + synthesized format satisfies the component. + """ + + types = {str(t).lower() for t in (file_types or [])} + + if not types or "audio" in types: + return self.audio(overrides) + + # Try the configured format first, then any other accepted audio format + accepted_audio = types & AUDIO_EXTS + if accepted_audio: + preferred = self.resolve("audio", overrides)["ext"].lower() + order = ([preferred] if preferred in accepted_audio else []) + \ + sorted(accepted_audio - {preferred}) + for candidate in order: + clip = self.audio(overrides, ext=candidate) + if clip is not None: + return clip + + accepted_midi = types & MIDI_EXTS + if accepted_midi: + preferred = self.resolve("midi", overrides)["ext"].lower() + return self.midi(overrides, + ext=preferred if preferred in accepted_midi + else sorted(accepted_midi)[0]) + + if ".json" in types: + return self.json + if types & {".txt", ".text", "text"}: + return self.text + + return None + + +def write_audio(path: Path, duration: float, sample_rate: int, + channels: int) -> Path: + """ + Write a sine sweep - a valid input for any audio model. + + Args: + path (Path): Destination path; its extension selects the format. + duration (float): Length of the sweep in seconds. + sample_rate (int): Sample rate in Hz. + channels (int): Number of (identical) channels to write. + + Returns: + path (Path): The written file, for chaining. + """ + + n = max(1, int(duration * sample_rate)) + t = numpy.arange(n) / sample_rate + # Sweep from 220 Hz up one octave over the clip + freq = 220.0 + 220.0 * numpy.arange(n) / n + mono = 0.5 * numpy.sin(2 * numpy.pi * freq * t) + soundfile.write(str(path), numpy.tile(mono[:, None], (1, channels)), sample_rate) + + return path + + +def write_midi(path: Path, num_notes: int, note_duration: float) -> Path: + """ + Write a standard MIDI file (format 0) of ascending notes from middle C. + + Args: + path (Path): Destination .mid path. + num_notes (int): Number of notes to write. + note_duration (float): Seconds each note sounds, at 120 BPM. + + Returns: + path (Path): The written file, for chaining. + """ + + ticks = max(1, int(round(note_duration / MIDI_SECONDS_PER_BEAT * MIDI_TICKS_PER_BEAT))) + + def delta(value): + """Encode a delta time as a MIDI variable-length quantity.""" + out = [value & 0x7F] + value >>= 7 + while value: + out.insert(0, (value & 0x7F) | 0x80) + value >>= 7 + return bytes(out) + + events = bytearray([0x00, 0xC0, 0x00]) # program change: grand piano + for i in range(max(0, num_notes)): + pitch = 60 + (i * 2) % 24 # ascending from middle C + events += bytes([0x00, 0x90, pitch, 0x64]) # note on + events += delta(ticks) + bytes([0x80, pitch, 0x40]) + events += bytes([0x00, 0xFF, 0x2F, 0x00]) # end of track + + header = b"MThd" + struct.pack(">IHHH", 6, 0, 1, MIDI_TICKS_PER_BEAT) + track = b"MTrk" + struct.pack(">I", len(events)) + bytes(events) + path.write_bytes(header + track) + + return path diff --git a/model_validation/src/audio.py b/model_validation/src/audio.py new file mode 100644 index 00000000..e71176f8 --- /dev/null +++ b/model_validation/src/audio.py @@ -0,0 +1,104 @@ +""" +Audio decoding for HARP model validation. + +Models return audio in whatever format their process function writes - +pyharp's save_audio() defaults to WAV, but a model passing its own output +path can produce FLAC, OGG, MP3, AIFF, and so on. All formats are decoded +through soundfile (libsndfile), which also frees the checks from caring +about bit depth: samples always arrive as floats in [-1, 1], so a level +threshold means the same thing for 16-bit, 24-bit, and float sources. +""" + +import math +import os + +import soundfile + + +__all__ = [ + 'read_audio_props', + 'supported_formats' +] + + +def supported_formats() -> list: + """ + List the audio formats this installation can decode. + + Returns: + formats (list): Sorted format names (e.g. WAV, FLAC, MP3, OGG). + """ + + try: + return sorted(soundfile.available_formats()) + except Exception: # noqa: BLE001 - diagnostics must never raise + return [] + + +def bit_depth_from_subtype(subtype: str): + """ + Map a libsndfile subtype to a PCM bit depth, when it has one. + + Args: + subtype (str): soundfile subtype, e.g. "PCM_16", "FLOAT", "VORBIS". + + Returns: + depth (int | None): Bits per sample for PCM/float encodings; None + for compressed encodings that have no fixed bit depth. + """ + + if subtype.startswith("PCM_"): + tail = subtype[len("PCM_"):] + if tail in ("S8", "U8"): + return 8 + if tail.isdigit(): + return int(tail) + return {"FLOAT": 32, "DOUBLE": 64}.get(subtype) + + +def read_audio_props(label: str, path: str) -> dict: + """ + Decode an audio file and measure the properties `expect` rules check. + + Args: + label (str): Output label, for error messages. + path (str): Path to the audio file. + + Returns: + props (dict): channels, sample_rate, duration (seconds), rms_db (RMS + level in dBFS; -inf for digital silence), subtype (libsndfile + encoding name), and bit_depth (int, or None for compressed + encodings). + + Raises: + AssertionError: If the file cannot be decoded or contains no audio. + """ + + try: + # A single header-and-data open: always_2d gives a consistent + # (frames, channels) shape, and float64 output normalizes any bit + # depth to [-1, 1] for the level measurement + with soundfile.SoundFile(path) as f: + sample_rate = f.samplerate + channels = f.channels + subtype = f.subtype + data = f.read(always_2d=True, dtype="float64") + except Exception as exc: # noqa: BLE001 - any decode failure + ext = os.path.splitext(path)[1] or "(no extension)" + raise AssertionError( + f"output '{label}' could not be decoded as audio (format {ext}): " + f"{exc}. Decodable formats: {', '.join(supported_formats())}") + + frames = data.shape[0] + assert frames, f"output '{label}' contains no audio frames" + + rms = math.sqrt(float((data * data).sum()) / data.size) + + return { + "channels": channels, + "sample_rate": sample_rate, + "duration": frames / sample_rate, + "rms_db": 20 * math.log10(rms) if rms > 0 else float("-inf"), + "subtype": subtype, + "bit_depth": bit_depth_from_subtype(subtype), + } diff --git a/model_validation/src/cases.py b/model_validation/src/cases.py new file mode 100644 index 00000000..90c7d0f1 --- /dev/null +++ b/model_validation/src/cases.py @@ -0,0 +1,499 @@ +""" +Test-case handling for HARP model validation. + +Covers the full input/output cycle of one /process test case: +synthesizing default inputs from a model's /controls spec, overlaying a +configured case's control values and input files, and checking the outputs +(structural validation plus optional per-case `expect` rules and custom +validators). +""" + +import os +from pathlib import Path + +from gradio_client import handle_file + +from assets import Assets +from audio import read_audio_props +from midi import read_midi_props +from validators import VALIDATORS, ValidatorNotApplicable + + +__all__ = [ + 'synthesize_default_args', + 'apply_case', + 'validate_outputs', + 'inspect_outputs', + 'EXPECT_RULES' +] + + +# Output component types that produce a file on disk +FILE_TYPES = {"audio_track", "midi_track", "generic_file"} +AUDIO, MIDI, JSON = {"audio_track"}, {"midi_track"}, {"json"} + +# The declarative `expect` vocabulary: rule name -> output types it applies to. +# Anything expressible as "read a property of one output and compare it" +# belongs here rather than in a custom validator (validators.py). Applying a +# rule to an output type it does not cover is a configuration error, not a +# model failure. config.yml documents what each rule asserts. +EXPECT_RULES = { + "ext": FILE_TYPES, + "min_bytes": FILE_TYPES, + "min_duration": AUDIO | MIDI, + "max_duration": AUDIO | MIDI, + "channels": AUDIO, + "sample_rate": AUDIO, + "bit_depth": AUDIO, + "min_rms_db": AUDIO, + "min_notes": MIDI, + "min_labels": JSON, +} + +# Rule groups, by what they need decoded. Duration rules are shared: they read +# from whichever of audio/MIDI props matches the output's type. +DURATION_RULES = {"min_duration", "max_duration"} +DECODED_RULES = { + "audio_track": {"channels", "sample_rate", "bit_depth", "min_rms_db"} | DURATION_RULES, + "midi_track": {"min_notes"} | DURATION_RULES, +} + +# Key selecting every compatible output rather than one named output +ALL_OUTPUTS = "*" + + +def synthesize_default_args(controls: dict, assets: Assets, + synth: dict | None = None) -> tuple: + """ + Build the positional argument list for /process from the /controls spec. + + Args: + controls (dict): The /controls payload (card, inputs, outputs). + assets (Assets): Synthesized input files to draw from. + synth (dict | None): Merged `synthesized_inputs` block controlling the + properties of the generated inputs (see config.yml). + + Returns: + args (list): One argument per input component, in declaration order. + missing (dict): Label -> reason for every input that could NOT be + synthesized. Such inputs get a None placeholder; a test case can + still run if its controls/files overrides cover them all. + """ + + args, missing = [], {} + + for spec in controls.get("inputs", []): + ctype = spec.get("type") + label = spec.get("label") + required = spec.get("required", True) + + if ctype in FILE_TYPES: + # Optional file inputs are left unsupplied, exercising the model's + # own handling of their absence + if not required: + args.append(None) + continue + if ctype == "audio_track": + path = assets.audio(synth) + elif ctype == "midi_track": + path = assets.midi(synth) + else: + path = assets.for_file_types(spec.get("file_types"), synth) + if path is None: + missing[label] = (f"cannot synthesize a {ctype} input for " + f"file_types={spec.get('file_types')}; supply " + f"one via a test case's `files` entry") + args.append(handle_file(str(path)) if path is not None else None) + elif ctype in ("slider", "number_box"): + value = spec.get("value") + if value is None: + value = spec.get("minimum", 0) + args.append(value) + elif ctype == "text_box": + # An empty string is a legitimate declared default, so only fall + # back when no default was declared at all + value = spec.get("value") + args.append(value if value is not None else "test") + elif ctype == "toggle": + args.append(bool(spec.get("value", False))) + elif ctype == "dropdown": + value = spec.get("value") + if value is None: + choices = spec.get("choices") or [] + if not choices: + missing[label] = "dropdown has no choices" + else: + # Choices arrive as [label, value] pairs or plain values + first = choices[0] + value = first[1] if isinstance(first, (list, tuple)) and len(first) > 1 else first + args.append(value) + else: + missing[label] = f"unsupported input control type '{ctype}'" + args.append(None) + + return args, missing + + +def apply_case(args: list, controls: dict, case: dict, config_dir: Path) -> list: + """ + Overlay a configured test case onto the default argument list. + + Args: + args (list): Default arguments from synthesize_default_args(). + controls (dict): The /controls payload, used to match labels. + case (dict): Test case entry from config.yml. May contain: + controls: {: } - override scalar values. + files: {: } - override track/file inputs + (paths relative to config.yml). + config_dir (Path): Directory containing config.yml, the base for + relative file paths. + + Returns: + args (list): A new argument list with the overrides applied. + + Raises: + ValueError: If a label does not match any input, or a file is missing. + """ + + labels = [spec.get("label") for spec in controls.get("inputs", [])] + args = list(args) + + for label, value in (case.get("controls") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"control '{label}' (available: {labels})") + args[labels.index(label)] = value + + for label, rel_path in (case.get("files") or {}).items(): + if label not in labels: + raise ValueError(f"test case '{case.get('name')}' references unknown " + f"input '{label}' (available: {labels})") + path = (config_dir / rel_path).resolve() + if not path.exists(): + raise ValueError(f"test case '{case.get('name')}': file not found: {path}") + args[labels.index(label)] = handle_file(str(path)) + + return args + + +def as_output_list(result, specs: list) -> list: + """ + Normalize a /process result to one value per output spec. + + With a single output the raw value is used as-is, so a JSON output + returning a list is not mistaken for multiple outputs. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (list): One value per output spec. + """ + + if len(specs) <= 1: + return [result] + + return list(result) if isinstance(result, (list, tuple)) else [result] + + +def validate_outputs(result, controls: dict) -> str | None: + """ + Structurally check the outputs every model must satisfy. + + File outputs must be present, on disk, and non-empty. JSON outputs carry + optional data such as a pyharp LabelList: None/absent is valid (labels + are optional), but a present value must be JSON-shaped and a LabelList + must be well-formed. + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + + Returns: + error (str | None): A description of the first problem found, or + None when the outputs look structurally sound. + """ + + specs = controls.get("outputs", []) + outputs = as_output_list(result, specs) + + if len(specs) > 1 and len(outputs) != len(specs): + return f"expected {len(specs)} outputs, got {len(outputs)}" + + for spec, out in zip(specs, outputs): + if spec.get("type") == "json": + if out is None: + continue + if not isinstance(out, (dict, list)): + return (f"JSON output '{spec.get('label')}' is not valid JSON " + f"data: {str(out)[:100]}") + if isinstance(out, dict) and "labels" in out and \ + not isinstance(out["labels"], list): + return f"label list output '{spec.get('label')}' is malformed" + continue + + if out is None: + return f"output '{spec.get('label')}' is None" + + # gradio_client downloads file outputs and returns local paths + path = out.get("path") if isinstance(out, dict) and "path" in out else out + if isinstance(path, str) and os.path.isfile(path) and os.path.getsize(path) == 0: + return f"output file for '{spec.get('label')}' is empty" + + return None + + +def outputs_by_label(result, specs: list) -> dict: + """ + Map output labels to values for inspection by validators. + + Args: + result: The raw value returned by gradio_client for /process. + specs (list): Output component specs from /controls. + + Returns: + outputs (dict): Label -> local file path (file outputs) or decoded + object (JSON outputs). + """ + + outputs = as_output_list(result, specs) + mapped = {} + + for spec, out in zip(specs, outputs): + value = out.get("path") if isinstance(out, dict) and "path" in out else out + mapped[spec.get("label")] = value + + return mapped + + +def require_file(label: str, value) -> str: + """ + Assert an output is a file on disk and return its path. + + Args: + label (str): Output label, for the error message. + value: The mapped output value. + + Returns: + path (str): The verified filesystem path. + + Raises: + AssertionError: If the output is not an existing file. + """ + + if not (isinstance(value, str) and os.path.exists(value)): + raise AssertionError(f"output '{label}' is not a file on disk: {value}") + + return value + + +def resolve_expect_targets(expect: dict, out_types: dict) -> list: + """ + Resolve an `expect` block into concrete (label, rules) pairs. + + Keys are output labels, or "*" to apply rules to every output the rule + is compatible with (e.g. `min_rms_db` under "*" reaches only the audio + outputs). A "*" rule that matches no output is simply dropped, so a + generic case can safely target output types a given model lacks. An + explicit label, by contrast, must exist and must accept the rule - + otherwise it is a configuration error, not a model failure. + + Args: + expect (dict): The case's `expect` block. + out_types (dict): Output label -> component type from /controls. + + Returns: + targets (list): (label, rules) pairs to check. + + Raises: + ValueError: If a label, rule name, or rule/output pairing is invalid. + """ + + targets = [] + + for key, rules in (expect or {}).items(): + rules = rules or {} + + unknown = set(rules) - set(EXPECT_RULES) + if unknown: + raise ValueError(f"unknown expect rule(s) {sorted(unknown)} for " + f"'{key}'; supported rules: {sorted(EXPECT_RULES)}") + + if key == ALL_OUTPUTS: + # Fan each rule out to the outputs whose type it covers; a rule + # matching nothing is dropped (lenient, so generic cases apply) + per_label = {} + for rule, value in rules.items(): + applicable = EXPECT_RULES[rule] + for label, otype in out_types.items(): + if otype in applicable: + per_label.setdefault(label, {})[rule] = value + targets.extend(per_label.items()) + continue + + if key not in out_types: + raise ValueError(f"expect references unknown output '{key}' " + f"(available: {list(out_types)})") + + for rule in rules: + applicable = EXPECT_RULES[rule] + if out_types[key] not in applicable: + raise ValueError( + f"expect rule '{rule}' does not apply to output '{key}' " + f"of type '{out_types[key]}'; it applies to " + f"{sorted(applicable)} outputs") + + targets.append((key, rules)) + + return targets + + +def check_duration(label: str, duration, rules: dict) -> None: + """ + Apply the shared min_duration / max_duration rules to a decoded length. + + Args: + label (str): Output label, for error messages. + duration (float | None): Length in seconds, or None if undetermined. + rules (dict): The output's rules (only duration keys are read). + + Raises: + AssertionError: If a duration bound is not met, or duration is + required but could not be determined. + """ + + if not (set(rules) & DURATION_RULES): + return + + assert duration is not None, \ + f"output '{label}': could not determine its duration" + + if "min_duration" in rules: + assert duration >= rules["min_duration"], \ + (f"output '{label}' is {duration:.2f}s, expected at least " + f"{rules['min_duration']}s") + + if "max_duration" in rules: + assert duration <= rules["max_duration"], \ + (f"output '{label}' is {duration:.2f}s, expected at most " + f"{rules['max_duration']}s") + + +def check_expectations(label: str, otype: str, value, rules: dict) -> None: + """ + Apply one output's declarative `expect` rules. + + Rule names and their applicability to this output are validated upstream + by resolve_expect_targets(). Audio/MIDI files are decoded in a single + pass, and only when a rule that needs the decoded properties is present. + + Args: + label (str): Output label the rules apply to. + otype (str): The output's component type from /controls. + value: The mapped output value (file path or decoded JSON). + rules (dict): Rule name -> expected value. + + Raises: + AssertionError: If any rule is not satisfied. + """ + + # --- Any file output ----------------------------------------------------- + if "ext" in rules: + allowed = rules["ext"] + allowed = [allowed] if isinstance(allowed, str) else list(allowed) + assert isinstance(value, str) and \ + any(value.lower().endswith(e.lower()) for e in allowed), \ + f"output '{label}' is not a {' or '.join(allowed)} file: {value}" + + if "min_bytes" in rules: + size = os.path.getsize(require_file(label, value)) + assert size >= rules["min_bytes"], \ + f"output file '{label}' is {size} bytes, expected at least {rules['min_bytes']}" + + # --- Decoded audio/MIDI properties --------------------------------------- + # Decode once, and only when a rule actually needs the decoded properties + props = None + if set(rules) & DECODED_RULES.get(otype, set()): + reader = read_audio_props if otype == "audio_track" else read_midi_props + props = reader(label, require_file(label, value)) + + if props is not None and otype == "audio_track": + if "channels" in rules: + assert props["channels"] == rules["channels"], \ + (f"output '{label}': expected {rules['channels']} channel(s), " + f"got {props['channels']}") + + if "sample_rate" in rules: + assert props["sample_rate"] == rules["sample_rate"], \ + (f"output '{label}': expected {rules['sample_rate']} Hz, " + f"got {props['sample_rate']} Hz") + + if "bit_depth" in rules: + assert props["bit_depth"] is not None, \ + (f"output '{label}' is a compressed format ({props['subtype']}) " + f"with no PCM bit depth to check") + assert props["bit_depth"] == rules["bit_depth"], \ + (f"output '{label}': expected {rules['bit_depth']}-bit audio, " + f"got {props['bit_depth']}-bit ({props['subtype']})") + + if "min_rms_db" in rules: + assert props["rms_db"] >= rules["min_rms_db"], \ + (f"output '{label}' appears silent (RMS {props['rms_db']:.1f} dBFS " + f"< {rules['min_rms_db']} dBFS)") + + if props is not None and otype == "midi_track": + if "min_notes" in rules: + assert props["num_notes"] >= rules["min_notes"], \ + (f"output '{label}' has {props['num_notes']} note(s), expected " + f"at least {rules['min_notes']}") + + if props is not None: + check_duration(label, props["duration"], rules) + + # --- JSON / LabelList outputs -------------------------------------------- + if "min_labels" in rules: + labels = value.get("labels") if isinstance(value, dict) else None + assert isinstance(labels, list), \ + f"output '{label}' does not contain a pyharp LabelList: {value}" + assert len(labels) >= rules["min_labels"], \ + (f"output '{label}' has {len(labels)} label(s), expected at least " + f"{rules['min_labels']}") + + +def inspect_outputs(result, controls: dict, case: dict) -> None: + """ + Apply a test case's deeper output checks: its `expect` rules (declarative, + per output - see EXPECT_RULES) and its `validators` (custom Python for + checks spanning several outputs - see validators.py). Both are optional; + without either, outputs are still subject to validate_outputs(). + + Args: + result: The raw value returned by gradio_client for /process. + controls (dict): The /controls payload. + case (dict): Test case entry from config.yml. + + Raises: + AssertionError: If an expectation or validator check fails. + ValueError: If the case references an unknown output, rule, or + validator, or applies a rule to an incompatible output. + """ + + specs = controls.get("outputs", []) + out_map = outputs_by_label(result, specs) + out_types = {spec.get("label"): spec.get("type") for spec in specs} + + for label, rules in resolve_expect_targets(case.get("expect"), out_types): + check_expectations(label, out_types[label], out_map[label], rules) + + for name, params in (case.get("validators") or {}).items(): + if name not in VALIDATORS: + raise ValueError(f"unknown validator '{name}' (available: " + f"{sorted(VALIDATORS)}); register it in " + f"validators.py") + try: + VALIDATORS[name](out_map, controls, params or {}) + except ValidatorNotApplicable: + # The model lacks the outputs this validator needs; skip it so a + # common case's validator does not fail on models it does not fit + continue diff --git a/model_validation/src/harness.py b/model_validation/src/harness.py new file mode 100644 index 00000000..0e1ea43d --- /dev/null +++ b/model_validation/src/harness.py @@ -0,0 +1,640 @@ +""" +The core validation harness for HARP model deployments: the code that sets +up a connection to a live model, drives it through /controls and /process, +and reports what happened, independent of where the model is running. + +Both validation tiers funnel into run_endpoint_tests(), which performs the +identical black-box checks against any live HARP gradio app: + + 1. the /controls and /process endpoints exist; + 2. /controls returns a well-formed model card and component spec; + 3. every configured /process test case produces valid outputs. + +test_space() wraps this for a remote Hugging Face Space (runtime stage +checks, optional restart, connection with wake-up retries) and +test_local_example() for a pyharp example app launched locally. +""" + +import argparse +import contextlib +import os +import subprocess +import sys +import time +import traceback +import urllib.request +from pathlib import Path + +from gradio_client import Client + +from assets import Assets, merge_specs +from cases import synthesize_default_args, apply_case, validate_outputs, inspect_outputs +from quota import is_zerogpu +from results import ModelResult, CaseResult, PASS, FAIL, SKIP +from utils import run_with_timeout, scrub, describe_exception + + +__all__ = [ + 'test_space', + 'test_local_example' +] + + +# Space runtime stages that indicate a hard failure (no point connecting) +DEAD_STAGES = {"BUILD_ERROR", "RUNTIME_ERROR", "CONFIG_ERROR", "DELETING"} +# Stages that resolve on their own if we wait (or wake on first request) +TRANSIENT_STAGES = {"BUILDING", "RUNNING_BUILDING", "APP_STARTING", "SLEEPING"} + +# Fixed sub-timeouts, in seconds. Unlike connect_timeout / process_timeout, +# these bound quick metadata calls whose duration does not vary by model, so +# they are module constants rather than per-model config. +CONTROLS_TIMEOUT = 120 # fetch the /controls spec (model already loaded) +CLIENT_CLOSE_TIMEOUT = 10 # best-effort gradio_client shutdown +SERVER_PROBE_INTERVAL = 2 # poll cadence while a local example boots +SERVER_PROBE_TIMEOUT = 5 # per-probe HTTP timeout against a local example +PROCESS_TERMINATE_TIMEOUT = 15 # grace period before killing a local example +LOADING_RETRY_INTERVAL = 15 # wait between retries while a model warms up +JOB_POLL_INTERVAL = 1 # cadence for polling a /process job's status + +# gradio job status codes that mean the job is still waiting in the ZeroGPU +# queue (not yet executing on the GPU) +QUEUE_STATUS_CODES = {"IN_QUEUE", "JOINING_QUEUE", "QUEUE_FULL"} + +# Substrings marking a "model is still loading" response - a ZeroGPU space +# waking its GPU worker returns this immediately rather than blocking, so it +# is retried (within connect_timeout) instead of treated as a failure. +LOADING_MARKERS = ("still loading", "loading, please wait", "is loading", + "currently loading", "warming up") + +# Substrings marking a ZeroGPU quota-exhausted response. Hitting this on one +# model means every other ZeroGPU model would hit it too, so the rest are +# skipped rather than retried. +QUOTA_EXHAUSTED_MARKERS = ("exceeded your gpu quota", "gpu quota exceeded", + "zerogpu quota", "quota exceeded", "gpu quota") + + +def is_loading_error(exc: Exception) -> bool: + """ + Whether an exception is a transient "model still loading" response. + + Args: + exc (Exception): The exception raised by a gradio call. + + Returns: + loading (bool): True if the model reported it was still loading. + """ + + message = str(exc).lower() + return any(marker in message for marker in LOADING_MARKERS) + + +def is_quota_exhausted(text: str) -> bool: + """ + Whether an error message indicates the ZeroGPU allowance is exhausted. + + Args: + text (str): An error message to inspect. + + Returns: + exhausted (bool): True if it looks like a ZeroGPU quota error. + """ + + lowered = text.lower() + return any(marker in lowered for marker in QUOTA_EXHAUSTED_MARKERS) + + +def call_through_loading(fn, deadline: float, what: str): + """ + Call fn(), retrying while the model reports it is still loading. + + A ZeroGPU space that must start its GPU worker answers the first request + with a "still loading" error immediately; retrying until the deadline + tolerates that start-up interval instead of failing on it. + + Args: + fn (callable): Zero-argument call to make. + deadline (float): time.time() value to stop retrying at. + what (str): Short description for the timeout message. + + Returns: + result: Whatever fn() returns once the model is ready. + + Raises: + Exception: fn()'s error, once it is not a loading error or the + deadline has passed. + """ + + while True: + try: + return fn() + except Exception as exc: # noqa: BLE001 - inspected below, else re-raised + if is_loading_error(exc) and time.time() + LOADING_RETRY_INTERVAL < deadline: + time.sleep(LOADING_RETRY_INTERVAL) + continue + raise + + +def job_status_code(job) -> str | None: + """ + Best-effort read of a gradio job's current status code name. + + Args: + job: A gradio_client Job handle. + + Returns: + code (str | None): The status code name (e.g. "IN_QUEUE", + "PROCESSING"), or None if the status cannot be read. + """ + + try: + status = job.status() + return getattr(getattr(status, "code", None), "name", None) + except Exception: # noqa: BLE001 - status polling is best-effort + return None + + +def wait_out_queue(job, queue_deadline: float, api_name: str) -> None: + """ + Block until a /process job leaves the ZeroGPU queue and begins executing. + + A ZeroGPU job waits in a shared queue before running; that wait is not + part of the model's execution and is not charged against quota, so it is + bounded by queue_deadline (the connect timeout) rather than by the shorter + execution timeout the caller applies afterwards. + + Args: + job: A gradio_client Job handle. + queue_deadline (float): time.time() by which the job must leave the + queue, or it is cancelled. + api_name (str): Endpoint name, for the timeout message. + + Raises: + TimeoutError: If the job never leaves the queue by queue_deadline. + """ + + while not job.done() and job_status_code(job) in QUEUE_STATUS_CODES: + if time.time() >= queue_deadline: + job.cancel() + raise TimeoutError(f"{api_name} still queued after waiting for the " + f"connect timeout") + time.sleep(JOB_POLL_INTERVAL) + + +def close_client(client) -> None: + """ + Shut down a gradio_client instance without letting cleanup errors + (or hangs in its heartbeat machinery) affect the result. + + Args: + client (Client | None): The client to close; None is a no-op. + """ + + if client is None: + return + + try: + run_with_timeout(client.close, CLIENT_CLOSE_TIMEOUT, "client close") + except Exception: # noqa: BLE001 - cleanup is best-effort + pass + + +def run_case(client: Client, case: dict, controls: dict, assets: Assets, + overrides: dict, config_dir: Path, process_timeout: float, + connect_timeout: float) -> CaseResult: + """ + Run one /process test case and check its outputs. + + The whole case sits inside an error boundary: any failure is recorded on + the returned record rather than raised, so remaining cases still run. + + Args: + client (Client): Connected gradio client for the deployment. + case (dict): Test case entry from config.yml. + controls (dict): The /controls payload. + assets (Assets): Synthesized input files. + overrides (dict): This model's entry from config.yml `overrides`. + config_dir (Path): Directory containing config.yml. + process_timeout (float): Seconds allowed for /process execution. + connect_timeout (float): Seconds allowed for the queue wait and for + retrying a model that is still loading. + + Returns: + case_result (CaseResult): The completed case record. + """ + + case_result = CaseResult(name=case.get("name", "unnamed")) + start = time.time() + # Set inside run_process so `executed` is recorded even when the job leaves + # the queue (and so reserves GPU) but then fails or times out + state = {"executed": False} + + try: + synth = merge_specs(overrides.get("synthesized_inputs"), + case.get("synthesized_inputs")) + default_args, missing = synthesize_default_args(controls, assets, synth) + + # Inputs we could not synthesize are fine if this case supplies them + supplied = set(case.get("controls") or {}) | set(case.get("files") or {}) + unsatisfied = {k: v for k, v in missing.items() if k not in supplied} + if unsatisfied: + case_result.error = "skipped: " + "; ".join( + f"'{k}': {v}" for k, v in unsatisfied.items()) + return case_result + + args = apply_case(default_args, controls, case, config_dir) + + def run_process(): + job = client.submit(*args, api_name="/process") + # The queue wait is bounded by connect_timeout; only once the job + # is dequeued does the (shorter) execution timeout apply + wait_out_queue(job, time.time() + connect_timeout, "/process") + state["executed"] = True + return job.result(timeout=process_timeout) + + output = call_through_loading( + run_process, time.time() + connect_timeout, "/process") + + error = validate_outputs(output, controls) + if error: + case_result.ok = False + case_result.error = f"/process output invalid: {error}" + else: + inspect_outputs(output, controls, case) + case_result.ok = True + except Exception as exc: # noqa: BLE001 - any exception fails the case + case_result.ok = False + case_result.error = describe_exception(exc) + finally: + case_result.duration = round(time.time() - start, 1) + case_result.executed = state["executed"] + + return case_result + + +def run_endpoint_tests(client: Client, result: ModelResult, assets: Assets, + overrides: dict, opts: argparse.Namespace) -> ModelResult: + """ + Verify /controls and run every configured /process test case. + + Each case runs inside its own error boundary, so a failing case does not + stop the remaining cases. + + Args: + client (Client): Connected gradio client for the deployment. + result (ModelResult): Result record to fill in. + assets (Assets): Synthesized input files. + overrides (dict): This model's entry from config.yml `overrides`. + opts (argparse.Namespace): Parsed command-line options. + + Returns: + result (ModelResult): The same record, completed. + """ + + # ZeroGPU models default to a lower process timeout (their jobs are short + # once running); a per-model process_timeout override still wins + default_process_timeout = (opts.zerogpu_process_timeout + if is_zerogpu(result.hardware) + else opts.process_timeout) + process_timeout = overrides.get("process_timeout", default_process_timeout) + connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) + load_only = opts.load_only or overrides.get("load_only", False) + config_dir = opts.config.parent.resolve() + + # --- Endpoint presence --------------------------------------------------- + endpoints = client.view_api(return_format="dict", print_info=False) or {} + named = endpoints.get("named_endpoints", {}) + if "/controls" not in named or "/process" not in named: + result.error = (f"missing HARP endpoints (found: {sorted(named)}); " + f"deployment may use an outdated pyharp") + return result + + # --- /controls (retry through a warming-up model) ------------------------ + controls = call_through_loading( + lambda: run_with_timeout( + lambda: client.predict(api_name="/controls"), CONTROLS_TIMEOUT, "/controls"), + time.time() + connect_timeout, "/controls") + if not isinstance(controls, dict) or "card" not in controls or "inputs" not in controls: + result.error = f"/controls returned malformed data: {str(controls)[:200]}" + return result + result.controls_ok = True + result.model_name = controls.get("card", {}).get("name", "") + + if load_only: + result.status = PASS + return result + + # --- /process test cases ------------------------------------------------- + # Common cases (generic, applied to every model) run alongside this + # model's own cases; their names are namespaced so both are legible in + # the report. A model can opt out with `skip_common_cases`. + own_cases = overrides.get("test_cases") or [{"name": "default"}] + if overrides.get("skip_common_cases"): + common_cases = [] + else: + common_cases = [dict(c, name=f"common:{c.get('name', 'unnamed')}") + for c in (opts.common_test_cases or [])] + cases = common_cases + own_cases + + for case in cases: + result.cases.append(run_case( + client, case, controls, assets, overrides, config_dir, + case.get("process_timeout", process_timeout), connect_timeout)) + + if any(c.ok is False for c in result.cases): + result.error = "; ".join( + f"[{c.name}] {c.error}" for c in result.cases if c.ok is False) + result.status = FAIL + else: + result.status = PASS + + return result + + +# --------------------------------------------------------------------------- +# Remote Hugging Face Spaces +# --------------------------------------------------------------------------- + +def wait_for_runtime(api, space_id: str, deadline: float): + """ + Poll a space's runtime until its stage settles or the deadline passes. + + Args: + api (HfApi): Authenticated Hugging Face API client. + space_id (str): The space to poll. + deadline (float): time.time() value to stop polling at. + + Returns: + runtime (SpaceRuntime): The last observed runtime (stage, hardware). + """ + + runtime = api.get_space_runtime(space_id) + + # SLEEPING resolves on first request rather than by waiting + while runtime.stage in TRANSIENT_STAGES - {"SLEEPING"} and time.time() < deadline: + time.sleep(10) + runtime = api.get_space_runtime(space_id) + + return runtime + + +def test_space(space_id: str, token: str, assets: Assets, + opts: argparse.Namespace, overrides: dict, + quota_exhausted=None, zerogpu_limiter=None) -> ModelResult: + """ + Validate one remote Hugging Face Space end-to-end. + + Args: + space_id (str): The space to validate (e.g. "teamup-tech/foo"). + token (str): Hugging Face access token. + assets (Assets): Synthesized input files. + opts (argparse.Namespace): Parsed command-line options. + overrides (dict): This space's entry from config.yml `overrides`. + quota_exhausted (threading.Event | None): Shared flag; when set, + ZeroGPU models are skipped, and this model sets it if its own run + reveals the allowance is exhausted. + zerogpu_limiter (threading.Semaphore | None): Caps how many ZeroGPU + models make GPU calls at once. Held only around the endpoint + tests, so slow restarts and connections still overlap. + + Returns: + result (ModelResult): The completed validation record. + """ + + # Imported lazily so the examples tier works without huggingface_hub + from huggingface_hub import HfApi + + result = ModelResult(target=space_id, kind="space") + start = time.time() + api = HfApi(token=token) + connect_timeout = overrides.get("connect_timeout", opts.connect_timeout) + client = None + + try: + # --- Runtime stage (restart crashed/stopped spaces by default) ------- + deadline = time.time() + connect_timeout + runtime = wait_for_runtime(api, space_id, deadline) + stage = runtime.stage + result.stage = stage + # requested_hardware reflects the space's configuration even while + # it is sleeping or stopped (hardware itself is only set when live) + result.hardware = runtime.requested_hardware or runtime.hardware or "" + zerogpu = is_zerogpu(result.hardware) + # Hardware is occasionally unreported for a sleeping or starting space. + # Throttle those as if they were ZeroGPU: being wrong only costs run + # time, whereas leaving a real ZeroGPU model unthrottled overlaps GPU + # reservations. Skipping, by contrast, stays strict - wrongly skipping + # a CPU model would silently drop it from validation. + throttle = zerogpu or not result.hardware + + # Skip ZeroGPU models before doing any work that would spend quota + if zerogpu and opts.skip_zerogpu: + result.status = SKIP + result.error = "skipped: ZeroGPU hardware (--skip-zerogpu)" + return result + if zerogpu and quota_exhausted is not None and quota_exhausted.is_set(): + result.status = SKIP + result.error = "skipped: ZeroGPU allowance already exhausted this run" + return result + + if stage in DEAD_STAGES or stage in ("STOPPED", "PAUSED"): + if opts.restart_failed and stage != "DELETING": + print(f" [{space_id}] stage={stage}, requesting restart...") + try: + api.restart_space(space_id) + except Exception as exc: # noqa: BLE001 - e.g. read-only token + result.error = scrub( + f"space is not running (stage={stage}) and restart " + f"failed: {describe_exception(exc)}", token) + return result + runtime = wait_for_runtime(api, space_id, time.time() + connect_timeout) + stage = runtime.stage + result.stage = stage + if stage not in ("RUNNING", "SLEEPING"): + result.error = f"space is not running (stage={stage})" + return result + + # --- Connect (wakes sleeping spaces; retry through the wake-up) ------ + last_exc = None + deadline = time.time() + connect_timeout + while time.time() < deadline: + try: + client = run_with_timeout( + lambda: Client(space_id, hf_token=token, verbose=False), + max(10.0, deadline - time.time()), "connect") + break + except Exception as exc: # noqa: BLE001 - space may still be waking + last_exc = exc + time.sleep(15) + if client is None: + raise RuntimeError(f"could not connect: {last_exc}") + # Connecting has woken the space; reflect that rather than the stale + # SLEEPING/STARTING stage seen before the wake-up, and fill in the + # hardware if it was not reported while the space was asleep + if result.stage != "RUNNING": + result.stage = "RUNNING" + if not result.hardware: + try: + awake = api.get_space_runtime(space_id) + result.hardware = awake.hardware or awake.requested_hardware or "" + zerogpu = is_zerogpu(result.hardware) + except Exception: # noqa: BLE001 - hardware stays unknown + pass + + # Serialize ZeroGPU models: concurrent GPU calls each reserve their + # declared duration, so overlapping them ties up allowance that is not + # actually being used + limiter = zerogpu_limiter if (throttle and zerogpu_limiter is not None) \ + else contextlib.nullcontext() + with limiter: + # The allowance may have run out while waiting for a slot + if zerogpu and quota_exhausted is not None and quota_exhausted.is_set(): + result.status = SKIP + result.error = "skipped: ZeroGPU allowance already exhausted this run" + return result + + run_endpoint_tests(client, result, assets, overrides, opts) + + # If a ZeroGPU model failed because the allowance is gone, trip the + # guard so the remaining ZeroGPU models are skipped + if zerogpu and quota_exhausted is not None and result.status == FAIL \ + and is_quota_exhausted(result.error): + quota_exhausted.set() + + return result + + except Exception as exc: # noqa: BLE001 - any failure means invalid + result.error = scrub(describe_exception(exc), token) + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + close_client(client) + + +# --------------------------------------------------------------------------- +# Local pyharp examples +# --------------------------------------------------------------------------- + +def wait_for_local_server(port: int, proc: subprocess.Popen, timeout: float) -> None: + """ + Block until a locally launched gradio app starts serving. + + Args: + port (int): The port the app was told to bind (GRADIO_SERVER_PORT). + proc (subprocess.Popen): The app process, watched for early exit. + timeout (float): Seconds to wait before giving up. + + Raises: + RuntimeError: If the app process exits before serving. + TimeoutError: If the app is not serving within the timeout. + """ + + deadline = time.time() + timeout + url = f"http://127.0.0.1:{port}/config" + + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"app exited early with code {proc.returncode}") + try: + with urllib.request.urlopen(url, timeout=SERVER_PROBE_TIMEOUT): + return + except Exception: # noqa: BLE001 - server not up yet + time.sleep(SERVER_PROBE_INTERVAL) + + raise TimeoutError(f"local app did not become ready within {int(timeout)}s") + + +def read_app_traceback(log_path: Path, max_chars: int = 400) -> str: + """ + Extract the final exception line from a local app's captured log. + + Args: + log_path (Path): The app's stdout/stderr log. + max_chars (int): Cap on the returned text. + + Returns: + detail (str): The last traceback's exception line, or "" if the log + holds no traceback (or cannot be read). + """ + + try: + lines = log_path.read_text(errors="replace").splitlines() + except OSError: + return "" + + # The exception line is the last non-indented line after the last "Traceback" + starts = [i for i, line in enumerate(lines) if line.startswith("Traceback")] + if not starts: + return "" + tail = [line for line in lines[starts[-1] + 1:] + if line.strip() and not line.startswith((" ", "\t"))] + + return tail[-1].strip()[:max_chars] if tail else "" + + +def test_local_example(app_dir: Path, port: int, assets: Assets, + opts: argparse.Namespace, overrides: dict) -> ModelResult: + """ + Validate one pyharp example app by launching it locally. + + The app's stdout/stderr are captured to /.log for + debugging failures. + + Args: + app_dir (Path): Example directory containing app.py. + port (int): Local port to launch the app on. + assets (Assets): Synthesized input files. + opts (argparse.Namespace): Parsed command-line options. + overrides (dict): This example's entry from config.yml `overrides` + (keyed "examples/"). + + Returns: + result (ModelResult): The completed validation record. + """ + + target = f"examples/{app_dir.name}" + result = ModelResult(target=target, kind="local", stage="LOCAL") + start = time.time() + + env = os.environ.copy() + env["GRADIO_SERVER_NAME"] = "127.0.0.1" + env["GRADIO_SERVER_PORT"] = str(port) + env.pop("HF_TOKEN", None) # local examples must not need credentials + + log_path = opts.output_dir / f"{app_dir.name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + proc = None + client = None + + try: + with open(log_path, "w") as log: + proc = subprocess.Popen( + [sys.executable, "app.py"], cwd=app_dir, env=env, + stdout=log, stderr=subprocess.STDOUT) + wait_for_local_server(port, proc, overrides.get( + "connect_timeout", opts.connect_timeout)) + client = Client(f"http://127.0.0.1:{port}", verbose=False) + run_endpoint_tests(client, result, assets, overrides, opts) + return result + except Exception as exc: # noqa: BLE001 - any failure means invalid + result.error = f"{describe_exception(exc)} (see {log_path.name})" + if opts.verbose: + traceback.print_exc() + return result + finally: + result.duration = round(time.time() - start, 1) + close_client(client) + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=PROCESS_TERMINATE_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + # A local app logs the real traceback even when the client is only + # told "an error occurred", so fold it into the report rather than + # leaving it in a file the reader has to go find + if result.error: + detail = read_app_traceback(log_path) + if detail and detail not in result.error: + result.error = f"{result.error} | app log: {detail}" diff --git a/model_validation/src/midi.py b/model_validation/src/midi.py new file mode 100644 index 00000000..d72af2d5 --- /dev/null +++ b/model_validation/src/midi.py @@ -0,0 +1,53 @@ +""" +MIDI decoding for HARP model validation. + +Parallels audio.py: reads a MIDI output and measures the handful of +properties `expect` rules check. Uses mido, a small pure-Python parser, so +it handles running status and tempo maps correctly without a native +dependency. +""" + +import mido + + +__all__ = [ + 'read_midi_props' +] + + +def read_midi_props(label: str, path: str) -> dict: + """ + Parse a MIDI file and measure the properties `expect` rules check. + + Args: + label (str): Output label, for error messages. + path (str): Path to the MIDI file. + + Returns: + props (dict): num_tracks, num_notes (note-on events with non-zero + velocity), and duration (seconds, or None when it cannot be + determined, e.g. an asynchronous format-2 file). + + Raises: + AssertionError: If the file cannot be parsed as MIDI. + """ + + try: + midi = mido.MidiFile(path) + except Exception as exc: # noqa: BLE001 - any parse failure + raise AssertionError(f"output '{label}' could not be parsed as MIDI: {exc}") + + num_notes = sum(1 for track in midi.tracks for msg in track + if msg.type == "note_on" and msg.velocity > 0) + + try: + duration = midi.length + except (ValueError, KeyError): + # length is undefined for asynchronous (format 2) files + duration = None + + return { + "num_tracks": len(midi.tracks), + "num_notes": num_notes, + "duration": duration, + } diff --git a/model_validation/src/quota.py b/model_validation/src/quota.py new file mode 100644 index 00000000..02426d9f --- /dev/null +++ b/model_validation/src/quota.py @@ -0,0 +1,89 @@ +""" +ZeroGPU usage tracking for HARP model validation. + +ZeroGPU allowances are consumed per account, so a long validation run draws on +the day's quota. Hugging Face publishes no reliable API for the remaining +allowance, so this module reports the work attributable to the run itself. +""" + +import threading + + +__all__ = [ + 'ZeroGPUTracker', + 'is_zerogpu' +] + + +def is_zerogpu(hardware: str) -> bool: + """ + Whether a space's hardware consumes ZeroGPU quota. + + ZeroGPU hardware ids start with "zero" (e.g. "zero-a10g"); anything + else (cpu-basic, t4-small, ...) does not draw on the shared allowance. + + Args: + hardware (str): HF hardware id, possibly empty. + + Returns: + zerogpu (bool): True when the hardware is a ZeroGPU tier. + """ + + return bool(hardware) and hardware.lower().startswith("zero") + + +class ZeroGPUTracker: + """ + Tracks ZeroGPU work done during a run. + + ZeroGPU bills dynamically: each call reserves its declared + `@spaces.GPU(duration=...)` time up front, then refunds the unused portion + once the function returns - so the account's usage rises during a run and + settles lower afterwards, and the exact billed amount is not readable from + the gradio client. Two figures are tracked instead: + + - the number of /process calls that reached the GPU - exact, and the + most reliable signal of how much of the allowance a run will use; + - the total /process wall time of those calls (queue plus execution) - + an over-estimate of the settled bill, in the range of the mid-run + reservation peak, not the amount that remains after refunds. + + Only calls that reached the GPU count; queued or input-skipped cases do + not. CPU and dedicated-hardware models never contribute. + """ + + def __init__(self, budget: float | None): + self.budget = budget + self.calls = 0 + self.wall_seconds = 0.0 + self._lock = threading.Lock() + + def add(self, calls: int, wall_seconds: float) -> None: + """ + Record ZeroGPU work from a completed model validation. + + Args: + calls (int): Number of /process calls that ran on the GPU. + wall_seconds (float): Total /process wall time of those calls. + """ + + with self._lock: + self.calls += calls + self.wall_seconds += wall_seconds + + def summary(self) -> str: + """ + Format the ZeroGPU work done so far for a console line. + + Returns: + summary (str): e.g. + "ZeroGPU: 25 calls, ~340s wall (approx)". + """ + + with self._lock: + calls, wall = self.calls, int(self.wall_seconds) + + budget = f"/{int(self.budget)}s" if self.budget else "" + + return (f"ZeroGPU: {calls} call{'s' if calls != 1 else ''}, " + f"~{wall}s{budget} wall (approx)") diff --git a/model_validation/src/results.py b/model_validation/src/results.py new file mode 100644 index 00000000..77639ec0 --- /dev/null +++ b/model_validation/src/results.py @@ -0,0 +1,172 @@ +""" +Result records and report generation for HARP model validation. +""" + +import dataclasses +import json +import time +from pathlib import Path + + +__all__ = [ + 'PASS', + 'FAIL', + 'SKIP', + 'CaseResult', + 'ModelResult', + 'status_emoji', + 'case_emoji', + 'write_reports' +] + + +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" + + +@dataclasses.dataclass +class CaseResult: + """Outcome of a single /process test case.""" + + name: str + ok: bool | None = None # None => skipped + duration: float = 0.0 # total /process wall time (queue + execution) + executed: bool = False # the /process job left the queue and ran on GPU + error: str = "" + + +@dataclasses.dataclass +class ModelResult: + """Outcome of validating one model deployment.""" + + target: str # space id or "examples/" + kind: str = "space" # "space" | "local" + status: str = FAIL + stage: str = "" # HF runtime stage, or "LOCAL" + hardware: str = "" # HF hardware (e.g. "zero-a10g", "cpu-basic") + controls_ok: bool = False + cases: list = dataclasses.field(default_factory=list) + duration: float = 0.0 + error: str = "" + model_name: str = "" # from the model card + + +def status_emoji(result: ModelResult) -> str: + """ + Symbol used for a result in console output and reports. + + Args: + result (ModelResult): The result to represent. + + Returns: + emoji (str): One of the pass/fail/skip symbols. + """ + + return {PASS: "✅", FAIL: "❌", SKIP: "⏭️"}[result.status] + + +def case_emoji(ok: bool | None) -> str: + """ + Symbol for one test case's outcome. + + Args: + ok (bool | None): True passed, False failed, None skipped. + + Returns: + emoji (str): The corresponding symbol. + """ + + return {True: "✅", False: "❌", None: "⏭️"}[ok] + + +def write_reports(results: list, out_dir: Path, label: str, + command: str = "", options: dict | None = None) -> None: + """ + Write the machine-readable and human-readable reports. + + Args: + results (list): ModelResult objects for every validated model. + out_dir (Path): Directory receiving report.json and report.md. + label (str): Report heading (e.g. "teamup-tech spaces"). + command (str): The command line the run was invoked with. + options (dict | None): The resolved run options, recorded in the + report so a run's parameters are reproducible from it. + """ + + out_dir.mkdir(parents=True, exist_ok=True) + + passed = sum(r.status == PASS for r in results) + failed = sum(r.status == FAIL for r in results) + skipped = sum(r.status == SKIP for r in results) + validated = len(results) - skipped # denominator excludes skipped models + + payload = { + "label": label, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "command": command, + "options": options or {}, + "total": len(results), + "validated": validated, + "passed": passed, + "failed": failed, + "skipped": skipped, + "results": [dataclasses.asdict(r) for r in results], + } + (out_dir / "report.json").write_text(json.dumps(payload, indent=2)) + + (out_dir / "report.md").write_text(render_markdown(results, payload)) + + +def render_markdown(results: list, payload: dict) -> str: + """ + Render the human-readable report: a summary table plus full error text. + + Args: + results (list): ModelResult objects for every validated model. + payload (dict): The report.json payload (headline counts, command). + + Returns: + markdown (str): The rendered report. + """ + + # Failures first, then alphabetical, so problems are visible at a glance + ranked = sorted(results, key=lambda r: (r.status != FAIL, r.target)) + + headline = f"**{payload['passed']}/{payload['validated']} models passed**" + if payload["skipped"]: + headline += f", {payload['skipped']} skipped" + + lines = [f"# HARP Model Validation Report - {payload['label']}", ""] + lines += [f"{headline} ({payload['timestamp']})", ""] + if payload["command"]: + lines += [f"Command: `{payload['command']}`", ""] + lines += ["| Model | Status | Stage | Hardware | Controls | Cases | Time (s) | Detail |", + "|---|---|---|---|---|---|---|---|"] + + for r in ranked: + cases = ", ".join(f"{c.name} {case_emoji(c.ok)}" for c in r.cases) or "—" + link = (f"[{r.target}](https://huggingface.co/spaces/{r.target})" + if r.kind == "space" else f"`{r.target}`") + # Keep the table scannable; the untruncated text follows below + detail = r.error.replace("|", "\\|") if r.error else "" + if len(detail) > 200: + detail = detail[:200] + " […]" + lines.append(f"| {link} | {status_emoji(r)} {r.status} | {r.stage} " + f"| {r.hardware or '—'} | {'✅' if r.controls_ok else '❌'} " + f"| {cases} | {r.duration} | {detail} |") + + # Full, untruncated text wherever something was reported - including a + # skipped case on an otherwise passing model + problems = [r for r in ranked + if r.error or r.status == FAIL or any(c.error for c in r.cases)] + if problems: + lines += ["", "## Details", ""] + for r in problems: + lines += [f"### {r.target} — {r.status}", ""] + if r.error and not r.cases: + lines += ["```", r.error, "```", ""] + for c in r.cases: + if c.error: + lines += [f"- **{c.name}** {case_emoji(c.ok)}", "", + "```", c.error, "```", ""] + + return "\n".join(lines) + "\n" diff --git a/model_validation/src/utils.py b/model_validation/src/utils.py new file mode 100644 index 00000000..78eac5b4 --- /dev/null +++ b/model_validation/src/utils.py @@ -0,0 +1,191 @@ +""" +Shared utilities for HARP model validation: credentials, error rendering, +configuration, model discovery, and timeout handling. +""" + +import os +import sys +import threading +from pathlib import Path + +import yaml + + +__all__ = [ + 'get_token', + 'scrub', + 'describe_exception', + 'run_with_timeout', + 'load_config', + 'get_excluded', + 'discover_spaces' +] + + +def describe_exception(exc: BaseException) -> str: + """ + Render an exception with as much detail as it carries. + + Beyond the usual "Type: message", this surfaces a gradio AppError's + `title` when the upstream app set an informative one, and follows the + exception chain so the underlying cause is not lost. Note that a Space + launched with `show_error=False` deliberately returns only the exception + class name, so for those there is genuinely nothing further to report. + + Args: + exc (BaseException): The exception to describe. + + Returns: + description (str): A single-line description of the exception. + """ + + parts = [f"{type(exc).__name__}: {exc}".strip()] + + # gradio's AppError carries a title alongside the message + title = getattr(exc, "title", None) + if title and title not in ("Error", str(exc)): + parts.append(f"[{title}]") + + # Follow the chain so the root cause survives (bounded to stay one line) + seen, cause = {id(exc)}, exc.__cause__ or exc.__context__ + while cause is not None and id(cause) not in seen and len(parts) < 4: + seen.add(id(cause)) + parts.append(f"caused by {type(cause).__name__}: {cause}".strip()) + cause = cause.__cause__ or cause.__context__ + + return " | ".join(parts) + + +def get_token(required: bool) -> str: + """ + Read the Hugging Face access token from the HF_TOKEN environment variable. + + The token is intentionally never accepted as a command-line argument + (argv is visible in process listings) and must never be committed. + + Args: + required (bool): Exit with code 2 if the token is missing. + + Returns: + token (str): The token, or an empty string when absent and optional. + """ + + token = os.environ.get("HF_TOKEN", "").strip() + + if required and not token: + print("ERROR: HF_TOKEN environment variable is not set.", file=sys.stderr) + print("Set it locally (export HF_TOKEN=...) or as a GitHub Actions secret.", + file=sys.stderr) + sys.exit(2) + + return token + + +def scrub(text: str, token: str) -> str: + """ + Remove the token from any string that might get printed or reported. + + Args: + text (str): Text that may contain the token (e.g. an error message). + token (str): The token to redact; empty string disables scrubbing. + + Returns: + text (str): The text with any token occurrence replaced. + """ + + return text.replace(token, "***HF_TOKEN***") if token else text + + +def run_with_timeout(fn, timeout: float, what: str): + """ + Run fn() in a daemon thread; raise TimeoutError if it exceeds timeout. + + A daemon thread (not a ThreadPoolExecutor) is essential here: executor + workers are non-daemon and are joined at interpreter shutdown, so a hung + call leaked by a timeout would make the whole process hang after the + final summary prints. + + Args: + fn (callable): Zero-argument function to execute. + timeout (float): Seconds to wait before giving up. + what (str): Short description used in the timeout message. + + Returns: + result: Whatever fn() returns. + + Raises: + TimeoutError: If fn() does not finish within timeout seconds. + Exception: Whatever fn() raised, re-raised in the calling thread. + """ + + outcome = {} + + def target(): + try: + outcome["result"] = fn() + except BaseException as exc: # noqa: BLE001 - re-raised in caller + outcome["error"] = exc + + thread = threading.Thread(target=target, daemon=True, name=f"timeout-{what}") + thread.start() + thread.join(timeout) + + if thread.is_alive(): + raise TimeoutError(f"{what} timed out after {int(timeout)}s") + if "error" in outcome: + raise outcome["error"] + + return outcome.get("result") + + +def load_config(path: Path) -> dict: + """ + Load the validation configuration (see config.yml for the schema). + + Args: + path (Path): Path to the YAML configuration file. + + Returns: + config (dict): Parsed configuration; empty when the file is absent. + """ + + if not path.exists(): + return {} + + return yaml.safe_load(path.read_text()) or {} + + +def get_excluded(config: dict, cli_exclude: list | None) -> set: + """ + Combine the models excluded from validation. + + Args: + config (dict): Parsed configuration (its `exclude` list is used). + cli_exclude (list | None): Models passed via --exclude, if any. + + Returns: + excluded (set): Model keys (space ids or "examples/"). + """ + + return set(config.get("exclude", [])) | set(cli_exclude or []) + + +def discover_spaces(api, org: str, config: dict, excluded: set) -> list: + """ + Enumerate the Hugging Face Spaces to validate. + + Args: + api (HfApi): Authenticated Hugging Face API client. + org (str): Organization whose spaces are discovered. + config (dict): Parsed configuration (`include_extra` adds spaces + outside the organization). + excluded (set): Model keys to leave out. + + Returns: + space_ids (list): Sorted space ids to validate. + """ + + spaces = [s.id for s in api.list_spaces(author=org)] + spaces += [s for s in config.get("include_extra", []) if s not in spaces] + + return sorted(s for s in spaces if s not in excluded) diff --git a/model_validation/src/validate_models.py b/model_validation/src/validate_models.py new file mode 100644 index 00000000..8eebf255 --- /dev/null +++ b/model_validation/src/validate_models.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +""" +HARP model validation - command-line entry point. + +Validates HARP deployments in two tiers, using the same black-box harness +(see harness.py): + +1. Spaces (default): discovers all Spaces under an organization (default: + teamup-tech), verifies each is running (restarting crashed/stopped spaces + by default), exposes the HARP gradio endpoints (/controls and /process), + and processes test inputs end-to-end. ZeroGPU spaces require an + authenticated request for GPU quota, so a token must be provided via the + HF_TOKEN environment variable - never on the command line or in the + repository. + +2. Examples (--local-examples): launches each app under pyharp/examples/ on + a local port and runs the identical endpoint tests. The examples tier + exercises pyharp itself with no Hugging Face infrastructure in the loop, + so a failure indicates a pyharp/gradio-level breakage rather than a + deployment-specific one. + +Per-model test cases are declared in config.yml (see README.md for a full +walkthrough). When a model has no configured cases, a single "default" case +runs with inputs synthesized automatically from the /controls spec. + +Usage: + HF_TOKEN=... python model_validation/src/validate_models.py + HF_TOKEN=... python model_validation/src/validate_models.py --spaces teamup-tech/pitch_shifter + HF_TOKEN=... python model_validation/src/validate_models.py --load-only --workers 8 + HF_TOKEN=... python model_validation/src/validate_models.py --no-restart-failed + python model_validation/src/validate_models.py --local-examples + +Exit codes: + 0 - all validated models passed (or were explicitly skipped) + 1 - at least one model failed + 2 - infrastructure/configuration error (bad token, no spaces found, ...) +""" + +import argparse +import concurrent.futures +import os +import sys +import threading +import time +from pathlib import Path + +from assets import Assets +from harness import test_space, test_local_example +from quota import ZeroGPUTracker, is_zerogpu +from results import PASS, FAIL, SKIP, status_emoji, write_reports +from utils import get_token, scrub, load_config, get_excluded, discover_spaces + + +DEFAULT_ORG = "teamup-tech" +SCRIPT_DIR = Path(__file__).parent # model_validation/src +MODEL_VALIDATION_DIR = SCRIPT_DIR.parent # model_validation +REPO_ROOT = MODEL_VALIDATION_DIR.parent +DEFAULT_CONFIG = MODEL_VALIDATION_DIR / "config.yml" +DEFAULT_EXAMPLES_DIR = REPO_ROOT / "pyharp" / "examples" +DEFAULT_OUTPUT_DIR = MODEL_VALIDATION_DIR / "reports" + +LOCAL_PORT_BASE = 7861 + + +def parse_args() -> argparse.Namespace: + """ + Define and parse the command-line interface. + + Returns: + opts (argparse.Namespace): Parsed options. + """ + + parser = argparse.ArgumentParser(description="Validate HARP model deployments.") + parser.add_argument("--org", default=DEFAULT_ORG, help="HF organization to scan") + parser.add_argument("--spaces", nargs="*", default=None, + help="Explicit space ids to validate (skips discovery)") + parser.add_argument("--exclude", nargs="*", default=None, metavar="MODEL", + help="Models to exclude from validation (space ids, or " + "examples/); merged with the config " + "`exclude` list") + parser.add_argument("--local-examples", nargs="*", default=None, metavar="DIR", + help="Validate local pyharp example apps instead of remote " + "spaces (the examples tier). With no DIRs given, tests " + "every app under pyharp/examples/.") + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG, + help="Optional YAML config (excludes, per-model overrides/cases)") + parser.add_argument("--load-only", action="store_true", + help="Only verify availability and /controls, do not run inference") + parser.add_argument("--skip-zerogpu", action="store_true", + help="Skip models on ZeroGPU hardware, to avoid spending " + "the shared ZeroGPU allowance") + parser.add_argument("--restart-failed", action=argparse.BooleanOptionalAction, + default=True, + help="Attempt to restart spaces found in an error/stopped state; " + "enabled by default (requires a token with write access), " + "disable with --no-restart-failed") + parser.add_argument("--workers", type=int, default=4, + help="Number of spaces to validate concurrently (spaces tier only)") + parser.add_argument("--zerogpu-workers", type=int, default=1, + help="How many ZeroGPU models may make GPU calls at once. " + "Each concurrent call reserves its declared duration, " + "so 1 (the default) keeps overlapping reservations " + "from tying up the allowance") + parser.add_argument("--connect-timeout", type=float, default=420, + help="Seconds to wait for a deployment to build/wake/start") + parser.add_argument("--process-timeout", type=float, default=600, + help="Seconds to wait for /process on non-ZeroGPU models") + parser.add_argument("--zerogpu-process-timeout", type=float, default=120, + help="Seconds allowed for /process EXECUTION on ZeroGPU " + "models once they leave the queue (queue wait is " + "bounded separately by --connect-timeout; default 120)") + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR, + help=f"Directory for reports, synthesized assets, and " + f"example logs (default: {DEFAULT_OUTPUT_DIR})") + parser.add_argument("--verbose", action="store_true") + + return parser.parse_args() + + +def result_line(result, extra: str = "", token: str = "") -> str: + """ + Format one model's console line. + + Args: + result (ModelResult): The completed validation record. + extra (str): Text inserted before the error note (e.g. hardware). + token (str): Token to scrub from the error text. + + Returns: + line (str): The formatted line. + """ + + note = f" - {result.error}" if result.error else "" + + return (f"{status_emoji(result)} {result.status:4s} {result.target} " + f"({result.duration}s){extra}{scrub(note, token)}") + + +def validate_examples(opts: argparse.Namespace, config: dict, excluded: set, + assets: Assets) -> list: + """ + Run the examples tier: launch and validate each local pyharp example. + + Examples run sequentially so concurrent model loads cannot exhaust the + machine's memory. + + Args: + opts (argparse.Namespace): Parsed command-line options. + config (dict): Parsed configuration. + excluded (set): Model keys to leave out. + assets (Assets): Synthesized input files. + + Returns: + results (list): ModelResult objects, or None when no examples exist. + """ + + overrides = config.get("overrides", {}) + + if opts.local_examples: + app_dirs = [Path(d) for d in opts.local_examples] + else: + app_dirs = sorted(d for d in DEFAULT_EXAMPLES_DIR.iterdir() + if (d / "app.py").exists()) + app_dirs = [d for d in app_dirs if f"examples/{d.name}" not in excluded] + + if not app_dirs: + print("ERROR: no local examples found", file=sys.stderr) + return None + + print(f"Validating {len(app_dirs)} pyharp examples\n") + + results = [] + for i, app_dir in enumerate(app_dirs): + r = test_local_example(app_dir, LOCAL_PORT_BASE + i, assets, opts, + overrides.get(f"examples/{app_dir.name}", {})) + results.append(r) + print(result_line(r)) + + return results + + +def validate_spaces(opts: argparse.Namespace, config: dict, excluded: set, + assets: Assets, token: str) -> list: + """ + Run the spaces tier: validate remote Hugging Face Spaces concurrently. + + Args: + opts (argparse.Namespace): Parsed command-line options. + config (dict): Parsed configuration. + excluded (set): Model keys to leave out. + assets (Assets): Synthesized input files. + token (str): Hugging Face access token. + + Returns: + results (list): ModelResult objects, or None when no spaces exist. + """ + + from huggingface_hub import HfApi + + overrides = config.get("overrides", {}) + api = HfApi(token=token) + + if opts.spaces: + space_ids = [s for s in opts.spaces if s not in excluded] + else: + space_ids = discover_spaces(api, opts.org, config, excluded) + + if not space_ids: + print(f"ERROR: no spaces found for org '{opts.org}'", file=sys.stderr) + return None + + tracker = ZeroGPUTracker(config.get("zerogpu_budget_seconds")) + quota_exhausted = threading.Event() + zerogpu_limiter = threading.Semaphore(max(1, opts.zerogpu_workers)) + print(f"Validating {len(space_ids)} spaces with {opts.workers} workers " + f"({opts.zerogpu_workers} concurrent on ZeroGPU; " + f"process test: {'OFF' if opts.load_only else 'ON'})\n") + + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=opts.workers) as pool: + futures = [pool.submit(test_space, sid, token, assets, opts, + overrides.get(sid, {}), quota_exhausted, zerogpu_limiter) + for sid in space_ids] + for future in concurrent.futures.as_completed(futures): + r = future.result() + results.append(r) + # Show the hardware for every model; ZeroGPU work only accrues for + # ZeroGPU models. Count each /process call that reached the GPU and + # its total wall time - a queued or input-skipped case never ran + # and is excluded + info = r.hardware or "?" + if is_zerogpu(r.hardware): + ran = [c for c in r.cases if c.executed] + tracker.add(len(ran), sum(c.duration for c in ran)) + info = f"{info} | {tracker.summary()}" + print(result_line(r, f" [{info}]", token)) + + return results + + +def main() -> int: + """ + Entry point: run the selected tier and write reports. + + Returns: + code (int): Process exit code (see module docstring). + """ + + opts = parse_args() + config = load_config(opts.config) + excluded = get_excluded(config, opts.exclude) + # Generic cases applied to every model, on top of its own (see config.yml) + opts.common_test_cases = config.get("common_test_cases", []) + + # Each run writes to its own timestamped directory so runs never overwrite + # each other; assets, logs, and reports all live under it. Local time is + # used for the directory name (on a UTC CI runner this is naturally UTC). + run_stamp = time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime()) + opts.output_dir = opts.output_dir / run_stamp + assets = Assets(opts.output_dir / "assets", + config.get("synthesized_inputs")) + + if opts.local_examples is not None: + token = "" + label = "pyharp examples" + results = validate_examples(opts, config, excluded, assets) + else: + token = get_token(required=True) + label = f"{opts.org} spaces" + results = validate_spaces(opts, config, excluded, assets, token) + + if results is None: + return 2 + + # Record how the run was invoked (argv holds no secrets - the token comes + # from HF_TOKEN) plus the resolved options, so the report is reproducible + command = " ".join(sys.argv) + options = {k: str(v) if isinstance(v, Path) else v + for k, v in vars(opts).items()} + write_reports(results, opts.output_dir, label, command, options) + + passed = [r for r in results if r.status == PASS] + failed = [r for r in results if r.status == FAIL] + skipped = [r for r in results if r.status == SKIP] + validated = len(results) - len(skipped) # exclude skipped from the total + summary = f"\n{len(passed)}/{validated} models passed" + if skipped: + summary += f", {len(skipped)} skipped" + print(f"{summary}. Reports written to {opts.output_dir}/") + if failed: + print("\nFailed models:") + for r in sorted(failed, key=lambda r: r.target): + print(f" - {r.target}: {scrub(r.error, token)}") + + return 1 if failed else 0 + + +if __name__ == "__main__": + code = main() + # Hard-exit rather than sys.exit: gradio_client and timed-out calls can + # leave non-daemon threads behind, and a normal interpreter shutdown + # would join them - hanging the process after all results are printed. + # Reports are already flushed to disk at this point. + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py new file mode 100644 index 00000000..90e0e59b --- /dev/null +++ b/model_validation/src/validators.py @@ -0,0 +1,124 @@ +""" +Custom output validators for HARP model validation. + +Most output checks do not belong here: anything expressible as "read a +property of one output and compare it" (extension, size, channel count, +sample rate, duration, signal level, label count) is a declarative `expect` +rule in config.yml - see EXPECT_RULES in cases.py. + +A validator covers the checks that cannot be written that way: relationships +spanning several outputs, or logic needing real computation. Register one +with the @validator decorator and reference it from a test case, with its +parameters nested beneath the name: + + overrides: + teamup-tech/some_transcriber: + test_cases: + - name: labels-line-up + validators: + labels_within_audio: + tolerance: 0.1 + +Each validator receives: + outputs (dict): output label -> value. File outputs (audio, MIDI, ...) + are local filesystem paths downloaded by gradio_client; JSON outputs + (e.g. pyharp LabelList) are the decoded objects (dict/list) or None. + controls (dict): the full /controls payload (card, inputs, outputs). + params (dict): the parameters nested under this validator's name, keeping + its settings separate from the test case's own fields. + +Validators signal failure by raising AssertionError with a message naming the +output and what was wrong. +""" + +from audio import read_audio_props + + +__all__ = [ + 'VALIDATORS', + 'validator', + 'ValidatorNotApplicable' +] + + +class ValidatorNotApplicable(Exception): + """ + Raised by a validator when the current model lacks the outputs it needs. + + Treated as a skip, not a failure. Raising this (rather than asserting) + for absent outputs lets a validator run in a common test case: it applies + to models that have the relevant outputs and is skipped on those that do + not, mirroring the leniency of a "*" expect rule. + """ + + +VALIDATORS = {} + + +def validator(name): + """ + Register a validator function under a config-referenceable name. + + Args: + name (str): The name test cases use in their `validator` entry. + + Returns: + register (callable): Decorator that records the function. + """ + + def register(fn): + VALIDATORS[name] = fn + return fn + return register + + +@validator("labels_within_audio") +def labels_within_audio(outputs, controls, params): + """ + Assert every returned label falls inside the audio output's timespan. + + This check warrants a validator rather than an `expect` rule because it + relates two different outputs to each other, which no per-output property + comparison can express. + + Params: + tolerance (float): seconds a label may exceed the audio duration + before it is treated as out of bounds (default 0.05). + + Raises: + AssertionError: If a label lies outside the audio output. + ValidatorNotApplicable: If the model has no audio output or no + LabelList output for the comparison. + """ + + tolerance = params.get("tolerance", 0.05) + + # Identify the audio output from the component spec rather than by file + # extension, since models may return any format soundfile can decode + audio_labels = [spec.get("label") for spec in controls.get("outputs", []) + if spec.get("type") == "audio_track"] + + duration = None + for label in audio_labels: + value = outputs.get(label) + if isinstance(value, str): + duration = read_audio_props(label, value)["duration"] + break + if duration is None: + raise ValidatorNotApplicable("no audio output to compare labels against") + + for label, value in outputs.items(): + if not (isinstance(value, dict) and isinstance(value.get("labels"), list)): + continue + for entry in value["labels"]: + start = entry.get("t", 0.0) + end = start + entry.get("duration", 0.0) + assert -tolerance <= start <= duration + tolerance, \ + (f"'{label}': label \"{entry.get('label')}\" starts at {start:.2f}s, " + f"outside the {duration:.2f}s audio output") + assert end <= duration + tolerance, \ + (f"'{label}': label \"{entry.get('label')}\" ends at {end:.2f}s, " + f"past the {duration:.2f}s audio output") + return + + raise ValidatorNotApplicable("no pyharp LabelList output to check") diff --git a/model_validation/test_data/.gitkeep b/model_validation/test_data/.gitkeep new file mode 100644 index 00000000..e69de29b