From 223a68778749eea166b394657a771ebc4dd781fd Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:14:14 -0700 Subject: [PATCH] refactor(integrations): move harbor out of the hud package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The integration contract belongs in core; its implementations do not. `hud/integrations/harbor/` shipped inside the `hud` wheel, contradicting the repository map, so it moves to `integrations/harbor/`. Core keeps `hud.environment.Integration` and imports no implementation — the built wheel now contains the contract and nothing else. An adapted image can no longer get the integration from `pip install hud`, so the adaptation layer copies it in from the build context and drops it into the serving venv. The code serving an image is then the revision that adapted it, rather than whatever the index resolves to at build time. The interpreter version the venv and that path share is one constant instead of being spelled out in the install script alone. Also trims `Integration.name` to what it actually backs: the format identifier and the taskset origin scheme. There is no format registry and no `--format` flag for it to key. --- docs/v6/advanced/harbor-convert.mdx | 8 +-- hud/environment/integration.py | 9 +-- hud/integrations/__init__.py | 9 --- integrations/__init__.py | 11 ++++ .../harbor/__init__.py | 0 .../harbor/_adapt.py | 57 ++++++++++++++----- .../harbor/_export.py | 0 .../harbor/_load.py | 4 +- .../harbor}/tests/__init__.py | 0 .../harbor}/tests/conftest.py | 0 .../harbor}/tests/test_contract.py | 22 +++---- .../harbor}/tests/test_harbor.py | 4 +- pyproject.toml | 3 +- 13 files changed, 80 insertions(+), 47 deletions(-) delete mode 100644 hud/integrations/__init__.py create mode 100644 integrations/__init__.py rename {hud/integrations => integrations}/harbor/__init__.py (100%) rename {hud/integrations => integrations}/harbor/_adapt.py (93%) rename {hud/integrations => integrations}/harbor/_export.py (100%) rename {hud/integrations => integrations}/harbor/_load.py (99%) rename {hud/integrations => integrations/harbor}/tests/__init__.py (100%) rename {hud/integrations => integrations/harbor}/tests/conftest.py (100%) rename {hud/integrations => integrations/harbor}/tests/test_contract.py (97%) rename {hud/integrations => integrations/harbor}/tests/test_harbor.py (99%) diff --git a/docs/v6/advanced/harbor-convert.mdx b/docs/v6/advanced/harbor-convert.mdx index eb2887344..4750d12d7 100644 --- a/docs/v6/advanced/harbor-convert.mdx +++ b/docs/v6/advanced/harbor-convert.mdx @@ -10,11 +10,11 @@ task dirs - is a *frontend* that loads into the same primitives (`Environment`, roundtrip to run foreign tasks. Each one implements [`hud.environment.Integration`](https://github.com/hud-evals/hud-python/blob/main/hud/environment/integration.py) - `load(ref) -> Taskset` and `environment(ref) -> Environment` - and ships as -`hud.integrations.`. +the repository's `integrations/`. Harbor's agent works *inside* its container, so its environment constructor is only meaningful in there. `adapt()` packages it: one HUD-speaking image per -distinct build context, whose CMD serves `hud.integrations.harbor:environment`. +distinct build context, whose CMD serves `harbor:environment`. The same rows then run on any container placement. ## Prerequisites @@ -29,7 +29,7 @@ directly - one row per task dir (`id` = the dir name), sharing one declarative `Environment` per distinct `environment/` build context: ```python -from hud.integrations import harbor +from integrations import harbor assert harbor.detect("./terminal-bench") taskset = harbor.load("./terminal-bench") @@ -86,7 +86,7 @@ that must run as *different* users (one image, one `USER`). `env.py`) into self-contained Harbor task folders: ```python -from hud.integrations.harbor import export +from integrations.harbor import export created = await export("tasks.py", "harbor_tasks") ``` diff --git a/hud/environment/integration.py b/hud/environment/integration.py index eacdd3db4..8f69c578b 100644 --- a/hud/environment/integration.py +++ b/hud/environment/integration.py @@ -14,8 +14,9 @@ such images is a format extra (e.g. ``harbor.adapt``), not part of the contract. -Implementations live outside core (e.g. :mod:`hud.integrations`, or any -installable package); core knows only this interface. +Implementations live outside core — this repository's ``integrations/``, or +any installable package; core knows only this interface, and imports none of +them. """ from __future__ import annotations @@ -34,8 +35,8 @@ class Integration(ABC): """One foreign format's frontend. See module docstring for the contract.""" - #: The format's identifier: its origin scheme (``:`` on loaded - #: tasksets), CLI ``--format`` value, and registry key. + #: The format's identifier, and the scheme its loaded tasksets are + #: origin-stamped with (``:``). name: ClassVar[str] @abstractmethod diff --git a/hud/integrations/__init__.py b/hud/integrations/__init__.py deleted file mode 100644 index 667456f3d..000000000 --- a/hud/integrations/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Interop frontends: foreign benchmark formats as HUD primitives. - -Each format implements :class:`hud.environment.Integration` — see its module -docstring for the contract (``load`` / ``environment``). Format -modules also keep an ergonomic function surface (``harbor.load(...)``) and -format extras: ``harbor.detect`` recognizes the layout, ``harbor.adapt`` -packages the constructor into container images, ``harbor.export`` is the -reverse direction. -""" diff --git a/integrations/__init__.py b/integrations/__init__.py new file mode 100644 index 000000000..4085fdc87 --- /dev/null +++ b/integrations/__init__.py @@ -0,0 +1,11 @@ +"""Interop frontends: foreign benchmark formats as HUD primitives. + +These live outside the ``hud`` package and ship no part of it — core knows +only :class:`hud.environment.Integration` and imports no implementation. + +Each format implements that contract (``load`` / ``environment``) and keeps +an ergonomic function surface (``harbor.load(...)``) plus format extras: +``harbor.detect`` recognizes the layout, ``harbor.adapt`` packages the +constructor into container images, ``harbor.export`` is the reverse +direction. +""" diff --git a/hud/integrations/harbor/__init__.py b/integrations/harbor/__init__.py similarity index 100% rename from hud/integrations/harbor/__init__.py rename to integrations/harbor/__init__.py diff --git a/hud/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py similarity index 93% rename from hud/integrations/harbor/_adapt.py rename to integrations/harbor/_adapt.py index e2e925667..daa75870f 100644 --- a/hud/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -11,7 +11,7 @@ CMD[ "hud", "serve", - "hud.integrations.harbor:environment", + "harbor:environment", "--arg", "ref=/hud/tasks", "--arg", @@ -77,10 +77,21 @@ _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") +#: The interpreter the serving venv is built on. The layer copies this +#: integration into that venv's site-packages by path, so the two must agree. +SERVING_PYTHON = "3.12" + #: Build-context entries never copied into adapted contexts. -_CONTEXT_IGNORE = shutil.ignore_patterns( - "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache" +_CONTEXT_IGNORE_NAMES = ( + "__pycache__", + "*.pyc", + ".git", + ".venv", + "venv", + "*.egg-info", + ".pytest_cache", ) +_CONTEXT_IGNORE = shutil.ignore_patterns(*_CONTEXT_IGNORE_NAMES) _INSTALL_SH = """\ #!/bin/sh @@ -110,24 +121,28 @@ # The interpreter must live under /hud: uv's default install dir is the # invoking user's home (root here), unreachable once the image's USER applies. export UV_PYTHON_INSTALL_DIR=/hud/python -uv python install 3.12 -uv venv /hud/venv --python 3.12 +uv python install __PYTHON__ +uv venv /hud/venv --python __PYTHON__ uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__ """ _LAYER = """ # ─── HUD adaptation layer: serve the control channel from inside ─── -# (generated by hud.integrations.harbor.adapt) +# (generated by harbor.adapt) # Installed as root: the base stage may end on a non-root USER that could not # write /hud. The task's own declared user, if any, is restored below. USER root COPY _hud /hud RUN sh /hud/install.sh +# This integration ships outside the hud distribution, so it comes from the +# build context rather than the index — which also means the code serving the +# image is the revision that adapted it, not whatever the index resolves to. +COPY _hud_harbor /hud/venv/lib/python__PYTHON__/site-packages/harbor EXPOSE 8765 __DECLARED__ENTRYPOINT [] CMD ["/hud/venv/bin/hud", "serve", \ - "hud.integrations.harbor:environment", "--arg", "ref=/hud/tasks", \ + "harbor:environment", "--arg", "ref=/hud/tasks", \ "--arg", "name=__ENV_NAME__", "--host", "0.0.0.0", "--port", "8765"] """ @@ -186,7 +201,7 @@ async def adapt( """Write adapted build contexts for every env group; build and push them. Returns ``{env_name: image_ref}`` — pass it to - :func:`~hud.integrations.harbor.load` as ``images=`` so the rows carry + :func:`~harbor.load` as ``images=`` so the rows carry their image. *push* is a registry prefix (``registry.io/acme``); without it images stay local, which serves ``DockerRuntime`` but not cloud placements. ``build=False`` writes the contexts under ``.hud-adapt/`` and @@ -248,7 +263,10 @@ def _write_context( if dockerignore.is_file(): # The task's ignore rules were written for its own build; they must # not exclude the adaptation layer from this one. - _write(dockerignore, dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n") + _write( + dockerignore, + dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n!_hud_harbor\n!_hud_harbor/**\n", + ) hud_dir = context / "_hud" hud_dir.mkdir(parents=True) @@ -257,6 +275,11 @@ def _write_context( if wheel.suffix == ".whl" and wheel.is_file(): shutil.copy2(wheel, hud_dir / wheel.name) requirement = f"/hud/{wheel.name}" + shutil.copytree( + Path(__file__).parent, + context / "_hud_harbor", + ignore=shutil.ignore_patterns("tests", ".hud-adapt", *_CONTEXT_IGNORE_NAMES), + ) for task_dir in group_dirs: target = hud_dir / "tasks" / task_dir.name target.mkdir(parents=True) @@ -265,12 +288,18 @@ def _write_context( _copy_task_content(task_dir / "tests", target / "tests") _write( hud_dir / "install.sh", - _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)), + _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)).replace( + "__PYTHON__", SERVING_PYTHON + ), ) - layer = _LAYER.replace("__ENV_NAME__", env_name).replace( - # One env serves one policy, so the group's tasks agree on these. - "__DECLARED__", - _declared_directives(group_dirs[0], final_stage(dockerfile).user), + layer = ( + _LAYER.replace("__ENV_NAME__", env_name) + .replace("__PYTHON__", SERVING_PYTHON) + .replace( + # One env serves one policy, so the group's tasks agree on these. + "__DECLARED__", + _declared_directives(group_dirs[0], final_stage(dockerfile).user), + ) ) _write(context / "Dockerfile", dockerfile + layer) return context diff --git a/hud/integrations/harbor/_export.py b/integrations/harbor/_export.py similarity index 100% rename from hud/integrations/harbor/_export.py rename to integrations/harbor/_export.py diff --git a/hud/integrations/harbor/_load.py b/integrations/harbor/_load.py similarity index 99% rename from hud/integrations/harbor/_load.py rename to integrations/harbor/_load.py index 2befb8e8f..8ced662f4 100644 --- a/hud/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -137,7 +137,7 @@ def load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset: the dataset name. Each row carries the task's declared launch requirements (:func:`runtime_config`: cpu/memory/gpu and time budgets), plus the adapted image ref once - :func:`~hud.integrations.harbor.adapt` has produced it, so it runs on any + :func:`~harbor.adapt` has produced it, so it runs on any container placement:: await harbor.adapt(path) @@ -391,7 +391,7 @@ def grouped(root: str | Path) -> list[tuple[str, list[Path]]]: One env name per group (the dataset slug, ``-gN``-suffixed when there are several): the join key between :func:`load`'s rows and - :func:`~hud.integrations.harbor.adapt`'s images. + :func:`~harbor.adapt`'s images. """ resolved = Path(root).resolve() dataset_name = resolved.parent.name if is_harbor_task(resolved) else resolved.name diff --git a/hud/integrations/tests/__init__.py b/integrations/harbor/tests/__init__.py similarity index 100% rename from hud/integrations/tests/__init__.py rename to integrations/harbor/tests/__init__.py diff --git a/hud/integrations/tests/conftest.py b/integrations/harbor/tests/conftest.py similarity index 100% rename from hud/integrations/tests/conftest.py rename to integrations/harbor/tests/conftest.py diff --git a/hud/integrations/tests/test_contract.py b/integrations/harbor/tests/test_contract.py similarity index 97% rename from hud/integrations/tests/test_contract.py rename to integrations/harbor/tests/test_contract.py index dd6b62e45..bb9c045a8 100644 --- a/hud/integrations/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -13,8 +13,8 @@ import pytest -from hud.integrations import harbor -from hud.integrations.harbor import _load as harbor_load +from integrations import harbor +from integrations.harbor import _load as harbor_load if TYPE_CHECKING: from pathlib import Path @@ -59,7 +59,7 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: context = tmp_path / ".hud-adapt" / contexts[0] dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") # The CMD serves the contract constructor by module reference — no baked env.py. - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile assert f'"name={context.name}"' in dockerfile assert "EXPOSE 8765" in dockerfile assert not (context / "_hud" / "env.py").exists() @@ -100,7 +100,7 @@ async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None assert sorted(env.tasks) == ["task-a", "task-b"] # The adapted CMD serves exactly this constructor. dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile def test_harbor_implements_the_integration_contract() -> None: @@ -254,7 +254,7 @@ def test_adapted_cmd_serves_the_contract_constructor(tmp_path) -> None: (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> None: @@ -263,7 +263,7 @@ async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> No import asyncio import json - from hud.integrations.harbor._adapt import _grade_with_verifier + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -334,7 +334,7 @@ def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None: def test_rewards_are_finite_numbers_not_booleans(tmp_path) -> None: import json as jsonlib - from hud.integrations.harbor._adapt import _read_reward + from integrations.harbor._adapt import _read_reward logs = tmp_path / "verifier" logs.mkdir() @@ -460,7 +460,7 @@ def test_final_stage_reads_only_what_the_shipped_image_declares() -> None: is not the shipped image's; ``user[:group]`` is a legal operand; and a heredoc body is data, not instructions. """ - from hud.integrations.harbor._load import final_stage + from integrations.harbor._load import final_stage multistage = final_stage( 'FROM golang:1.22 AS build\nUSER builder\nENTRYPOINT ["/tool"]\nRUN true\n' @@ -534,8 +534,8 @@ async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None: # would block the writer forever and score a finished script as a timeout. import asyncio - from hud.integrations.harbor._adapt import _grade_with_verifier from hud.utils.process import create_process_group_exec + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -609,8 +609,8 @@ async def test_a_cancelled_grade_leaves_nothing_running(tmp_path) -> None: # process group and the pipe readers are released. import asyncio - from hud.integrations.harbor._adapt import _grade_with_verifier from hud.utils.process import create_process_group_exec + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -645,7 +645,7 @@ async def run_tests(): def test_image_tags_do_not_depend_on_host_state(tmp_path) -> None: # Links are copied as links, so hashing must read the link — not what it # points at — or the same dataset tags differently on another machine. - from hud.integrations.harbor._load import hash_directory + from integrations.harbor._load import hash_directory target = tmp_path / "outside.txt" target.write_text("first", encoding="utf-8") diff --git a/hud/integrations/tests/test_harbor.py b/integrations/harbor/tests/test_harbor.py similarity index 99% rename from hud/integrations/tests/test_harbor.py rename to integrations/harbor/tests/test_harbor.py index 8e06bbb65..72a258945 100644 --- a/hud/integrations/tests/test_harbor.py +++ b/integrations/harbor/tests/test_harbor.py @@ -8,7 +8,7 @@ import pytest -from hud.integrations.harbor import detect, export, load +from integrations.harbor import detect, export, load from .conftest import make_harbor_task, make_multi_step_task @@ -317,7 +317,7 @@ async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: - from hud.integrations.harbor._export import _safe_component + from integrations.harbor._export import _safe_component assert "/" not in _safe_component("suite/fix") assert _safe_component("../escape") == "escape" diff --git a/pyproject.toml b/pyproject.toml index b46dd294f..b0e2abafe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ path = "hud/version.py" exclude = [ "docs/", "cookbooks/", + "integrations/", "hud-python/", "**/checkpoints/", "**/*.safetensors", @@ -261,7 +262,7 @@ omit = [ [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" -testpaths = ["hud"] +testpaths = ["hud", "integrations"] addopts = "" markers = [ "integration: marks tests as integration tests (require HUD_API_KEY, network access)",