diff --git a/docs/v6/advanced/harbor-convert.mdx b/docs/v6/advanced/harbor-convert.mdx index eb2887344..4750d12d7 100644 --- a/docs/v6/advanced/harbor-convert.mdx +++ b/docs/v6/advanced/harbor-convert.mdx @@ -10,11 +10,11 @@ task dirs - is a *frontend* that loads into the same primitives (`Environment`, roundtrip to run foreign tasks. Each one implements [`hud.environment.Integration`](https://github.com/hud-evals/hud-python/blob/main/hud/environment/integration.py) - `load(ref) -> Taskset` and `environment(ref) -> Environment` - and ships as -`hud.integrations.`. +the repository's `integrations/`. Harbor's agent works *inside* its container, so its environment constructor is only meaningful in there. `adapt()` packages it: one HUD-speaking image per -distinct build context, whose CMD serves `hud.integrations.harbor:environment`. +distinct build context, whose CMD serves `harbor:environment`. The same rows then run on any container placement. ## Prerequisites @@ -29,7 +29,7 @@ directly - one row per task dir (`id` = the dir name), sharing one declarative `Environment` per distinct `environment/` build context: ```python -from hud.integrations import harbor +from integrations import harbor assert harbor.detect("./terminal-bench") taskset = harbor.load("./terminal-bench") @@ -86,7 +86,7 @@ that must run as *different* users (one image, one `USER`). `env.py`) into self-contained Harbor task folders: ```python -from hud.integrations.harbor import export +from integrations.harbor import export created = await export("tasks.py", "harbor_tasks") ``` diff --git a/hud/environment/integration.py b/hud/environment/integration.py index eacdd3db4..8f69c578b 100644 --- a/hud/environment/integration.py +++ b/hud/environment/integration.py @@ -14,8 +14,9 @@ such images is a format extra (e.g. ``harbor.adapt``), not part of the contract. -Implementations live outside core (e.g. :mod:`hud.integrations`, or any -installable package); core knows only this interface. +Implementations live outside core — this repository's ``integrations/``, or +any installable package; core knows only this interface, and imports none of +them. """ from __future__ import annotations @@ -34,8 +35,8 @@ class Integration(ABC): """One foreign format's frontend. See module docstring for the contract.""" - #: The format's identifier: its origin scheme (``:`` on loaded - #: tasksets), CLI ``--format`` value, and registry key. + #: The format's identifier, and the scheme its loaded tasksets are + #: origin-stamped with (``:``). name: ClassVar[str] @abstractmethod diff --git a/hud/integrations/__init__.py b/hud/integrations/__init__.py deleted file mode 100644 index 667456f3d..000000000 --- a/hud/integrations/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Interop frontends: foreign benchmark formats as HUD primitives. - -Each format implements :class:`hud.environment.Integration` — see its module -docstring for the contract (``load`` / ``environment``). Format -modules also keep an ergonomic function surface (``harbor.load(...)``) and -format extras: ``harbor.detect`` recognizes the layout, ``harbor.adapt`` -packages the constructor into container images, ``harbor.export`` is the -reverse direction. -""" diff --git a/integrations/__init__.py b/integrations/__init__.py new file mode 100644 index 000000000..4085fdc87 --- /dev/null +++ b/integrations/__init__.py @@ -0,0 +1,11 @@ +"""Interop frontends: foreign benchmark formats as HUD primitives. + +These live outside the ``hud`` package and ship no part of it — core knows +only :class:`hud.environment.Integration` and imports no implementation. + +Each format implements that contract (``load`` / ``environment``) and keeps +an ergonomic function surface (``harbor.load(...)``) plus format extras: +``harbor.detect`` recognizes the layout, ``harbor.adapt`` packages the +constructor into container images, ``harbor.export`` is the reverse +direction. +""" diff --git a/hud/integrations/harbor/__init__.py b/integrations/harbor/__init__.py similarity index 100% rename from hud/integrations/harbor/__init__.py rename to integrations/harbor/__init__.py diff --git a/hud/integrations/harbor/_adapt.py b/integrations/harbor/_adapt.py similarity index 93% rename from hud/integrations/harbor/_adapt.py rename to integrations/harbor/_adapt.py index e2e925667..daa75870f 100644 --- a/hud/integrations/harbor/_adapt.py +++ b/integrations/harbor/_adapt.py @@ -11,7 +11,7 @@ CMD[ "hud", "serve", - "hud.integrations.harbor:environment", + "harbor:environment", "--arg", "ref=/hud/tasks", "--arg", @@ -77,10 +77,21 @@ _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*") +#: The interpreter the serving venv is built on. The layer copies this +#: integration into that venv's site-packages by path, so the two must agree. +SERVING_PYTHON = "3.12" + #: Build-context entries never copied into adapted contexts. -_CONTEXT_IGNORE = shutil.ignore_patterns( - "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache" +_CONTEXT_IGNORE_NAMES = ( + "__pycache__", + "*.pyc", + ".git", + ".venv", + "venv", + "*.egg-info", + ".pytest_cache", ) +_CONTEXT_IGNORE = shutil.ignore_patterns(*_CONTEXT_IGNORE_NAMES) _INSTALL_SH = """\ #!/bin/sh @@ -110,24 +121,28 @@ # The interpreter must live under /hud: uv's default install dir is the # invoking user's home (root here), unreachable once the image's USER applies. export UV_PYTHON_INSTALL_DIR=/hud/python -uv python install 3.12 -uv venv /hud/venv --python 3.12 +uv python install __PYTHON__ +uv venv /hud/venv --python __PYTHON__ uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__ """ _LAYER = """ # ─── HUD adaptation layer: serve the control channel from inside ─── -# (generated by hud.integrations.harbor.adapt) +# (generated by harbor.adapt) # Installed as root: the base stage may end on a non-root USER that could not # write /hud. The task's own declared user, if any, is restored below. USER root COPY _hud /hud RUN sh /hud/install.sh +# This integration ships outside the hud distribution, so it comes from the +# build context rather than the index — which also means the code serving the +# image is the revision that adapted it, not whatever the index resolves to. +COPY _hud_harbor /hud/venv/lib/python__PYTHON__/site-packages/harbor EXPOSE 8765 __DECLARED__ENTRYPOINT [] CMD ["/hud/venv/bin/hud", "serve", \ - "hud.integrations.harbor:environment", "--arg", "ref=/hud/tasks", \ + "harbor:environment", "--arg", "ref=/hud/tasks", \ "--arg", "name=__ENV_NAME__", "--host", "0.0.0.0", "--port", "8765"] """ @@ -186,7 +201,7 @@ async def adapt( """Write adapted build contexts for every env group; build and push them. Returns ``{env_name: image_ref}`` — pass it to - :func:`~hud.integrations.harbor.load` as ``images=`` so the rows carry + :func:`~harbor.load` as ``images=`` so the rows carry their image. *push* is a registry prefix (``registry.io/acme``); without it images stay local, which serves ``DockerRuntime`` but not cloud placements. ``build=False`` writes the contexts under ``.hud-adapt/`` and @@ -248,7 +263,10 @@ def _write_context( if dockerignore.is_file(): # The task's ignore rules were written for its own build; they must # not exclude the adaptation layer from this one. - _write(dockerignore, dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n") + _write( + dockerignore, + dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n!_hud_harbor\n!_hud_harbor/**\n", + ) hud_dir = context / "_hud" hud_dir.mkdir(parents=True) @@ -257,6 +275,11 @@ def _write_context( if wheel.suffix == ".whl" and wheel.is_file(): shutil.copy2(wheel, hud_dir / wheel.name) requirement = f"/hud/{wheel.name}" + shutil.copytree( + Path(__file__).parent, + context / "_hud_harbor", + ignore=shutil.ignore_patterns("tests", ".hud-adapt", *_CONTEXT_IGNORE_NAMES), + ) for task_dir in group_dirs: target = hud_dir / "tasks" / task_dir.name target.mkdir(parents=True) @@ -265,12 +288,18 @@ def _write_context( _copy_task_content(task_dir / "tests", target / "tests") _write( hud_dir / "install.sh", - _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)), + _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)).replace( + "__PYTHON__", SERVING_PYTHON + ), ) - layer = _LAYER.replace("__ENV_NAME__", env_name).replace( - # One env serves one policy, so the group's tasks agree on these. - "__DECLARED__", - _declared_directives(group_dirs[0], final_stage(dockerfile).user), + layer = ( + _LAYER.replace("__ENV_NAME__", env_name) + .replace("__PYTHON__", SERVING_PYTHON) + .replace( + # One env serves one policy, so the group's tasks agree on these. + "__DECLARED__", + _declared_directives(group_dirs[0], final_stage(dockerfile).user), + ) ) _write(context / "Dockerfile", dockerfile + layer) return context diff --git a/hud/integrations/harbor/_export.py b/integrations/harbor/_export.py similarity index 100% rename from hud/integrations/harbor/_export.py rename to integrations/harbor/_export.py diff --git a/hud/integrations/harbor/_load.py b/integrations/harbor/_load.py similarity index 99% rename from hud/integrations/harbor/_load.py rename to integrations/harbor/_load.py index 2befb8e8f..8ced662f4 100644 --- a/hud/integrations/harbor/_load.py +++ b/integrations/harbor/_load.py @@ -137,7 +137,7 @@ def load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset: the dataset name. Each row carries the task's declared launch requirements (:func:`runtime_config`: cpu/memory/gpu and time budgets), plus the adapted image ref once - :func:`~hud.integrations.harbor.adapt` has produced it, so it runs on any + :func:`~harbor.adapt` has produced it, so it runs on any container placement:: await harbor.adapt(path) @@ -391,7 +391,7 @@ def grouped(root: str | Path) -> list[tuple[str, list[Path]]]: One env name per group (the dataset slug, ``-gN``-suffixed when there are several): the join key between :func:`load`'s rows and - :func:`~hud.integrations.harbor.adapt`'s images. + :func:`~harbor.adapt`'s images. """ resolved = Path(root).resolve() dataset_name = resolved.parent.name if is_harbor_task(resolved) else resolved.name diff --git a/hud/integrations/tests/__init__.py b/integrations/harbor/tests/__init__.py similarity index 100% rename from hud/integrations/tests/__init__.py rename to integrations/harbor/tests/__init__.py diff --git a/hud/integrations/tests/conftest.py b/integrations/harbor/tests/conftest.py similarity index 100% rename from hud/integrations/tests/conftest.py rename to integrations/harbor/tests/conftest.py diff --git a/hud/integrations/tests/test_contract.py b/integrations/harbor/tests/test_contract.py similarity index 97% rename from hud/integrations/tests/test_contract.py rename to integrations/harbor/tests/test_contract.py index dd6b62e45..bb9c045a8 100644 --- a/hud/integrations/tests/test_contract.py +++ b/integrations/harbor/tests/test_contract.py @@ -13,8 +13,8 @@ import pytest -from hud.integrations import harbor -from hud.integrations.harbor import _load as harbor_load +from integrations import harbor +from integrations.harbor import _load as harbor_load if TYPE_CHECKING: from pathlib import Path @@ -59,7 +59,7 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None: context = tmp_path / ".hud-adapt" / contexts[0] dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") # The CMD serves the contract constructor by module reference — no baked env.py. - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile assert f'"name={context.name}"' in dockerfile assert "EXPOSE 8765" in dockerfile assert not (context / "_hud" / "env.py").exists() @@ -100,7 +100,7 @@ async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None assert sorted(env.tasks) == ["task-a", "task-b"] # The adapted CMD serves exactly this constructor. dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile def test_harbor_implements_the_integration_contract() -> None: @@ -254,7 +254,7 @@ def test_adapted_cmd_serves_the_contract_constructor(tmp_path) -> None: (context,) = sorted((tmp_path / ".hud-adapt").iterdir()) dockerfile = (context / "Dockerfile").read_text(encoding="utf-8") - assert "hud.integrations.harbor:environment" in dockerfile + assert "harbor:environment" in dockerfile async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> None: @@ -263,7 +263,7 @@ async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> No import asyncio import json - from hud.integrations.harbor._adapt import _grade_with_verifier + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -334,7 +334,7 @@ def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None: def test_rewards_are_finite_numbers_not_booleans(tmp_path) -> None: import json as jsonlib - from hud.integrations.harbor._adapt import _read_reward + from integrations.harbor._adapt import _read_reward logs = tmp_path / "verifier" logs.mkdir() @@ -460,7 +460,7 @@ def test_final_stage_reads_only_what_the_shipped_image_declares() -> None: is not the shipped image's; ``user[:group]`` is a legal operand; and a heredoc body is data, not instructions. """ - from hud.integrations.harbor._load import final_stage + from integrations.harbor._load import final_stage multistage = final_stage( 'FROM golang:1.22 AS build\nUSER builder\nENTRYPOINT ["/tool"]\nRUN true\n' @@ -534,8 +534,8 @@ async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None: # would block the writer forever and score a finished script as a timeout. import asyncio - from hud.integrations.harbor._adapt import _grade_with_verifier from hud.utils.process import create_process_group_exec + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -609,8 +609,8 @@ async def test_a_cancelled_grade_leaves_nothing_running(tmp_path) -> None: # process group and the pipe readers are released. import asyncio - from hud.integrations.harbor._adapt import _grade_with_verifier from hud.utils.process import create_process_group_exec + from integrations.harbor._adapt import _grade_with_verifier task = _write_harbor_task(tmp_path, "task-a") logs = tmp_path / "logs" @@ -645,7 +645,7 @@ async def run_tests(): def test_image_tags_do_not_depend_on_host_state(tmp_path) -> None: # Links are copied as links, so hashing must read the link — not what it # points at — or the same dataset tags differently on another machine. - from hud.integrations.harbor._load import hash_directory + from integrations.harbor._load import hash_directory target = tmp_path / "outside.txt" target.write_text("first", encoding="utf-8") diff --git a/hud/integrations/tests/test_harbor.py b/integrations/harbor/tests/test_harbor.py similarity index 99% rename from hud/integrations/tests/test_harbor.py rename to integrations/harbor/tests/test_harbor.py index 8e06bbb65..72a258945 100644 --- a/hud/integrations/tests/test_harbor.py +++ b/integrations/harbor/tests/test_harbor.py @@ -8,7 +8,7 @@ import pytest -from hud.integrations.harbor import detect, export, load +from integrations.harbor import detect, export, load from .conftest import make_harbor_task, make_multi_step_task @@ -317,7 +317,7 @@ async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: - from hud.integrations.harbor._export import _safe_component + from integrations.harbor._export import _safe_component assert "/" not in _safe_component("suite/fix") assert _safe_component("../escape") == "escape" diff --git a/pyproject.toml b/pyproject.toml index b46dd294f..b0e2abafe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ path = "hud/version.py" exclude = [ "docs/", "cookbooks/", + "integrations/", "hud-python/", "**/checkpoints/", "**/*.safetensors", @@ -261,7 +262,7 @@ omit = [ [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" -testpaths = ["hud"] +testpaths = ["hud", "integrations"] addopts = "" markers = [ "integration: marks tests as integration tests (require HUD_API_KEY, network access)",