refactor(integrations): move harbor out of the hud package

docs/v6/advanced/harbor-convert.mdx

@@ -10,11 +10,11 @@ task dirs - is a *frontend* that loads into the same primitives (`Environment`,
 roundtrip to run foreign tasks. Each one implements
 [`hud.environment.Integration`](https://github.com/hud-evals/hud-python/blob/main/hud/environment/integration.py)
 - `load(ref) -> Taskset` and `environment(ref) -> Environment` - and ships as
-`hud.integrations.<format>`.
+the repository's `integrations/<format>`.
 
 Harbor's agent works *inside* its container, so its environment constructor is
 only meaningful in there. `adapt()` packages it: one HUD-speaking image per
-distinct build context, whose CMD serves `hud.integrations.harbor:environment`.
+distinct build context, whose CMD serves `harbor:environment`.
 The same rows then run on any container placement.
 
 ## Prerequisites
@@ -29,7 +29,7 @@ directly - one row per task dir (`id` = the dir name), sharing one declarative
 `Environment` per distinct `environment/` build context:
 
 ```python
-from hud.integrations import harbor
+from integrations import harbor
 
 assert harbor.detect("./terminal-bench")
 taskset = harbor.load("./terminal-bench")
@@ -86,7 +86,7 @@ that must run as *different* users (one image, one `USER`).
 `env.py`) into self-contained Harbor task folders:
 
 ```python
-from hud.integrations.harbor import export
+from integrations.harbor import export
 
 created = await export("tasks.py", "harbor_tasks")
 ```

hud/environment/integration.py

@@ -14,8 +14,9 @@
 such images is a format extra (e.g. ``harbor.adapt``), not part of the
 contract.
 
-Implementations live outside core (e.g. :mod:`hud.integrations`, or any
-installable package); core knows only this interface.
+Implementations live outside core — this repository's ``integrations/``, or
+any installable package; core knows only this interface, and imports none of
+them.
 """
 
 from __future__ import annotations
@@ -34,8 +35,8 @@
 class Integration(ABC):
     """One foreign format's frontend. See module docstring for the contract."""
 
-    #: The format's identifier: its origin scheme (``<name>:<ref>`` on loaded
-    #: tasksets), CLI ``--format`` value, and registry key.
+    #: The format's identifier, and the scheme its loaded tasksets are
+    #: origin-stamped with (``<name>:<ref>``).
     name: ClassVar[str]
 
     @abstractmethod

integrations/__init__.py

@@ -0,0 +1,11 @@
+"""Interop frontends: foreign benchmark formats as HUD primitives.
+
+These live outside the ``hud`` package and ship no part of it — core knows
+only :class:`hud.environment.Integration` and imports no implementation.
+
+Each format implements that contract (``load`` / ``environment``) and keeps
+an ergonomic function surface (``harbor.load(...)``) plus format extras:
+``harbor.detect`` recognizes the layout, ``harbor.adapt`` packages the
+constructor into container images, ``harbor.export`` is the reverse
+direction.
+"""

hud/integrations/harbor/_adapt.py → integrations/harbor/_adapt.py

@@ -11,7 +11,7 @@
     CMD[
         "hud",
         "serve",
-        "hud.integrations.harbor:environment",
+        "harbor:environment",
         "--arg",
         "ref=/hud/tasks",
         "--arg",
@@ -77,10 +77,21 @@
 _DOCKER_ENV_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
 _DOCKER_USER = re.compile(r"[A-Za-z0-9_.][A-Za-z0-9_.-]*")
 
+#: The interpreter the serving venv is built on. The layer copies this
+#: integration into that venv's site-packages by path, so the two must agree.
+SERVING_PYTHON = "3.12"
+
 #: Build-context entries never copied into adapted contexts.
-_CONTEXT_IGNORE = shutil.ignore_patterns(
-    "__pycache__", "*.pyc", ".git", ".venv", "venv", "*.egg-info", ".pytest_cache"
+_CONTEXT_IGNORE_NAMES = (
+    "__pycache__",
+    "*.pyc",
+    ".git",
+    ".venv",
+    "venv",
+    "*.egg-info",
+    ".pytest_cache",
 )
+_CONTEXT_IGNORE = shutil.ignore_patterns(*_CONTEXT_IGNORE_NAMES)
 
 _INSTALL_SH = """\
 #!/bin/sh
@@ -110,24 +121,28 @@
 # The interpreter must live under /hud: uv's default install dir is the
 # invoking user's home (root here), unreachable once the image's USER applies.
 export UV_PYTHON_INSTALL_DIR=/hud/python
-uv python install 3.12
-uv venv /hud/venv --python 3.12
+uv python install __PYTHON__
+uv venv /hud/venv --python __PYTHON__
 uv pip install --python /hud/venv/bin/python __HUD_REQUIREMENT__
 """
 
 _LAYER = """
 
 # ─── HUD adaptation layer: serve the control channel from inside ───
-# (generated by hud.integrations.harbor.adapt)
+# (generated by harbor.adapt)
 # Installed as root: the base stage may end on a non-root USER that could not
 # write /hud. The task's own declared user, if any, is restored below.
 USER root
 COPY _hud /hud
 RUN sh /hud/install.sh
+# This integration ships outside the hud distribution, so it comes from the
+# build context rather than the index — which also means the code serving the
+# image is the revision that adapted it, not whatever the index resolves to.
+COPY _hud_harbor /hud/venv/lib/python__PYTHON__/site-packages/harbor
 EXPOSE 8765
 __DECLARED__ENTRYPOINT []
 CMD ["/hud/venv/bin/hud", "serve", \
-     "hud.integrations.harbor:environment", "--arg", "ref=/hud/tasks", \
+     "harbor:environment", "--arg", "ref=/hud/tasks", \
      "--arg", "name=__ENV_NAME__", "--host", "0.0.0.0", "--port", "8765"]
 """
 
@@ -186,7 +201,7 @@ async def adapt(
     """Write adapted build contexts for every env group; build and push them.
 
     Returns ``{env_name: image_ref}`` — pass it to
-    :func:`~hud.integrations.harbor.load` as ``images=`` so the rows carry
+    :func:`~harbor.load` as ``images=`` so the rows carry
     their image. *push* is a registry prefix (``registry.io/acme``); without it
     images stay local, which serves ``DockerRuntime`` but not cloud
     placements. ``build=False`` writes the contexts under ``.hud-adapt/`` and
@@ -248,7 +263,10 @@ def _write_context(
     if dockerignore.is_file():
         # The task's ignore rules were written for its own build; they must
         # not exclude the adaptation layer from this one.
-        _write(dockerignore, dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n")
+        _write(
+            dockerignore,
+            dockerignore.read_text("utf-8") + "\n!_hud\n!_hud/**\n!_hud_harbor\n!_hud_harbor/**\n",
+        )
 
     hud_dir = context / "_hud"
     hud_dir.mkdir(parents=True)
@@ -257,6 +275,11 @@ def _write_context(
     if wheel.suffix == ".whl" and wheel.is_file():
         shutil.copy2(wheel, hud_dir / wheel.name)
         requirement = f"/hud/{wheel.name}"
+    shutil.copytree(
+        Path(__file__).parent,
+        context / "_hud_harbor",
+        ignore=shutil.ignore_patterns("tests", ".hud-adapt", *_CONTEXT_IGNORE_NAMES),
+    )
     for task_dir in group_dirs:
         target = hud_dir / "tasks" / task_dir.name
         target.mkdir(parents=True)
@@ -265,12 +288,18 @@ def _write_context(
         _copy_task_content(task_dir / "tests", target / "tests")
     _write(
         hud_dir / "install.sh",
-        _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)),
+        _INSTALL_SH.replace("__HUD_REQUIREMENT__", shlex.quote(requirement)).replace(
+            "__PYTHON__", SERVING_PYTHON
+        ),
     )
-    layer = _LAYER.replace("__ENV_NAME__", env_name).replace(
-        # One env serves one policy, so the group's tasks agree on these.
-        "__DECLARED__",
-        _declared_directives(group_dirs[0], final_stage(dockerfile).user),
+    layer = (
+        _LAYER.replace("__ENV_NAME__", env_name)
+        .replace("__PYTHON__", SERVING_PYTHON)
+        .replace(
+            # One env serves one policy, so the group's tasks agree on these.
+            "__DECLARED__",
+            _declared_directives(group_dirs[0], final_stage(dockerfile).user),
+        )
     )
     _write(context / "Dockerfile", dockerfile + layer)
     return context

hud/integrations/harbor/_load.py → integrations/harbor/_load.py

@@ -137,7 +137,7 @@ def load(path: str | Path, *, images: dict[str, str] | None = None) -> Taskset:
     the dataset name. Each row carries the task's declared launch
     requirements (:func:`runtime_config`: cpu/memory/gpu and time budgets),
     plus the adapted image ref once
-    :func:`~hud.integrations.harbor.adapt` has produced it, so it runs on any
+    :func:`~harbor.adapt` has produced it, so it runs on any
     container placement::
 
         await harbor.adapt(path)
@@ -391,7 +391,7 @@ def grouped(root: str | Path) -> list[tuple[str, list[Path]]]:
 
     One env name per group (the dataset slug, ``-gN``-suffixed when there are
     several): the join key between :func:`load`'s rows and
-    :func:`~hud.integrations.harbor.adapt`'s images.
+    :func:`~harbor.adapt`'s images.
     """
     resolved = Path(root).resolve()
     dataset_name = resolved.parent.name if is_harbor_task(resolved) else resolved.name

hud/integrations/tests/test_contract.py → integrations/harbor/tests/test_contract.py

@@ -13,8 +13,8 @@
 
 import pytest
 
-from hud.integrations import harbor
-from hud.integrations.harbor import _load as harbor_load
+from integrations import harbor
+from integrations.harbor import _load as harbor_load
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -59,7 +59,7 @@ async def test_adapt_contexts_bake_the_serving_layer(tmp_path) -> None:
     context = tmp_path / ".hud-adapt" / contexts[0]
     dockerfile = (context / "Dockerfile").read_text(encoding="utf-8")
     # The CMD serves the contract constructor by module reference — no baked env.py.
-    assert "hud.integrations.harbor:environment" in dockerfile
+    assert "harbor:environment" in dockerfile
     assert f'"name={context.name}"' in dockerfile
     assert "EXPOSE 8765" in dockerfile
     assert not (context / "_hud" / "env.py").exists()
@@ -100,7 +100,7 @@ async def test_environment_serves_the_baked_tasks(tmp_path, monkeypatch) -> None
     assert sorted(env.tasks) == ["task-a", "task-b"]
     # The adapted CMD serves exactly this constructor.
     dockerfile = (context / "Dockerfile").read_text(encoding="utf-8")
-    assert "hud.integrations.harbor:environment" in dockerfile
+    assert "harbor:environment" in dockerfile
 
 
 def test_harbor_implements_the_integration_contract() -> None:
@@ -254,7 +254,7 @@ def test_adapted_cmd_serves_the_contract_constructor(tmp_path) -> None:
     (context,) = sorted((tmp_path / ".hud-adapt").iterdir())
     dockerfile = (context / "Dockerfile").read_text(encoding="utf-8")
 
-    assert "hud.integrations.harbor:environment" in dockerfile
+    assert "harbor:environment" in dockerfile
 
 
 async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> None:
@@ -263,7 +263,7 @@ async def test_planted_reward_files_are_discarded_before_grading(tmp_path) -> No
     import asyncio
     import json
 
-    from hud.integrations.harbor._adapt import _grade_with_verifier
+    from integrations.harbor._adapt import _grade_with_verifier
 
     task = _write_harbor_task(tmp_path, "task-a")
     logs = tmp_path / "logs"
@@ -334,7 +334,7 @@ def test_declared_uid_zero_beats_the_source_user(tmp_path) -> None:
 def test_rewards_are_finite_numbers_not_booleans(tmp_path) -> None:
     import json as jsonlib
 
-    from hud.integrations.harbor._adapt import _read_reward
+    from integrations.harbor._adapt import _read_reward
 
     logs = tmp_path / "verifier"
     logs.mkdir()
@@ -460,7 +460,7 @@ def test_final_stage_reads_only_what_the_shipped_image_declares() -> None:
     is not the shipped image's; ``user[:group]`` is a legal operand; and a
     heredoc body is data, not instructions.
     """
-    from hud.integrations.harbor._load import final_stage
+    from integrations.harbor._load import final_stage
 
     multistage = final_stage(
         'FROM golang:1.22 AS build\nUSER builder\nENTRYPOINT ["/tool"]\nRUN true\n'
@@ -534,8 +534,8 @@ async def test_a_chatty_verifier_does_not_deadlock(tmp_path) -> None:
     # would block the writer forever and score a finished script as a timeout.
     import asyncio
 
-    from hud.integrations.harbor._adapt import _grade_with_verifier
     from hud.utils.process import create_process_group_exec
+    from integrations.harbor._adapt import _grade_with_verifier
 
     task = _write_harbor_task(tmp_path, "task-a")
     logs = tmp_path / "logs"
@@ -609,8 +609,8 @@ async def test_a_cancelled_grade_leaves_nothing_running(tmp_path) -> None:
     # process group and the pipe readers are released.
     import asyncio
 
-    from hud.integrations.harbor._adapt import _grade_with_verifier
     from hud.utils.process import create_process_group_exec
+    from integrations.harbor._adapt import _grade_with_verifier
 
     task = _write_harbor_task(tmp_path, "task-a")
     logs = tmp_path / "logs"
@@ -645,7 +645,7 @@ async def run_tests():
 def test_image_tags_do_not_depend_on_host_state(tmp_path) -> None:
     # Links are copied as links, so hashing must read the link — not what it
     # points at — or the same dataset tags differently on another machine.
-    from hud.integrations.harbor._load import hash_directory
+    from integrations.harbor._load import hash_directory
 
     target = tmp_path / "outside.txt"
     target.write_text("first", encoding="utf-8")

hud/integrations/tests/test_harbor.py → integrations/harbor/tests/test_harbor.py

@@ -8,7 +8,7 @@
 
 import pytest
 
-from hud.integrations.harbor import detect, export, load
+from integrations.harbor import detect, export, load
 
 from .conftest import make_harbor_task, make_multi_step_task
 
@@ -317,7 +317,7 @@ async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None:
 
 
 async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None:
-    from hud.integrations.harbor._export import _safe_component
+    from integrations.harbor._export import _safe_component
 
     assert "/" not in _safe_component("suite/fix")
     assert _safe_component("../escape") == "escape"

pyproject.toml

@@ -60,6 +60,7 @@ path = "hud/version.py"
 exclude = [
     "docs/",
     "cookbooks/",
+    "integrations/",
     "hud-python/",
     "**/checkpoints/",
     "**/*.safetensors",
@@ -261,7 +262,7 @@ omit = [
 [tool.pytest.ini_options]
 asyncio_default_fixture_loop_scope = "function"
 asyncio_mode = "auto"
-testpaths = ["hud"]
+testpaths = ["hud", "integrations"]
 addopts = ""
 markers = [
     "integration: marks tests as integration tests (require HUD_API_KEY, network access)",