[2.0] Add new Frontier-CS 2.0 problem: nanoslm_hybrid_arch_design

.gitignore

@@ -2,6 +2,8 @@ build
 dist
 *.egg-info
 **/__pycache__/
+# Baked corpus / large local assets (staged on Modal volumes, never committed)
+**/.assets/
 *.log
 **/*.lic
 .vscode

2.0/README.md

@@ -127,6 +127,18 @@ guardrail (so drift can't be bought with more compute). A history-stabilization
 reference reliably beats baseline (validated t≈2.5/22 clips); beating it
 substantially is the open challenge.
 
+## NanoSLM Hybrid Architecture Design
+
+This task formulates the hybrid language model architecture design as a scored task
+at 200M scale (capped at 400M parameters), framed on *Olmo Hybrid: From Theory to
+Practice and Back* (arXiv:2604.03444). Its problem ID is `nanoslm_hybrid_arch_design`.
+Agents submit a single `/app/model.py` defining `build_model(config)` (or a
+`class NanoSLM`), with full freedom over the model definition. The judge trains
+it from scratch under a fixed wall-clock budget `T` on one H100 and scores the
+**absolute** reduction in held-out **bits-per-byte** (`val_bpb`, normalized by
+bytes so it is tokenizer-independent) against a baseline pure-attention
+`olmo3_190M` baseline. 
+
 ## Structured-LWE Public Witness Recovery
 
 This cryptanalysis task publishes 200 structured-LWE instances spanning ten

2.0/problems/nanoslm_hybrid_arch_design/config.yaml

@@ -0,0 +1,65 @@
+tag: systems
+runtime:
+  # Submission is a single Python file, the model definition (/app/model.py).
+  # `language: python` keeps the extension/CLI conventions standard.
+  language: python
+  timeout_seconds: 86400
+  environment: >-
+    Full-freedom PyTorch model.py submitted against a locked
+    training harness over a dolma2-BPE-tokenized FineWeb-Edu corpus; a single
+    Modal H100 trains the model from scratch for a fixed wall-clock budget; the
+    judge scores held-out validation bits-per-byte vs a locked baseline
+    architecture trained under the same budget.
+  apt_packages:
+    - bash
+    - ca-certificates
+    - curl
+    - git
+    - python3
+    - python3-pip
+  judge_apt_packages:
+    - bash
+    - ca-certificates
+    - curl
+    - git
+    - python3
+    - python3-pip
+  judge_pip_packages:
+    - modal
+  docker:
+    image: frontiercs/nanoslm-hybrid-arch-design-agent:experimental-v0
+    judge_image: frontiercs/nanoslm-hybrid-arch-design-judge:experimental-v0
+environment:
+  cpus: 8
+  memory_mb: 32768
+  storage_mb: 32768
+  build_timeout_seconds: 7200
+evaluation:
+  gpu: H100
+  tokenizer: allenai/dolma2-tokenizer
+  # 100352 = dolma2's 100278 real ids padded up (Olmo 3 convention; ids on
+  # disk stay < 100278). > 65535, so token streams are uint32.
+  vocab_size: 100352
+  dataset: HuggingFaceFW/fineweb-edu:sample-10BT
+  block_size: 8192
+  eval_block_size: 8192
+  train_seconds: 1800                 # fixed wall-clock budget T per training run (30 min)
+  # LR cosine horizon: the FULL 6h schedule. A 30-min run traverses only its
+  # first 1/12 (prefix-of-a-long-run behavior), not a compressed anneal.
+  lr_schedule_seconds: 21600
+  max_train_seconds: 3600
+  val_tokens: 1048576
+  # The scored metric is bits per byte: nll_nats / (val_bytes * ln2). val_bytes
+  # is measured by docker/prep_assets.py and shipped in the judge image's
+  # manifest.json -- normalizing by the token count would make the metric
+  # tokenizer-dependent.
+  metric: val_bpb
+  # The score IS the submission's held-out val_bpb, raw and unscaled -- LOWER
+  # IS BETTER. No clipping, no baseline-relative mapping (bpb_score_scale was
+  # removed); the baseline's bpb and the gain are reported for context only.
+  param_cap: 400000000
+  seed: 1337
+submission:
+  kind: file
+  path: /app/model.py
+  max_queue_size: 2

2.0/problems/nanoslm_hybrid_arch_design/docker/agent/Dockerfile

@@ -0,0 +1,59 @@
+# Agent workspace image for nanoslm_hybrid_arch_design.
+#
+# CPU-ONLY. The agent designs an architecture and submits /app/model.py; the
+# JUDGE trains it (on a Modal H100). So this image ships no torch, no CUDA and
+# no corpus -- an agent cannot and need not train here.
+#
+# What it DOES ship is the judge's own static policy gate, so
+# `bash /app/public_test.sh` runs the exact accept/reject rules the judge
+# applies, instead of the agent discovering them by burning submissions.
+#
+# CRITICALLY: /opt/nanoslm_arch/data/val.bin -- the held-out stream -- is NOT
+# here and must never be. build_images.sh asserts its absence post-build.
+FROM python:3.11-slim
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PIP_NO_CACHE_DIR=1 \
+    PYTHONUNBUFFERED=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        bash ca-certificates curl git ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+# Parity with the adapter template's tooling.
+RUN pip install --no-cache-dir "numpy<2" \
+    && curl -fsSL https://claude.ai/install.sh | bash 2>/dev/null || true
+
+WORKDIR /app
+
+# Workspace scaffolding: starter model.py, README, public_test.
+COPY task_ctx/harbor_app/ /app/
+
+# The judge's static gate, shipped verbatim so the local check and the judge's
+# check cannot drift. policy.py is settings-free and names no hidden value:
+# it contains the DENYLIST (which is public by design and stated in the readme)
+# but no thresholds, no paths, and no held-out identifiers. build_images.sh
+# greps it before copying.
+COPY task_ctx/agent_task/policy.py /app/policy.py
+RUN chmod +x /app/*.sh 2>/dev/null || true
+
+# Fail the build if the starter submission does not pass the shipped gate --
+# an agent must never open a workspace whose default file is already rejected.
+#
+# Assert through public_test.sh, the SAME entrypoint the agent runs, not by
+# importing check_source directly. A direct `sys.path`+import once passed here
+# while `bash /app/public_test.sh` crashed, because public_test.py loads
+# policy.py via spec_from_file_location and that path had its own bug. Testing
+# the real entrypoint is what makes this assertion mean what it claims.
+# The __pycache__ removal is load-bearing, not tidiness: running the gate
+# compiles policy.py, and the resulting .pyc embeds the denylist strings
+# ("val.bin", "/opt/"). build_images.sh greps every file under /app except
+# policy.py itself, so leaving the byte-compiled twin behind trips that check.
+RUN bash /app/public_test.sh \
+    || (echo 'FATAL: shipped starter model.py fails the policy gate' && exit 1) \
+    && rm -rf /app/__pycache__
+
+# Assert the held-out stream is absent (defence in depth; build_images.sh also
+# checks the built image).
+RUN test ! -e /opt/nanoslm_arch/data/val.bin || \
+    (echo 'FATAL: held-out val.bin present in the AGENT image' && exit 1)

2.0/problems/nanoslm_hybrid_arch_design/docker/build_images.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# Build the agent + judge images for nanoslm_hybrid_arch_design.
+#
+#   bash docker/build_images.sh [tag]
+#
+# NOTE ON THE THREE-IMAGE ARCHITECTURE. This script builds TWO images; a third
+# is built remotely by Modal from harness/modal_app.py and is never produced
+# here:
+#
+#   agent  (this script, CPU)  - workspace: starter model.py + the static gate
+#   judge  (this script, CPU)  - dispatches to Modal, scores, holds val.bin
+#   Modal  (modal deploy, GPU) - torch 2.6.0 / triton 3.2.0 / fla 0.5.1
+#
+# The GPU stack lives ONLY in the Modal image. Keep its pins in
+# harness/modal_app.py in sync: that exact combination is
+# the one whose GDN backward kernel compiles (triton 3.1.0 hung for 30+ min;
+# fla 0.4.1 failed to lower the backward).
+set -euo pipefail
+
+TAG="${1:-experimental-v0}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROB="$(cd "$HERE/.." && pwd)"
+ASSETS="${NANOSLM_ARCH_ASSETS:-$PROB/.assets}"
+AGENT_IMG="frontiercs/nanoslm-hybrid-arch-design-agent:$TAG"
+JUDGE_IMG="frontiercs/nanoslm-hybrid-arch-design-judge:$TAG"
+
+# --- assets -------------------------------------------------------------------
+if [ ! -s "$ASSETS/train.bin" ] || [ ! -s "$ASSETS/val.bin" ] \
+   || [ ! -s "$ASSETS/manifest.json" ]; then
+  echo "[build] staging tokenized corpus into $ASSETS (cold cache)"
+  NANOSLM_ARCH_ASSETS="$ASSETS" python3 "$HERE/prep_assets.py"
+fi
+# manifest.json is NOT optional metadata: it carries `val_bytes`, the denominator
+# of the scored bits-per-byte. Without it harness/data.py raises rather than
+# silently normalizing by a token count -- so a judge image built without it
+# would fail every run. See docker/prep_assets.py.
+for f in train.bin val.bin manifest.json; do
+  [ -s "$ASSETS/$f" ] || { echo "FATAL: missing $ASSETS/$f"; exit 1; }
+done
+MANIFEST="$ASSETS/manifest.json" python3 - <<'PY'
+import json, os, sys
+m = json.load(open(os.environ["MANIFEST"]))
+b, t = m.get("val_bytes", 0), m.get("val_target_tokens", 0)
+if not (b > 0 and t > 0):
+    sys.exit("FATAL: manifest has no positive val_bytes/val_target_tokens")
+print("[check] val_bytes=%d over %d target tokens (%.2f B/tok)" % (b, t, b / t))
+PY
+
+# --- build context ------------------------------------------------------------
+CTX="$(mktemp -d)"
+trap 'rm -rf "$CTX"' EXIT
+mkdir -p "$CTX/task_ctx/task_pkg" "$CTX/task_ctx/agent_task" "$CTX/task_ctx/assets"
+
+cp -r "$PROB/harness"      "$CTX/task_ctx/task_pkg/harness"
+cp    "$PROB/evaluator.py" "$CTX/task_ctx/evaluator.py"
+cp -r "$PROB/harbor/app"   "$CTX/task_ctx/harbor_app"
+cp    "$PROB/harness/policy.py" "$CTX/task_ctx/agent_task/policy.py"
+cp    "$ASSETS/train.bin" "$ASSETS/val.bin" "$ASSETS/manifest.json" \
+      "$CTX/task_ctx/assets/"
+
+# modal_app.py declares the GPU image and is judge-side only; it is already in
+# task_pkg via harness/. Nothing else needs to move.
+
+# --- leak grep: what ships to the AGENT ---------------------------------------
+# policy.py legitimately CONTAINS the denylist, which names the metric and
+# held-out identifiers -- those strings are public by design (the readme states
+# them) and are what the gate matches on. So grep for hidden VALUES and real
+# PATHS, not for the denylist's own words.
+POLICY="$CTX/task_ctx/agent_task/policy.py"
+#
+# TWO greps, not one. policy.py's DENY tuples literally contain "val.bin",
+# "/opt/", "holdout" etc -- that IS the denylist, it is public by design, and a
+# naive single grep fails the build on the gate's own source every time. (This
+# exact trap already exists in lm_arch_discovery/docker/build_images.sh and in
+# align_overopt_stability's; it is easy to re-introduce.)
+#
+#   (1) hidden VALUES over the whole file. These are strictly more specific
+#       than anything the denylist contains, so they cannot collide with it.
+#   (2) asset identifiers with the DENY tuples EXCISED, which catches a real
+#       path leaking in via a default or a stray constant while letting the
+#       denylist keep naming them.
+# `bpb_score_scale` replaced the old `r_target`; both are matched so a stale
+# checkout cannot slip the old name through this gate.
+if grep -qiE "r_target|bpb_score_scale|train_seconds|param_cap|baseline_ppl|[0-9]{6,}" "$POLICY"; then
+  echo "FATAL: policy.py would leak a hidden VALUE to the agent image"
+  exit 1
+fi
+if sed -e '/^POLICY_DENY_TOKENS[A-Z_]*: tuple/,/^)/d' -e '/^[[:space:]]*#/d' "$POLICY" \
+     | grep -qiE "/opt/nanoslm_arch|val\.bin"; then
+  echo "FATAL: policy.py names a judge asset outside its denylist block"
+  exit 1
+fi
+# The starter model.py must not name the held-out stream either.
+if grep -qiE "/opt/nanoslm_arch|val\.bin" "$CTX/task_ctx/harbor_app/model.py"; then
+  echo "FATAL: starter model.py references judge assets"; exit 1
+fi
+
+# --- build --------------------------------------------------------------------
+echo "[build] agent -> $AGENT_IMG"
+docker build -f "$HERE/agent/Dockerfile" -t "$AGENT_IMG" "$CTX"
+echo "[build] judge -> $JUDGE_IMG"
+docker build -f "$HERE/judge/Dockerfile" -t "$JUDGE_IMG" "$CTX"
+
+# --- post-build isolation ------------------------------------------------------
+echo "[check] held-out stream must be unreachable from the AGENT image"
+# policy.py is EXEMPT from the content grep and only from it: its denylist
+# names "val.bin" by design, which is the third place in this script where that
+# distinction matters. EVERY other file under /app is grepped whole -- the loop
+# enumerates with `find`, not a fixed list, so a leak in a file added to the
+# workspace later still fails the build. (It used to name four files explicitly,
+# which made the policy.py exemption below dead code and left anything new
+# unchecked.) File PRESENCE is checked unconditionally -- no exemption there.
+if docker run --rm --entrypoint bash "$AGENT_IMG" -c '
+      ls /opt/nanoslm_arch/data/val.bin 2>/dev/null;
+      ls /opt/nanoslm_arch 2>/dev/null;
+      find /app -type f | while read -r f; do
+        [ "$f" = /app/policy.py ] && continue;
+        grep -il "val\.bin\|/opt/nanoslm_arch" "$f" 2>/dev/null;
+      done
+   ' | grep -q .; then
+  echo "FATAL: held-out data or its identity is reachable from the AGENT image"
+  exit 1
+fi
+
+echo "[check] judge must NOT carry a GPU stack (it dispatches to Modal)"
+if docker run --rm --entrypoint bash "$JUDGE_IMG" -c \
+     'python3 -c "import torch" 2>/dev/null && echo present'  | grep -q present; then
+  echo "FATAL: torch present in the CPU-only judge image -- a silent LOCAL"
+  echo "       fallback would try to train ~190M at ctx 8192 on CPU"
+  exit 1
+fi
+
+echo "[check] judge must hold both token streams AND the byte-count manifest"
+docker run --rm --entrypoint bash "$JUDGE_IMG" -c \
+  'test -s /opt/nanoslm_arch/data/train.bin \
+   && test -s /opt/nanoslm_arch/data/val.bin \
+   && test -s /opt/nanoslm_arch/data/manifest.json' \
+  || { echo "FATAL: judge image missing a token stream or manifest.json"; exit 1; }
+
+cat <<EOF
+
+built:
+  $AGENT_IMG
+  $JUDGE_IMG
+
+REMINDER: the GPU image is NOT built here. Deploy it separately and keep its
+pins (torch 2.6.0 / triton 3.2.0 / flash-linear-attention 0.5.1) in sync:
+
+  modal deploy harness/modal_app.py
+EOF

2.0/problems/nanoslm_hybrid_arch_design/docker/judge/Dockerfile

@@ -0,0 +1,73 @@
+# Judge image for nanoslm_hybrid_arch_design.
+#
+# CPU-ONLY, AND DELIBERATELY SO. The H100 is served on Modal
+# (harness/modal_app.py); this container dispatches to it and scores the
+# returned metrics. It therefore installs NO torch, NO CUDA and NO
+# flash-linear-attention -- those live in the Modal image, which Modal builds
+# remotely from its own declarative spec.
+#
+# That split is load-bearing rather than incidental: `evaluator._backend()`
+# picks MODAL whenever a Modal token is present, and `run_pair_remote` returns
+# only plain builtins so this container can deserialize the result without
+# torch. Installing a GPU stack here would be dead weight AND would mask a
+# misconfiguration -- a judge that quietly falls back to LOCAL would try to
+# train a ~190M model at ctx 8192 on CPU.
+#
+# The versions the GPU side must use are pinned
+# in harness/modal_app.py:
+#     torch 2.6.0+cu124 / triton 3.2.0 / flash-linear-attention 0.5.1
+# That exact combination is the one whose GDN BACKWARD kernel compiles; on
+# triton 3.1.0 every call hung 30+ min, and on fla 0.4.1 the backward failed to
+# lower. Do not drift them independently.
+FROM python:3.11-slim
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PIP_NO_CACHE_DIR=1 \
+    PYTHONUNBUFFERED=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        bash ca-certificates curl git \
+    && rm -rf /var/lib/apt/lists/*
+
+# modal: the GPU dispatch client. numpy: harness/data.py reads the uint32 token
+# streams even on the judge side for shape/sanity checks. No torch by design.
+RUN pip install --no-cache-dir "modal>=1.0" "numpy<2"
+
+WORKDIR /opt/nanoslm_arch
+
+# The task package. Preserve the DIRECTORY layout: evaluator.py does
+# `from harness import policy, scoring, settings` and the harness modules use
+# relative imports, so flattening breaks both.
+COPY task_ctx/task_pkg/harness/ /opt/nanoslm_arch/task/harness/
+COPY task_ctx/evaluator.py       /opt/nanoslm_arch/task/evaluator.py
+
+# Hidden assets. val.bin is the HELD-OUT stream and exists ONLY here -- it is
+# never copied into the agent image, and build_images.sh asserts that. Both
+# streams are flat uint32 dolma2 token ids (vocab 100278), not raw bytes.
+#
+# manifest.json is REQUIRED, not documentation: it carries `val_bytes`, the
+# denominator of the scored bits-per-byte. harness/data.py raises rather than
+# normalizing by a token count, so a judge without it fails every run.
+COPY task_ctx/assets/train.bin     /opt/nanoslm_arch/data/train.bin
+COPY task_ctx/assets/val.bin       /opt/nanoslm_arch/data/val.bin
+COPY task_ctx/assets/manifest.json /opt/nanoslm_arch/data/manifest.json
+RUN mkdir -p /opt/nanoslm_arch/baseline
+
+# Paths must match harness/settings.py TaskConfig defaults.
+ENV PYTHONPATH=/opt/nanoslm_arch/task \
+    NANOSLM_ARCH_ASSETS=/opt/nanoslm_arch/data
+
+# Fail the build if the judge cannot import its own entrypoint, or if it can
+# import torch (which would mean the CPU-only contract was broken and a silent
+# LOCAL fallback is possible).
+RUN python3 -c "\
+import sys; sys.path.insert(0, '/opt/nanoslm_arch/task');\
+import evaluator, harness.policy, harness.scoring, harness.settings;\
+print('judge imports OK');\
+"
+# MUST stay ONE physical line. Docker splices a trailing `\` by deleting the
+# newline, so a multi-line try/except collapses into `...)except ImportError:`
+# -- a SyntaxError, which exits non-zero and thus "passed" for the wrong reason
+# while never testing torch at all. find_spec also beats `import torch` here: it
+# flags a present-but-broken torch, which a bare ImportError would swallow.
+RUN python3 -c "import importlib.util, sys; sys.exit('FATAL: torch present in the CPU-only judge image') if importlib.util.find_spec('torch') else print('torch absent as intended')"