From bbc23f7b751c44c4c46fede3333a1cc27fcccaa2 Mon Sep 17 00:00:00 2001 From: muhtasham Date: Tue, 5 May 2026 13:23:09 +0200 Subject: [PATCH] Add ABIDES arena --- README.md | 6 +- codeclash/arenas/__init__.py | 2 + codeclash/arenas/abides/ABIDES.Dockerfile | 30 ++ codeclash/arenas/abides/__init__.py | 3 + codeclash/arenas/abides/abides.py | 150 +++++++ codeclash/arenas/abides/constraints.txt | 21 + codeclash/arenas/abides/runtime/.gitignore | 4 + codeclash/arenas/abides/runtime/README.md | 14 + .../arenas/abides/runtime/abides_agent.py | 5 + codeclash/arenas/abides/runtime/run_abides.py | 380 ++++++++++++++++++ configs/examples/ABIDES__dummy__r1__s2.yaml | 30 ++ docs/reference/arenas/abides.md | 111 +++++ mkdocs.yml | 1 + tests/arenas/test_abides.py | 175 ++++++++ 14 files changed, 931 insertions(+), 1 deletion(-) create mode 100644 codeclash/arenas/abides/ABIDES.Dockerfile create mode 100644 codeclash/arenas/abides/__init__.py create mode 100644 codeclash/arenas/abides/abides.py create mode 100644 codeclash/arenas/abides/constraints.txt create mode 100644 codeclash/arenas/abides/runtime/.gitignore create mode 100644 codeclash/arenas/abides/runtime/README.md create mode 100644 codeclash/arenas/abides/runtime/abides_agent.py create mode 100644 codeclash/arenas/abides/runtime/run_abides.py create mode 100644 configs/examples/ABIDES__dummy__r1__s2.yaml create mode 100644 docs/reference/arenas/abides.md create mode 100644 tests/arenas/test_abides.py diff --git a/README.md b/README.md index 6dda5e5..f783e9a 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,11 @@ The winner is the LM agent who wins the most rounds. ## 🧩 Available Arenas CodeClash includes competitive programming games and simulation-backed arenas, including BattleSnake, -CoreWar, CybORG, Halite, HuskyBench, RoboCode, RobotRumble, and SCML. +ABIDES, CoreWar, CybORG, Halite, HuskyBench, RoboCode, RobotRumble, and SCML. + +ABIDES is a financial-market simulation arena based on the Agent-Based Interactive Discrete Event +Simulation environment. Agents edit a Python `abides_agent.py` implementation and compete to +maximize mark-to-market profit across compact simulated limit-order-book markets. SCML is a supply-chain negotiation arena based on the ANAC Supply Chain Management League OneShot track. Agents edit a Python `scml_agent.py` implementation and compete to maximize average profit diff --git a/codeclash/arenas/__init__.py b/codeclash/arenas/__init__.py index 700f87d..773fbd6 100644 --- a/codeclash/arenas/__init__.py +++ b/codeclash/arenas/__init__.py @@ -1,3 +1,4 @@ +from codeclash.arenas.abides.abides import ABIDESArena from codeclash.arenas.arena import CodeArena from codeclash.arenas.battlecode23.battlecode23 import BattleCode23Arena from codeclash.arenas.battlecode24.battlecode24 import BattleCode24Arena @@ -19,6 +20,7 @@ from codeclash.arenas.scml.scml import SCMLOneShotArena ARENAS = [ + ABIDESArena, BattleCode23Arena, BattleCode24Arena, BattleCode25Arena, diff --git a/codeclash/arenas/abides/ABIDES.Dockerfile b/codeclash/arenas/abides/ABIDES.Dockerfile new file mode 100644 index 0000000..eff4ed9 --- /dev/null +++ b/codeclash/arenas/abides/ABIDES.Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.11-slim-bookworm + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONPATH=/opt/abides + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates git build-essential jq \ + && rm -rf /var/lib/apt/lists/* + +COPY codeclash/arenas/abides/constraints.txt /tmp/abides-constraints.txt + +RUN python -m pip install pip==26.1.1 \ + && git clone https://github.com/abides-sim/abides.git /opt/abides \ + && cd /opt/abides \ + && git checkout c4bf157678928934417aba6073eb0651aeaf6d15 \ + && python -c "from pathlib import Path; p = Path('/opt/abides/util/OrderBook.py'); s = p.read_text(); p.write_text(s.replace('from pandas.io.json import json_normalize', 'from pandas import json_normalize'))" \ + && python -m pip install -e /opt/abides -c /tmp/abides-constraints.txt + +WORKDIR /workspace + +COPY codeclash/arenas/abides/runtime/ /workspace/ + +RUN git init \ + && git config user.email "player@codeclash.com" \ + && git config user.name "Player" \ + && git add . \ + && git commit -m "Initial ABIDES workspace" diff --git a/codeclash/arenas/abides/__init__.py b/codeclash/arenas/abides/__init__.py new file mode 100644 index 0000000..08efeeb --- /dev/null +++ b/codeclash/arenas/abides/__init__.py @@ -0,0 +1,3 @@ +from codeclash.arenas.abides.abides import ABIDESArena + +__all__ = ["ABIDESArena"] diff --git a/codeclash/arenas/abides/abides.py b/codeclash/arenas/abides/abides.py new file mode 100644 index 0000000..e9152a7 --- /dev/null +++ b/codeclash/arenas/abides/abides.py @@ -0,0 +1,150 @@ +import json +import shlex +import subprocess + +from codeclash.agents.player import Player +from codeclash.arenas.arena import CodeArena, RoundStats +from codeclash.constants import RESULT_TIE +from codeclash.utils.environment import assert_zero_exit_code + +RESULTS_JSON = "abides_results.json" +CRASH_SCORE = -1_000_000.0 + + +class ABIDESArena(CodeArena): + name: str = "ABIDES" + submission: str = "abides_agent.py" + description: str = """ABIDES is an agent-based market simulator for financial-market research. + +Your bot is a Python file named `abides_agent.py` that defines a class named `MyAgent`. +`MyAgent` should be an ABIDES trading agent class, for example: + + from agent.ValueAgent import ValueAgent as MyAgent + +Each round runs several compact ABIDES market simulations. Every submitted agent is evaluated in +identical seeded market worlds with the same exchange, market maker, and background traders. The +objective is to maximize average mark-to-market profit across all simulations in the round. +""" + default_args: dict = { + "sims_per_round": 3, + "market_minutes": 5, + "background_agents": 3, + "timeout": 240, + } + + def _game_arg(self, key: str): + return self.game_config.get("args", {}).get(key, self.game_config.get(key, self.default_args[key])) + + def validate_code(self, agent: Player) -> tuple[bool, str | None]: + quoted_submission = shlex.quote(self.submission) + file_check = agent.environment.execute(f"test -f {quoted_submission} && echo exists") + if "exists" not in file_check["output"]: + return False, f"Submission file `{self.submission}` not found in the workspace root" + + content = agent.environment.execute(f"cat {quoted_submission}")["output"] + if not content.strip(): + return False, f"`{self.submission}` is empty" + + syntax_check = agent.environment.execute(f"python -m py_compile {quoted_submission}") + if syntax_check["returncode"] != 0: + return False, f"Python syntax error in `{self.submission}`:\n{syntax_check['output']}" + + import_check = agent.environment.execute( + "python - <<'PY'\n" + "import importlib.util\n" + "import numpy as np\n" + "from agent.TradingAgent import TradingAgent\n" + f"spec = importlib.util.spec_from_file_location('submission_agent', {self.submission!r})\n" + "module = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(module)\n" + "assert hasattr(module, 'MyAgent'), 'MyAgent class not found'\n" + "assert issubclass(module.MyAgent, TradingAgent), 'MyAgent must inherit from an ABIDES TradingAgent class'\n" + "module.MyAgent(\n" + " id=1,\n" + " name='validation',\n" + " type='ValidationAgent',\n" + " symbol='JPM',\n" + " starting_cash=10000000,\n" + " log_orders=False,\n" + " random_state=np.random.RandomState(seed=1),\n" + ")\n" + "PY" + ) + if import_check["returncode"] != 0: + return ( + False, + f"Could not import and instantiate `MyAgent` from `{self.submission}`:\n{import_check['output']}", + ) + + return True, None + + def execute_round(self, agents: list[Player]) -> None: + agent_args = [] + for agent in agents: + agent_args.extend(["--agent", f"{agent.name}=/{agent.name}/{self.submission}"]) + + cmd = [ + "python", + "run_abides.py", + "--sims", + str(self.game_config.get("sims_per_round", self.default_args["sims_per_round"])), + "--market-minutes", + str(self._game_arg("market_minutes")), + "--background-agents", + str(self._game_arg("background_agents")), + "--output", + str(self.log_env / RESULTS_JSON), + *agent_args, + ] + full_cmd = " ".join(shlex.quote(part) for part in cmd) + self.logger.info(f"Running game: {full_cmd}") + try: + response = self.environment.execute(full_cmd, timeout=int(self._game_arg("timeout"))) + except subprocess.TimeoutExpired as exc: + raise RuntimeError("ABIDES round timed out") from exc + assert_zero_exit_code(response, logger=self.logger) + + def get_results(self, agents: list[Player], round_num: int, stats: RoundStats): + result_file = self.log_round(round_num) / RESULTS_JSON + if not result_file.exists(): + self.logger.error(f"Missing result file: {result_file}") + stats.winner = RESULT_TIE + for agent in agents: + stats.scores[agent.name] = 0.0 + stats.player_stats[agent.name].score = 0.0 + return + + with open(result_file) as f: + result = json.load(f) + + scores = {agent.name: 0.0 for agent in agents} + for player, score in result.get("average_scores", {}).items(): + if player in scores: + scores[player] = float(score) + missing_players = sorted(set(scores) - set(result.get("average_scores", {}))) + for player in missing_players: + scores[player] = CRASH_SCORE + stats.details.append( + json.dumps( + { + "player": player, + "score": CRASH_SCORE, + "status": "error", + "error": "missing ABIDES score", + }, + sort_keys=True, + ) + ) + + stats.scores = scores + stats.details.extend(result.get("details", [])) + for player, score in scores.items(): + stats.player_stats[player].score = score + + if not scores: + stats.winner = RESULT_TIE + return + + top_score = max(scores.values()) + winners = [player for player, score in scores.items() if score == top_score] + stats.winner = winners[0] if len(winners) == 1 else RESULT_TIE diff --git a/codeclash/arenas/abides/constraints.txt b/codeclash/arenas/abides/constraints.txt new file mode 100644 index 0000000..61b1d91 --- /dev/null +++ b/codeclash/arenas/abides/constraints.txt @@ -0,0 +1,21 @@ +contourpy==1.3.3 +cycler==0.12.1 +fonttools==4.62.1 +joblib==1.5.3 +jsons==1.6.3 +kiwisolver==1.5.0 +matplotlib==3.10.9 +numpy==2.4.4 +packaging==26.2 +pandas==3.0.2 +pillow==12.2.0 +pprofile==2.2.0 +psutil==7.2.2 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +pytz==2026.2 +scipy==1.17.1 +seaborn==0.13.2 +six==1.17.0 +tqdm==4.67.3 +typish==1.9.3 diff --git a/codeclash/arenas/abides/runtime/.gitignore b/codeclash/arenas/abides/runtime/.gitignore new file mode 100644 index 0000000..ae978d4 --- /dev/null +++ b/codeclash/arenas/abides/runtime/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +log/ +logs/ diff --git a/codeclash/arenas/abides/runtime/README.md b/codeclash/arenas/abides/runtime/README.md new file mode 100644 index 0000000..71c1067 --- /dev/null +++ b/codeclash/arenas/abides/runtime/README.md @@ -0,0 +1,14 @@ +# ABIDES CodeClash Workspace + +Edit `abides_agent.py`. + +Your file must define `MyAgent`, an ABIDES trading-agent class. A safe starting point is: + +```python +from agent.ValueAgent import ValueAgent as MyAgent +``` + +The arena runs compact ABIDES market simulations and scores agents by average mark-to-market profit +across identical seeded market worlds. +Some upstream ABIDES agents keep default behavior behind exact-class checks. If you subclass one of +those agents, override the relevant hooks instead of relying on an empty subclass. diff --git a/codeclash/arenas/abides/runtime/abides_agent.py b/codeclash/arenas/abides/runtime/abides_agent.py new file mode 100644 index 0000000..c829e31 --- /dev/null +++ b/codeclash/arenas/abides/runtime/abides_agent.py @@ -0,0 +1,5 @@ +from agent.ValueAgent import ValueAgent + +MyAgent = ValueAgent + +__all__ = ["MyAgent"] diff --git a/codeclash/arenas/abides/runtime/run_abides.py b/codeclash/arenas/abides/runtime/run_abides.py new file mode 100644 index 0000000..e535447 --- /dev/null +++ b/codeclash/arenas/abides/runtime/run_abides.py @@ -0,0 +1,380 @@ +import argparse +import contextlib +import importlib.util +import json +import os +import random +import re +import tempfile +import traceback +from pathlib import Path + +import numpy as np +import pandas as pd +from agent.ExchangeAgent import ExchangeAgent +from agent.market_makers.MarketMakerAgent import MarketMakerAgent +from agent.TradingAgent import TradingAgent +from agent.ZeroIntelligenceAgent import ZeroIntelligenceAgent +from Kernel import Kernel +from util import util +from util.oracle.SparseMeanRevertingOracle import SparseMeanRevertingOracle +from util.order import LimitOrder + +CRASH_SCORE = -1_000_000.0 +SYMBOL = "JPM" +STARTING_CASH = 10_000_000 +PLAYER_LAMBDA_A = 7e-11 +GUARDED_METHOD_DEFAULTS = { + "kernelInitializing": None, + "kernelStarting": None, + "kernelStopping": None, + "wakeup": False, + "receiveMessage": None, +} + + +def safe_module_name(player_name: str, sim_idx: int | None = None) -> str: + safe = re.sub(r"\W+", "_", player_name) + if not safe or safe[0].isdigit(): + safe = f"player_{safe}" + suffix = "" if sim_idx is None else f"_sim_{sim_idx}" + return f"codeclash_abides_{safe.lower()}{suffix}" + + +def load_agent_class(player_name: str, path: str, *, sim_idx: int | None = None): + spec = importlib.util.spec_from_file_location(safe_module_name(player_name, sim_idx), path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load module spec from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not hasattr(module, "MyAgent"): + raise RuntimeError(f"{path} does not define MyAgent") + agent_class = module.MyAgent + if not issubclass(agent_class, TradingAgent): + raise RuntimeError(f"{path} MyAgent must inherit from ABIDES TradingAgent") + return agent_class + + +def make_random_state() -> np.random.RandomState: + return np.random.RandomState(seed=np.random.randint(low=0, high=2**32, dtype="uint64")) + + +def make_player_agent(agent_class: type, player_name: str, agent_id: int): + agent = agent_class( + id=agent_id, + name=player_name, + type=f"PLAYER:{player_name}", + symbol=SYMBOL, + starting_cash=STARTING_CASH, + log_orders=False, + random_state=make_random_state(), + ) + if hasattr(agent, "lambda_a"): + agent.lambda_a = min(float(agent.lambda_a), PLAYER_LAMBDA_A) + return guard_player_agent(agent) + + +def guard_player_agent(agent: TradingAgent) -> TradingAgent: + agent._codeclash_error = None + agent._codeclash_traceback = None + + for method_name, fallback in GUARDED_METHOD_DEFAULTS.items(): + original = getattr(agent, method_name, None) + if original is None: + continue + + def guarded(*args, _original=original, _method_name=method_name, _fallback=fallback, **kwargs): + if getattr(agent, "_codeclash_error", None): + return _fallback + try: + return _original(*args, **kwargs) + except Exception as exc: + agent._codeclash_error = f"{type(exc).__name__} in {_method_name}: {exc}" + agent._codeclash_traceback = traceback.format_exc(limit=5) + return _fallback + + setattr(agent, method_name, guarded) + + return agent + + +def make_world_agents(agent_classes: dict[str, type], *, sim_idx: int, market_minutes: int, background_agents: int): + historical_date = pd.to_datetime("2019-06-28") + mkt_open = historical_date + pd.to_timedelta("09:30:00") + mkt_close = mkt_open + pd.to_timedelta(market_minutes, unit="m") + symbols = { + SYMBOL: { + "r_bar": 100000, + "kappa": 1.67e-12, + "agent_kappa": 1.67e-15, + "sigma_s": 0, + "fund_vol": 1e-8, + "megashock_lambda_a": 2.77778e-13, + "megashock_mean": 1e3, + "megashock_var": 5e4, + "random_state": make_random_state(), + } + } + oracle = SparseMeanRevertingOracle(mkt_open, mkt_close, symbols) + + agents = [] + agent_id = 0 + exchange = ExchangeAgent( + id=agent_id, + name="EXCHANGE_AGENT", + type="ExchangeAgent", + mkt_open=mkt_open, + mkt_close=mkt_close, + symbols=[SYMBOL], + log_orders=False, + book_freq=None, + pipeline_delay=0, + computation_delay=0, + stream_history=10, + random_state=make_random_state(), + ) + agents.append(exchange) + agent_id += 1 + + agents.append( + MarketMakerAgent( + id=agent_id, + name="MARKET_MAKER_AGENT", + type="MarketMakerAgent", + symbol=SYMBOL, + starting_cash=STARTING_CASH, + min_size=20, + max_size=50, + wake_up_freq="30s", + log_orders=False, + random_state=make_random_state(), + ) + ) + agent_id += 1 + + symbol_config = symbols[SYMBOL] + for idx in range(background_agents): + agents.append( + ZeroIntelligenceAgent( + id=agent_id, + name=f"ZI_AGENT_{idx}", + type="ZeroIntelligenceAgent", + symbol=SYMBOL, + starting_cash=STARTING_CASH, + sigma_n=10000, + r_bar=symbol_config["r_bar"], + kappa=symbol_config["agent_kappa"], + sigma_s=symbol_config["fund_vol"], + q_max=10, + sigma_pv=5e4, + R_min=0, + R_max=100, + eta=1, + lambda_a=1e-10, + log_orders=False, + random_state=make_random_state(), + ) + ) + agent_id += 1 + + player_agents = {} + player_names = list(agent_classes.keys()) + if player_names: + offset = sim_idx % len(player_names) + ordered_names = player_names[offset:] + player_names[:offset] + for player_name in ordered_names: + agent = make_player_agent(agent_classes[player_name], player_name, agent_id) + agents.append(agent) + player_agents[player_name] = agent + agent_id += 1 + + for agent in agents: + agent.log_to_file = False + + return { + "agents": agents, + "player_agents": player_agents, + "exchange": exchange, + "historical_date": historical_date, + "mkt_close": mkt_close, + "oracle": oracle, + } + + +def score_player(agent: TradingAgent, final_price: int) -> tuple[float, dict]: + if getattr(agent, "_codeclash_error", None): + return CRASH_SCORE, { + "status": "error", + "error": agent._codeclash_error, + "traceback": agent._codeclash_traceback, + } + + try: + cash = int(agent.holdings.get("CASH", 0)) + shares = int(agent.holdings.get(SYMBOL, 0)) + score = float(cash + shares * final_price - STARTING_CASH) + except Exception as exc: + return CRASH_SCORE, { + "status": "error", + "error": f"{type(exc).__name__} while scoring: {exc}", + "traceback": traceback.format_exc(limit=5), + } + + return score, {"status": "ok", "cash": cash, "shares": shares} + + +def run_player_market( + player: str, + agent_class: type, + *, + sim_idx: int, + market_minutes: int, + background_agents: int, +) -> dict: + seed = 9200 + sim_idx + random.seed(seed) + np.random.seed(seed) + util.silent_mode = True + LimitOrder.silent_mode = True + + world = make_world_agents( + {player: agent_class}, + sim_idx=sim_idx, + market_minutes=market_minutes, + background_agents=background_agents, + ) + kernel = Kernel("CodeClash ABIDES Kernel", random_state=make_random_state()) + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.devnull, "w") as devnull, contextlib.chdir(tmpdir), contextlib.redirect_stdout(devnull): + kernel.runner( + agents=world["agents"], + startTime=world["historical_date"], + stopTime=world["mkt_close"] + pd.to_timedelta("00:01:00"), + defaultComputationDelay=50, + defaultLatency=1, + oracle=world["oracle"], + skip_log=True, + log_dir=None, + ) + + final_price = int(world["exchange"].order_books[SYMBOL].last_trade) + agent = world["player_agents"][player] + score, score_detail = score_player(agent, final_price) + return { + "score": score, + "detail": { + "sim": sim_idx, + "player": player, + "score": score, + "final_price": final_price, + **score_detail, + }, + } + + +def run_market(agent_paths: dict[str, str], *, sim_idx: int, market_minutes: int, background_agents: int) -> dict: + scores = {} + details = [] + for player, path in agent_paths.items(): + try: + agent_class = load_agent_class(player, path, sim_idx=sim_idx) + result = run_player_market( + player, + agent_class, + sim_idx=sim_idx, + market_minutes=market_minutes, + background_agents=background_agents, + ) + scores[player] = result["score"] + details.append(result["detail"]) + except Exception as exc: + scores[player] = CRASH_SCORE + details.append( + { + "sim": sim_idx, + "player": player, + "score": CRASH_SCORE, + "status": "error", + "error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc(limit=5), + } + ) + + return {"scores": scores, "details": details} + + +def parse_agent_arg(value: str) -> tuple[str, str]: + if "=" not in value: + raise argparse.ArgumentTypeError("--agent values must be NAME=/path/to/abides_agent.py") + name, path = value.split("=", 1) + if not name: + raise argparse.ArgumentTypeError("agent name cannot be empty") + if not Path(path).exists(): + raise argparse.ArgumentTypeError(f"agent path does not exist: {path}") + return name, path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--agent", action="append", type=parse_agent_arg, required=True) + parser.add_argument("--sims", type=int, default=3) + parser.add_argument("--market-minutes", type=int, default=5) + parser.add_argument("--background-agents", type=int, default=3) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + if args.sims < 1: + parser.error("--sims must be at least 1") + if args.market_minutes < 1: + parser.error("--market-minutes must be at least 1") + if args.background_agents < 0: + parser.error("--background-agents cannot be negative") + + agent_names = [name for name, _ in args.agent] + if len(agent_names) != len(set(agent_names)): + parser.error("--agent names must be unique") + + agent_paths = dict(args.agent) + totals = {name: 0.0 for name in agent_paths} + details = [] + + for sim_idx in range(args.sims): + try: + result = run_market( + agent_paths, + sim_idx=sim_idx, + market_minutes=args.market_minutes, + background_agents=args.background_agents, + ) + except Exception as exc: + result = { + "scores": {name: CRASH_SCORE for name in agent_paths}, + "details": [ + { + "sim": sim_idx, + "player": name, + "score": CRASH_SCORE, + "status": "error", + "error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc(limit=5), + } + for name in agent_paths + ], + } + for player, score in result["scores"].items(): + totals[player] += score + details.extend(result["details"]) + + averages = {player: score / args.sims for player, score in totals.items()} + output = { + "average_scores": averages, + "total_scores": totals, + "sims": args.sims, + "details": [json.dumps(item, sort_keys=True) for item in details], + } + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(json.dumps(output, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/configs/examples/ABIDES__dummy__r1__s2.yaml b/configs/examples/ABIDES__dummy__r1__s2.yaml new file mode 100644 index 0000000..e401238 --- /dev/null +++ b/configs/examples/ABIDES__dummy__r1__s2.yaml @@ -0,0 +1,30 @@ +tournament: + rounds: 1 +game: + name: ABIDES + sims_per_round: 2 + args: + market_minutes: 5 + background_agents: 3 + timeout: 240 +players: +- agent: dummy + name: alpha +- agent: dummy + name: beta +prompts: + game_description: |- + You are a software developer ({{player_id}}) competing in CodeClash's ABIDES arena. + + The game is played in {{total_rounds}} rounds. For every round, you and your competitors edit + code that controls a trading agent in a simulated limit-order-book market. This is round {{round}}. + + Your task: improve `abides_agent.py`, located in {{working_dir}}. + All commands run from {{working_dir}}. + + Your file must define `MyAgent`, an ABIDES TradingAgent subclass. A valid starting point is: + + from agent.ValueAgent import ValueAgent as MyAgent + + The arena runs compact ABIDES market simulations. Your objective is to maximize average + mark-to-market profit across identical seeded market worlds. diff --git a/docs/reference/arenas/abides.md b/docs/reference/arenas/abides.md new file mode 100644 index 0000000..91a2f2f --- /dev/null +++ b/docs/reference/arenas/abides.md @@ -0,0 +1,111 @@ +# ABIDES + +Financial-market simulation arena based on the ABIDES agent-based interactive discrete event +simulation environment. + +## Overview + +ABIDES simulates trading agents interacting through a discrete-event market simulator and a +limit-order-book exchange. The CodeClash arena uses compact generated-market simulations so agents +can compete on trading strategy without requiring proprietary market data. + +Each CodeClash player edits an ABIDES trading agent. A round evaluates every player in identical +seeded market worlds and scores each player by average mark-to-market profit. + +## Resources + +- [ABIDES GitHub Repository](https://github.com/abides-sim/abides) +- [ABIDES Paper](https://arxiv.org/abs/1904.12066) +- [ABIDES Wiki](https://github.com/abides-sim/abides/wiki) + +## Implementation + +::: codeclash.arenas.abides.abides.ABIDESArena + options: + show_root_heading: true + heading_level: 2 + +## Agent Interface + +Your bot must be a Python file named `abides_agent.py` that defines `MyAgent`. + +`MyAgent` must be an ABIDES `TradingAgent` subclass and accept the standard ABIDES constructor +arguments used by the arena. A valid starting point is: + +```python +from agent.ValueAgent import ValueAgent as MyAgent +``` + +Agents can use the ABIDES APIs exposed by the upstream `abides-sim/abides` repository. The package +is installed in the ABIDES arena Docker image, not in CodeClash's core Python environment. + +Some upstream ABIDES agents, including `ValueAgent`, keep default behavior behind exact-class +checks. If you subclass one of those agents, override the relevant `wakeup` and `receiveMessage` +hooks instead of relying on `pass`. + +## Configuration Example + +```yaml +tournament: + rounds: 1 +game: + name: ABIDES + sims_per_round: 2 + args: + market_minutes: 5 + background_agents: 3 + timeout: 240 +players: + - agent: dummy + name: alpha + - agent: dummy + name: beta +``` + +## Scoring + +The arena runs `sims_per_round` independent ABIDES market seeds. For each seed, every submitted +CodeClash trading agent is evaluated in its own matching ABIDES market world with an exchange, a +market maker, and background zero-intelligence traders. The final CodeClash score is the player's +average mark-to-market profit across simulations. + +## Smoke Test + +From the repository root, run the dummy-player example: + +```bash +uv run python main.py configs/examples/ABIDES__dummy__r1__s2.yaml -o /tmp/codeclash-abides-smoke +``` + +Use a fresh `-o` directory when rerunning the smoke check. + +Expected shape: + +- the command exits with status 0; +- both players pass submission validation; +- stdout includes `In round 0, the winner is ...` and `In round 1, the winner is ...`; +- each round summary contains floating-point mark-to-market scores for `alpha` and `beta`; +- per-simulation details have `status: "ok"`, `cash`, and `shares` fields; +- the output directory contains `metadata.json`, `game.log`, `tournament.log`, and + `rounds/round_0.tar.gz` / `rounds/round_1.tar.gz`. + +A representative `metadata.json` round contains a `scores` object with one floating-point profit +score per player: + +```json +"scores": { + "alpha": -1736.0, + "beta": -2297.0 +} +``` + +Exact values can change with simulator randomness and configuration; the smoke check is meant to +verify the Docker/runtime adapter path, player-name mapping, and score/log artifact shape. + +The exact tournament directory name includes a timestamp, so inspect the metadata with: + +```bash +find /tmp/codeclash-abides-smoke -maxdepth 3 -name metadata.json -print +``` + +--8<-- "docs/_footer.md" diff --git a/mkdocs.yml b/mkdocs.yml index 1024a13..36808e5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,6 +70,7 @@ nav: - "reference/index.md" - Arenas: - "CodeGame (Abstract)": "reference/arenas/game.md" + - "ABIDES": "reference/arenas/abides.md" - "BattleCode": "reference/arenas/battlecode.md" - "BattleSnake": "reference/arenas/battlesnake.md" - "Bridge": "reference/arenas/bridge.md" diff --git a/tests/arenas/test_abides.py b/tests/arenas/test_abides.py new file mode 100644 index 0000000..6406674 --- /dev/null +++ b/tests/arenas/test_abides.py @@ -0,0 +1,175 @@ +import json +from pathlib import Path + +from codeclash.arenas.abides.abides import CRASH_SCORE, ABIDESArena +from codeclash.arenas.arena import RoundStats +from codeclash.constants import RESULT_TIE + +from .conftest import MockEnvironment, MockPlayer + + +class TestABIDESValidation: + def test_valid_agent(self, mock_player_factory): + arena = ABIDESArena.__new__(ABIDESArena) + arena.submission = "abides_agent.py" + player = mock_player_factory( + name="Alice", + files={"abides_agent.py": "from agent.ValueAgent import ValueAgent as MyAgent\n"}, + command_outputs={ + "test -f abides_agent.py && echo exists": {"output": "exists\n", "returncode": 0}, + "cat abides_agent.py": { + "output": "from agent.ValueAgent import ValueAgent as MyAgent\n", + "returncode": 0, + }, + "python -m py_compile abides_agent.py": {"output": "", "returncode": 0}, + "python - <<'PY'": {"output": "", "returncode": 0}, + }, + ) + + valid, error = arena.validate_code(player) + + assert valid is True + assert error is None + import_command = next(cmd for cmd in player.environment._executed_commands if cmd.startswith("python - <<'PY'")) + assert import_command.index("from agent.TradingAgent import TradingAgent") < import_command.index( + "spec.loader.exec_module(module)" + ) + + def test_missing_myagent(self, mock_player_factory): + arena = ABIDESArena.__new__(ABIDESArena) + arena.submission = "abides_agent.py" + player = mock_player_factory( + name="Alice", + files={"abides_agent.py": "class OtherAgent:\n pass\n"}, + command_outputs={ + "test -f abides_agent.py && echo exists": {"output": "exists\n", "returncode": 0}, + "cat abides_agent.py": {"output": "class OtherAgent:\n pass\n", "returncode": 0}, + "python -m py_compile abides_agent.py": {"output": "", "returncode": 0}, + "python - <<'PY'": {"output": "MyAgent class not found", "returncode": 1}, + }, + ) + + valid, error = arena.validate_code(player) + + assert valid is False + assert "Could not import and instantiate" in error + + def test_import_failure(self, mock_player_factory): + arena = ABIDESArena.__new__(ABIDESArena) + arena.submission = "abides_agent.py" + player = mock_player_factory( + name="Alice", + files={"abides_agent.py": "class MyAgent:\n pass\n"}, + command_outputs={ + "test -f abides_agent.py && echo exists": {"output": "exists\n", "returncode": 0}, + "cat abides_agent.py": {"output": "class MyAgent:\n pass\n", "returncode": 0}, + "python -m py_compile abides_agent.py": {"output": "", "returncode": 0}, + "python - <<'PY'": {"output": "ImportError", "returncode": 1}, + }, + ) + + valid, error = arena.validate_code(player) + + assert valid is False + assert "Could not import and instantiate" in error + + +class TestABIDESResults: + def test_parse_winner(self, tmp_log_dir): + arena = ABIDESArena.__new__(ABIDESArena) + arena.log_local = tmp_log_dir + arena.logger = type("Logger", (), {"error": lambda self, msg: None})() + round_dir = tmp_log_dir / "rounds" / "1" + round_dir.mkdir(parents=True) + (round_dir / "abides_results.json").write_text( + json.dumps( + { + "average_scores": {"Alice": 125.0, "Bob": 75.0}, + "details": ['{"sim": 0, "player": "Alice", "score": 125.0}'], + } + ) + ) + + agents = [MockPlayer("Alice"), MockPlayer("Bob")] + stats = RoundStats(round_num=1, agents=agents) + + arena.get_results(agents, 1, stats) + + assert stats.winner == "Alice" + assert stats.scores == {"Alice": 125.0, "Bob": 75.0} + assert stats.player_stats["Alice"].score == 125.0 + assert stats.details == ['{"sim": 0, "player": "Alice", "score": 125.0}'] + + def test_parse_tie(self, tmp_log_dir): + arena = ABIDESArena.__new__(ABIDESArena) + arena.log_local = tmp_log_dir + arena.logger = type("Logger", (), {"error": lambda self, msg: None})() + round_dir = tmp_log_dir / "rounds" / "1" + round_dir.mkdir(parents=True) + (round_dir / "abides_results.json").write_text(json.dumps({"average_scores": {"Alice": 1, "Bob": 1}})) + + agents = [MockPlayer("Alice"), MockPlayer("Bob")] + stats = RoundStats(round_num=1, agents=agents) + + arena.get_results(agents, 1, stats) + + assert stats.winner == RESULT_TIE + assert stats.scores == {"Alice": 1.0, "Bob": 1.0} + + def test_missing_score_is_penalized(self, tmp_log_dir): + arena = ABIDESArena.__new__(ABIDESArena) + arena.log_local = tmp_log_dir + arena.logger = type("Logger", (), {"error": lambda self, msg: None})() + round_dir = tmp_log_dir / "rounds" / "1" + round_dir.mkdir(parents=True) + (round_dir / "abides_results.json").write_text(json.dumps({"average_scores": {"Bob": -5.0}})) + + agents = [MockPlayer("Alice"), MockPlayer("Bob")] + stats = RoundStats(round_num=1, agents=agents) + + arena.get_results(agents, 1, stats) + + assert stats.winner == "Bob" + assert stats.scores == {"Alice": CRASH_SCORE, "Bob": -5.0} + assert "missing ABIDES score" in stats.details[0] + + +class TestABIDESExecution: + def test_execute_round_uses_nested_game_args(self): + arena = ABIDESArena.__new__(ABIDESArena) + arena.submission = "abides_agent.py" + arena.config = { + "game": { + "sims_per_round": 5, + "args": { + "market_minutes": 11, + "background_agents": 13, + "timeout": 17, + }, + } + } + arena.log_env = Path("/logs") + arena.logger = type("Logger", (), {"info": lambda self, msg: None, "error": lambda self, msg: None})() + + class CapturingEnvironment(MockEnvironment): + def __init__(self): + super().__init__() + self.timeout = None + + def execute(self, cmd, cwd=None, timeout=None): + self._executed_commands.append(cmd) + self.timeout = timeout + return {"output": "", "returncode": 0} + + arena.environment = CapturingEnvironment() + + arena.execute_round([MockPlayer("Alice"), MockPlayer("Bob")]) + + cmd = arena.environment._executed_commands[0] + assert "--sims 5" in cmd + assert "--market-minutes 11" in cmd + assert "--background-agents 13" in cmd + assert "--output /logs/abides_results.json" in cmd + assert "--agent Alice=/Alice/abides_agent.py" in cmd + assert "--agent Bob=/Bob/abides_agent.py" in cmd + assert arena.environment.timeout == 17